From 7a565a69df6977eed77d90cf2bc4b588b486f304 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 31 Mar 2016 16:44:37 -0700 Subject: [PATCH] scoutfs: add percpu coutners with sysfs files Add percpu counters that will let us track all manner of things. To report them we add a sysfs directory full of attribute files in a sysfs dir for each mount: # (cd /sys/fs/scoutfs/loop0/counters && grep . *) skip_delete:0 skip_insert:3218 skip_lookup:8439 skip_next:1190 skip_search:156 The implementation is careful to define each counter in only one place. We don't have to make sure that a bunch of defintions and arrays are in sync. This builds off of Ben's initial patches that added sysfs dirs. Signed-off-by: Zach Brown Signed-off-by: Ben McClelland --- kmod/src/Makefile | 4 +- kmod/src/counters.c | 131 ++++++++++++++++++++++++++++++++++++++++++++ kmod/src/counters.h | 47 ++++++++++++++++ kmod/src/skip.c | 9 +++ kmod/src/super.c | 51 ++++++++++++----- kmod/src/super.h | 6 ++ 6 files changed, 233 insertions(+), 15 deletions(-) create mode 100644 kmod/src/counters.c create mode 100644 kmod/src/counters.h diff --git a/kmod/src/Makefile b/kmod/src/Makefile index 31093000..da32ee19 100644 --- a/kmod/src/Makefile +++ b/kmod/src/Makefile @@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include -scoutfs-y += block.o bloom.o chunk.o crc.o dir.o filerw.o inode.o manifest.o \ - msg.o ring.o scoutfs_trace.o segment.o skip.o super.o +scoutfs-y += block.o bloom.o counters.o chunk.o crc.o dir.o filerw.o inode.o \ + manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o super.o diff --git a/kmod/src/counters.c b/kmod/src/counters.c new file mode 100644 index 00000000..fe8b247c --- /dev/null +++ b/kmod/src/counters.c @@ -0,0 +1,131 @@ +/* + * Copyright (C) 2016 Versity Software, Inc. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include +#include +#include +#include + +#include "super.h" +#include "counters.h" + +/* + * Maintain simple percpu counters which are always ticking. sysfs + * makes this a whole lot more noisy than it needs to be. + */ + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) { .name = __stringify(which), .mode = 0644 }, +static struct attribute scoutfs_counter_attrs[] = { + EXPAND_EACH_COUNTER +}; + +/* zero BSS and + 1 makes this null terminated */ +#define NR_ATTRS ARRAY_SIZE(scoutfs_counter_attrs) +static struct attribute *scoutfs_counter_attr_ptrs[NR_ATTRS + 1]; + +static ssize_t scoutfs_counter_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + size_t index; + + /* use the index in the _attrs array to discover the pcpu pointer */ + counters = container_of(kobj, struct scoutfs_counters, kobj); + index = attr - scoutfs_counter_attrs; + pcpu = &counters->FIRST_COUNTER + index; + + return snprintf(buf, PAGE_SIZE, "%lld\n", percpu_counter_sum(pcpu)); +} + +static void scoutfs_counters_kobj_release(struct kobject *kobj) +{ + struct scoutfs_counters *counters; + + counters = container_of(kobj, struct scoutfs_counters, kobj); + + complete(&counters->comp); +} + +static const struct sysfs_ops scoutfs_counter_attr_ops = { + .show = scoutfs_counter_attr_show, +}; + +static struct kobj_type scoutfs_counters_ktype = { + .default_attrs = scoutfs_counter_attr_ptrs, + .sysfs_ops = &scoutfs_counter_attr_ops, + .release = scoutfs_counters_kobj_release, +}; + +int scoutfs_setup_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters; + struct percpu_counter *pcpu; + int ret; + + counters = kzalloc(sizeof(struct scoutfs_counters), GFP_KERNEL); + if (!counters) + return -ENOMEM; + sbi->counters = counters; + + scoutfs_foreach_counter(sb, pcpu) { + ret = percpu_counter_init(pcpu, 0, GFP_KERNEL); + if (ret) + return ret; + } + + counters->kobj.kset = sbi->kset; + init_completion(&counters->comp); + ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype, + NULL, "counters"); + if (ret) { + /* tear down partial to avoid destroying null kobjs */ + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + kfree(counters); + sbi->counters = NULL; + } + + return ret; +} + +void scoutfs_destroy_counters(struct super_block *sb) +{ + struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb); + struct scoutfs_counters *counters = sbi->counters; + struct percpu_counter *pcpu; + + /* this only destroys fully initialized counters */ + if (!counters) + return; + + kobject_del(&counters->kobj); + kobject_put(&counters->kobj); + wait_for_completion(&counters->comp); + + scoutfs_foreach_counter(sb, pcpu) + percpu_counter_destroy(pcpu); + + kfree(counters); + sbi->counters = NULL; +} + +void __init scoutfs_init_counters(void) +{ + int i; + + /* not ARRAY_SIZE because that would clobber null term */ + for (i = 0; i < NR_ATTRS; i++) + scoutfs_counter_attr_ptrs[i] = &scoutfs_counter_attrs[i]; +} diff --git a/kmod/src/counters.h b/kmod/src/counters.h new file mode 100644 index 00000000..1f6c1843 --- /dev/null +++ b/kmod/src/counters.h @@ -0,0 +1,47 @@ +#ifndef _SCOUTFS_COUNTERS_H_ +#define _SCOUTFS_COUNTERS_H_ + +#include +#include +#include + +#include "super.h" + +/* + * We only have to define each counter here and it'll be enumerated in + * other places by this macro. Don't forget to update LAST_COUNTER. + */ +#define EXPAND_EACH_COUNTER \ + EXPAND_COUNTER(skip_lookup) \ + EXPAND_COUNTER(skip_insert) \ + EXPAND_COUNTER(skip_search) \ + EXPAND_COUNTER(skip_delete) \ + EXPAND_COUNTER(skip_next) \ + +#define FIRST_COUNTER skip_lookup +#define LAST_COUNTER skip_next + +#undef EXPAND_COUNTER +#define EXPAND_COUNTER(which) struct percpu_counter which; + +struct scoutfs_counters { + /* $sysfs/fs/scoutfs/$id/counters/ */ + struct kobject kobj; + struct completion comp; + + EXPAND_EACH_COUNTER +}; + +#define scoutfs_foreach_counter(sb, pcpu) \ + for (pcpu = &SCOUTFS_SB(sb)->counters->FIRST_COUNTER; \ + pcpu <= &SCOUTFS_SB(sb)->counters->LAST_COUNTER; \ + pcpu++) + +#define scoutfs_inc_counter(sb, which) \ + percpu_counter_inc(&SCOUTFS_SB(sb)->counters->which) + +void __init scoutfs_init_counters(void); +int scoutfs_setup_counters(struct super_block *sb); +void scoutfs_destroy_counters(struct super_block *sb); + +#endif diff --git a/kmod/src/skip.c b/kmod/src/skip.c index c855ae49..d320e2a7 100644 --- a/kmod/src/skip.c +++ b/kmod/src/skip.c @@ -19,6 +19,7 @@ #include "key.h" #include "block.h" #include "skip.h" +#include "counters.h" /* * The items in a log segment block are sorted by their keys in a skip @@ -216,6 +217,8 @@ int scoutfs_skip_insert(struct super_block *sb, u64 blkno, WARN_ON_ONCE(item->skip_height > SCOUTFS_SKIP_HEIGHT)) return -EINVAL; + scoutfs_inc_counter(sb, skip_insert); + ret = skip_search(sb, blkno, &path, &item->key, &cmp); if (ret == 0) { if (cmp == 0) { @@ -261,6 +264,7 @@ int scoutfs_skip_lookup(struct super_block *sb, u64 blkno, struct scoutfs_key *key, struct buffer_head **bh, struct scoutfs_item **item) { + scoutfs_inc_counter(sb, skip_lookup); return skip_lookup(sb, blkno, key, bh, item, true); } @@ -271,6 +275,7 @@ int scoutfs_skip_search(struct super_block *sb, u64 blkno, struct scoutfs_key *key, struct buffer_head **bh, struct scoutfs_item **item) { + scoutfs_inc_counter(sb, skip_search); return skip_lookup(sb, blkno, key, bh, item, false); } @@ -284,6 +289,8 @@ int scoutfs_skip_delete(struct super_block *sb, u64 blkno, int ret; int i; + scoutfs_inc_counter(sb, skip_delete); + ret = skip_search(sb, blkno, &path, key, &cmp); if (ret == 0) { if (*path.next[0] && cmp) { @@ -316,6 +323,8 @@ int scoutfs_skip_next(struct super_block *sb, u64 blkno, if (!(*bh)) return -ENOENT; + scoutfs_inc_counter(sb, skip_next); + next = (*item)->skip_next[0]; brelse(*bh); diff --git a/kmod/src/super.c b/kmod/src/super.c index f0c2d6c6..95159ca3 100644 --- a/kmod/src/super.c +++ b/kmod/src/super.c @@ -27,8 +27,11 @@ #include "manifest.h" #include "ring.h" #include "segment.h" +#include "counters.h" #include "scoutfs_trace.h" +static struct kset *scoutfs_kset; + static const struct super_operations scoutfs_super_ops = { .alloc_inode = scoutfs_alloc_inode, .destroy_inode = scoutfs_destroy_inode, @@ -178,15 +181,15 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent) return -EINVAL; } - ret = read_supers(sb); - if (ret) - return ret; + /* XXX can have multiple mounts of a device, need mount id */ + sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj); + if (!sbi->kset) + return -ENOMEM; - ret = scoutfs_setup_manifest(sb); - if (ret) - return ret; - - ret = scoutfs_replay_ring(sb); + ret = scoutfs_setup_counters(sb) ?: + read_supers(sb) ?: + scoutfs_setup_manifest(sb) ?: + scoutfs_replay_ring(sb); if (ret) return ret; @@ -218,6 +221,9 @@ static void scoutfs_kill_sb(struct super_block *sb) /* kill block super should have synced */ WARN_ON_ONCE(sbi->dirty_blkno); scoutfs_destroy_manifest(sb); + scoutfs_destroy_counters(sb); + if (sbi->kset) + kset_unregister(sbi->kset); kfree(sbi); } } @@ -230,19 +236,38 @@ static struct file_system_type scoutfs_fs_type = { .fs_flags = FS_REQUIRES_DEV, }; +/* safe to call at any failure point in _init */ +static void teardown_module(void) +{ + scoutfs_dir_exit(); + scoutfs_inode_exit(); + if (scoutfs_kset) + kset_unregister(scoutfs_kset); +} + static int __init scoutfs_module_init(void) { - return scoutfs_inode_init() ?: - scoutfs_dir_init() ?: - register_filesystem(&scoutfs_fs_type); + int ret; + + scoutfs_init_counters(); + + scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj); + if (!scoutfs_kset) + return -ENOMEM; + + ret = scoutfs_inode_init() ?: + scoutfs_dir_init() ?: + register_filesystem(&scoutfs_fs_type); + if (ret) + teardown_module(); + return ret; } module_init(scoutfs_module_init) static void __exit scoutfs_module_exit(void) { unregister_filesystem(&scoutfs_fs_type); - scoutfs_dir_exit(); - scoutfs_inode_exit(); + teardown_module(); } module_exit(scoutfs_module_exit) diff --git a/kmod/src/super.h b/kmod/src/super.h index 1663034d..0857faf0 100644 --- a/kmod/src/super.h +++ b/kmod/src/super.h @@ -5,6 +5,7 @@ #include "format.h" struct scoutfs_manifest; +struct scoutfs_counters; struct scoutfs_sb_info { struct scoutfs_super_block super; @@ -31,6 +32,11 @@ struct scoutfs_sb_info { u64 dirty_blkno; int dirty_item_off; int dirty_val_off; + + /* $sysfs/fs/scoutfs/$id/ */ + struct kset *kset; + + struct scoutfs_counters *counters; }; static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb)