scoutfs: add percpu coutners with sysfs files

Add percpu counters that will let us track all manner of things.

To report them we add a sysfs directory full of attribute files in a
sysfs dir for each mount:

    # (cd /sys/fs/scoutfs/loop0/counters && grep . *)
    skip_delete:0
    skip_insert:3218
    skip_lookup:8439
    skip_next:1190
    skip_search:156

The implementation is careful to define each counter in only one place.
We don't have to make sure that a bunch of defintions and arrays are in
sync.

This builds off of Ben's initial patches that added sysfs dirs.

Signed-off-by: Zach Brown <zab@versity.com>
Signed-off-by: Ben McClelland <ben.mcclelland@versity.com>

kmod/src/Makefile

@@ -2,5 +2,5 @@ obj-$(CONFIG_SCOUTFS_FS) := scoutfs.o
 
 CFLAGS_scoutfs_trace.o = -I$(src) # define_trace.h double include
  
-scoutfs-y += block.o bloom.o chunk.o crc.o dir.o filerw.o inode.o manifest.o \
-	     msg.o ring.o scoutfs_trace.o segment.o skip.o super.o
+scoutfs-y += block.o bloom.o counters.o chunk.o crc.o dir.o filerw.o inode.o \
+	     manifest.o msg.o ring.o scoutfs_trace.o segment.o skip.o super.o

kmod/src/counters.c

@@ -0,0 +1,131 @@
+/*
+ * Copyright (C) 2016 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/fs.h>
+#include <linux/percpu_counter.h>
+
+#include "super.h"
+#include "counters.h"
+
+/*
+ * Maintain simple percpu counters which are always ticking.  sysfs
+ * makes this a whole lot more noisy than it needs to be.
+ */
+
+#undef EXPAND_COUNTER
+#define EXPAND_COUNTER(which) { .name = __stringify(which), .mode = 0644 },
+static struct attribute scoutfs_counter_attrs[] = {
+	EXPAND_EACH_COUNTER
+};
+
+/* zero BSS and + 1 makes this null terminated */
+#define NR_ATTRS ARRAY_SIZE(scoutfs_counter_attrs)
+static struct attribute *scoutfs_counter_attr_ptrs[NR_ATTRS + 1];
+
+static ssize_t scoutfs_counter_attr_show(struct kobject *kobj,
+				         struct attribute *attr, char *buf)
+{
+	struct scoutfs_counters *counters;
+	struct percpu_counter *pcpu;
+	size_t index;
+
+	/* use the index in the _attrs array to discover the pcpu pointer */
+	counters = container_of(kobj, struct scoutfs_counters, kobj);
+	index = attr - scoutfs_counter_attrs;
+	pcpu = &counters->FIRST_COUNTER + index;
+
+	return snprintf(buf, PAGE_SIZE, "%lld\n", percpu_counter_sum(pcpu));
+}
+
+static void scoutfs_counters_kobj_release(struct kobject *kobj)
+{
+	struct scoutfs_counters *counters;
+
+	counters = container_of(kobj, struct scoutfs_counters, kobj);
+
+	complete(&counters->comp);
+}
+
+static const struct sysfs_ops scoutfs_counter_attr_ops = {
+	.show   = scoutfs_counter_attr_show,
+};
+
+static struct kobj_type scoutfs_counters_ktype = {
+	.default_attrs  = scoutfs_counter_attr_ptrs,
+	.sysfs_ops      = &scoutfs_counter_attr_ops,
+	.release        = scoutfs_counters_kobj_release,
+};
+
+int scoutfs_setup_counters(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_counters *counters;
+	struct percpu_counter *pcpu;
+	int ret;
+
+	counters = kzalloc(sizeof(struct scoutfs_counters), GFP_KERNEL);
+	if (!counters)
+		return -ENOMEM;
+	sbi->counters = counters;
+
+	scoutfs_foreach_counter(sb, pcpu) {
+		ret = percpu_counter_init(pcpu, 0, GFP_KERNEL);
+		if (ret)
+			return ret;
+	}
+
+	counters->kobj.kset = sbi->kset;
+	init_completion(&counters->comp);
+	ret = kobject_init_and_add(&counters->kobj, &scoutfs_counters_ktype,
+				    NULL, "counters");
+	if (ret) {
+		/* tear down partial to avoid destroying null kobjs */
+		scoutfs_foreach_counter(sb, pcpu)
+			percpu_counter_destroy(pcpu);
+		kfree(counters);
+		sbi->counters = NULL;
+	}
+
+	return ret;
+}
+
+void scoutfs_destroy_counters(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct scoutfs_counters *counters = sbi->counters;
+	struct percpu_counter *pcpu;
+
+	/* this only destroys fully initialized counters */
+	if (!counters)
+		return;
+
+	kobject_del(&counters->kobj);
+	kobject_put(&counters->kobj);
+	wait_for_completion(&counters->comp);
+
+	scoutfs_foreach_counter(sb, pcpu)
+		percpu_counter_destroy(pcpu);
+
+	kfree(counters);
+	sbi->counters = NULL;
+}
+
+void __init scoutfs_init_counters(void)
+{
+	int i;
+
+	/* not ARRAY_SIZE because that would clobber null term */
+	for (i = 0; i < NR_ATTRS; i++)
+		scoutfs_counter_attr_ptrs[i] = &scoutfs_counter_attrs[i];
+}

kmod/src/counters.h

@@ -0,0 +1,47 @@
+#ifndef _SCOUTFS_COUNTERS_H_
+#define _SCOUTFS_COUNTERS_H_
+
+#include <linux/kobject.h>
+#include <linux/completion.h>
+#include <linux/percpu_counter.h>
+
+#include "super.h"
+
+/*
+ * We only have to define each counter here and it'll be enumerated in
+ * other places by this macro.  Don't forget to update LAST_COUNTER.
+ */
+#define EXPAND_EACH_COUNTER 		\
+	EXPAND_COUNTER(skip_lookup)	\
+	EXPAND_COUNTER(skip_insert)	\
+	EXPAND_COUNTER(skip_search)	\
+	EXPAND_COUNTER(skip_delete)	\
+	EXPAND_COUNTER(skip_next)	\
+
+#define FIRST_COUNTER skip_lookup
+#define LAST_COUNTER skip_next
+
+#undef EXPAND_COUNTER
+#define EXPAND_COUNTER(which) struct percpu_counter which;
+
+struct scoutfs_counters {
+	/* $sysfs/fs/scoutfs/$id/counters/ */
+	struct kobject kobj;
+	struct completion comp;
+
+	EXPAND_EACH_COUNTER
+};
+
+#define scoutfs_foreach_counter(sb, pcpu) 			\
+	for (pcpu = &SCOUTFS_SB(sb)->counters->FIRST_COUNTER;	\
+	     pcpu <= &SCOUTFS_SB(sb)->counters->LAST_COUNTER;	\
+	     pcpu++)
+
+#define scoutfs_inc_counter(sb, which) \
+	percpu_counter_inc(&SCOUTFS_SB(sb)->counters->which)
+
+void __init scoutfs_init_counters(void);
+int scoutfs_setup_counters(struct super_block *sb);
+void scoutfs_destroy_counters(struct super_block *sb);
+
+#endif

kmod/src/skip.c

@@ -19,6 +19,7 @@
 #include "key.h"
 #include "block.h"
 #include "skip.h"
+#include "counters.h"
 
 /*
  * The items in a log segment block are sorted by their keys in a skip
@@ -216,6 +217,8 @@ int scoutfs_skip_insert(struct super_block *sb, u64 blkno,
 	    WARN_ON_ONCE(item->skip_height > SCOUTFS_SKIP_HEIGHT))
 		return -EINVAL;
 
+	scoutfs_inc_counter(sb, skip_insert);
+
 	ret = skip_search(sb, blkno, &path, &item->key, &cmp);
 	if (ret == 0) {
 		if (cmp == 0) {
@@ -261,6 +264,7 @@ int scoutfs_skip_lookup(struct super_block *sb, u64 blkno,
 			struct scoutfs_key *key, struct buffer_head **bh,
 			struct scoutfs_item **item)
 {
+	scoutfs_inc_counter(sb, skip_lookup);
 	return skip_lookup(sb, blkno, key, bh, item, true);
 }
 
@@ -271,6 +275,7 @@ int scoutfs_skip_search(struct super_block *sb, u64 blkno,
 			struct scoutfs_key *key, struct buffer_head **bh,
 			struct scoutfs_item **item)
 {
+	scoutfs_inc_counter(sb, skip_search);
 	return skip_lookup(sb, blkno, key, bh, item, false);
 }
 
@@ -284,6 +289,8 @@ int scoutfs_skip_delete(struct super_block *sb, u64 blkno,
 	int ret;
 	int i;
 
+	scoutfs_inc_counter(sb, skip_delete);
+
 	ret = skip_search(sb, blkno, &path, key, &cmp);
 	if (ret == 0) {
 		if (*path.next[0] && cmp) {
@@ -316,6 +323,8 @@ int scoutfs_skip_next(struct super_block *sb, u64 blkno,
 	if (!(*bh))
 		return -ENOENT;
 
+	scoutfs_inc_counter(sb, skip_next);
+
 	next = (*item)->skip_next[0];
 	brelse(*bh);
 

kmod/src/super.c

@@ -27,8 +27,11 @@
 #include "manifest.h"
 #include "ring.h"
 #include "segment.h"
+#include "counters.h"
 #include "scoutfs_trace.h"
 
+static struct kset *scoutfs_kset;
+
 static const struct super_operations scoutfs_super_ops = {
 	.alloc_inode = scoutfs_alloc_inode,
 	.destroy_inode = scoutfs_destroy_inode,
@@ -178,15 +181,15 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 		return -EINVAL;
 	}
 
-	ret = read_supers(sb);
-	if (ret)
-		return ret;
+	/* XXX can have multiple mounts of a  device, need mount id */
+	sbi->kset = kset_create_and_add(sb->s_id, NULL, &scoutfs_kset->kobj);
+	if (!sbi->kset)
+		return -ENOMEM;
 
-	ret = scoutfs_setup_manifest(sb);
-	if (ret)
-		return ret;
-
-	ret = scoutfs_replay_ring(sb);
+	ret = scoutfs_setup_counters(sb) ?:
+	      read_supers(sb) ?:
+	      scoutfs_setup_manifest(sb) ?:
+	      scoutfs_replay_ring(sb);
 	if (ret)
 		return ret;
 
@@ -218,6 +221,9 @@ static void scoutfs_kill_sb(struct super_block *sb)
 		/* kill block super should have synced */
 		WARN_ON_ONCE(sbi->dirty_blkno);
 		scoutfs_destroy_manifest(sb);
+		scoutfs_destroy_counters(sb);
+		if (sbi->kset)
+			kset_unregister(sbi->kset);
 		kfree(sbi);
 	}
 }
@@ -230,19 +236,38 @@ static struct file_system_type scoutfs_fs_type = {
 	.fs_flags	= FS_REQUIRES_DEV,
 };
 
+/* safe to call at any failure point in _init */
+static void teardown_module(void)
+{
+	scoutfs_dir_exit();
+	scoutfs_inode_exit();
+	if (scoutfs_kset)
+		kset_unregister(scoutfs_kset);
+}
+
 static int __init scoutfs_module_init(void)
 {
-	return scoutfs_inode_init() ?:
+	int ret;
+
+	scoutfs_init_counters();
+
+	scoutfs_kset = kset_create_and_add("scoutfs", NULL, fs_kobj);
+	if (!scoutfs_kset)
+		return -ENOMEM;
+
+	ret = scoutfs_inode_init() ?:
 	      scoutfs_dir_init() ?:
 	      register_filesystem(&scoutfs_fs_type);
+	if (ret)
+		teardown_module();
+	return ret;
 }
 module_init(scoutfs_module_init)
 
 static void __exit scoutfs_module_exit(void)
 {
 	unregister_filesystem(&scoutfs_fs_type);
-	scoutfs_dir_exit();
-	scoutfs_inode_exit();
+	teardown_module();
 }
 module_exit(scoutfs_module_exit)
 

kmod/src/super.h

@@ -5,6 +5,7 @@
 #include "format.h"
 
 struct scoutfs_manifest;
+struct scoutfs_counters;
 
 struct scoutfs_sb_info {
 	struct scoutfs_super_block super;
@@ -31,6 +32,11 @@ struct scoutfs_sb_info {
 	u64 dirty_blkno;
 	int dirty_item_off;
 	int dirty_val_off;
+
+	/* $sysfs/fs/scoutfs/$id/ */
+	struct kset *kset;
+
+	struct scoutfs_counters *counters;
 };
 
 static inline struct scoutfs_sb_info *SCOUTFS_SB(struct super_block *sb)