Add volume options super, server, and sysfs

Introduce global volume options.  They're stored in the superblock and
can be seen in sysfs files that use network commands to get and
set the options on the server.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/Makefile

@@ -42,6 +42,7 @@ scoutfs-y +=			\
 	trans.o			\
 	triggers.o		\
 	tseq.o			\
+	volopt.o		\
 	xattr.o
 
 #

kmod/src/client.c

@@ -249,6 +249,33 @@ int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr,
 					&args, sizeof(args), map, sizeof(*map));
 }
 
+/* The client is asking the server for the current volume options */
+int scoutfs_client_get_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+
+	return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_GET_VOLOPT,
+					NULL, 0, volopt, sizeof(*volopt));
+}
+
+/* The client is asking the server to update volume options */
+int scoutfs_client_set_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+
+	return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_SET_VOLOPT,
+					volopt, sizeof(*volopt), NULL, 0);
+}
+
+/* The client is asking the server to clear volume options */
+int scoutfs_client_clear_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt)
+{
+	struct client_info *client = SCOUTFS_SB(sb)->client_info;
+
+	return scoutfs_net_sync_request(sb, client->conn, SCOUTFS_NET_CMD_CLEAR_VOLOPT,
+					volopt, sizeof(*volopt), NULL, 0);
+}
+
 /* The client is receiving a invalidation request from the server */
 static int client_lock(struct super_block *sb,
 		       struct scoutfs_net_connection *conn, u8 cmd, u64 id,

kmod/src/client.h

@@ -26,6 +26,9 @@ int scoutfs_client_send_omap_response(struct super_block *sb, u64 id,
 				      struct scoutfs_open_ino_map *map);
 int scoutfs_client_open_ino_map(struct super_block *sb, u64 group_nr,
 				struct scoutfs_open_ino_map *map);
+int scoutfs_client_get_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt);
+int scoutfs_client_set_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt);
+int scoutfs_client_clear_volopt(struct super_block *sb, struct scoutfs_volume_options *volopt);
 
 int scoutfs_client_setup(struct super_block *sb);
 void scoutfs_client_destroy(struct super_block *sb);

kmod/src/format.h

@@ -626,6 +626,29 @@ struct scoutfs_quorum_block {
 
 #define SCOUTFS_QUORUM_BLOCK_LEADER (1 << 0)
 
+/*
+ * Tunable options that apply to the entire system.  They can be set in
+ * mkfs or in sysfs files which send an rpc to the server to make the
+ * change.  The super version defines the options that exist.
+ *
+ * @set_bits: bits for each 64bit starting offset after set_bits
+ * indicate which logical option is set.
+ */
+struct scoutfs_volume_options {
+	__le64 set_bits;
+	__le64 __future_expansion[64];
+};
+
+#define scoutfs_volopt_nr(field)							\
+	((offsetof(struct scoutfs_volume_options, field) -				\
+	  (offsetof(struct scoutfs_volume_options, set_bits) +				\
+	   member_sizeof(struct scoutfs_volume_options, set_bits))) / sizeof(__le64))
+#define scoutfs_volopt_bit(field)							\
+	(1ULL << scoutfs_volopt_nr(field))
+
+#define SCOUTFS_VOLOPT_EXPANSION_BITS \
+	(~(scoutfs_volopt_bit(__future_expansion) - 1))
+
 #define SCOUTFS_FLAG_IS_META_BDEV 0x01
 
 struct scoutfs_super_block {
@@ -652,6 +675,7 @@ struct scoutfs_super_block {
 	struct scoutfs_btree_root trans_seqs;
 	struct scoutfs_btree_root mounted_clients;
 	struct scoutfs_btree_root srch_root;
+	struct scoutfs_volume_options volopt;
 };
 
 #define SCOUTFS_ROOT_INO 1
@@ -841,6 +865,9 @@ enum scoutfs_net_cmd {
 	SCOUTFS_NET_CMD_SRCH_GET_COMPACT,
 	SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT,
 	SCOUTFS_NET_CMD_OPEN_INO_MAP,
+	SCOUTFS_NET_CMD_GET_VOLOPT,
+	SCOUTFS_NET_CMD_SET_VOLOPT,
+	SCOUTFS_NET_CMD_CLEAR_VOLOPT,
 	SCOUTFS_NET_CMD_FAREWELL,
 	SCOUTFS_NET_CMD_UNKNOWN,
 };

kmod/src/server.c

@@ -99,6 +99,11 @@ struct server_info {
 	seqcount_t roots_seqcount;
 	struct scoutfs_net_roots roots;
 
+	/* serializing and get and set volume options */
+	seqcount_t volopt_seqcount;
+	struct mutex volopt_mutex;
+	struct scoutfs_volume_options volopt;
+
 	/* recovery timeout fences from work */
 	struct work_struct fence_pending_recov_work;
 };
@@ -114,6 +119,38 @@ struct server_client_info {
 	struct list_head head;
 };
 
+static __le64 *first_valopt(struct scoutfs_volume_options *valopt)
+{
+	return &valopt->set_bits + 1;
+}
+
+/*
+ * A server caller wants to know if a volume option is set and wants to
+ * know it's value.  This is quite early in the file to make it
+ * available to all of the server paths.
+ */
+static bool get_volopt_val(struct server_info *server, int nr, u64 *val)
+{
+	u64 bit = 1ULL << nr;
+	__le64 *opt = first_valopt(&server->volopt) + nr;
+	bool is_set = false;
+	unsigned seq;
+
+	do {
+		seq = read_seqcount_begin(&server->volopt_seqcount);
+		if ((le64_to_cpu(server->volopt.set_bits) & bit)) {
+			is_set = true;
+			*val = le64_to_cpup(opt);
+		} else {
+			is_set = false;
+			*val = 0;
+		};
+	} while (read_seqcount_retry(&server->volopt_seqcount, seq));
+
+	return is_set;
+}
+
+
 struct commit_waiter {
 	struct completion comp;
 	struct llist_node node;
@@ -1075,6 +1112,125 @@ out:
 	return 0;
 }
 
+/* The server is receiving a request for the current volume options */
+static int server_get_volopt(struct super_block *sb, struct scoutfs_net_connection *conn,
+			     u8 cmd, u64 id, void *arg, u16 arg_len)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	struct scoutfs_volume_options volopt;
+	unsigned seq;
+	int ret = 0;
+
+	if (arg_len != 0) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	do {
+		seq = read_seqcount_begin(&server->volopt_seqcount);
+		volopt = server->volopt;
+	} while (read_seqcount_retry(&server->volopt_seqcount, seq));
+
+out:
+	return scoutfs_net_response(sb, conn, cmd, id, ret, &volopt, sizeof(volopt));
+}
+
+/*
+ * The server is receiving a request to update volume options.
+ *
+ * The in-memory options that readers use is updated only once the
+ * updated options are written in the super block.
+ */
+static int server_set_volopt(struct super_block *sb, struct scoutfs_net_connection *conn,
+			     u8 cmd, u64 id, void *arg, u16 arg_len)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
+	struct scoutfs_volume_options *volopt;
+	int ret = 0;
+
+	if (arg_len != sizeof(struct scoutfs_volume_options)) {
+		ret = -EINVAL;
+		goto out;
+	}
+	volopt = arg;
+
+	if (le64_to_cpu(volopt->set_bits) & SCOUTFS_VOLOPT_EXPANSION_BITS) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mutex_lock(&server->volopt_mutex);
+
+	ret = scoutfs_server_hold_commit(sb);
+	if (ret)
+		goto unlock;
+
+	ret = scoutfs_server_apply_commit(sb, ret);
+
+	write_seqcount_begin(&server->volopt_seqcount);
+	if (ret == 0)
+		server->volopt = super->volopt;
+	else
+		super->volopt = server->volopt;
+	write_seqcount_end(&server->volopt_seqcount);
+
+unlock:
+	mutex_unlock(&server->volopt_mutex);
+out:
+	return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0);
+}
+
+static int server_clear_volopt(struct super_block *sb, struct scoutfs_net_connection *conn,
+			       u8 cmd, u64 id, void *arg, u16 arg_len)
+{
+	DECLARE_SERVER_INFO(sb, server);
+	struct scoutfs_super_block *super = &SCOUTFS_SB(sb)->super;
+	struct scoutfs_volume_options *volopt;
+	__le64 *opt;
+	u64 bit;
+	int ret = 0;
+	int i;
+
+	if (arg_len != sizeof(struct scoutfs_volume_options)) {
+		ret = -EINVAL;
+		goto out;
+	}
+	volopt = arg;
+
+	if (le64_to_cpu(volopt->set_bits) & SCOUTFS_VOLOPT_EXPANSION_BITS) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	mutex_lock(&server->volopt_mutex);
+
+	ret = scoutfs_server_hold_commit(sb);
+	if (ret)
+		goto unlock;
+
+	for (i = 0, bit = 1, opt = first_valopt(&super->volopt); i < 64; i++, bit <<= 1, opt++) {
+		if (le64_to_cpu(volopt->set_bits) & bit) {
+			super->volopt.set_bits &= ~cpu_to_le64(bit);
+			*opt = 0;
+		}
+	}
+
+	ret = scoutfs_server_apply_commit(sb, ret);
+
+	write_seqcount_begin(&server->volopt_seqcount);
+	if (ret == 0)
+		server->volopt = super->volopt;
+	else
+		super->volopt = server->volopt;
+	write_seqcount_end(&server->volopt_seqcount);
+
+unlock:
+	mutex_unlock(&server->volopt_mutex);
+out:
+	return scoutfs_net_response(sb, conn, cmd, id, ret, NULL, 0);
+}
+
 static void init_mounted_client_key(struct scoutfs_key *key, u64 rid)
 {
 	*key = (struct scoutfs_key) {
@@ -1565,6 +1721,9 @@ static scoutfs_net_request_t server_req_funcs[] = {
 	[SCOUTFS_NET_CMD_SRCH_GET_COMPACT]	= server_srch_get_compact,
 	[SCOUTFS_NET_CMD_SRCH_COMMIT_COMPACT]	= server_srch_commit_compact,
 	[SCOUTFS_NET_CMD_OPEN_INO_MAP]		= server_open_ino_map,
+	[SCOUTFS_NET_CMD_GET_VOLOPT]		= server_get_volopt,
+	[SCOUTFS_NET_CMD_SET_VOLOPT]		= server_set_volopt,
+	[SCOUTFS_NET_CMD_CLEAR_VOLOPT]		= server_clear_volopt,
 	[SCOUTFS_NET_CMD_FAREWELL]		= server_farewell,
 };
 
@@ -1784,6 +1943,11 @@ static void scoutfs_server_worker(struct work_struct *work)
 	if (ret < 0)
 		goto shutdown;
 
+	/* update volume options early, possibly for use during startup */
+	write_seqcount_begin(&server->volopt_seqcount);
+	server->volopt = super->volopt;
+	write_seqcount_end(&server->volopt_seqcount);
+
 	set_roots(server, &super->fs_root, &super->logs_root,
 		  &super->srch_root);
 	scoutfs_block_writer_init(sb, &server->wri);
@@ -1932,6 +2096,8 @@ int scoutfs_server_setup(struct super_block *sb)
 	mutex_init(&server->srch_mutex);
 	mutex_init(&server->mounted_clients_mutex);
 	seqcount_init(&server->roots_seqcount);
+	seqcount_init(&server->volopt_seqcount);
+	mutex_init(&server->volopt_mutex);
 	INIT_WORK(&server->fence_pending_recov_work, fence_pending_recov_worker);
 
 	server->wq = alloc_workqueue("scoutfs_server",

kmod/src/super.c

@@ -46,6 +46,7 @@
 #include "alloc.h"
 #include "recov.h"
 #include "omap.h"
+#include "volopt.h"
 #include "scoutfs_trace.h"
 
 static struct dentry *scoutfs_debugfs_root;
@@ -253,6 +254,7 @@ static void scoutfs_put_super(struct super_block *sb)
 	scoutfs_lock_shutdown(sb);
 
 	scoutfs_shutdown_trans(sb);
+	scoutfs_volopt_destroy(sb);
 	scoutfs_client_destroy(sb);
 	scoutfs_inode_destroy(sb);
 	scoutfs_item_destroy(sb);
@@ -601,6 +603,7 @@ static int scoutfs_fill_super(struct super_block *sb, void *data, int silent)
 	      scoutfs_server_setup(sb) ?:
 	      scoutfs_quorum_setup(sb) ?:
 	      scoutfs_client_setup(sb) ?:
+	      scoutfs_volopt_setup(sb) ?:
 	      scoutfs_lock_rid(sb, SCOUTFS_LOCK_WRITE, 0, sbi->rid,
 				   &sbi->rid_lock) ?:
 	      scoutfs_trans_get_log_trees(sb) ?:

kmod/src/super.h

@@ -28,6 +28,7 @@ struct forest_info;
 struct srch_info;
 struct recov_info;
 struct omap_info;
+struct volopt_info;
 
 struct scoutfs_sb_info {
 	struct super_block *sb;
@@ -51,6 +52,7 @@ struct scoutfs_sb_info {
 	struct forest_info *forest_info;
 	struct srch_info *srch_info;
 	struct omap_info *omap_info;
+	struct volopt_info *volopt_info;
 	struct item_cache_info *item_cache_info;
 
 	wait_queue_head_t trans_hold_wq;

kmod/src/volopt.c

@@ -0,0 +1,187 @@
+/*
+ * Copyright (C) 2021 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/kobject.h>
+#include <linux/sysfs.h>
+
+#include "super.h"
+#include "client.h"
+#include "volopt.h"
+
+/*
+ * Volume options are exposed through a sysfs directory.  Getting and
+ * setting the values sends rpcs to the server who owns the options in
+ * the super block.
+ */
+
+struct volopt_info {
+	struct super_block *sb;
+	struct scoutfs_sysfs_attrs ssa;
+};
+
+#define DECLARE_VOLOPT_INFO(sb, name) \
+	struct volopt_info *name = SCOUTFS_SB(sb)->volopt_info
+#define DECLARE_VOLOPT_INFO_KOBJ(kobj, name) \
+	DECLARE_VOLOPT_INFO(SCOUTFS_SYSFS_ATTRS_SB(kobj), name)
+
+/*
+ * attribute arrays need to be dense but the options we export could
+ * well become sparse over time.  .store and .load are generic and we
+ * have a lookup table to map the attributes array indexes to the number
+ * and name of the option.
+ */
+static struct volopt_nr_name {
+	int nr;
+	char *name;
+} volopt_table[] = {
+};
+
+/* initialized by setup, pointer array is null terminated */
+static struct kobj_attribute volopt_attrs[ARRAY_SIZE(volopt_table)];
+static struct attribute *volopt_attr_ptrs[ARRAY_SIZE(volopt_table) + 1];
+
+static void get_opt_data(struct kobj_attribute *attr, struct scoutfs_volume_options *volopt,
+			 u64 *bit, __le64 **opt)
+{
+	size_t index = attr - &volopt_attrs[0];
+	int nr = volopt_table[index].nr;
+
+	*bit = 1ULL << nr;
+	*opt = &volopt->set_bits + 1 + nr;
+}
+
+static ssize_t volopt_attr_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	DECLARE_VOLOPT_INFO_KOBJ(kobj, vinf);
+	struct super_block *sb = vinf->sb;
+	struct scoutfs_volume_options volopt;
+	__le64 *opt;
+	u64 bit;
+	int ret;
+
+	ret = scoutfs_client_get_volopt(sb, &volopt);
+	if (ret < 0)
+		return ret;
+
+	get_opt_data(attr, &volopt, &bit, &opt);
+
+	if (le64_to_cpu(volopt.set_bits) & bit) {
+		return snprintf(buf, PAGE_SIZE, "%llu", le64_to_cpup(opt));
+	} else {
+		buf[0] = '\0';
+		return 0;
+	}
+}
+
+static ssize_t volopt_attr_store(struct kobject *kobj, struct kobj_attribute *attr,
+				 const char *buf, size_t count)
+{
+	DECLARE_VOLOPT_INFO_KOBJ(kobj, vinf);
+	struct super_block *sb = vinf->sb;
+	struct scoutfs_volume_options volopt = {0,};
+	u8 chars[32];
+	__le64 *opt;
+	u64 bit;
+	u64 val;
+	int ret;
+
+	if (count == 0)
+		return 0;
+	if (count > sizeof(chars) - 1)
+		return -ERANGE;
+
+	get_opt_data(attr, &volopt, &bit, &opt);
+
+	if (buf[0] == '\n' || buf[0] == '\r') {
+		volopt.set_bits = cpu_to_le64(bit);
+
+		ret = scoutfs_client_clear_volopt(sb, &volopt);
+	} else {
+		memcpy(chars, buf, count);
+		chars[count] = '\0';
+		ret = kstrtoull(chars, 0, &val);
+		if (ret < 0)
+			return ret;
+
+		volopt.set_bits = cpu_to_le64(bit);
+		*opt = cpu_to_le64(val);
+
+		ret = scoutfs_client_set_volopt(sb, &volopt);
+	}
+
+	if (ret == 0)
+		ret = count;
+	return ret;
+}
+
+/*
+ * The volume option sysfs files are slim shims around RPCs so this
+ * should be called after the client is setup and before it is torn
+ * down.
+ */
+int scoutfs_volopt_setup(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct volopt_info *vinf;
+	int ret;
+	int i;
+
+	/* persistent volume options are always a bitmap u64 then the 64 options */
+	BUILD_BUG_ON(sizeof(struct scoutfs_volume_options) != (1 + 64) * 8);
+
+	vinf = kzalloc(sizeof(struct volopt_info), GFP_KERNEL);
+	if (!vinf) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	scoutfs_sysfs_init_attrs(sb, &vinf->ssa);
+	vinf->sb = sb;
+	sbi->volopt_info = vinf;
+
+	for (i = 0; i < ARRAY_SIZE(volopt_table); i++) {
+		volopt_attrs[i] = (struct kobj_attribute) {
+			.attr = { .name = volopt_table[i].name, .mode = S_IWUSR | S_IRUGO },
+			.show = volopt_attr_show,
+			.store  = volopt_attr_store,
+		};
+		volopt_attr_ptrs[i] = &volopt_attrs[i].attr;
+	}
+
+	BUILD_BUG_ON(ARRAY_SIZE(volopt_table) != ARRAY_SIZE(volopt_attr_ptrs) - 1);
+	volopt_attr_ptrs[i] = NULL;
+
+	ret = scoutfs_sysfs_create_attrs(sb, &vinf->ssa, volopt_attr_ptrs, "volume_options");
+	if (ret < 0)
+		goto out;
+
+out:
+	if (ret)
+		scoutfs_volopt_destroy(sb);
+
+	return ret;
+}
+
+void scoutfs_volopt_destroy(struct super_block *sb)
+{
+	struct scoutfs_sb_info *sbi = SCOUTFS_SB(sb);
+	struct volopt_info *vinf = SCOUTFS_SB(sb)->volopt_info;
+
+	if (vinf) {
+		scoutfs_sysfs_destroy_attrs(sb, &vinf->ssa);
+		kfree(vinf);
+		sbi->volopt_info = NULL;
+	}
+}

kmod/src/volopt.h

@@ -0,0 +1,7 @@
+#ifndef _SCOUTFS_VOLOPT_H_
+#define _SCOUTFS_VOLOPT_H_
+
+int scoutfs_volopt_setup(struct super_block *sb);
+void scoutfs_volopt_destroy(struct super_block *sb);
+
+#endif

utils/man/scoutfs.5

@@ -34,6 +34,26 @@ the server for the filesystem if it is elected leader.
 The assigned number must match one of the slots defined with \-Q options
 when the filesystem was created with mkfs.  If the number assigned
 doesn't match a number created during mkfs then the mount will fail.
+.SH VOLUME OPTIONS
+Volume options are persistent options which are stored in the super
+block in the metadata device and which apply to all mounts of the volume.
+.sp
+Volume options may be initially specified as the volume is created
+as described in the mkfs command in
+.BR scoutfs (8).
+.sp
+Volume options may be changed at runtime by writing to files in sysfs
+while the volume is mounted.  Volume options are found in the
+volume_options/ directory with a file for each option.  Reading the
+file provides the current setting of the option and an empty string
+is returned if the option is not set.  To set the option, write
+the new value ofthe option to the file.  To clear the option, write
+a blank line with a newline to the file.  The write syscall will
+return an error if the set operation fails and a message will be written
+to the console.
+.sp
+The following volume options are supported:
+.TP
 .SH FURTHER READING
 A
 .B scoutfs

utils/src/print.c

@@ -900,6 +900,10 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 		le64_to_cpu(super->fs_root.ref.blkno),
 		le64_to_cpu(super->fs_root.ref.seq));
 
+	printf("  volume options:\n"
+	       "    set_bits: %016llx\n",
+		le64_to_cpu(super->volopt.set_bits));
+
 	printf("  quorum config version %llu\n",
 		le64_to_cpu(super->qconf.version));
 	for (i = 0; i < array_size(super->qconf.slots); i++) {