Add server_data_fill_target option

Add the server_data_fill_target mount option.  This replaces the
previous static target size of the data allocator that the server fills
for client transactions.  We calculate the previously static server and
client low water marks based on the option.

This first step doesn't change the default value while making it tunable
only to minimize risk.

Signed-off-by: Zach Brown <zab@versity.com>

kmod/src/alloc.h

@@ -35,10 +35,6 @@
 #define SCOUTFS_ALLOC_DATA_LG_THRESH \
 	(8ULL * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
 
-/* the client will force commits if data allocators get too low */
-#define SCOUTFS_ALLOC_DATA_REFILL_THRESH \
-	((256ULL * 1024 * 1024) >> SCOUTFS_BLOCK_SM_SHIFT)
-
 /*
  * Fill client alloc roots to the target when they fall below the lo
  * threshold.
@@ -51,11 +47,28 @@
 	SCOUTFS_ALLOC_LIST_MAX_BLOCKS
 #define SCOUTFS_SERVER_META_FILL_LO \
 	(SCOUTFS_ALLOC_LIST_MAX_BLOCKS / 2)
-#define SCOUTFS_SERVER_DATA_FILL_TARGET \
-	(4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
-#define SCOUTFS_SERVER_DATA_FILL_LO \
-	(1ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
 
+/*
+ * server_data_fill_target: During transaction commit processing, the
+ * server will try to fill the client transaction's data allocator with
+ * this many blocks when it decides to fill.
+ *
+ * server_data_fill_low: _target / 4.  The server will try to fill the
+ * client transaction's data allocator to _target if the current size
+ * falls under _low.
+ *
+ * client_data_fill_low:  _target / 16.  A client writing transaction
+ * holder will trigger a commit if the client transaction's data
+ * allocator falls under _low.  It's attempting to avoid returning ENOSPC
+ * as the allocator drains while there was still global free space at
+ * the server.
+ */
+#define SCOUTFS_SERVER_DATA_FILL_TARGET_DEFAULT \
+	(4ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
+#define SCOUTFS_SERVER_DATA_FILL_TARGET_MIN \
+	(1ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
+#define SCOUTFS_SERVER_DATA_FILL_TARGET_MAX \
+	(256ULL * 1024 * 1024 * 1024 >> SCOUTFS_BLOCK_SM_SHIFT)
 
 /*
  * Log merge meta allocations are only used for one request and will

kmod/src/options.c

@@ -31,6 +31,7 @@
 
 enum {
 	Opt_acl,
+	Opt_server_data_fill_target,
 	Opt_data_prealloc_blocks,
 	Opt_data_prealloc_contig_only,
 	Opt_ino_alloc_per_lock,
@@ -47,6 +48,7 @@ enum {
 
 static const match_table_t tokens = {
 	{Opt_acl, "acl"},
+	{Opt_server_data_fill_target, "server_data_fill_target=%s"},
 	{Opt_data_prealloc_blocks, "data_prealloc_blocks=%s"},
 	{Opt_data_prealloc_contig_only, "data_prealloc_contig_only=%s"},
 	{Opt_ino_alloc_per_lock, "ino_alloc_per_lock=%s"},
@@ -142,6 +144,7 @@ static void init_default_options(struct scoutfs_mount_options *opts)
 {
 	memset(opts, 0, sizeof(*opts));
 
+	opts->server_data_fill_target = SCOUTFS_SERVER_DATA_FILL_TARGET_DEFAULT;
 	opts->data_prealloc_blocks = SCOUTFS_DATA_PREALLOC_DEFAULT_BLOCKS;
 	opts->data_prealloc_contig_only = 1;
 	opts->ino_alloc_per_lock = SCOUTFS_LOCK_INODE_GROUP_NR;
@@ -213,6 +216,19 @@ static int verify_tcp_keepalive_timeout_ms(struct super_block *sb, int ret, int
 	return 0;
 }
 
+static bool invalid_server_data_fill_target(struct super_block *sb, bool force, u64 val)
+{
+	if (force || val < SCOUTFS_SERVER_DATA_FILL_TARGET_MIN ||
+	    val > SCOUTFS_SERVER_DATA_FILL_TARGET_MAX) {
+		scoutfs_err(sb, "invalid server_data_fill_target option, must be between %llu and %llu",
+			    SCOUTFS_SERVER_DATA_FILL_TARGET_MIN,
+			    SCOUTFS_SERVER_DATA_FILL_TARGET_MAX);
+		return true;
+	}
+
+	return false;
+}
+
 /*
  * Parse the option string into our options struct.   This can allocate
  * memory in the struct.  The caller is responsible for always calling
@@ -239,6 +255,13 @@ static int parse_options(struct super_block *sb, char *options, struct scoutfs_m
 			sb->s_flags |= SB_POSIXACL;
 			break;
 
+		case Opt_server_data_fill_target:
+			ret = match_u64(args, &nr64);
+			if (invalid_server_data_fill_target(sb, ret < 0, nr64))
+				return -EINVAL;
+			opts->server_data_fill_target = nr64;
+			break;
+
 		case Opt_data_prealloc_blocks:
 			ret = match_u64(args, &nr64);
 			if (ret < 0 ||
@@ -436,6 +459,7 @@ int scoutfs_options_show(struct seq_file *seq, struct dentry *root)
 
 	if (is_acl)
 		seq_puts(seq, ",acl");
+	seq_printf(seq, ",server_data_fill_target=%llu", opts.server_data_fill_target);
 	seq_printf(seq, ",data_prealloc_blocks=%llu", opts.data_prealloc_blocks);
 	seq_printf(seq, ",data_prealloc_contig_only=%u", opts.data_prealloc_contig_only);
 	seq_printf(seq, ",ino_alloc_per_lock=%u", opts.ino_alloc_per_lock);
@@ -740,6 +764,42 @@ static ssize_t quorum_slot_nr_show(struct kobject *kobj, struct kobj_attribute *
 }
 SCOUTFS_ATTR_RO(quorum_slot_nr);
 
+static ssize_t server_data_fill_target_show(struct kobject *kobj, struct kobj_attribute *attr,
+					    char *buf)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
+	return snprintf(buf, PAGE_SIZE, "%llu", opts.server_data_fill_target);
+}
+static ssize_t server_data_fill_target_store(struct kobject *kobj, struct kobj_attribute *attr,
+					     const char *buf, size_t count)
+{
+	struct super_block *sb = SCOUTFS_SYSFS_ATTRS_SB(kobj);
+	DECLARE_OPTIONS_INFO(sb, optinf);
+	char nullterm[30]; /* more than enough for octal -U64_MAX */
+	u64 val;
+	int len;
+	int ret;
+
+	len = min(count, sizeof(nullterm) - 1);
+	memcpy(nullterm, buf, len);
+	nullterm[len] = '\0';
+
+	ret = kstrtoll(nullterm, 0, &val);
+	if (invalid_server_data_fill_target(sb, ret < 0, val))
+		return -EINVAL;
+
+	write_seqlock(&optinf->seqlock);
+	optinf->opts.server_data_fill_target = val;
+	write_sequnlock(&optinf->seqlock);
+
+	return count;
+}
+SCOUTFS_ATTR_RW(server_data_fill_target);
+
 static struct attribute *options_attrs[] = {
 	SCOUTFS_ATTR_PTR(data_prealloc_blocks),
 	SCOUTFS_ATTR_PTR(data_prealloc_contig_only),
@@ -750,6 +810,7 @@ static struct attribute *options_attrs[] = {
 	SCOUTFS_ATTR_PTR(orphan_scan_delay_ms),
 	SCOUTFS_ATTR_PTR(quorum_heartbeat_timeout_ms),
 	SCOUTFS_ATTR_PTR(quorum_slot_nr),
+	SCOUTFS_ATTR_PTR(server_data_fill_target),
 	NULL,
 };
 

kmod/src/options.h

@@ -15,6 +15,7 @@ struct scoutfs_mount_options {
 	unsigned int orphan_scan_delay_ms;
 	int quorum_slot_nr;
 	u64 quorum_heartbeat_timeout_ms;
+	u64 server_data_fill_target;
 	int tcp_keepalive_timeout_ms;
 };
 
@@ -28,4 +29,14 @@ int scoutfs_options_setup(struct super_block *sb);
 void scoutfs_options_stop(struct super_block *sb);
 void scoutfs_options_destroy(struct super_block *sb);
 
+static inline u64 scoutfs_opt_server_data_fill_low(struct scoutfs_mount_options *opts)
+{
+	return opts->server_data_fill_target / 4;
+}
+
+static inline u64 scoutfs_opt_client_data_fill_low(struct scoutfs_mount_options *opts)
+{
+	return opts->server_data_fill_target / 16;
+}
+
 #endif	/* _SCOUTFS_OPTIONS_H_ */

kmod/src/server.c

@@ -1635,6 +1635,7 @@ static int server_get_log_trees(struct super_block *sb,
 	DECLARE_SERVER_INFO(sb, server);
 	__le64 exclusive[SCOUTFS_DATA_ALLOC_ZONE_LE64S];
 	__le64 vacant[SCOUTFS_DATA_ALLOC_ZONE_LE64S];
+	struct scoutfs_mount_options opts;
 	struct alloc_extent_cb_args cba;
 	struct scoutfs_log_trees lt;
 	struct scoutfs_key key;
@@ -1646,6 +1647,8 @@ static int server_get_log_trees(struct super_block *sb,
 	int ret;
 	int err;
 
+	scoutfs_options_read(sb, &opts);
+
 	if (arg_len != 0) {
 		ret = -EINVAL;
 		goto out;
@@ -1752,14 +1755,15 @@ static int server_get_log_trees(struct super_block *sb,
 		lt.meta_avail.flags &= ~cpu_to_le32(SCOUTFS_ALLOC_FLAG_LOW);
 
 	ret = alloc_move_refill_zoned(sb, &lt.data_avail, &super->data_alloc,
-				      SCOUTFS_SERVER_DATA_FILL_LO, SCOUTFS_SERVER_DATA_FILL_TARGET,
+				      scoutfs_opt_server_data_fill_low(&opts),
+				      opts.server_data_fill_target,
 				      exclusive, vacant, data_zone_blocks);
 	if (ret < 0) {
 		err_str = "refilling data_avail";
 		goto update;
 	}
 
-	if (le64_to_cpu(lt.data_avail.total_len) < SCOUTFS_SERVER_DATA_FILL_LO)
+	if (le64_to_cpu(lt.data_avail.total_len) < scoutfs_opt_server_data_fill_low(&opts))
 		lt.data_avail.flags |= cpu_to_le32(SCOUTFS_ALLOC_FLAG_LOW);
 	else
 		lt.data_avail.flags &= ~cpu_to_le32(SCOUTFS_ALLOC_FLAG_LOW);
@@ -3363,11 +3367,14 @@ static int server_set_volopt(struct super_block *sb, struct scoutfs_net_connecti
 	DECLARE_SERVER_INFO(sb, server);
 	struct scoutfs_super_block *super = DIRTY_SUPER_SB(sb);
 	struct scoutfs_volume_options *volopt;
+	struct scoutfs_mount_options opts;
 	COMMIT_HOLD(hold);
 	u64 opt;
 	u64 nr;
 	int ret = 0;
 
+	scoutfs_options_read(sb, &opts);
+
 	if (arg_len != sizeof(struct scoutfs_volume_options)) {
 		ret = -EINVAL;
 		goto out;
@@ -3385,9 +3392,9 @@ static int server_set_volopt(struct super_block *sb, struct scoutfs_net_connecti
 
 	if (le64_to_cpu(volopt->set_bits) & SCOUTFS_VOLOPT_DATA_ALLOC_ZONE_BLOCKS_BIT) {
 		opt = le64_to_cpu(volopt->data_alloc_zone_blocks);
-		if (opt < SCOUTFS_SERVER_DATA_FILL_TARGET) {
+		if (opt < opts.server_data_fill_target) {
 			scoutfs_err(sb, "setting data_alloc_zone_blocks to '%llu' failed, must be at least %llu mount data allocation target blocks",
-				    opt, SCOUTFS_SERVER_DATA_FILL_TARGET);
+				    opt, opts.server_data_fill_target);
 			ret = -EINVAL;
 			goto apply;
 		}

kmod/src/trans.c

@@ -443,6 +443,10 @@ static void release_holders(struct super_block *sb)
  */
 static bool commit_before_hold(struct super_block *sb, struct trans_info *tri)
 {
+	struct scoutfs_mount_options opts;
+
+	scoutfs_options_read(sb, &opts);
+
 	/*
 	 * In theory each dirty item page could be straddling two full
 	 * blocks, requiring 4 allocations for each item cache page.
@@ -470,7 +474,7 @@ static bool commit_before_hold(struct super_block *sb, struct trans_info *tri)
 	}
 
 	/* if we're low and can't refill then alloc could empty and return enospc */
-	if (scoutfs_data_alloc_should_refill(sb, SCOUTFS_ALLOC_DATA_REFILL_THRESH)) {
+	if (scoutfs_data_alloc_should_refill(sb, scoutfs_opt_client_data_fill_low(&opts))) {
 		scoutfs_inc_counter(sb, trans_commit_data_alloc_low);
 		return true;
 	}

utils/man/scoutfs.5

@@ -155,6 +155,23 @@ The assigned number must match one of the slots defined with \-Q options
 when the filesystem was created with mkfs.  If the number assigned
 doesn't match a number created during mkfs then the mount will fail.
 .TP
+.B server_data_fill_target=<number>
+Sets the number of 4KiB blocks that the server will try to make
+available to client transactions.  The server only tries to refill the
+data allocators to this number when the current data allocator size
+falls under 1/4 of this number.  The client also will force commits to
+refill the data allocators when the current data allocator size falls
+under 1/16th of this number.
+.sp
+This must not be set less than 1GiB in KiB blocks (524288) or greater
+than 256GiB (67108864).
+.sp
+Setting this too large can store free space in client transactions that
+is unavailable to others, possibly causing premature ENOSPC errors.
+Setting this too low can cause exessive commits of client transactions
+that tree to refill the data allocators, causing low performance of
+heavy writes.
+.TP
 .B tcp_keepalive_timeout_ms=<number>
 This option sets the amount of time, in milliseconds, that a client
 connection will wait for active TCP packets, before deciding that