Show last holder PID, inode in client_locks

Add last_user_pid[mode] and last_user_ino[mode] arrays to scoutfs_lock,
filled in at the granted-mode path alongside the existing counts.
The inode is passed by callers for all per-inode cases, and set to 0
for others. PID is from the current task.

The client_locks line is expanded with "ino: rd I wr I wo I pid: rd P
wr P wo P".  Existing users:/waiters: field positions are unchanged.

A simple test case demonstrates the functionality for the two simple
inode/non-inode case, and for a contended lock case (multiple rd/wr
lock holders).

Signed-off-by: Auke Kok <auke.kok@versity.com>

ReleaseNotes.md

@@ -1,6 +1,21 @@
 Versity ScoutFS Release Notes
 =============================
 
+---
+v1.31
+\
+*May 5, 2026*
+
+Fix race between modifying quota rules and internal reading of the rules
+that tripped an assertion.
+
+Fix a bug that could skip merging totl items under specific heavy write
+loads.  This could lead to merged totl items incorrectly tracking the
+sum of all the contributing totl xattrs.
+
+Fix many small low risk bugs in error paths that were found with code
+analysis and testing.
+
 ---
 v1.30
 \

kmod/src/block.c

@@ -218,6 +218,7 @@ static void block_free_work(struct work_struct *work)
 
 	llist_for_each_entry_safe(bp, tmp, deleted, free_node) {
 		block_free(sb, bp);
+		cond_resched();
 	}
 }
 
@@ -467,9 +468,6 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 	sector_t sector;
 	int ret = 0;
 
-	if (scoutfs_forcing_unmount(sb))
-		return -ENOLINK;
-
 	sector = bp->bl.blkno << (SCOUTFS_BLOCK_LG_SHIFT - 9);
 
 	WARN_ON_ONCE(bp->bl.blkno == U64_MAX);
@@ -480,6 +478,17 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 	set_bit(BLOCK_BIT_IO_BUSY, &bp->bits);
 	block_get(bp);
 
+	/*
+	 * A second thread may already be waiting on this block's completion
+	 * after this thread won the race to submit the block.  We exit through
+	 * the block_end_io error path which sets BLOCK_BIT_ERROR and assures
+	 * that other callers in the waitq get woken up.
+	 */
+	if (scoutfs_forcing_unmount(sb)) {
+		ret = -ENOLINK;
+		goto end_io;
+	}
+
 	blk_start_plug(&plug);
 
 	for (off = 0; off < SCOUTFS_BLOCK_LG_SIZE; off += PAGE_SIZE) {
@@ -517,6 +526,7 @@ static int block_submit_bio(struct super_block *sb, struct block_private *bp,
 
 	blk_finish_plug(&plug);
 
+end_io:
 	/* let racing end_io know we're done */
 	block_end_io(sb, opf, bp, ret);
 

kmod/src/inode.c

@@ -549,6 +549,7 @@ retry:
 				goto out;
 			if (scoutfs_data_wait_found(&dw)) {
 				scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE);
+				lock = NULL;
 
 				/* XXX callee locks instead? */
 				inode_unlock(inode);

kmod/src/ioctl.c

@@ -1739,6 +1739,43 @@ out:
 	return ret;
 }
 
+static long scoutfs_ioc_inject_totl_delta(struct file *file, unsigned long arg)
+{
+	struct super_block *sb = file_inode(file)->i_sb;
+	struct scoutfs_ioctl_inject_totl_delta __user *uitd = (void __user *)arg;
+	struct scoutfs_ioctl_inject_totl_delta itd;
+	struct scoutfs_xattr_totl_val tval;
+	struct scoutfs_lock *lock = NULL;
+	struct scoutfs_key key;
+	int ret;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (copy_from_user(&itd, uitd, sizeof(itd)))
+		return -EFAULT;
+
+	scoutfs_xattr_init_totl_key(&key, itd.name);
+	tval.total = cpu_to_le64((u64)itd.total);
+	tval.count = cpu_to_le64((u64)itd.count);
+
+	ret = scoutfs_lock_xattr_totl(sb, SCOUTFS_LOCK_WRITE_ONLY, 0, &lock);
+	if (ret < 0)
+		goto out;
+
+	ret = scoutfs_hold_trans(sb, true);
+	if (ret < 0)
+		goto unlock;
+
+	ret = scoutfs_item_delta(sb, &key, &tval, sizeof(tval), lock);
+
+	scoutfs_release_trans(sb);
+unlock:
+	scoutfs_unlock(sb, lock, SCOUTFS_LOCK_WRITE_ONLY);
+out:
+	return ret;
+}
+
 long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 {
 	switch (cmd) {
@@ -1790,6 +1827,8 @@ long scoutfs_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 		return scoutfs_ioc_read_xattr_index(file, arg);
 	case SCOUTFS_IOC_PUNCH_OFFLINE:
 		return scoutfs_ioc_punch_offline(file, arg);
+	case SCOUTFS_IOC_INJECT_TOTL_DELTA:
+		return scoutfs_ioc_inject_totl_delta(file, arg);
 	}
 
 	return -ENOTTY;

kmod/src/ioctl.h

@@ -876,4 +876,17 @@ struct scoutfs_ioctl_punch_offline {
 #define SCOUTFS_IOC_PUNCH_OFFLINE \
 	_IOW(SCOUTFS_IOCTL_MAGIC, 24, struct scoutfs_ioctl_punch_offline)
 
+/*
+ * Inject a signed (total, count) delta at the totl key @name (a, b, c
+ * match the trailing dotted u64s of a totl xattr name).
+ */
+struct scoutfs_ioctl_inject_totl_delta {
+	__u64	name[SCOUTFS_IOCTL_XATTR_TOTAL_NAME_NR];
+	__s64	total;
+	__s64	count;
+};
+
+#define SCOUTFS_IOC_INJECT_TOTL_DELTA \
+	_IOW(SCOUTFS_IOCTL_MAGIC, 25, struct scoutfs_ioctl_inject_totl_delta)
+
 #endif

kmod/src/lock.c

@@ -813,6 +813,7 @@ int scoutfs_lock_invalidate_request(struct super_block *sb, u64 net_id,
 
 out:
 	if (!lock) {
+		kfree(ireq);
 		ret = scoutfs_client_lock_response(sb, net_id, nl);
 		BUG_ON(ret); /* lock server doesn't fence timed out client requests */
 	}
@@ -979,7 +980,7 @@ static bool lock_flags_invalid(int flags)
  */
 static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
 			  struct scoutfs_key *start, struct scoutfs_key *end,
-			  struct scoutfs_lock **ret_lock)
+			  u64 ino, struct scoutfs_lock **ret_lock)
 {
 	DECLARE_LOCK_INFO(sb, linfo);
 	struct scoutfs_lock *lock;
@@ -1027,6 +1028,8 @@ static int lock_key_range(struct super_block *sb, enum scoutfs_lock_mode mode, i
 		/* the fast path where we can use the granted mode */
 		if (lock_modes_match(lock->mode, mode)) {
 			lock_inc_count(lock->users, mode);
+			lock->last_user_pid[mode] = task_pid_nr(current);
+			lock->last_user_ino[mode] = ino;
 			*ret_lock = lock;
 			ret = 0;
 			break;
@@ -1107,7 +1110,7 @@ int scoutfs_lock_ino(struct super_block *sb, enum scoutfs_lock_mode mode, int fl
 	end.sk_zone = SCOUTFS_FS_ZONE;
 	end.ski_ino = cpu_to_le64(ino | SCOUTFS_LOCK_INODE_GROUP_MASK);
 
-	return lock_key_range(sb, mode, flags, &start, &end, ret_lock);
+	return lock_key_range(sb, mode, flags, &start, &end, ino, ret_lock);
 }
 
 /*
@@ -1237,7 +1240,7 @@ int scoutfs_lock_rename(struct super_block *sb, enum scoutfs_lock_mode mode, int
 		.sk_type = SCOUTFS_RENAME_TYPE,
 	};
 
-	return lock_key_range(sb, mode, flags, &key, &key, lock);
+	return lock_key_range(sb, mode, flags, &key, &key, 0, lock);
 }
 
 /*
@@ -1285,7 +1288,7 @@ int scoutfs_lock_inode_index(struct super_block *sb, enum scoutfs_lock_mode mode
 
 	scoutfs_lock_get_index_item_range(type, major, ino, &start, &end);
 
-	return lock_key_range(sb, mode, 0, &start, &end, ret_lock);
+	return lock_key_range(sb, mode, 0, &start, &end, ino, ret_lock);
 }
 
 /*
@@ -1312,7 +1315,7 @@ int scoutfs_lock_orphan(struct super_block *sb, enum scoutfs_lock_mode mode, int
 	end.sko_ino = cpu_to_le64(U64_MAX);
 	end.sk_type = SCOUTFS_ORPHAN_TYPE;
 
-	return lock_key_range(sb, mode, flags, &start, &end, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, ino, lock);
 }
 
 int scoutfs_lock_xattr_totl(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1323,7 +1326,7 @@ int scoutfs_lock_xattr_totl(struct super_block *sb, enum scoutfs_lock_mode mode,
 
 	scoutfs_totl_set_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
 }
 
 int scoutfs_lock_xattr_indx(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1334,7 +1337,7 @@ int scoutfs_lock_xattr_indx(struct super_block *sb, enum scoutfs_lock_mode mode,
 
 	scoutfs_xattr_indx_get_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
 }
 
 int scoutfs_lock_quota(struct super_block *sb, enum scoutfs_lock_mode mode, int flags,
@@ -1345,7 +1348,7 @@ int scoutfs_lock_quota(struct super_block *sb, enum scoutfs_lock_mode mode, int
 
 	scoutfs_quota_get_lock_range(&start, &end);
 
-	return lock_key_range(sb, mode, flags, &start, &end, lock);
+	return lock_key_range(sb, mode, flags, &start, &end, 0, lock);
 }
 
 void scoutfs_unlock(struct super_block *sb, struct scoutfs_lock *lock, enum scoutfs_lock_mode mode)
@@ -1462,7 +1465,7 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
 	struct scoutfs_lock *lock =
 		container_of(ent, struct scoutfs_lock, tseq_entry);
 
-	seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u\n",
+	seq_printf(m, "start "SK_FMT" end "SK_FMT" refresh_gen %llu mode %d waiters: rd %u wr %u wo %u users: rd %u wr %u wo %u ino: rd %llu wr %llu wo %llu pid: rd %d wr %d wo %d\n",
 			   SK_ARG(&lock->start), SK_ARG(&lock->end),
 			   lock->refresh_gen, lock->mode,
 			   lock->waiters[SCOUTFS_LOCK_READ],
@@ -1470,7 +1473,13 @@ static void lock_tseq_show(struct seq_file *m, struct scoutfs_tseq_entry *ent)
 			   lock->waiters[SCOUTFS_LOCK_WRITE_ONLY],
 			   lock->users[SCOUTFS_LOCK_READ],
 			   lock->users[SCOUTFS_LOCK_WRITE],
-			   lock->users[SCOUTFS_LOCK_WRITE_ONLY]);
+			   lock->users[SCOUTFS_LOCK_WRITE_ONLY],
+			   lock->last_user_ino[SCOUTFS_LOCK_READ],
+			   lock->last_user_ino[SCOUTFS_LOCK_WRITE],
+			   lock->last_user_ino[SCOUTFS_LOCK_WRITE_ONLY],
+			   lock->last_user_pid[SCOUTFS_LOCK_READ],
+			   lock->last_user_pid[SCOUTFS_LOCK_WRITE],
+			   lock->last_user_pid[SCOUTFS_LOCK_WRITE_ONLY]);
 }
 
 /*

kmod/src/lock.h

@@ -42,6 +42,8 @@ struct scoutfs_lock {
 	enum scoutfs_lock_mode invalidating_mode;
 	unsigned int waiters[SCOUTFS_LOCK_NR_MODES];
 	unsigned int users[SCOUTFS_LOCK_NR_MODES];
+	pid_t last_user_pid[SCOUTFS_LOCK_NR_MODES];
+	u64 last_user_ino[SCOUTFS_LOCK_NR_MODES];
 
 	struct scoutfs_tseq_entry tseq_entry;
 

kmod/src/net.c

@@ -525,7 +525,7 @@ static int process_response(struct scoutfs_net_connection *conn,
 	struct super_block *sb = conn->sb;
 	struct message_send *msend;
 	scoutfs_net_response_t resp_func = NULL;
-	void *resp_data;
+	void *resp_data = NULL;
 
 	spin_lock(&conn->lock);
 
@@ -804,7 +804,7 @@ static void scoutfs_net_recv_worker(struct work_struct *work)
 			if (invalid_message(conn, nh)) {
 				scoutfs_inc_counter(sb, net_recv_invalid_message);
 				ret = -EBADMSG;
-				break;
+				goto out;
 			}
 
 			data_len = le16_to_cpu(nh->data_len);

kmod/src/quota.c

@@ -1114,6 +1114,7 @@ int scoutfs_quota_mod_rule(struct super_block *sb, bool is_add,
 			goto release;
 	}
 
+	wait_event(qtinf->waitq, !ruleset_is_busy(qtinf));
 	scoutfs_quota_invalidate(sb);
 	ret = 0;
 
@@ -1142,12 +1143,17 @@ void scoutfs_quota_get_lock_range(struct scoutfs_key *start, struct scoutfs_key
 }
 
 /*
- * This is called during cluster lock invalidation to indicate that the
- * ruleset is no longer protected by cluster locking and might have been
- * modified.  We mark the ruleset invalid and free it once all readers
- * drain.  The next check will acquire the cluster lock and read the
- * rules.  Because this is called during invalidation this is serialized
- * with write holders of cluster locks so we can never see -EBUSY here.
+ * Mark the cached ruleset invalid and free the previous one once readers
+ * drain.  Called from cluster lock invalidation and from quota rule
+ * modification.
+ *
+ * Cluster lock invalidation runs only after the lock layer has drained
+ * local READ users.  Since EBUSY is set only while a reader holds READ,
+ * the reader has already published by the time we run.
+ *
+ * Quota rule modification waits on the waitq for any in-flight reader
+ * to publish before calling here, so the next check rebuilds against
+ * the newly written rules rather than the reader's stale result.
  */
 void scoutfs_quota_invalidate(struct super_block *sb)
 {
@@ -1161,13 +1167,10 @@ void scoutfs_quota_invalidate(struct super_block *sb)
 
 	spin_lock(&qtinf->lock);
 	rs = rcu_dereference_protected(qtinf->ruleset, lockdep_is_held(&qtinf->lock));
-	if (rs != ERR_PTR(-EINVAL))
+	if (rs == ERR_PTR(-ENOENT) || !IS_ERR(rs))
 		rcu_assign_pointer(qtinf->ruleset, ERR_PTR(-EINVAL));
 	spin_unlock(&qtinf->lock);
 
-	/* cluster locking should have prevented this */
-	BUG_ON(rs == ERR_PTR(-EBUSY));
-
 	if (!IS_ERR(rs))
 		call_rcu(&rs->rcu, free_ruleset_rcu);
 

kmod/src/server.c

@@ -1077,8 +1077,7 @@ static int next_log_merge_range(struct super_block *sb, struct scoutfs_btree_roo
 	struct scoutfs_key key;
 	int ret;
 
-	key = *start;
-	key.sk_zone = SCOUTFS_LOG_MERGE_RANGE_ZONE;
+	init_log_merge_key(&key, SCOUTFS_LOG_MERGE_RANGE_ZONE, 0, 0);
 	scoutfs_key_set_ones(&rng->start);
 
 	do {

tests/.gitignore

@@ -12,3 +12,4 @@ src/o_tmpfile_umask
 src/o_tmpfile_linkat
 src/mmap_stress
 src/mmap_validate
+src/totl-delta-inject

tests/Makefile

@@ -15,7 +15,8 @@ BIN := src/createmany			\
 	src/o_tmpfile_umask		\
 	src/o_tmpfile_linkat		\
 	src/mmap_stress			\
-	src/mmap_validate
+	src/mmap_validate		\
+	src/totl-delta-inject
 
 DEPS := $(wildcard src/*.d)
 

tests/golden/lock-pid-ino

@@ -0,0 +1,6 @@
+== set up file
+== exercise read, write, and write-only modes
+== verify FS-zone lock recorded read and write ino+pid
+== verify orphan-zone lock recorded write-only ino+pid
+== contend on a single inode with concurrent read and write loops
+== verify both rd and wr slots populated by concurrent contention

tests/golden/quota-invalidate-race

@@ -0,0 +1,6 @@
+== setup
+== concurrent quota mod and check across mounts
+== verify quota rules are consistent after race
+== verify file creation still works under quota
+file visible on mount 1
+== cleanup

tests/golden/totl-delta-inject

@@ -0,0 +1,10 @@
+== setup three files contributing to totl 8888.0.0
+== merge baseline into fs_root
+8888.0.0 = 42, 3
+== inject (+128, +2) unbalances totl 8888.0.0
+8888.0.0 = 170, 5
+== unlink f3 (value 32) produces a -32/-1 delta
+8888.0.0 = 138, 4
+== inject (-128, -2) restores accounting for the remaining files
+8888.0.0 = 10, 2
+== cleanup

tests/sequence

@@ -29,7 +29,10 @@ totl-xattr-tag.sh
 basic-xattr-indx.sh
 quota.sh
 totl-merge-read.sh
+quota-invalidate-race.sh
+totl-delta-inject.sh
 lock-refleak.sh
+lock-pid-ino.sh
 lock-shrink-consistency.sh
 lock-shrink-read-race.sh
 lock-pr-cw-conflict.sh

tests/src/totl-delta-inject.c

@@ -0,0 +1,121 @@
+/*
+ * Test helper that calls SCOUTFS_IOC_INJECT_TOTL_DELTA to seed
+ * arbitrary totl deltas.
+ *
+ * Copyright (C) 2026 Versity Software, Inc.  All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License v2 as published by the Free Software Foundation.
+ */
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/ioctl.h>
+#include <linux/types.h>
+
+#include "ioctl.h"
+
+static void usage(const char *prog)
+{
+	fprintf(stderr,
+		"Usage: %s <mountpoint> <a>.<b>.<c> <total> <count>\n",
+		prog);
+	exit(2);
+}
+
+static int parse_s64(const char *s, int64_t *out)
+{
+	char *end;
+	int64_t v;
+
+	errno = 0;
+	v = strtoll(s, &end, 0);
+	if (errno || *end != '\0' || end == s)
+		return -1;
+	*out = v;
+	return 0;
+}
+
+/*
+ * Parse "<a>.<b>.<c>" into abc[0..2] (skxt_a, skxt_b, skxt_c).  Each
+ * component must be a non-empty unsigned base-0 integer.
+ */
+static int parse_dotted_name(const char *s, uint64_t abc[3])
+{
+	const char *p = s;
+	char *end;
+	int i;
+
+	for (i = 0; i < 3; i++) {
+		if (*p == '\0' || *p == '.')
+			return -1;
+		errno = 0;
+		abc[i] = strtoull(p, &end, 0);
+		if (errno || end == p)
+			return -1;
+
+		if (i < 2) {
+			if (*end != '.')
+				return -1;
+			p = end + 1;
+		} else {
+			if (*end != '\0')
+				return -1;
+		}
+	}
+	return 0;
+}
+
+int main(int argc, char **argv)
+{
+	struct scoutfs_ioctl_inject_totl_delta itd = {{0,}};
+	uint64_t abc[3];
+	int64_t total, count;
+	int fd;
+	int ret;
+
+	if (argc != 5)
+		usage(argv[0]);
+
+	if (parse_dotted_name(argv[2], abc) ||
+	    parse_s64(argv[3], &total) ||
+	    parse_s64(argv[4], &count)) {
+		fprintf(stderr, "could not parse arguments\n");
+		usage(argv[0]);
+	}
+
+	itd.name[0] = abc[0];
+	itd.name[1] = abc[1];
+	itd.name[2] = abc[2];
+	itd.total = total;
+	itd.count = count;
+
+	fd = open(argv[1], O_RDONLY | O_DIRECTORY);
+	if (fd < 0) {
+		fprintf(stderr, "open(%s): %s\n", argv[1], strerror(errno));
+		return 1;
+	}
+
+	ret = ioctl(fd, SCOUTFS_IOC_INJECT_TOTL_DELTA, &itd);
+	if (ret < 0) {
+		fprintf(stderr,
+			"INJECT_TOTL_DELTA(%" PRIu64 ".%" PRIu64 ".%" PRIu64
+			", total=%" PRId64 ", count=%" PRId64 "): %s\n",
+			abc[0], abc[1], abc[2], total, count, strerror(errno));
+		close(fd);
+		return 1;
+	}
+
+	close(fd);
+	return 0;
+}

tests/tests/lock-pid-ino.sh

@@ -0,0 +1,68 @@
+#
+# verify debugfs client_locks reports per-mode last-user PID and inode.
+#
+
+t_require_commands stat touch awk rm
+
+FILE="$T_D0/file"
+
+echo "== set up file"
+touch "$FILE"
+INO=$(stat -c %i "$FILE")
+GROUP_START=$(( INO & ~1023 ))
+
+echo "== exercise read, write, and write-only modes"
+t_quiet stat "$FILE"
+echo data > "$FILE"
+rm -f "$FILE"
+
+echo "== verify FS-zone lock recorded read and write ino+pid"
+ERR=$(awk -v group="$GROUP_START" -v ino="$INO" '
+	$2 == "16." group ".0.0.0.0" {
+		if ($25 != ino || $32 <= 0)
+			print "read mode: ino=" $25 " pid=" $32 " want ino=" ino " pid>0"
+		if ($27 != ino || $34 <= 0)
+			print "write mode: ino=" $27 " pid=" $34 " want ino=" ino " pid>0"
+		found = 1
+	}
+	END { if (!found) print "no FS-zone client_locks line for group " group }
+' < "$(t_debugfs_path)/client_locks")
+[ -n "$ERR" ] && t_fail "$ERR"
+
+echo "== verify orphan-zone lock recorded write-only ino+pid"
+ERR=$(awk -v ino="$INO" '
+	$2 == "8.0.4.0.0.0" {
+		if ($29 != ino || $36 <= 0)
+			print "write-only mode: ino=" $29 " pid=" $36 " want ino=" ino " pid>0"
+		found = 1
+	}
+	END { if (!found) print "no orphan-zone client_locks line" }
+' < "$(t_debugfs_path)/client_locks")
+[ -n "$ERR" ] && t_fail "$ERR"
+
+echo "== contend on a single inode with concurrent read and write loops"
+FILE2="$T_D0/file2"
+touch "$FILE2"
+INO2=$(stat -c %i "$FILE2")
+GROUP2=$(( INO2 & ~1023 ))
+
+for i in $(seq 1 5); do t_quiet stat "$FILE2"; done &
+RPID=$!
+for i in $(seq 1 5); do echo $i > "$FILE2"; done &
+WPID=$!
+wait $RPID $WPID
+
+echo "== verify both rd and wr slots populated by concurrent contention"
+ERR=$(awk -v group="$GROUP2" -v ino="$INO2" '
+	$2 == "16." group ".0.0.0.0" {
+		if ($25 != ino || $32 <= 0)
+			print "concurrent read: ino=" $25 " pid=" $32 " want ino=" ino " pid>0"
+		if ($27 != ino || $34 <= 0)
+			print "concurrent write: ino=" $27 " pid=" $34 " want ino=" ino " pid>0"
+		found = 1
+	}
+	END { if (!found) print "no FS-zone client_locks line for group " group }
+' < "$(t_debugfs_path)/client_locks")
+[ -n "$ERR" ] && t_fail "$ERR"
+
+t_pass

tests/tests/quota-invalidate-race.sh

@@ -0,0 +1,70 @@
+#
+# Regression for the BUG_ON in scoutfs_quota_invalidate when a concurrent
+# ruleset read on one mount races with a quota rule modification.
+#
+
+t_require_mounts 2
+
+TEST_UID=22222
+SET_UID="--ruid=$TEST_UID --euid=$TEST_UID"
+
+echo "== setup"
+mkdir -p "$T_D0/dir"
+chown --quiet $TEST_UID "$T_D0/dir"
+
+# totl xattr gives quota checks something to consult
+setfattr -n scoutfs.totl.test.1.1.1 -v 1 "$T_D0/dir"
+
+echo "== concurrent quota mod and check across mounts"
+
+(
+	for i in $(seq 1 20); do
+		scoutfs quota-add -p "$T_M0" \
+			-r "1 1,L,- 1,L,- $i,L,- I 999999 -" 2>/dev/null
+		scoutfs quota-del -p "$T_M0" \
+			-r "1 1,L,- 1,L,- $i,L,- I 999999 -" 2>/dev/null
+	done
+) &
+MOD_PID=$!
+
+# same mount as the mod: races local read against invalidate
+(
+	for i in $(seq 1 50); do
+		setpriv $SET_UID touch "$T_D0/dir/race0_$i" 2>/dev/null
+		rm -f "$T_D0/dir/race0_$i"
+	done
+) &
+CHECK0_PID=$!
+
+# other mount: drives cross-node lock traffic
+(
+	for i in $(seq 1 50); do
+		setpriv $SET_UID touch "$T_D1/dir/race1_$i" 2>/dev/null
+		rm -f "$T_D1/dir/race1_$i"
+	done
+) &
+CHECK1_PID=$!
+
+t_quiet wait $MOD_PID
+t_quiet wait $CHECK0_PID
+t_quiet wait $CHECK1_PID
+
+echo "== verify quota rules are consistent after race"
+scoutfs quota-wipe -p "$T_M0"
+scoutfs quota-list -p "$T_M0"
+
+echo "== verify file creation still works under quota"
+scoutfs quota-add -p "$T_M0" -r "1 1,L,- 1,L,- 1,L,- I 999999 -"
+sync
+echo 1 > $(t_debugfs_path)/drop_weak_item_cache
+echo 1 > $(t_debugfs_path)/drop_quota_check_cache
+setpriv $SET_UID touch "$T_D0/dir/verify_file"
+test -f "$T_D1/dir/verify_file" && echo "file visible on mount 1"
+rm -f "$T_D0/dir/verify_file"
+scoutfs quota-wipe -p "$T_M0"
+
+echo "== cleanup"
+setfattr -x scoutfs.totl.test.1.1.1 "$T_D0/dir"
+rm -rf "$T_D0/dir"
+
+t_pass

tests/tests/totl-delta-inject.sh

@@ -0,0 +1,43 @@
+#
+# Exercise the SCOUTFS_IOC_INJECT_TOTL_DELTA ioctl that injects totl
+# deltas directly via totl-delta-inject(1).
+#
+
+t_require_commands setfattr scoutfs sync rm touch totl-delta-inject
+
+# force a log merge then read-xattr-totals filtered to our own keys
+read_totals()
+{
+	t_force_log_merge
+	sync
+	echo 1 > $(t_debugfs_path)/drop_weak_item_cache
+	scoutfs read-xattr-totals -p "$T_M0" | \
+		grep -E '^8888\.' || true
+}
+
+echo "== setup three files contributing to totl 8888.0.0"
+touch "$T_D0/f1" "$T_D0/f2" "$T_D0/f3"
+setfattr -n scoutfs.totl.inj.8888.0.0 -v 2  "$T_D0/f1"
+setfattr -n scoutfs.totl.inj.8888.0.0 -v 8  "$T_D0/f2"
+setfattr -n scoutfs.totl.inj.8888.0.0 -v 32 "$T_D0/f3"
+
+echo "== merge baseline into fs_root"
+read_totals
+
+echo "== inject (+128, +2) unbalances totl 8888.0.0"
+totl-delta-inject "$T_M0" 8888.0.0 128 2
+read_totals
+
+echo "== unlink f3 (value 32) produces a -32/-1 delta"
+rm -f "$T_D0/f3"
+read_totals
+
+echo "== inject (-128, -2) restores accounting for the remaining files"
+totl-delta-inject "$T_M0" 8888.0.0 -128 -2
+read_totals
+
+echo "== cleanup"
+rm -f "$T_D0/f1" "$T_D0/f2"
+read_totals
+
+t_pass