Drop pending greetings from resend_queue in set_valid_greeting.

There's a race possible where set_valid_greeting() runs just as
another greeting reply sits on the resend_queue from a handshake:

  1. server_greeting() calls scoutfs_net_response() to queue the
     server's greeting reply on send_queue and kicks send_work.
  2. server_greeting() falls through to scoutfs_net_server_greeting()
     which calls set_valid_greeting() under conn->lock.
  3. If send_work already ran between steps 1 and 2 it moved the
     reply onto resend_queue.
  4. set_valid_greeting() splices resend_queue back into send_queue,
     re-queueing the greeting reply for a second send on the same
     socket.

The receiver treats a duplicate greeting as fatal: the receive path
invalid_message check on the saw_greeting flag tears the connection
down silently. The client never sees a normal disconnect, just a
stalled connection, and the server only notices once the 30s
recovery timeout fires and fences the client.

Fix by walking resend_queue in set_valid_greeting() and dropping any
queued greeting before splicing them in. Dropping them should be safe
and mimics what scoutfs_net_shutdown_worker() does.

Signed-off-by: Auke Kok <auke.kok@versity.com>

ReleaseNotes.md

@@ -1,6 +1,20 @@
 Versity ScoutFS Release Notes
 =============================
 
+---
+v1.32
+\
+*June 2, 2026*
+
+Fix writing POSIX ACLs over NFS mounts that export the scoutfs
+filesystem.
+
+Add support for kernels in the RHEL 9.8 minor release.
+
+Reduce unneeded block allocation when data\_prealloc\_contig\_only was
+set to 0. This will help achieve more efficient data space usage when
+writing small files.
+
 ---
 v1.31
 \

kmod/src/Makefile.kernelcompat

@@ -479,6 +479,16 @@ ifneq (,$(shell grep '^unsigned int stack_trace_save' include/linux/stacktrace.h
 ccflags-y += -DKC_STACK_TRACE_SAVE
 endif
 
+#
+# v3.14-rc1-7-g4e34e719e457
+#
+# .set_acl callback added to struct inode_operations.  Most kernels
+# we target have it, but el7 (3.10 base) does not, so detect.
+#
+ifneq (,$(shell grep 'int ..set_acl..struct' include/linux/fs.h))
+ccflags-y += -DKC_HAS_SET_ACL
+endif
+
 #
 # v6.1-rc1-2-g138060ba92b3
 #
@@ -496,3 +506,12 @@ endif
 ifneq (,$(shell grep 'struct posix_acl.*get_inode_acl' include/linux/fs.h))
 ccflags-y += -DKC_GET_INODE_ACL
 endif
+
+#
+# v6.15-13744-g41cb08555c41
+#
+# from_timer renamed to timer_container_of.
+#
+ifneq (,$(shell grep 'define timer_container_of' include/linux/timer.h))
+ccflags-y += -DKC_TIMER_CONTAINER_OF
+endif

kmod/src/acl.c

@@ -216,7 +216,8 @@ int scoutfs_set_acl(KC_VFS_NS_DEF
 {
 	struct inode *inode = dentry->d_inode;
 #else
-int scoutfs_set_acl(struct inode *inode, struct posix_acl *acl, int type)
+int scoutfs_set_acl(KC_VFS_NS_DEF
+		    struct inode *inode, struct posix_acl *acl, int type)
 {
 #endif
 	struct super_block *sb = inode->i_sb;
@@ -309,7 +310,7 @@ int scoutfs_acl_set_xattr(struct dentry *dentry, const char *name, const void *v
 #ifdef KC_SET_ACL_DENTRY
 	ret = scoutfs_set_acl(KC_VFS_INIT_NS dentry, acl, type);
 #else
-	ret = scoutfs_set_acl(dentry->d_inode, acl, type);
+	ret = scoutfs_set_acl(KC_VFS_INIT_NS dentry->d_inode, acl, type);
 #endif
 out:
 	posix_acl_release(acl);

kmod/src/acl.h

@@ -5,7 +5,8 @@
 int scoutfs_set_acl(KC_VFS_NS_DEF
 		    struct dentry *dentry, struct posix_acl *acl, int type);
 #else
-int scoutfs_set_acl(struct inode *inode, struct posix_acl *acl, int type);
+int scoutfs_set_acl(KC_VFS_NS_DEF
+		    struct inode *inode, struct posix_acl *acl, int type);
 #endif
 #ifdef KC_GET_INODE_ACL
 struct posix_acl *scoutfs_get_acl(struct inode *inode, int type, bool rcu);

kmod/src/data.c

@@ -422,6 +422,8 @@ static int alloc_block(struct super_block *sb, struct inode *inode,
 
 	mutex_lock(&datinf->mutex);
 
+	scoutfs_inode_get_onoff(inode, &online, &offline);
+
 	/* default to single allocation at the written block */
 	start = iblock;
 	count = 1;
@@ -444,7 +446,6 @@ static int alloc_block(struct super_block *sb, struct inode *inode,
 		 * the preallocation size to the number of online
 		 * blocks.
 		 */
-		scoutfs_inode_get_onoff(inode, &online, &offline);
 		if (iblock > 1 && iblock == online) {
 			ret = scoutfs_ext_next(sb, &data_ext_ops, &args,
 					       iblock, 1, &found);
@@ -486,6 +487,13 @@ static int alloc_block(struct super_block *sb, struct inode *inode,
 		/* trim count by next extent after iblock */
 		if (found.len && found.start > start && found.start < start + count)
 			count = (found.start - start);
+
+		/*
+		 * Ramp the aligned region size up proportionally with
+		 * the file's online block count rather than jumping to
+		 * the full prealloc size.
+		 */
+		count = max_t(u64, 1, min(count, online));
 	}
 
 	/* overall prealloc limit */

kmod/src/dir.c

@@ -2063,7 +2063,7 @@ const struct inode_operations scoutfs_dir_iops = {
 #else
 	.get_acl	= scoutfs_get_acl,
 #endif
-#ifdef KC_SET_ACL_DENTRY
+#ifdef KC_HAS_SET_ACL
 	.set_acl	= scoutfs_set_acl,
 #endif
 	.symlink	= scoutfs_symlink,

kmod/src/fence.c

@@ -222,7 +222,7 @@ static struct attribute *fence_attrs[] = {
 
 static void fence_timeout(struct timer_list *timer)
 {
-	struct pending_fence *fence = from_timer(fence, timer, timer);
+	struct pending_fence *fence = timer_container_of(fence, timer, timer);
 	struct super_block *sb = fence->sb;
 	DECLARE_FENCE_INFO(sb, fi);
 

kmod/src/inode.c

@@ -154,7 +154,7 @@ static const struct inode_operations scoutfs_file_iops = {
 #else
 	.get_acl	= scoutfs_get_acl,
 #endif
-#ifdef KC_SET_ACL_DENTRY
+#ifdef KC_HAS_SET_ACL
 	.set_acl	= scoutfs_set_acl,
 #endif
 	.fiemap		= scoutfs_data_fiemap,
@@ -174,7 +174,7 @@ static const struct inode_operations scoutfs_special_iops = {
 #else
 	.get_acl	= scoutfs_get_acl,
 #endif
-#ifdef KC_SET_ACL_DENTRY
+#ifdef KC_HAS_SET_ACL
 	.set_acl	= scoutfs_set_acl,
 #endif
 };

kmod/src/kernelcompat.h

@@ -489,4 +489,9 @@ static inline void stack_trace_print(unsigned long *entries, unsigned int nr_ent
 }
 #endif
 
+#ifndef KC_TIMER_CONTAINER_OF
+#define timer_container_of(var, callback_timer, timer_fieldname) \
+	from_timer(var, callback_timer, timer_fieldname)
+#endif
+
 #endif

kmod/src/lock_server.c

@@ -626,9 +626,14 @@ out:
 int scoutfs_lock_server_greeting(struct super_block *sb, u64 rid)
 {
 	struct scoutfs_key key;
+	bool pending;
 	int ret;
 
-	if (scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS)) {
+	pending = scoutfs_recov_is_pending(sb, rid, SCOUTFS_RECOV_LOCKS);
+
+	trace_scoutfs_lock_server_greeting(sb, rid, pending);
+
+	if (pending) {
 		scoutfs_key_set_zeros(&key);
 		ret = scoutfs_server_lock_recover_request(sb, rid, &key);
 	} else {

kmod/src/net.c

@@ -1730,12 +1730,30 @@ int scoutfs_net_connect(struct super_block *sb,
 
 static void set_valid_greeting(struct scoutfs_net_connection *conn)
 {
+	struct net_info *ninf = SCOUTFS_SB(conn->sb)->net_info;
+	struct message_send *msend;
+	struct message_send *tmp;
+
 	assert_spin_locked(&conn->lock);
 
 	/* recv should have dropped invalid duplicate greeting messages */
 	BUG_ON(test_conn_fl(conn, valid_greeting));
 
 	set_conn_fl(conn, valid_greeting);
+
+	/*
+	 * Drop greetings from the resend_queue before splicing it into
+	 * the send_queue. We might have a greeting left in the resend
+	 * queue at the moment that we reach this point. A duplicate
+	 * greeting is treated as fatal and causes a stall and fence.
+	 */
+	list_for_each_entry_safe(msend, tmp, &conn->resend_queue, head) {
+		if (msend->nh.cmd == SCOUTFS_NET_CMD_GREETING) {
+			msend->dead = 1;
+			free_msend(ninf, conn, msend);
+		}
+	}
+
 	list_splice_tail_init(&conn->resend_queue, &conn->send_queue);
 	queue_work(conn->workq, &conn->send_work);
 }

kmod/src/recov.c

@@ -21,6 +21,7 @@
 #include "super.h"
 #include "recov.h"
 #include "cmp.h"
+#include "scoutfs_trace.h"
 
 /*
  * There are a few server messages which can't be processed until they
@@ -103,7 +104,7 @@ int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which)
 	if (!alloc)
 		return -ENOMEM;
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 
 	pend = lookup_pending(recinf, rid, SCOUTFS_RECOV_ALL);
 	if (pend) {
@@ -116,9 +117,12 @@ int scoutfs_recov_prepare(struct super_block *sb, u64 rid, int which)
 		list_sort(NULL, &recinf->pending, cmp_pending_rid);
 	}
 
-	spin_unlock(&recinf->lock);
+	spin_unlock_bh(&recinf->lock);
 
 	kfree(alloc);
+
+	trace_scoutfs_recov_prepare(sb, rid, which);
+
 	return 0;
 }
 
@@ -134,7 +138,16 @@ static int recov_finished(struct recov_info *recinf)
 
 static void timer_callback(struct timer_list *timer)
 {
-	struct recov_info *recinf = from_timer(recinf, timer, timer);
+	struct recov_info *recinf = timer_container_of(recinf, timer, timer);
+	struct recov_pending *pend;
+	int nr = 0;
+
+	spin_lock(&recinf->lock);
+	list_for_each_entry(pend, &recinf->pending, head)
+		nr++;
+	spin_unlock(&recinf->lock);
+
+	trace_scoutfs_recov_timeout_fire(recinf->sb, nr);
 
 	recinf->timeout_fn(recinf->sb);
 }
@@ -153,7 +166,7 @@ int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_
 	DECLARE_RECOV_INFO(sb, recinf);
 	int ret;
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 
 	recinf->timeout_fn = timeout_fn;
 	recinf->timer.expires = jiffies + msecs_to_jiffies(timeout_ms);
@@ -161,7 +174,7 @@ int scoutfs_recov_begin(struct super_block *sb, void (*timeout_fn)(struct super_
 
 	ret = recov_finished(recinf);
 
-	spin_unlock(&recinf->lock);
+	spin_unlock_bh(&recinf->lock);
 
 	if (ret > 0)
 		del_timer_sync(&recinf->timer);
@@ -181,9 +194,11 @@ int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which)
 {
 	DECLARE_RECOV_INFO(sb, recinf);
 	struct recov_pending *pend;
+	struct recov_pending *iter;
+	int remaining = 0;
 	int ret = 0;
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 
 	pend = lookup_pending(recinf, rid, which);
 	if (pend) {
@@ -196,7 +211,12 @@ int scoutfs_recov_finish(struct super_block *sb, u64 rid, int which)
 		}
 	}
 
-	spin_unlock(&recinf->lock);
+	list_for_each_entry(iter, &recinf->pending, head)
+		remaining++;
+
+	spin_unlock_bh(&recinf->lock);
+
+	trace_scoutfs_recov_finish(sb, rid, which, remaining);
 
 	if (ret > 0)
 		del_timer_sync(&recinf->timer);
@@ -215,9 +235,9 @@ bool scoutfs_recov_is_pending(struct super_block *sb, u64 rid, int which)
 	DECLARE_RECOV_INFO(sb, recinf);
 	bool is_pending;
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 	is_pending = lookup_pending(recinf, rid, which) != NULL;
-	spin_unlock(&recinf->lock);
+	spin_unlock_bh(&recinf->lock);
 
 	return is_pending;
 }
@@ -236,10 +256,10 @@ u64 scoutfs_recov_next_pending(struct super_block *sb, u64 rid, int which)
 	DECLARE_RECOV_INFO(sb, recinf);
 	struct recov_pending *pend;
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 	pend = next_pending(recinf, rid, which);
 	rid = pend ? pend->rid : 0;
-	spin_unlock(&recinf->lock);
+	spin_unlock_bh(&recinf->lock);
 
 	return rid;
 }
@@ -257,10 +277,10 @@ void scoutfs_recov_shutdown(struct super_block *sb)
 
 	del_timer_sync(&recinf->timer);
 
-	spin_lock(&recinf->lock);
+	spin_lock_bh(&recinf->lock);
 	list_splice_init(&recinf->pending, &list);
 	recinf->timeout_fn = NULL;
-	spin_unlock(&recinf->lock);
+	spin_unlock_bh(&recinf->lock);
 
 	list_for_each_entry_safe(pend, tmp, &list, head) {
 		list_del(&pend->head);

kmod/src/scoutfs_trace.h

@@ -2121,6 +2121,110 @@ DEFINE_EVENT(scoutfs_server_client_count_class, scoutfs_server_client_down,
 	TP_ARGS(sb, rid, nr_clients)
 );
 
+TRACE_EVENT(scoutfs_recov_prepare,
+	TP_PROTO(struct super_block *sb, u64 rid, int which),
+
+	TP_ARGS(sb, rid, which),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, c_rid)
+		__field(int, which)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->c_rid = rid;
+		__entry->which = which;
+	),
+
+	TP_printk(SCSBF" rid %016llx which 0x%x",
+		  SCSB_TRACE_ARGS, __entry->c_rid, __entry->which)
+);
+
+TRACE_EVENT(scoutfs_recov_finish,
+	TP_PROTO(struct super_block *sb, u64 rid, int which, int remaining),
+
+	TP_ARGS(sb, rid, which, remaining),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, c_rid)
+		__field(int, which)
+		__field(int, remaining)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->c_rid = rid;
+		__entry->which = which;
+		__entry->remaining = remaining;
+	),
+
+	TP_printk(SCSBF" rid %016llx which 0x%x remaining %d",
+		  SCSB_TRACE_ARGS, __entry->c_rid, __entry->which,
+		  __entry->remaining)
+);
+
+TRACE_EVENT(scoutfs_recov_timeout_fire,
+	TP_PROTO(struct super_block *sb, int nr_pending),
+
+	TP_ARGS(sb, nr_pending),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(int, nr_pending)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->nr_pending = nr_pending;
+	),
+
+	TP_printk(SCSBF" nr_pending %d",
+		  SCSB_TRACE_ARGS, __entry->nr_pending)
+);
+
+TRACE_EVENT(scoutfs_recov_fence_rid,
+	TP_PROTO(struct super_block *sb, u64 rid),
+
+	TP_ARGS(sb, rid),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, c_rid)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->c_rid = rid;
+	),
+
+	TP_printk(SCSBF" rid %016llx",
+		  SCSB_TRACE_ARGS, __entry->c_rid)
+);
+
+TRACE_EVENT(scoutfs_lock_server_greeting,
+	TP_PROTO(struct super_block *sb, u64 rid, bool recov_pending),
+
+	TP_ARGS(sb, rid, recov_pending),
+
+	TP_STRUCT__entry(
+		SCSB_TRACE_FIELDS
+		__field(__u64, c_rid)
+		__field(bool, recov_pending)
+	),
+
+	TP_fast_assign(
+		SCSB_TRACE_ASSIGN(sb);
+		__entry->c_rid = rid;
+		__entry->recov_pending = recov_pending;
+	),
+
+	TP_printk(SCSBF" rid %016llx recov_pending %d",
+		  SCSB_TRACE_ARGS, __entry->c_rid, __entry->recov_pending)
+);
+
 DECLARE_EVENT_CLASS(scoutfs_server_commit_users_class,
         TP_PROTO(struct super_block *sb, int holding, int applying,
 		 int nr_holders, u32 budget,

kmod/src/server.c

@@ -4386,6 +4386,8 @@ static void fence_pending_recov_worker(struct work_struct *work)
 		scoutfs_err(sb, "%lu ms recovery timeout expired for client rid %016llx, fencing",
 			    SERVER_RECOV_TIMEOUT_MS, rid);
 
+		trace_scoutfs_recov_fence_rid(sb, rid);
+
 		ret = lookup_mounted_client_addr(sb, rid, &addr);
 		if (ret < 0) {
 			scoutfs_err(sb, "client rid addr lookup err %d, shutting down server", ret);

tests/funcs/filter.sh

@@ -171,6 +171,13 @@ t_filter_dmesg()
 	# orphan log trees reclaim is handled, not an error
 	re="$re|scoutfs .* reclaiming orphan log trees"
 
+	# nfs can emit a whole range of messages we can ignore
+	re="$re|Installing knfsd .*"
+	re="$re|nfsd: .*"
+	re="$re|NFSD: .*"
+	re="$re|RPC: .*"
+	re="$re|FS-Cache: .*"
+
 	# fencing tests force unmounts and trigger timeouts
 	re="$re|scoutfs .* forcing unmount"
 	re="$re|scoutfs .* reconnect timed out"

tests/golden/basic-nfs

@@ -0,0 +1,32 @@
+== write via NFS, read both sides
+== POSIX ACL set via NFS, read both sides
+user::rw-
+user:22222:rw-
+group::r--
+mask::rw-
+other::r--
+
+user::rw-
+user:22222:rw-
+group::r--
+mask::rw-
+other::r--
+
+== POSIX ACL set on scoutfs, read via NFS
+user::rw-
+user:22222:rw-
+group::r--
+group:44444:r--
+mask::rw-
+other::r--
+
+== default ACL inheritance via NFS
+user::rw-
+user:22222:rwx	#effective:rw-
+group::r-x	#effective:r--
+mask::rw-
+other::r--
+
+== NFS read demand-stages a released file
+1
+== cleanup

tests/golden/data-prealloc

@@ -8,10 +8,10 @@
 /mnt/test/test/data-prealloc/file-1: extents: 32
 /mnt/test/test/data-prealloc/file-2: extents: 32
 == any writes to region prealloc get full extents
-/mnt/test/test/data-prealloc/file-1: extents: 4
-/mnt/test/test/data-prealloc/file-2: extents: 4
-/mnt/test/test/data-prealloc/file-1: extents: 4
-/mnt/test/test/data-prealloc/file-2: extents: 4
+/mnt/test/test/data-prealloc/file-1: extents: 8
+/mnt/test/test/data-prealloc/file-2: extents: 8
+/mnt/test/test/data-prealloc/file-1: extents: 8
+/mnt/test/test/data-prealloc/file-2: extents: 8
 == streaming offline writes get full extents either way
 /mnt/test/test/data-prealloc/file-1: extents: 4
 /mnt/test/test/data-prealloc/file-2: extents: 4
@@ -20,8 +20,8 @@
 == goofy preallocation amounts work
 /mnt/test/test/data-prealloc/file-1: extents: 6
 /mnt/test/test/data-prealloc/file-2: extents: 6
-/mnt/test/test/data-prealloc/file-1: extents: 6
-/mnt/test/test/data-prealloc/file-2: extents: 6
+/mnt/test/test/data-prealloc/file-1: extents: 10
+/mnt/test/test/data-prealloc/file-2: extents: 10
 /mnt/test/test/data-prealloc/file-1: extents: 3
 /mnt/test/test/data-prealloc/file-2: extents: 3
 == block writes into region allocs hole

tests/sequence

@@ -3,6 +3,7 @@ basic-block-counts.sh
 basic-bad-mounts.sh
 basic-posix-acl.sh
 basic-acl-consistency.sh
+basic-nfs.sh
 inode-items-updated.sh
 simple-inode-index.sh
 simple-staging.sh

tests/tests/basic-nfs.sh

@@ -0,0 +1,86 @@
+#
+# Test basic scoutfs-nfs interactions:
+# - read/write
+# - stage/release and data wait
+# - nfs setacl/getacl mapping
+#
+
+t_require_commands scoutfs setfacl getfacl exportfs mount.nfs umount \
+		   stat dd cmp systemctl
+
+systemctl start nfs-server >> "$T_TMPDIR/nfs.log" 2>&1 || \
+	t_skip "nfs-server not available"
+
+# Keep file creation modes deterministic for the ACL golden output.
+umask 022
+
+EXPORT_OPTS="rw,async,no_root_squash,no_subtree_check,fsid=42"
+NFS_MNT="$T_TMP.nfs"
+NFS_DIR="$NFS_MNT/test/basic-nfs"
+
+filter() { sed "s@$T_TMPDIR@T_TMPDIR@g" | t_filter_fs; }
+gf() { getfacl -n --omit-header "$@" 2>/dev/null; }
+
+teardown_nfs()
+{
+	(
+		umount "$NFS_MNT"
+		exportfs -u "127.0.0.1:$T_M0"
+		exportfs -f
+		systemctl stop nfs-server
+		rmdir "$NFS_MNT"
+	) >> "$T_TMPDIR/nfs.log" 2>&1
+}
+trap teardown_nfs EXIT
+
+exportfs -u "127.0.0.1:$T_M0" >> "$T_TMPDIR/nfs.log" 2>&1 || true
+t_quiet mkdir -p "$NFS_MNT"
+exportfs -o "$EXPORT_OPTS" "127.0.0.1:$T_M0" >> "$T_TMPDIR/nfs.log" 2>&1
+mount.nfs -o vers=3,noac,actimeo=0 "127.0.0.1:$T_M0" "$NFS_MNT" >> "$T_TMPDIR/nfs.log" 2>&1
+
+test -d "$NFS_DIR" || t_fail "test dir $NFS_DIR not visible over NFS"
+
+echo "== write via NFS, read both sides"
+dd if=/dev/urandom bs=4096 count=1 of="$T_TMP.data" status=none
+cp "$T_TMP.data" "$NFS_DIR/file"
+cmp "$T_TMP.data" "$T_D0/file"
+cmp "$T_TMP.data" "$NFS_DIR/file"
+
+echo "== POSIX ACL set via NFS, read both sides"
+setfacl -m u:22222:rw "$NFS_DIR/file" 2>&1 | filter
+gf "$NFS_DIR/file"
+gf "$T_D0/file"
+
+echo "== POSIX ACL set on scoutfs, read via NFS"
+setfacl -m g:44444:r "$T_D0/file" 2>&1 | filter
+gf "$NFS_DIR/file"
+
+echo "== default ACL inheritance via NFS"
+mkdir "$NFS_DIR/d"
+setfacl -d -m u:22222:rwx "$NFS_DIR/d" 2>&1 | filter
+touch "$NFS_DIR/d/child"
+gf "$NFS_DIR/d/child"
+
+echo "== NFS read demand-stages a released file"
+dd if=/dev/urandom bs=4096 count=1 of="$T_TMP.big" status=none
+cp "$T_TMP.big" "$T_D0/big"
+sync
+vers=$(scoutfs stat -s data_version "$T_D0/big")
+t_quiet scoutfs release "$T_D0/big" -V "$vers" -o 0 -l 4K
+
+# NFS read against the offline file blocks in scoutfs_read waiting
+# for the data to come back online.
+cat "$NFS_DIR/big" > "$T_TMP.read" &
+read_pid=$!
+sleep 1
+scoutfs data-waiting -B 0 -I 0 -p "$T_D0" | wc -l
+
+t_quiet scoutfs stage "$T_TMP.big" "$T_D0/big" -V "$vers" -o 0 -l 4096
+wait "$read_pid"
+cmp "$T_TMP.big" "$T_TMP.read"
+
+echo "== cleanup"
+rm -f "$T_D0/file" "$T_D0/big"
+rm -rf "$T_D0/d"
+
+t_pass