Test export-lookup-evict-race in a loop with tracing.

This test hits the unmount hang consistently in our CI the most,
so run it in a tight loop and enable tracing. Discard traces when
the run succeeded.

This will blow up if a hung task timeout occurs, so we should
crash on panic and then extract traces from the crash. Make sure
we don't wait for an hour before doing so, then, too.

Signed-off-by: Auke Kok <auke.kok@versity.com>

kmod/src/server.c

@@ -3036,7 +3036,13 @@ static int server_commit_log_merge(struct super_block *sb,
 				  SCOUTFS_LOG_MERGE_STATUS_ZONE, 0, 0,
 				  &stat, sizeof(stat));
 	if (ret < 0) {
-		err_str = "getting merge status item";
+		/*
+		 * During a retransmission, it's possible that the server
+		 * already committed and resolved this log merge. ENOENT
+		 * is expected in that case.
+		 */
+		if (ret != -ENOENT)
+			err_str = "getting merge status item";
 		goto out;
 	}
 

tests/fenced-local-force-unmount.sh

@@ -9,7 +9,7 @@
 echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'"
 
 echo_fail() {
-	echo "$@" >> /dev/stderr
+	echo "$@" >&2
 	exit 1
 }
 
@@ -27,8 +27,7 @@ for fs in /sys/fs/scoutfs/*; do
 	nr="$(quiet_cat $fs/data_device_maj_min)"
 	[ ! -d "$fs" -o "$fs_rid" != "$rid" ] && continue
 
-	mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \
-		echo_fail "findmnt -t scoutfs -S $nr failed"
+	mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr)
 	[ -z "$mnt" ] && continue
 
 	if ! umount -qf "$mnt"; then

tests/run-tests.sh

@@ -92,10 +92,14 @@ done
 T_TRACE_DUMP="0"
 T_TRACE_PRINTK="0"
 T_PORT_START="19700"
-T_LOOP_ITER="1"
+T_LOOP_ITER="100"
 
 # array declarations to be able to use array ops
 declare -a T_TRACE_GLOB
+T_TRACE_GLOB=( "scoutfs*" )
+
+# CI sets this to 3600, but, for this case we want it very short
+echo 30 > /proc/sys/kernel/hung_task_timeout_secs
 
 while true; do
 	case $1 in
@@ -493,6 +497,11 @@ crash_monitor()
 			bad=1
 		fi
 
+		if dmesg | grep -q "blocked for more than"; then
+			echo "run-tests monitor saw blocked task message"
+			bad=1
+		fi
+
 		if dmesg | grep -q "error indicated by fence action" ; then
 			echo "run-tests monitor saw fence agent error message"
 			bad=1
@@ -504,6 +513,8 @@ crash_monitor()
 		fi
 
 		if [ "$bad" != 0 ]; then
+			sync & # maybe this gets logs synced...
+			sleep .1
 			echo "run-tests monitor triggering crash"
 			echo c > /proc/sysrq-trigger
 			exit 1
@@ -706,6 +717,8 @@ for t in $tests; do
 		# stop looping if we didn't pass
 		if [ "$sts" != "$T_PASS_STATUS" ]; then
 			break;
+		else
+			echo > /sys/kernel/debug/tracing/trace
 		fi
 	done
 

tests/sequence

@@ -1,60 +1 @@
-export-get-name-parent.sh
-basic-block-counts.sh
-basic-bad-mounts.sh
-basic-posix-acl.sh
-inode-items-updated.sh
-simple-inode-index.sh
-simple-staging.sh
-simple-release-extents.sh
-simple-readdir.sh
-get-referring-entries.sh
-fallocate.sh
-basic-truncate.sh
-data-prealloc.sh
-setattr_more.sh
-offline-extent-waiting.sh
-move-blocks.sh
-projects.sh
-large-fragmented-free.sh
-format-version-forward-back.sh
-enospc.sh
-mmap.sh
-srch-safe-merge-pos.sh
-srch-basic-functionality.sh
-simple-xattr-unit.sh
-retention-basic.sh
-totl-xattr-tag.sh
-quota.sh
-lock-refleak.sh
-lock-shrink-consistency.sh
-lock-shrink-read-race.sh
-lock-pr-cw-conflict.sh
-lock-revoke-getcwd.sh
-lock-recover-invalidate.sh
 export-lookup-evict-race.sh
-createmany-parallel.sh
-createmany-large-names.sh
-createmany-rename-large-dir.sh
-stage-release-race-alloc.sh
-stage-multi-part.sh
-o_tmpfile.sh
-basic-posix-consistency.sh
-dirent-consistency.sh
-mkdir-rename-rmdir.sh
-lock-ex-race-processes.sh
-cross-mount-data-free.sh
-persistent-item-vers.sh
-setup-error-teardown.sh
-resize-devices.sh
-change-devices.sh
-fence-and-reclaim.sh
-quorum-heartbeat-timeout.sh
-orphan-inodes.sh
-mount-unmount-race.sh
-client-unmount-recovery.sh
-createmany-parallel-mounts.sh
-archive-light-cycle.sh
-block-stale-reads.sh
-inode-deletion.sh
-renameat2-noreplace.sh
-xfstests.sh

tests/tests/renameat2-noreplace.sh

@@ -8,19 +8,19 @@ t_require_mounts 2
 echo "=== renameat2 noreplace flag test"
 
 # give each mount their own dir (lock group) to minimize create contention
-mkdir $T_M0/dir0
-mkdir $T_M1/dir1
+mkdir $T_D0/dir0
+mkdir $T_D1/dir1
 
 echo "=== run two asynchronous calls to renameat2 NOREPLACE"
 for i in $(seq 0 100); do
         # prepare inputs in isolation
-        touch "$T_M0/dir0/old0"
-        touch "$T_M1/dir1/old1"
+        touch "$T_D0/dir0/old0"
+        touch "$T_D1/dir1/old1"
 
         # race doing noreplace renames, both can't succeed
-        dumb_renameat2 -n "$T_M0/dir0/old0" "$T_M0/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_D0/dir0/old0" "$T_D0/dir0/sharednew" 2> /dev/null &
         pid0=$!
-        dumb_renameat2 -n "$T_M1/dir1/old1" "$T_M1/dir0/sharednew" 2> /dev/null &
+        dumb_renameat2 -n "$T_D1/dir1/old1" "$T_D1/dir0/sharednew" 2> /dev/null &
         pid1=$!
 
         wait $pid0
@@ -31,7 +31,7 @@ for i in $(seq 0 100); do
         test "$rc0" == 0 -a "$rc1" == 0 && t_fail "both renames succeeded"
 
         # blow away possible files for either race outcome
-        rm -f "$T_M0/dir0/old0" "$T_M1/dir1/old1" "$T_M0/dir0/sharednew" "$T_M1/dir1/sharednew"
+        rm -f "$T_D0/dir0/old0" "$T_D1/dir1/old1" "$T_D0/dir0/sharednew" "$T_D1/dir1/sharednew"
 done
 
 t_pass

utils/fenced/scoutfs-fenced

@@ -7,7 +7,7 @@ message_output()
 
 error_message()
 {
-	message_output "$@" >> /dev/stderr
+	message_output "$@" >&2
 }
 
 error_exit()

utils/man/scoutfs.8

@@ -402,39 +402,25 @@ before destroying an old empty data device.
 .PD
 
 .TP
-.BI "print {-a|--allocs} {-i|--items ITEMS} {-r|--roots ROOTS} {-S|--skip-likely-huge} META-DEVICE"
+.BI "print {-S|--skip-likely-huge} META-DEVICE"
 .sp
-Prints out some or all of the metadata in the file system.  This makes no effort
+Prints out all of the metadata in the file system.  This makes no effort
 to ensure that the structures are consistent as they're traversed and
 can present structures that seem corrupt as they change as they're
 output.
-.sp
-Structures that are related to the number of mounts and are maintained at a
-relatively reasonable size are always printed. These include per-mount log
-trees, srch files, allocators, and the metadata allocators used by server
-commits. Other btrees and their items can be selected as desired.
 .RS 1.0i
 .PD 0
+.TP
 .sp
-.TP
-.B "-a, --allocs"
-Print the metadata and data allocators. Enabled by default.
-.TP
-.B "-r, --roots ROOTS"
-This option can be used to select which btrees are traversed. It is a comma-separated list containing one or more of the following btree roots: logs, srch, fs. Default is all roots.
-.TP
-.B "-i, --items ITEMS"
-This option can be used to choose which btree items are printed from the
-selected btree roots. It is a comma-separated list containing one or
-more of the following items: inode, xattr, dirent, symlink, backref, extent.
-Default is all items.
-.TP
 .B "-S, --skip-likely-huge"
 Skip printing structures that are likely to be very large.  The
 structures that are skipped tend to be global and whose size tends to be
 related to the size of the volume.   Examples of skipped structures include
 the global fs items, srch files, and metadata and data
-allocators.
+allocators.  Similar structures that are not skipped are related to the
+number of mounts and are maintained at a relatively reasonable size.
+These include per-mount log trees, srch files, allocators, and the
+metadata allocators used by server commits.
 .sp
 Skipping the larger structures limits the print output to a relatively
 constant size rather than being a large multiple of the used metadata

utils/src/print.c

@@ -29,42 +29,6 @@
 #include "leaf_item_hash.h"
 #include "dev.h"
 
-struct print_args {
-	char *meta_device;
-	bool skip_likely_huge;
-	bool roots_requested;
-	bool items_requested;
-	bool allocs_requested;
-	bool walk_allocs;
-	bool walk_logs_root;
-	bool walk_fs_root;
-	bool walk_srch_root;
-	bool print_inodes;
-	bool print_xattrs;
-	bool print_dirents;
-	bool print_symlinks;
-	bool print_backrefs;
-	bool print_extents;
-};
-
-static struct print_args print_args = {
-	.meta_device	  = NULL,
-	.skip_likely_huge = false,
-	.roots_requested  = false,
-	.items_requested  = false,
-	.allocs_requested = false,
-	.walk_allocs	  = true,
-	.walk_logs_root	  = true,
-	.walk_fs_root	  = true,
-	.walk_srch_root	  = true,
-	.print_inodes	  = true,
-	.print_xattrs	  = true,
-	.print_dirents	  = true,
-	.print_symlinks	  = true,
-	.print_backrefs	  = true,
-	.print_extents	  = true
-};
-
 static void print_block_header(struct scoutfs_block_header *hdr, int size)
 {
 	u32 crc = crc_block(hdr, size);
@@ -231,7 +195,7 @@ static void print_inode_index(struct scoutfs_key *key, void *val, int val_len)
 
 typedef void (*print_func_t)(struct scoutfs_key *key, void *val, int val_len);
 
-static print_func_t find_printer(u8 zone, u8 type, bool *suppress)
+static print_func_t find_printer(u8 zone, u8 type)
 {
 	if (zone == SCOUTFS_INODE_INDEX_ZONE &&
 	    type >= SCOUTFS_INODE_INDEX_META_SEQ_TYPE  &&
@@ -254,34 +218,13 @@ static print_func_t find_printer(u8 zone, u8 type, bool *suppress)
 
 	if (zone == SCOUTFS_FS_ZONE) {
 		switch(type) {
-			case SCOUTFS_INODE_TYPE:
-				if (!print_args.print_inodes)
-					*suppress = true;
-				return print_inode;
-			case SCOUTFS_XATTR_TYPE:
-				if (!print_args.print_xattrs)
-					*suppress = true;
-				return print_xattr;
-			case SCOUTFS_DIRENT_TYPE:
-				if (!print_args.print_dirents)
-					*suppress = true;
-				return print_dirent;
-			case SCOUTFS_READDIR_TYPE:
-				if (!print_args.print_dirents)
-					*suppress = true;
-				return print_dirent;
-			case SCOUTFS_SYMLINK_TYPE:
-				if (!print_args.print_symlinks)
-					*suppress = true;
-				return print_symlink;
-			case SCOUTFS_LINK_BACKREF_TYPE:
-				if (!print_args.print_backrefs)
-					*suppress = true;
-				return print_dirent;
-			case SCOUTFS_DATA_EXTENT_TYPE:
-				if (!print_args.print_extents)
-					*suppress = true;
-				return print_data_extent;
+			case SCOUTFS_INODE_TYPE: return print_inode;
+			case SCOUTFS_XATTR_TYPE: return print_xattr;
+			case SCOUTFS_DIRENT_TYPE: return print_dirent;
+			case SCOUTFS_READDIR_TYPE: return print_dirent;
+			case SCOUTFS_SYMLINK_TYPE: return print_symlink;
+			case SCOUTFS_LINK_BACKREF_TYPE: return print_dirent;
+			case SCOUTFS_DATA_EXTENT_TYPE: return print_data_extent;
 		}
 	}
 
@@ -301,16 +244,12 @@ static int print_fs_item(struct scoutfs_key *key, u64 seq, u8 flags, void *val,
 
 	/* only items in leaf blocks have values */
 	if (val != NULL && !(flags & SCOUTFS_ITEM_FLAG_DELETION)) {
-		bool suppress = false;
-
-		printer = find_printer(key->sk_zone, key->sk_type, &suppress);
-		if (printer) {
-			if (!suppress)
-				printer(key, val, val_len);
-		} else {
+		printer = find_printer(key->sk_zone, key->sk_type);
+		if (printer)
+			printer(key, val, val_len);
+		else
 			printf("      (unknown zone %u type %u)\n",
 			       key->sk_zone, key->sk_type);
-		}
 	}
 
 	return 0;
@@ -1098,7 +1037,12 @@ static void print_super_block(struct scoutfs_super_block *super, u64 blkno)
 	}
 }
 
-static int print_volume(int fd)
+struct print_args {
+	char *meta_device;
+	bool skip_likely_huge;
+};
+
+static int print_volume(int fd, struct print_args *args)
 {
 	struct scoutfs_super_block *super = NULL;
 	struct print_recursion_args pa;
@@ -1148,7 +1092,7 @@ static int print_volume(int fd)
 			ret = err;
 	}
 
-	if (print_args.walk_allocs) {
+	if (!args->skip_likely_huge) {
 		for (i = 0; i < array_size(super->meta_alloc); i++) {
 			snprintf(str, sizeof(str), "meta_alloc[%u]", i);
 			err = print_btree(fd, super, str, &super->meta_alloc[i].root,
@@ -1175,21 +1119,18 @@ static int print_volume(int fd)
 
 	pa.super = super;
 	pa.fd = fd;
-	if (print_args.walk_srch_root) {
+	if (!args->skip_likely_huge) {
 		err = print_btree_leaf_items(fd, super, &super->srch_root.ref,
 					     print_srch_root_files, &pa);
 		if (err && !ret)
 			ret = err;
 	}
+	err = print_btree_leaf_items(fd, super, &super->logs_root.ref,
+				     print_log_trees_roots, &pa);
+	if (err && !ret)
+		ret = err;
 
-	if (print_args.walk_logs_root) {
-		err = print_btree_leaf_items(fd, super, &super->logs_root.ref,
-					     print_log_trees_roots, &pa);
-		if (err && !ret)
-			ret = err;
-	}
-
-	if (print_args.walk_fs_root) {
+	if (!args->skip_likely_huge) {
 		err = print_btree(fd, super, "fs_root", &super->fs_root,
 				  print_fs_item, NULL);
 		if (err && !ret)
@@ -1202,16 +1143,16 @@ out:
 	return ret;
 }
 
-static int do_print(void)
+static int do_print(struct print_args *args)
 {
 	int ret;
 	int fd;
 
-	fd = open(print_args.meta_device, O_RDONLY);
+	fd = open(args->meta_device, O_RDONLY);
 	if (fd < 0) {
 		ret = -errno;
 		fprintf(stderr, "failed to open '%s': %s (%d)\n",
-			print_args.meta_device, strerror(errno), errno);
+			args->meta_device, strerror(errno), errno);
 		return ret;
 	}
 
@@ -1219,169 +1160,30 @@ static int do_print(void)
 	if (ret < 0)
 		goto out;
 
-	ret = print_volume(fd);
+	ret = print_volume(fd, args);
 out:
 	close(fd);
 	return ret;
 };
 
-enum {
-	LOGS_OPT = 0,
-	FS_OPT,
-	SRCH_OPT
-};
-
-static char *const root_tokens[] = {
-	[LOGS_OPT] = "logs",
-	[FS_OPT] =   "fs",
-	[SRCH_OPT] = "srch",
-	NULL
-};
-
-enum {
-	INODE_OPT = 0,
-	XATTR_OPT,
-	DIRENT_OPT,
-	SYMLINK_OPT,
-	BACKREF_OPT,
-	EXTENT_OPT
-};
-
-static char *const item_tokens[] = {
-	[INODE_OPT] =   "inode",
-	[XATTR_OPT] =   "xattr",
-	[DIRENT_OPT] =  "dirent",
-	[SYMLINK_OPT] = "symlink",
-	[BACKREF_OPT] = "backref",
-	[EXTENT_OPT] =  "extent",
-	NULL
-};
-
-static void clear_items(void)
-{
-	print_args.print_inodes = false;
-	print_args.print_xattrs = false;
-	print_args.print_dirents = false;
-	print_args.print_symlinks = false;
-	print_args.print_backrefs = false;
-	print_args.print_extents = false;
-}
-
-static void clear_roots(void)
-{
-	print_args.walk_logs_root = false;
-	print_args.walk_fs_root = false;
-	print_args.walk_srch_root = false;
-}
-
 static int parse_opt(int key, char *arg, struct argp_state *state)
 {
 	struct print_args *args = state->input;
-	char *subopts;
-	char *value;
-	bool parse_err = false;
 
 	switch (key) {
 	case 'S':
 		args->skip_likely_huge = true;
 		break;
-
-	case 'a':
-		args->allocs_requested = true;
-		args->walk_allocs = true;
-		break;
-
-	case 'i':
-		/* Specific items being requested- clear them all to start */
-		if (!args->items_requested) {
-			clear_items();
-			if (!args->allocs_requested)
-				args->walk_allocs = false;
-			args->items_requested = true;
-		}
-
-		subopts = arg;
-		while (*subopts != '\0' && !parse_err) {
-			switch (getsubopt(&subopts, item_tokens, &value)) {
-			case INODE_OPT:
-				args->print_inodes = true;
-				break;
-			case XATTR_OPT:
-				args->print_xattrs = true;
-				break;
-			case DIRENT_OPT:
-				args->print_dirents = true;
-				break;
-			case SYMLINK_OPT:
-				args->print_symlinks = true;
-				break;
-			case BACKREF_OPT:
-				args->print_backrefs = true;
-				break;
-			case EXTENT_OPT:
-				args->print_extents = true;
-				break;
-			default:
-				argp_usage(state);
-				parse_err = true;
-				break;
-			}
-		}
-		break;
-
-	case 'r':
-		/* Specific roots being requested- clear them all to start */
-		if (!args->roots_requested) {
-			clear_roots();
-			if (!args->allocs_requested)
-				args->walk_allocs = false;
-			args->roots_requested = true;
-		}
-
-		subopts = arg;
-		while (*subopts != '\0' && !parse_err) {
-			switch (getsubopt(&subopts, root_tokens, &value)) {
-			case LOGS_OPT:
-				args->walk_logs_root = true;
-				break;
-			case FS_OPT:
-				args->walk_fs_root = true;
-				break;
-			case SRCH_OPT:
-				args->walk_srch_root = true;
-				break;
-			default:
-				argp_usage(state);
-				parse_err = true;
-				break;
-			}
-		}
-		break;
-
 	case ARGP_KEY_ARG:
 		if (!args->meta_device)
 			args->meta_device = strdup_or_error(state, arg);
 		else
 			argp_error(state, "more than one argument given");
 		break;
-
 	case ARGP_KEY_FINI:
 		if (!args->meta_device)
 			argp_error(state, "no metadata device argument given");
-
-		/*
-		 * For backwards compatibility, translate -S. Should we warn if
-		 * this conflicts with other explicit options?
-		 */
-		if (args->skip_likely_huge) {
-			if (!args->allocs_requested)
-				args->walk_allocs = false;
-			args->walk_fs_root = false;
-			args->walk_srch_root = false;
-		}
-
 		break;
-
 	default:
 		break;
 	}
@@ -1390,10 +1192,7 @@ static int parse_opt(int key, char *arg, struct argp_state *state)
 }
 
 static struct argp_option options[] = {
-	{ "allocs", 'a', NULL, 0, "Print metadata and data alloc lists" },
-	{ "items", 'i', "ITEMS", 0, "Item(s) to print (inode, xattr, dirent, symlink, backref, extent)" },
-	{ "roots", 'r', "ROOTS", 0, "Tree root(s) to walk (logs, srch, fs)" },
-	{ "skip-likely-huge", 'S', NULL, 0, "Skip allocs, srch root and fs root to minimize output size" },
+	{ "skip-likely-huge", 'S', NULL, 0, "Skip large structures to minimize output size"},
 	{ NULL }
 };
 
@@ -1406,15 +1205,17 @@ static struct argp argp = {
 
 static int print_cmd(int argc, char **argv)
 {
+	struct print_args print_args = {NULL};
 	int ret;
 
 	ret = argp_parse(&argp, argc, argv, 0, NULL, &print_args);
 	if (ret)
 		return ret;
 
-	return do_print();
+	return do_print(&print_args);
 }
 
+
 static void __attribute__((constructor)) print_ctor(void)
 {
 	cmd_register_argp("print", &argp, GROUP_DEBUG, print_cmd);