scoutfs/tests/run-tests.sh

#!/usr/bin/bash

# Force system tools to use ASCII quotes
export LC_ALL=C

#
# XXX
#  - could have helper functions for waiting for pids
#  - *always* be gathering traces?  just slow ones?
#  - would be nice to show running resource consumption
#  - sample quorum from super instead of option (wrong w/o -m mkfs)
#  - tracing options are not great, should be smarter
#

msg() {
	echo "[== $@ ==]"
}

die() {
	msg "$@, exiting"
	exit 1
}

timestamp()
{
	date '+%F %T.%N'
}

# output a message with a timestamp to the run.log
log()
{
	echo "[$(timestamp)] $*" >> "$T_RESULTS/run.log"
}

# run a logged command, exiting if it fails
cmd() {
	log "$*"
	"$@" >> "$T_RESULTS/run.log" 2>&1 || \
		die "cmd failed (check the run.log)"
}

show_help()
{
cat << EOF
$(basename $0) options:
    -a        | Abort after the first test failure, leave fs mounted.
    -D <file> | Specify the filesystem's data device path that contains the
              | file system to be tested.  Will be clobbered by -m mkfs.
    -E <re>   | Exclude tests whose file name matches the regular expression.
              | Can be provided multiple times
    -e <file> | Specify an extra storage data device for testing.  Will be clobbered.
    -f <file> | Specify an extra storage meta device for testing.  Will be clobbered.
    -F        | Dump accumulated ftrace buffer to the console on oops.
    -I <re>   | Include tests whose file name matches the regular expression.
              | By default all tests are run.  If this is provided then
              | only tests matching will be run.  Can be provided multiple
              | times
    -i        | Force removing and inserting the built scoutfs.ko module.
    -l <nr>   | Loop each test <nr> times while passing, last run counts.
    -M <file> | Specify the filesystem's meta data device path that contains
              | the file system to be tested.  Will be clobbered by -m mkfs.
    -m        | Run mkfs on the device before mounting and running
              | tests.  Implies unmounting existing mounts first.
    -n <nr>   | The number of devices and mounts to test.
    -o <opts> | Add option string to all mounts during all tests.
    -P        | Enable trace_printk.
    -p        | Exit script after preparing mounts only, don't run tests.
    -q <nr>   | The first <nr> mounts will be quorum members.  Must be
              | at least 1 and no greater than -n number of mounts.
    -r <dir>  | Specify the directory in which to store results of
              | test runs.  The directory will be created if it doesn't
              | exist.  Previous results will be deleted as each test runs.
    -R        | shuffle the test order randomly using shuf
    -s        | Skip git repo checkouts.
    -t        | Enabled trace events that match the given glob argument.
              | Multiple options enable multiple globbed events.
    -T <nr>   | Multiply the original trace buffer size by nr during the run.
    -V <nr>   | Set mkfs device format version.
    -X        | xfstests git repo. Used by tests/xfstests.sh.
    -x        | xfstests git branch to checkout and track.
    -y        | xfstests ./check additional args
    -z <nr>   | set data-alloc-zone-blocks in mkfs
EOF
}

# unset all the T_ variables
for v in ${!T_*}; do
	eval unset $v
done

# set some T_ defaults
T_TRACE_DUMP="0"
T_TRACE_PRINTK="0"
T_PORT_START="19700"
T_LOOP_ITER="30"
T_INCLUDE="quorum-heartbeat-timeout"

# array declarations to be able to use array ops
declare -a T_TRACE_GLOB

while true; do
	case $1 in
	-a)
		T_ABORT="1"
		;;
	-D)
		test -n "$2" || die "-d must have device file argument"
		T_DATA_DEVICE="$2"
		shift
		;;
	-E)
		test -n "$2" || die "-E must have test exclusion regex argument"
		T_EXCLUDE+="-e '$2' "
		shift
		;;
	-e)
		test -n "$2" || die "-e must have extra device file argument"
		T_EX_DATA_DEV="$2"
		shift
		;;
	-f)
		test -n "$2" || die "-e must have extra device file argument"
		T_EX_META_DEV="$2"
		shift
		;;
	-F)
		T_TRACE_DUMP="1"
		;;
	-I)
		test -n "$2" || die "-I must have test incusion regex argument"
		T_INCLUDE+="-e '$2' "
		shift
		;;
	-i)
		T_INSMOD="1"
		;;
	-l)
	        test -n "$2" || die "-l must have a nr iterations argument"
		test "$2" -eq "$2" 2>/dev/null || die "-l <nr> argument must be an integer"
		T_LOOP_ITER="$2"
		shift
		;;
	-M)
	        test -n "$2" || die "-z must have meta device file argument"
	        T_META_DEVICE="$2"
		shift
		;;
	-m)
		T_MKFS="1"
		;;
	-n)
		test -n "$2" || die "-n must have nr mounts argument"
		T_NR_MOUNTS="$2"
		shift
		;;
	-o)
		test -n "$2" || die "-o must have option string argument"
		# always appending to existing options
		T_MNT_OPTIONS+=",$2"
		shift
		;;
	-P)
		T_TRACE_PRINTK="1"
		;;
	-p)
		T_PREPARE="1"
		;;
	-q)
		test -n "$2" || die "-q must have quorum count argument"
		T_QUORUM="$2"
		shift
		;;
	-r)
		test -n "$2" || die "-r must have results dir argument"
		T_RESULTS="$2"
		shift
		;;
	-R)
		T_SHUF="1"
		;;
	-s)
	        T_SKIP_CHECKOUT="1"
		;;
	-t)
		test -n "$2" || die "-t must have trace glob argument"
		T_TRACE_GLOB+=("$2")
		shift
		;;
	-T)
		test -n "$2" || die "-T must have trace buffer size multiplier argument"
		T_TRACE_MULT="$2"
		shift
		;;
	-V)
		test -n "$2" || die "-V must have a format version argument"
		T_MKFS_FORMAT_VERSION="-V $2"
		shift
		;;
	-X)
		test -n "$2" || die "-X requires xfstests git repo dir argument"
		T_XFSTESTS_REPO="$2"
		shift
		;;
	-x)
		test -n "$2" || die "-x requires xfstests git branch argument"
		T_XFSTESTS_BRANCH="$2"
		shift
		;;
	-y)
		test -n "$2" || die "-x requires xfstests ./check args argument"
		T_XFSTESTS_ARGS="$2"
		shift
		;;
	-z)
		test -n "$2" || die "-z must have nr mounts argument"
		T_DATA_ALLOC_ZONE_BLOCKS="-z $2"
		shift
		;;
	-h|-\?|--help)
		show_help
		exit 1
		;;
	--)
		break
		;;
	-?*)
		printf 'WARN: Unknown option: %s\n' "$1" >&2
		show_help
		exit 1
		;;
	*)
		break
		;;
	esac

	shift
done

test -n "$T_DATA_DEVICE" || die "must specify -D data device"
test -e "$T_DATA_DEVICE" || die "data device -D '$T_DATA_DEVICE' doesn't exist"
test -n "$T_META_DEVICE" || die "must specify -M meta device"
test -e "$T_META_DEVICE" || die "meta device -M '$T_META_DEVICE' doesn't exist"

test -n "$T_EX_META_DEV" || die "must specify -f extra meta device"
test -e "$T_EX_META_DEV" || die "extra meta device -f '$T_EX_META_DEV' doesn't exist"
test -n "$T_EX_DATA_DEV" || die "must specify -e extra data device"
test -e "$T_EX_DATA_DEV" || die "extra data device -e '$T_EX_DATA_DEV' doesn't exist"

test -n "$T_RESULTS" || die "must specify -r results dir"
test -n "$T_XFSTESTS_REPO" -a -z "$T_XFSTESTS_BRANCH" -a -z "$T_SKIP_CHECKOUT" && \
	die "-X xfstests repo requires -x xfstests branch"
test -n "$T_XFSTESTS_BRANCH" -a -z "$T_XFSTESTS_REPO" -a -z "$T_SKIP_CHECKOUT" && \
	die "-X xfstests branch requires -x xfstests repo"

test -n "$T_NR_MOUNTS" || die "must specify -n nr mounts"
test "$T_NR_MOUNTS" -ge 1 -a "$T_NR_MOUNTS" -le 8 || \
	 die "-n nr mounts must be >= 1 and <= 8"
test -n "$T_QUORUM" || \
	 die "must specify -q number of mounts that are quorum members"
test "$T_QUORUM" -ge "1" || \
	 die "-q quorum mmembers must be at least 1"
test "$T_QUORUM" -le "$T_NR_MOUNTS" || \
	 die "-q quorum mmembers must not be greater than -n mounts"

# top level paths
T_TESTS=$(realpath "$(dirname $0)")
T_KMOD=$(realpath "$T_TESTS/../kmod")
T_UTILS=$(realpath "$T_TESTS/../utils")

test -d "$T_KMOD" || die "kmod/ repo dir $T_KMOD not directory"
test -d "$T_UTILS" || die "utils/ repo dir $T_UTILS not directory"

# canonicalize paths
for e in T_META_DEVICE T_DATA_DEVICE T_EX_META_DEV T_EX_DATA_DEV T_KMOD T_RESULTS T_UTILS T_XFSTESTS_REPO; do
	eval $e=\"$(readlink -f "${!e}")\"
done

# try and check ports, but not necessary
T_TEST_PORT="$T_PORT_START"
T_SCRATCH_PORT="$((T_PORT_START + 100))"
T_DEV_PORT="$((T_PORT_START + 200))"
read local_start local_end < /proc/sys/net/ipv4/ip_local_port_range
if [ -n "$local_start" -a -n "$local_end" -a "$local_start" -lt "$local_end" ]; then
	if [ ! "$T_DEV_PORT" -lt "$local_start" -a ! "$T_TEST_PORT" -gt "$local_end" ]; then
		die "listening port range $T_TEST_PORT - $T_DEV_PORT is within local dynamic port range $local_start - $local_end in /proc/sys/net/ipv4/ip_local_port_range"
	fi
fi

# permute sequence?
T_SEQUENCE=sequence
if [ -n "$T_SHUF" ]; then
	msg "shuffling test order"
	shuf sequence -o sequence.shuf
	# keep xfstests at the end
	if grep -q 'xfstests.sh' sequence.shuf ; then
		sed -i '/xfstests.sh/d' sequence.shuf
		echo "xfstests.sh" >> sequence.shuf
	fi
	T_SEQUENCE=sequence.shuf
fi

# include everything by default
test -z "$T_INCLUDE" && T_INCLUDE="-e '.*'"
# (quickly) exclude nothing by default
test -z "$T_EXCLUDE" && T_EXCLUDE="-e '\Zx'"

# eval to strip re ticks but not expand
tests=$(grep -v "^#" $T_SEQUENCE |
	eval grep "$T_INCLUDE" | eval grep -v "$T_EXCLUDE")
test -z "$tests" && \
	die "no tests found by including $T_INCLUDE and excluding $T_EXCLUDE"

# create results dir 
test -e "$T_RESULTS" || mkdir -p "$T_RESULTS"
test -d "$T_RESULTS" || \
	 die "$T_RESULTS dir is not a directory"

# might as well build our stuff with all cpus, assuming idle system
MAKE_ARGS="-j $(getconf _NPROCESSORS_ONLN)"

# build kernel module
msg "building kmod/ dir $T_KMOD"
cmd cd "$T_KMOD"
cmd make $MAKE_ARGS
cmd sync
cmd cd -

# build utils
msg "building utils/ dir $T_UTILS"
cmd cd "$T_UTILS"
cmd make $MAKE_ARGS
cmd sync
cmd cd -

# we can now run the built scoutfs binary, prefer over installed
PATH="$T_UTILS/src:$PATH"

# verify xfstests branch
if [ -n "$T_XFSTESTS_REPO" ] && [ -z "$T_SKIP_CHECKOUT" ]; then
	msg "verifying xfstests repo $T_XFSTESTS_REPO branch $T_XFSTESTS_BRANCH"
	cmd cd "$T_XFSTESTS_REPO"
	cmd git rev-parse --verify "$T_XFSTESTS_BRANCH"
	cmd cd -
fi

# building our test binaries
msg "building test binaries"
cmd make $MAKE_ARGS

# set any options implied by others 
test -n "$T_MKFS" && T_UNMOUNT=1
test -n "$T_INSMOD" && T_UNMOUNT=1

#
# unmount concurrently because the final quorum can only unmount once
# they're all unmounting.  We unmount all mounts because we might be
# removing the module.
#
unmount_all() {
	msg "unmounting all scoutfs mounts"
	pids=""
	for m in $(findmnt -t scoutfs -o TARGET); do
		if [ -d "$m" ]; then
			cmd umount "$m" &
			p="$!"
			pids="$pids $!"
		fi
	done
	for p in $pids; do
		cmd wait $p
	done

	# delete all temp devices
	for dev in /dev/mapper/_scoutfs_test_*; do
		if [ -b "$dev" ]; then
			cmd dmsetup remove $dev
		fi
	done
}
if [ -n "$T_UNMOUNT" ]; then
	unmount_all
fi

quo=""
if [ -n "$T_MKFS" ]; then
	for i in $(seq -0 $((T_QUORUM - 1))); do
		quo="$quo -Q $i,127.0.0.1,$((T_TEST_PORT + i))"
	done

	msg "making new filesystem with $T_QUORUM quorum members"
	cmd scoutfs mkfs -f $quo $T_DATA_ALLOC_ZONE_BLOCKS $T_MKFS_FORMAT_VERSION \
		"$T_META_DEVICE" "$T_DATA_DEVICE"
fi

if [ -n "$T_INSMOD" ]; then
	msg "removing and reinserting scoutfs module"
	test -e /sys/module/scoutfs && cmd rmmod scoutfs
	cmd modprobe libcrc32c
	T_MODULE="$T_KMOD/src/scoutfs.ko"
	cmd insmod "$T_MODULE"
fi

if [ -n "$T_TRACE_MULT" ]; then
	orig_trace_size=$(cat /sys/kernel/debug/tracing/buffer_size_kb)
	mult_trace_size=$((orig_trace_size * T_TRACE_MULT))
	msg "increasing trace buffer size from $orig_trace_size KiB to $mult_trace_size KiB"
	echo $mult_trace_size > /sys/kernel/debug/tracing/buffer_size_kb
fi

nr_globs=${#T_TRACE_GLOB[@]}
if [ $nr_globs -gt 0 ]; then
	echo 0 > /sys/kernel/debug/tracing/events/scoutfs/enable

	for g in "${T_TRACE_GLOB[@]}"; do
		for e in /sys/kernel/debug/tracing/events/scoutfs/$g/enable; do
			if test -w "$e"; then
				echo 1 > "$e"
			else
				die "-t glob '$g' matched no scoutfs events"
			fi
		done
	done

	nr_events=$(cat /sys/kernel/debug/tracing/set_event | wc -l)
	msg "enabled $nr_events trace events from $nr_globs -t globs"
fi

if [ -n "$T_TRACE_PRINTK" ]; then
	echo "$T_TRACE_PRINTK" > /sys/kernel/debug/tracing/options/trace_printk
fi

if [ -n "$T_TRACE_DUMP" ]; then
	echo "$T_TRACE_DUMP" > /proc/sys/kernel/ftrace_dump_on_oops
fi

# always describe tracing in the logs
cmd cat /sys/kernel/debug/tracing/set_event
cmd grep .  /sys/kernel/debug/tracing/options/trace_printk \
	    /sys/kernel/debug/tracing/buffer_size_kb \
	    /proc/sys/kernel/ftrace_dump_on_oops

# we can record pids to kill as we exit, we kill in reverse added order
atexit_kill_pids=""
add_atexit_kill_pid()
{
	atexit_kill_pids="$1 $atexit_kill_pids"
}
atexit_kill()
{
	local pid

	# suppress bg function exited messages
	exec {ERR}>&2 2>/dev/null

	for pid in $atexit_kill_pids; do
		if test -e "/proc/$pid/status" ; then
			kill "$pid"
			wait "$pid"
		fi
	done

	exec 2>&$ERR {ERR}>&-
}
trap atexit_kill EXIT

#
# Build a fenced config that runs scripts out of the repository rather
# than the default system directory
#
conf="$T_RESULTS/scoutfs-fenced.conf"
cat > $conf << EOF
SCOUTFS_FENCED_DELAY=1
SCOUTFS_FENCED_RUN=$T_TESTS/fenced-local-force-unmount.sh
SCOUTFS_FENCED_RUN_ARGS="ignored run args"
EOF
export SCOUTFS_FENCED_CONFIG_FILE="$conf"
T_FENCED_LOG="$T_RESULTS/fenced.log"

$T_UTILS/fenced/scoutfs-fenced > "$T_FENCED_LOG" 2>&1 &
fenced_pid=$!
add_atexit_kill_pid $fenced_pid

#
# some critical failures will cause fs operations to hang.  We can watch
# for evidence of them and cause the system to crash, at least.
#
crash_monitor()
{
	local bad=0

	while sleep 1; do
		if dmesg | grep -q "inserting extent.*overlaps existing"; then
			echo "run-tests monitor saw overlapping extent message"
			bad=1
		fi

		if dmesg | grep -q "error indicated by fence action" ; then
			echo "run-tests monitor saw fence agent error message"
			bad=1
		fi

		if [ ! -e "/proc/${fenced_pid}/status" ]; then
			echo "run-tests monitor didn't see fenced pid $fenced_pid /proc dir"
			bad=1
		fi

		if [ "$bad" != 0 ]; then
			echo "run-tests monitor triggering crash"
			echo c > /proc/sysrq-trigger
			exit 1
		fi
	done
}
crash_monitor &
add_atexit_kill_pid $!

# setup dm tables
echo "0 $(blockdev --getsz $T_META_DEVICE) linear $T_META_DEVICE 0" > \
	$T_RESULTS/dmtable.meta
echo "0 $(blockdev --getsz $T_DATA_DEVICE) linear $T_DATA_DEVICE 0" > \
	$T_RESULTS/dmtable.data

#
# mount concurrently so that a quorum is present to elect the leader and
# start a server.
#
msg "mounting $T_NR_MOUNTS mounts on meta $T_META_DEVICE data $T_DATA_DEVICE"
pids=""
for i in $(seq 0 $((T_NR_MOUNTS - 1))); do

	name="_scoutfs_test_meta_$i"
	cmd dmsetup create "$name" --table "$(cat $T_RESULTS/dmtable.meta)"
	meta_dev="/dev/mapper/$name"

	name="_scoutfs_test_data_$i"
	cmd dmsetup create "$name" --table "$(cat $T_RESULTS/dmtable.data)"
	data_dev="/dev/mapper/$name"

	dir="/mnt/test.$i"
	test -d "$dir" || cmd mkdir -p "$dir"

	opts="-o metadev_path=$meta_dev"
	if [ "$i" -lt "$T_QUORUM" ]; then
		opts="$opts,quorum_slot_nr=$i"
	fi
	opts="${opts}${T_MNT_OPTIONS}"

	msg "mounting $meta_dev|$data_dev on $dir"
	cmd mount -t scoutfs $opts "$data_dev" "$dir" &

	p="$!"
	pids="$pids $!"
	log "background mount $i pid $p"

	eval T_O$i=\"$opts\"
	T_O[$i]="$opts"
	T_OS+="$opts "

	eval T_MB$i=$meta_dev
	T_MB[$i]=$meta_dev
	T_MBS+="$meta_dev "

	eval T_DB$i=$data_dev
	T_DB[$i]=$data_dev
	T_DBS+="$data_dev "

	eval T_M$i=\"$dir\"
	T_M[$i]=$dir
	T_MS+="$dir "

done
for p in $pids; do
	log "waiting for background mount pid $p"
	cmd wait $p
done

if [ -n "$T_PREPARE" ]; then
	findmnt -t scoutfs
	msg "-p given, exiting after preparing mounts"
	exit 0
fi

# we need the STATUS definitions and filters
. funcs/exec.sh
. funcs/filter.sh

# give tests access to built binaries in src/, prefer over installed
export PATH="$PWD/src:$PATH"

msg "running tests"
> "$T_RESULTS/skip.log"
> "$T_RESULTS/fail.log"

# generate a test ID to make sure we can de-duplicate TAP results in aggregation
. funcs/tap.sh
t_tap_header $(uuidgen)

testcount=0
passed=0
skipped=0
failed=0
skipped_permitted=0
for t in $tests; do
	# tests has basenames from sequence, get path and name
	t="tests/$t"
	test_name=$(basename "$t" | sed -e 's/.sh$//')

	# get stats from previous pass
	last="$T_RESULTS/last-passed-test-stats"
	stats=$(grep -s "^$test_name " "$last" | cut -d " " -f 2-)
	test -n "$stats" && stats="last: $stats"
	printf "  %-30s $stats" "$test_name"

	# mark in dmesg as to what test we are running
	echo "run scoutfs test $test_name" > /dev/kmsg

	# let the test get at its extra files
	T_EXTRA="$T_TESTS/extra/$test_name"

	for iter in $(seq 1 $T_LOOP_ITER); do

		# create a temporary dir and file path for the test
		T_TMPDIR="$T_RESULTS/tmp/$test_name"
		T_TMP="$T_TMPDIR/tmp"
		cmd rm -rf "$T_TMPDIR"
		cmd mkdir -p "$T_TMPDIR"

		# create a test name dir in the fs, clean up old data as needed
		T_DS=""
		for i in $(seq 0 $((T_NR_MOUNTS - 1))); do
			dir="${T_M[$i]}/test/$test_name"

			test $i == 0 && (
				test -d "$dir" && cmd rm -rf "$dir"
				cmd mkdir -p "$dir"
			)

			eval T_D$i=$dir
			T_D[$i]=$dir
			T_DS+="$dir "
		done

		# export all our T_ variables
		for v in ${!T_*}; do
			eval export $v
		done

		# prepare to compare output to golden output
		test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output"
		out="$T_RESULTS/output/$test_name"
		> "$T_TMPDIR/status.msg"
		golden="golden/$test_name"

		# record dmesg before
		dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.before"

		# give tests stdout and compared output on specific fds
		exec 6>&1
		exec 7>$out

		# run the test with access to our functions
		start_secs=$SECONDS
		bash -c "for f in funcs/*.sh; do . \$f; done; . $t" >&7 2>&1
		sts="$?"
		log "test $t exited with status $sts"
		stats="$((SECONDS - start_secs))s"

		# close our weird descriptors
		exec 6>&-
		exec 7>&-

		# compare output if the test returned passed status
		if [ "$sts" == "$T_PASS_STATUS" ]; then
			if [ ! -e "$golden" ]; then
				message="no golden output"
				sts=$T_FAIL_STATUS
			elif ! cmp -s "$golden" "$out"; then 
				message="output differs"
				sts=$T_FAIL_STATUS
				diff -u "$golden" "$out" >> "$T_RESULTS/fail.log"
			fi
		else
			# get message from t_*() functions
			message=$(cat "$T_TMPDIR/status.msg")
		fi

		# see if anything unexpected was added to dmesg
		if [ "$sts" == "$T_PASS_STATUS" ]; then
			dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.after"
			diff --old-line-format="" --unchanged-line-format="" \
				"$T_TMPDIR/dmesg.before" "$T_TMPDIR/dmesg.after" > \
				"$T_TMPDIR/dmesg.new"

			if [ -s "$T_TMPDIR/dmesg.new" ]; then
				message="unexpected messages in dmesg"
				sts=$T_FAIL_STATUS
				cat "$T_TMPDIR/dmesg.new" >> "$T_RESULTS/fail.log"
			fi
		fi

		# record unknown exit status
		if [ "$sts" -lt "$T_FIRST_STATUS" -o "$sts" -gt "$T_LAST_STATUS" ]; then
			message="unknown status: $sts"
			sts=$T_FAIL_STATUS
		fi

		# stop looping if we didn't pass
		if [ "$sts" != "$T_PASS_STATUS" ]; then
			break;
		fi
	done

	# show and record the result of the test
	if [ "$sts" == "$T_PASS_STATUS" ]; then
		echo "  passed: $stats"
		((passed++))
		# save stats for passed test
		grep -s -v "^$test_name " "$last" > "$last.tmp"
		echo "$test_name $stats" >> "$last.tmp"
		mv -f "$last.tmp" "$last"
	elif [ "$sts" == "$T_SKIP_PERMITTED_STATUS" ]; then
		echo "  [ skipped (permitted): $message ]"
		echo "$test_name skipped (permitted) $message " >> "$T_RESULTS/skip.log"
		((skipped_permitted++))
	elif [ "$sts" == "$T_SKIP_STATUS" ]; then
		echo "  [ skipped: $message ]"
		echo "$test_name $message" >> "$T_RESULTS/skip.log"
		((skipped++))
	elif [ "$sts" == "$T_FAIL_STATUS" ]; then
		echo "  [ failed: $message ]"
		echo "$test_name $message" >> "$T_RESULTS/fail.log"
		((failed++))

		test -n "$T_ABORT" && die "aborting after first failure"
	fi

	# record results for TAP format output
	t_tap_progress $test_name $sts
	((testcount++))

done

msg "all tests run: $passed passed, $skipped skipped, $skipped_permitted skipped (permitted), $failed failed"


if [ -n "$T_TRACE_GLOB" -o -n "$T_TRACE_PRINTK" ]; then
	msg "saving traces and disabling tracing"
	echo 0 > /sys/kernel/debug/tracing/events/scoutfs/enable
	echo 0 > /sys/kernel/debug/tracing/options/trace_printk
	cat /sys/kernel/debug/tracing/trace > "$T_RESULTS/traces"
	if [ -n "$orig_trace_size" ]; then
		echo $orig_trace_size > /sys/kernel/debug/tracing/buffer_size_kb
	fi
fi

if [ "$skipped" == 0 -a "$failed" == 0 ]; then
	msg "all tests passed"
	unmount_all
	exit 0
fi

if [ "$skipped" != 0 ]; then
	msg "$skipped tests skipped, check skip.log, still mounted"
fi
if [ "$failed" != 0 ]; then
	msg "$failed tests failed, check fail.log, still mounted"
fi
exit 1