From 24031cde1d545545791f2e83932e8add662f791b Mon Sep 17 00:00:00 2001
From: Auke Kok <auke.kok@versity.com>
Date: Mon, 14 Apr 2025 15:11:55 -0700
Subject: [PATCH] TAP formatted output.

Stored as `results/scoutfs.tap`, this file contains TAP format 14
generated test results.

Embedded in the output are some metadata so that these files can be
aggregated and stored in an unique and deduplicating way, but using a
generated UUID at the start of testing. The file itself also catches git
ID, date, and kernel version, as well as the (possibly altered) test
sequence used.

Any test that has diff or dmesg output will be considered failed, and a
copy of the relevant data is included as comments.

Signed-off-by: Auke Kok <auke.kok@versity.com>
---
 tests/funcs/tap.sh | 88 ++++++++++++++++++++++++++++++++++++++++++++++
 tests/run-tests.sh | 10 ++++++
 2 files changed, 98 insertions(+)
 create mode 100644 tests/funcs/tap.sh

diff --git a/tests/funcs/tap.sh b/tests/funcs/tap.sh
new file mode 100644
index 00000000..04f9b8bc
--- /dev/null
+++ b/tests/funcs/tap.sh
@@ -0,0 +1,88 @@
+
+#
+# Generate TAP format test results
+#
+
+t_tap_header()
+{
+	local runid=$1
+	local sequence=( $(echo $tests) )
+	local count=${#sequence[@]}
+
+	# avoid recreating the same TAP result over again - harness sets this
+	[[ -z "$runid" ]] && runid="*test*"
+
+	cat > $T_RESULTS/scoutfs.tap <<TAPEOF
+TAP version 14
+1..${count}
+#
+# TAP results for run ${runid}
+#
+# host/run info:
+#
+#   hostname: ${HOSTNAME}
+#   test start time: $(date --utc)
+#   uname -r: $(uname -r)
+#   scoutfs commit id: $(git describe --tags)
+#
+# sequence for this run:
+#
+TAPEOF
+
+	# Sequence
+	for t in ${tests}; do
+		 echo ${t/.sh/}
+	done | cat -n | expand | column -c 120 | expand | sed 's/^ /#/' >> $T_RESULTS/scoutfs.tap
+	echo "#" >> $T_RESULTS/scoutfs.tap
+}
+
+t_tap_progress()
+{
+(
+	local i=$(( testcount + 1 ))
+	local testname=$1
+	local result=$2
+
+	local diff=""
+	local dmsg=""
+
+	if [[ -s "$T_RESULTS/tmp/${testname}/dmesg.new" ]]; then
+		dmsg="1"
+	fi
+
+	if ! cmp -s golden/${testname} $T_RESULTS/output/${testname}; then
+		diff="1"
+	fi
+
+	if [[ "${result}" == "100" ]] && [[ -z "${dmsg}" ]] && [[ -z "${diff}" ]]; then
+		echo "ok ${i} - ${testname}"
+	elif [[ "${result}" == "103" ]]; then
+		echo "ok ${i} - ${testname}"
+		echo "# ${testname} ** skipped - permitted **"
+	else
+		echo "not ok ${i} - ${testname}"
+		case ${result} in
+		101)
+			echo "# ${testname} ** skipped **"
+			;;
+		102)
+			echo "# ${testname} ** failed **"
+			;;
+		esac
+
+		if [[ -n "${diff}" ]]; then
+			echo "#"
+			echo "# diff:"
+			echo "#"
+			diff -u golden/${testname} $T_RESULTS/output/${testname} | expand | sed 's/^/#   /'
+		fi
+
+		if [[ -n "${dmsg}" ]]; then
+			echo "#"
+			echo "# dmesg:"
+			echo "#"
+			cat "$T_RESULTS/tmp/${testname}/dmesg.new" | sed 's/^/#   /'
+		fi
+	fi
+) >> $T_RESULTS/scoutfs.tap
+}
diff --git a/tests/run-tests.sh b/tests/run-tests.sh
index 891f7dfb..3bc494c0 100755
--- a/tests/run-tests.sh
+++ b/tests/run-tests.sh
@@ -512,6 +512,11 @@ msg "running tests"
 > "$T_RESULTS/skip.log"
 > "$T_RESULTS/fail.log"
 
+# generate a test ID to make sure we can de-duplicate TAP results in aggregation
+. funcs/tap.sh
+t_tap_header $(uuidgen)
+
+testcount=0
 passed=0
 skipped=0
 failed=0
@@ -637,6 +642,11 @@ for t in $tests; do
 
 		test -n "$T_ABORT" && die "aborting after first failure"
 	fi
+
+	# record results for TAP format output
+	t_tap_progress $test_name $sts
+	((testcount++))
+
 done
 
 msg "all tests run: $passed passed, $skipped skipped, $skipped_permitted skipped (permitted), $failed failed"