From 7d14b57b2d74d1114fd5f16a634007856d1562e1 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 6 Nov 2025 11:02:38 -0800 Subject: [PATCH 01/12] Export PATH once in run-tests Might as well just export the PATH once as we change it, no need to export it in every test iteration. Signed-off-by: Zach Brown --- tests/run-tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/run-tests.sh b/tests/run-tests.sh index f3351775..74215002 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -535,7 +535,7 @@ fi . funcs/filter.sh # give tests access to built binaries in src/, prefer over installed -PATH="$PWD/src:$PATH" +export PATH="$PWD/src:$PATH" msg "running tests" > "$T_RESULTS/skip.log" @@ -580,7 +580,6 @@ for t in $tests; do for v in ${!T_*}; do eval export $v done - export PATH # give test access to scoutfs binary # prepare to compare output to golden output test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output" From 23aaa994dfb334d20faa7f825d56fbcf51bf17fb Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 6 Nov 2025 11:03:21 -0800 Subject: [PATCH 02/12] Add -l to run-tests for looping over tests Add an option to run-tests to have it loop over each test that will be run a number of times. Looping stops if the test doesn't pass. Most of the change in the per-test execution is indenting as we add the for loop block. The stats and kmsg output are lifted up before of the loop. Signed-off-by: Zach Brown --- tests/run-tests.sh | 167 ++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 76 deletions(-) diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 74215002..cbfad39e 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -56,6 +56,7 @@ $(basename $0) options: | only tests matching will be run. Can be provided multiple | times -i | Force removing and inserting the built scoutfs.ko module. + -l | Loop each test times while passing, last run counts. -M | Specify the filesystem's meta data device path that contains | the file system to be tested. Will be clobbered by -m mkfs. -m | Run mkfs on the device before mounting and running @@ -91,6 +92,7 @@ done T_TRACE_DUMP="0" T_TRACE_PRINTK="0" T_PORT_START="19700" +T_LOOP_ITER="1" # array declarations to be able to use array ops declare -a T_TRACE_GLOB @@ -131,6 +133,12 @@ while true; do -i) T_INSMOD="1" ;; + -l) + test -n "$2" || die "-l must have a nr iterations argument" + test "$2" -eq "$2" 2>/dev/null || die "-l argument must be an integer" + T_LOOP_ITER="$2" + shift + ;; -M) test -n "$2" || die "-z must have meta device file argument" T_META_DEVICE="$2" @@ -555,100 +563,107 @@ for t in $tests; do t="tests/$t" test_name=$(basename "$t" | sed -e 's/.sh$//') - # create a temporary dir and file path for the test - T_TMPDIR="$T_RESULTS/tmp/$test_name" - T_TMP="$T_TMPDIR/tmp" - cmd rm -rf "$T_TMPDIR" - cmd mkdir -p "$T_TMPDIR" - - # create a test name dir in the fs, clean up old data as needed - T_DS="" - for i in $(seq 0 $((T_NR_MOUNTS - 1))); do - dir="${T_M[$i]}/test/$test_name" - - test $i == 0 && ( - test -d "$dir" && cmd rm -rf "$dir" - cmd mkdir -p "$dir" - ) - - eval T_D$i=$dir - T_D[$i]=$dir - T_DS+="$dir " - done - - # export all our T_ variables - for v in ${!T_*}; do - eval export $v - done - - # prepare to compare output to golden output - test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output" - out="$T_RESULTS/output/$test_name" - > "$T_TMPDIR/status.msg" - golden="golden/$test_name" - # get stats from previous pass last="$T_RESULTS/last-passed-test-stats" stats=$(grep -s "^$test_name " "$last" | cut -d " " -f 2-) test -n "$stats" && stats="last: $stats" - printf " %-30s $stats" "$test_name" # mark in dmesg as to what test we are running echo "run scoutfs test $test_name" > /dev/kmsg - # record dmesg before - dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.before" + for iter in $(seq 1 $T_LOOP_ITER); do - # give tests stdout and compared output on specific fds - exec 6>&1 - exec 7>$out + # create a temporary dir and file path for the test + T_TMPDIR="$T_RESULTS/tmp/$test_name" + T_TMP="$T_TMPDIR/tmp" + cmd rm -rf "$T_TMPDIR" + cmd mkdir -p "$T_TMPDIR" - # run the test with access to our functions - start_secs=$SECONDS - bash -c "for f in funcs/*.sh; do . \$f; done; . $t" >&7 2>&1 - sts="$?" - log "test $t exited with status $sts" - stats="$((SECONDS - start_secs))s" + # create a test name dir in the fs, clean up old data as needed + T_DS="" + for i in $(seq 0 $((T_NR_MOUNTS - 1))); do + dir="${T_M[$i]}/test/$test_name" - # close our weird descriptors - exec 6>&- - exec 7>&- + test $i == 0 && ( + test -d "$dir" && cmd rm -rf "$dir" + cmd mkdir -p "$dir" + ) - # compare output if the test returned passed status - if [ "$sts" == "$T_PASS_STATUS" ]; then - if [ ! -e "$golden" ]; then - message="no golden output" - sts=$T_FAIL_STATUS - elif ! cmp -s "$golden" "$out"; then - message="output differs" - sts=$T_FAIL_STATUS - diff -u "$golden" "$out" >> "$T_RESULTS/fail.log" + eval T_D$i=$dir + T_D[$i]=$dir + T_DS+="$dir " + done + + # export all our T_ variables + for v in ${!T_*}; do + eval export $v + done + + # prepare to compare output to golden output + test -e "$T_RESULTS/output" || cmd mkdir -p "$T_RESULTS/output" + out="$T_RESULTS/output/$test_name" + > "$T_TMPDIR/status.msg" + golden="golden/$test_name" + + # record dmesg before + dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.before" + + # give tests stdout and compared output on specific fds + exec 6>&1 + exec 7>$out + + # run the test with access to our functions + start_secs=$SECONDS + bash -c "for f in funcs/*.sh; do . \$f; done; . $t" >&7 2>&1 + sts="$?" + log "test $t exited with status $sts" + stats="$((SECONDS - start_secs))s" + + # close our weird descriptors + exec 6>&- + exec 7>&- + + # compare output if the test returned passed status + if [ "$sts" == "$T_PASS_STATUS" ]; then + if [ ! -e "$golden" ]; then + message="no golden output" + sts=$T_FAIL_STATUS + elif ! cmp -s "$golden" "$out"; then + message="output differs" + sts=$T_FAIL_STATUS + diff -u "$golden" "$out" >> "$T_RESULTS/fail.log" + fi + else + # get message from t_*() functions + message=$(cat "$T_TMPDIR/status.msg") fi - else - # get message from t_*() functions - message=$(cat "$T_TMPDIR/status.msg") - fi - # see if anything unexpected was added to dmesg - if [ "$sts" == "$T_PASS_STATUS" ]; then - dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.after" - diff --old-line-format="" --unchanged-line-format="" \ - "$T_TMPDIR/dmesg.before" "$T_TMPDIR/dmesg.after" > \ - "$T_TMPDIR/dmesg.new" + # see if anything unexpected was added to dmesg + if [ "$sts" == "$T_PASS_STATUS" ]; then + dmesg | t_filter_dmesg > "$T_TMPDIR/dmesg.after" + diff --old-line-format="" --unchanged-line-format="" \ + "$T_TMPDIR/dmesg.before" "$T_TMPDIR/dmesg.after" > \ + "$T_TMPDIR/dmesg.new" - if [ -s "$T_TMPDIR/dmesg.new" ]; then - message="unexpected messages in dmesg" - sts=$T_FAIL_STATUS - cat "$T_TMPDIR/dmesg.new" >> "$T_RESULTS/fail.log" + if [ -s "$T_TMPDIR/dmesg.new" ]; then + message="unexpected messages in dmesg" + sts=$T_FAIL_STATUS + cat "$T_TMPDIR/dmesg.new" >> "$T_RESULTS/fail.log" + fi fi - fi - # record unknown exit status - if [ "$sts" -lt "$T_FIRST_STATUS" -o "$sts" -gt "$T_LAST_STATUS" ]; then - message="unknown status: $sts" - sts=$T_FAIL_STATUS - fi + # record unknown exit status + if [ "$sts" -lt "$T_FIRST_STATUS" -o "$sts" -gt "$T_LAST_STATUS" ]; then + message="unknown status: $sts" + sts=$T_FAIL_STATUS + fi + + # stop looping if we didn't pass + if [ "$sts" != "$T_PASS_STATUS" ]; then + break; + fi + done # show and record the result of the test if [ "$sts" == "$T_PASS_STATUS" ]; then From a077104531f12f33fd13fcd877f68a722db889f8 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 6 Nov 2025 11:54:40 -0800 Subject: [PATCH 03/12] Add crash monitor to run-tests Add a little background function that runs during the test which triggers a crash if it finds catastrophic failure conditions. This is the second bg task we want to kill and we can only have one function run on the EXIT trap, so we create a generic process killing trap function. We feed it the fenced pid as well. run-tests didn't log much of value into the fenced log, and we're not logging the kills into anymore, so we just remove run-tests fenced logging. Signed-off-by: Zach Brown --- tests/run-tests.sh | 77 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/tests/run-tests.sh b/tests/run-tests.sh index cbfad39e..f7e6d354 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -438,6 +438,30 @@ cmd grep . /sys/kernel/debug/tracing/options/trace_printk \ /sys/kernel/debug/tracing/buffer_size_kb \ /proc/sys/kernel/ftrace_dump_on_oops +# we can record pids to kill as we exit, we kill in reverse added order +atexit_kill_pids="" +add_atexit_kill_pid() +{ + atexit_kill_pids="$1 $atexit_kill_pids" +} +atexit_kill() +{ + local pid + + # suppress bg function exited messages + exec {ERR}>&2 2>/dev/null + + for pid in $atexit_kill_pids; do + if test -e "/proc/$pid/status" ; then + kill "$pid" + wait "$pid" + fi + done + + exec 2>&$ERR {ERR}>&- +} +trap atexit_kill EXIT + # # Build a fenced config that runs scripts out of the repository rather # than the default system directory @@ -451,26 +475,43 @@ EOF export SCOUTFS_FENCED_CONFIG_FILE="$conf" T_FENCED_LOG="$T_RESULTS/fenced.log" -# -# Run the agent in the background, log its output, an kill it if we -# exit -# -fenced_log() -{ - echo "[$(timestamp)] $*" >> "$T_FENCED_LOG" -} -fenced_pid="" -kill_fenced() -{ - if test -n "$fenced_pid" -a -d "/proc/$fenced_pid" ; then - fenced_log "killing fenced pid $fenced_pid" - kill "$fenced_pid" - fi -} -trap kill_fenced EXIT $T_UTILS/fenced/scoutfs-fenced > "$T_FENCED_LOG" 2>&1 & fenced_pid=$! -fenced_log "started fenced pid $fenced_pid in the background" +add_atexit_kill_pid $fenced_pid + +# +# some critical failures will cause fs operations to hang. We can watch +# for evidence of them and cause the system to crash, at least. +# +crash_monitor() +{ + local bad=0 + + while sleep 1; do + if dmesg | grep -q "inserting extent.*overlaps existing"; then + echo "run-tests monitor saw overlapping extent message" + bad=1 + fi + + if dmesg | grep -q "error indicated by fence action" ; then + echo "run-tests monitor saw fence agent error message" + bad=1 + fi + + if [ ! -e "/proc/${fenced_pid}/status" ]; then + echo "run-tests monitor didn't see fenced pid $fenced_pid /proc dir" + bad=1 + fi + + if [ "$bad" != 0 ]; then + echo "run-tests monitor triggering crash" + echo c > /proc/sysrq-trigger + exit 1 + fi + done +} +crash_monitor & +add_atexit_kill_pid $! # setup dm tables echo "0 $(blockdev --getsz $T_META_DEVICE) linear $T_META_DEVICE 0" > \ From 8484a58dd6a7c80e8aa6cf78da65df9673c45985 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 6 Nov 2025 12:23:44 -0800 Subject: [PATCH 04/12] Have xfstest pass when using args The xfstests's golden output includes the full set of tests we expect to run when no args are specified. If we specify args then the set of tests can change and the test will always fail when they do. This fixes that by having the test check the set of tests itself, rather than relying on golden output. If args are specified then our xfstest only fails if any of the executed xfstest tests failed. Without args, we perform the same scraping of the check output and compare it against the expected results ourself. It would have been a bit much to put that large file inline in the test file, so we add a dir of per-test files in revision control. We can also put the list of exclusions there. We can also clean up the output redirection helper functions to make them more clear. After xfstests has executed we want to redirect output back to the compared output so that we can catch any unexpected output. Signed-off-by: Zach Brown --- tests/README.md | 1 + tests/extra/xfstests/expected-results | 882 ++++++++++++++++++++++++++ tests/extra/xfstests/local.exclude | 44 ++ tests/funcs/exec.sh | 20 +- tests/golden/xfstests | 882 -------------------------- tests/run-tests.sh | 3 + tests/tests/xfstests.sh | 92 +-- 7 files changed, 978 insertions(+), 946 deletions(-) create mode 100644 tests/extra/xfstests/expected-results create mode 100644 tests/extra/xfstests/local.exclude diff --git a/tests/README.md b/tests/README.md index 4acbdb50..8701b6f2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -117,6 +117,7 @@ used during the test. | T\_NR\_MOUNTS | number of mounts | -n | 3 | | T\_O[0-9] | mount options | created per run | -o server\_addr= | | T\_QUORUM | quorum count | -q | 2 | +| T\_EXTRA | per-test file dir | revision ctled | tests/extra/t | | T\_TMP | per-test tmp prefix | made for test | results/tmp/t/tmp | | T\_TMPDIR | per-test tmp dir dir | made for test | results/tmp/t | diff --git a/tests/extra/xfstests/expected-results b/tests/extra/xfstests/expected-results new file mode 100644 index 00000000..c4032ca9 --- /dev/null +++ b/tests/extra/xfstests/expected-results @@ -0,0 +1,882 @@ +Ran: +generic/001 +generic/002 +generic/004 +generic/005 +generic/006 +generic/007 +generic/008 +generic/009 +generic/011 +generic/012 +generic/013 +generic/014 +generic/015 +generic/016 +generic/018 +generic/020 +generic/021 +generic/022 +generic/023 +generic/024 +generic/025 +generic/026 +generic/028 +generic/029 +generic/030 +generic/031 +generic/032 +generic/033 +generic/034 +generic/035 +generic/037 +generic/039 +generic/040 +generic/041 +generic/050 +generic/052 +generic/053 +generic/056 +generic/057 +generic/058 +generic/059 +generic/060 +generic/061 +generic/062 +generic/063 +generic/064 +generic/065 +generic/066 +generic/067 +generic/069 +generic/070 +generic/071 +generic/073 +generic/076 +generic/078 +generic/079 +generic/080 +generic/081 +generic/082 +generic/084 +generic/086 +generic/087 +generic/088 +generic/090 +generic/091 +generic/092 +generic/094 +generic/096 +generic/097 +generic/098 +generic/099 +generic/101 +generic/104 +generic/105 +generic/106 +generic/107 +generic/110 +generic/111 +generic/113 +generic/114 +generic/115 +generic/116 +generic/117 +generic/118 +generic/119 +generic/120 +generic/121 +generic/122 +generic/123 +generic/124 +generic/126 +generic/128 +generic/129 +generic/130 +generic/131 +generic/134 +generic/135 +generic/136 +generic/138 +generic/139 +generic/140 +generic/141 +generic/142 +generic/143 +generic/144 +generic/145 +generic/146 +generic/147 +generic/148 +generic/149 +generic/150 +generic/151 +generic/152 +generic/153 +generic/154 +generic/155 +generic/156 +generic/157 +generic/158 +generic/159 +generic/160 +generic/161 +generic/162 +generic/163 +generic/169 +generic/171 +generic/172 +generic/173 +generic/174 +generic/177 +generic/178 +generic/179 +generic/180 +generic/181 +generic/182 +generic/183 +generic/184 +generic/185 +generic/188 +generic/189 +generic/190 +generic/191 +generic/193 +generic/194 +generic/195 +generic/196 +generic/197 +generic/198 +generic/199 +generic/200 +generic/201 +generic/202 +generic/203 +generic/205 +generic/206 +generic/207 +generic/210 +generic/211 +generic/212 +generic/214 +generic/215 +generic/216 +generic/217 +generic/218 +generic/219 +generic/220 +generic/221 +generic/222 +generic/223 +generic/225 +generic/227 +generic/228 +generic/229 +generic/230 +generic/235 +generic/236 +generic/237 +generic/238 +generic/240 +generic/244 +generic/245 +generic/246 +generic/247 +generic/248 +generic/249 +generic/250 +generic/252 +generic/253 +generic/254 +generic/255 +generic/256 +generic/257 +generic/258 +generic/259 +generic/260 +generic/261 +generic/262 +generic/263 +generic/264 +generic/265 +generic/266 +generic/267 +generic/268 +generic/271 +generic/272 +generic/276 +generic/277 +generic/278 +generic/279 +generic/281 +generic/282 +generic/283 +generic/284 +generic/286 +generic/287 +generic/288 +generic/289 +generic/290 +generic/291 +generic/292 +generic/293 +generic/294 +generic/295 +generic/296 +generic/301 +generic/302 +generic/303 +generic/304 +generic/305 +generic/306 +generic/307 +generic/308 +generic/309 +generic/312 +generic/313 +generic/314 +generic/315 +generic/316 +generic/317 +generic/319 +generic/322 +generic/324 +generic/325 +generic/326 +generic/327 +generic/328 +generic/329 +generic/330 +generic/331 +generic/332 +generic/335 +generic/336 +generic/337 +generic/341 +generic/342 +generic/343 +generic/346 +generic/348 +generic/353 +generic/355 +generic/358 +generic/359 +generic/360 +generic/361 +generic/362 +generic/363 +generic/364 +generic/365 +generic/366 +generic/367 +generic/368 +generic/369 +generic/370 +generic/371 +generic/372 +generic/373 +generic/374 +generic/375 +generic/376 +generic/377 +generic/378 +generic/379 +generic/380 +generic/381 +generic/382 +generic/383 +generic/384 +generic/385 +generic/386 +generic/389 +generic/391 +generic/392 +generic/393 +generic/394 +generic/395 +generic/396 +generic/397 +generic/398 +generic/400 +generic/401 +generic/402 +generic/403 +generic/404 +generic/406 +generic/407 +generic/408 +generic/412 +generic/413 +generic/414 +generic/417 +generic/419 +generic/420 +generic/421 +generic/422 +generic/424 +generic/425 +generic/426 +generic/427 +generic/428 +generic/436 +generic/437 +generic/439 +generic/440 +generic/443 +generic/445 +generic/446 +generic/448 +generic/449 +generic/450 +generic/451 +generic/452 +generic/453 +generic/454 +generic/456 +generic/458 +generic/460 +generic/462 +generic/463 +generic/465 +generic/466 +generic/468 +generic/469 +generic/470 +generic/471 +generic/474 +generic/477 +generic/478 +generic/479 +generic/480 +generic/481 +generic/483 +generic/485 +generic/486 +generic/487 +generic/488 +generic/489 +generic/490 +generic/491 +generic/492 +generic/498 +generic/499 +generic/501 +generic/502 +generic/503 +generic/504 +generic/505 +generic/506 +generic/507 +generic/508 +generic/509 +generic/510 +generic/511 +generic/512 +generic/513 +generic/514 +generic/515 +generic/516 +generic/517 +generic/518 +generic/519 +generic/520 +generic/523 +generic/524 +generic/525 +generic/526 +generic/527 +generic/528 +generic/529 +generic/530 +generic/531 +generic/533 +generic/534 +generic/535 +generic/536 +generic/537 +generic/538 +generic/539 +generic/540 +generic/541 +generic/542 +generic/543 +generic/544 +generic/545 +generic/546 +generic/547 +generic/548 +generic/549 +generic/550 +generic/552 +generic/553 +generic/555 +generic/556 +generic/557 +generic/566 +generic/567 +generic/571 +generic/572 +generic/573 +generic/574 +generic/575 +generic/576 +generic/577 +generic/578 +generic/580 +generic/581 +generic/582 +generic/583 +generic/584 +generic/586 +generic/587 +generic/588 +generic/591 +generic/592 +generic/593 +generic/594 +generic/595 +generic/596 +generic/597 +generic/598 +generic/599 +generic/600 +generic/601 +generic/602 +generic/603 +generic/604 +generic/605 +generic/606 +generic/607 +generic/608 +generic/609 +generic/610 +generic/611 +generic/612 +generic/613 +generic/614 +generic/618 +generic/621 +generic/623 +generic/624 +generic/625 +generic/626 +generic/628 +generic/629 +generic/630 +generic/632 +generic/634 +generic/635 +generic/637 +generic/638 +generic/639 +generic/640 +generic/644 +generic/645 +generic/646 +generic/647 +generic/651 +generic/652 +generic/653 +generic/654 +generic/655 +generic/657 +generic/658 +generic/659 +generic/660 +generic/661 +generic/662 +generic/663 +generic/664 +generic/665 +generic/666 +generic/667 +generic/668 +generic/669 +generic/673 +generic/674 +generic/675 +generic/676 +generic/677 +generic/678 +generic/679 +generic/680 +generic/681 +generic/682 +generic/683 +generic/684 +generic/685 +generic/686 +generic/687 +generic/688 +generic/689 +shared/002 +shared/032 +Not +run: +generic/008 +generic/009 +generic/012 +generic/015 +generic/016 +generic/018 +generic/021 +generic/022 +generic/025 +generic/026 +generic/031 +generic/033 +generic/050 +generic/052 +generic/058 +generic/059 +generic/060 +generic/061 +generic/063 +generic/064 +generic/078 +generic/079 +generic/081 +generic/082 +generic/091 +generic/094 +generic/096 +generic/110 +generic/111 +generic/113 +generic/114 +generic/115 +generic/116 +generic/118 +generic/119 +generic/121 +generic/122 +generic/123 +generic/128 +generic/130 +generic/134 +generic/135 +generic/136 +generic/138 +generic/139 +generic/140 +generic/142 +generic/143 +generic/144 +generic/145 +generic/146 +generic/147 +generic/148 +generic/149 +generic/150 +generic/151 +generic/152 +generic/153 +generic/154 +generic/155 +generic/156 +generic/157 +generic/158 +generic/159 +generic/160 +generic/161 +generic/162 +generic/163 +generic/171 +generic/172 +generic/173 +generic/174 +generic/177 +generic/178 +generic/179 +generic/180 +generic/181 +generic/182 +generic/183 +generic/185 +generic/188 +generic/189 +generic/190 +generic/191 +generic/193 +generic/194 +generic/195 +generic/196 +generic/197 +generic/198 +generic/199 +generic/200 +generic/201 +generic/202 +generic/203 +generic/205 +generic/206 +generic/207 +generic/210 +generic/211 +generic/212 +generic/214 +generic/216 +generic/217 +generic/218 +generic/219 +generic/220 +generic/222 +generic/223 +generic/225 +generic/227 +generic/229 +generic/230 +generic/235 +generic/238 +generic/240 +generic/244 +generic/250 +generic/252 +generic/253 +generic/254 +generic/255 +generic/256 +generic/259 +generic/260 +generic/261 +generic/262 +generic/263 +generic/264 +generic/265 +generic/266 +generic/267 +generic/268 +generic/271 +generic/272 +generic/276 +generic/277 +generic/278 +generic/279 +generic/281 +generic/282 +generic/283 +generic/284 +generic/287 +generic/288 +generic/289 +generic/290 +generic/291 +generic/292 +generic/293 +generic/295 +generic/296 +generic/301 +generic/302 +generic/303 +generic/304 +generic/305 +generic/312 +generic/314 +generic/316 +generic/317 +generic/324 +generic/326 +generic/327 +generic/328 +generic/329 +generic/330 +generic/331 +generic/332 +generic/353 +generic/355 +generic/358 +generic/359 +generic/361 +generic/362 +generic/363 +generic/364 +generic/365 +generic/366 +generic/367 +generic/368 +generic/369 +generic/370 +generic/371 +generic/372 +generic/373 +generic/374 +generic/378 +generic/379 +generic/380 +generic/381 +generic/382 +generic/383 +generic/384 +generic/385 +generic/386 +generic/391 +generic/392 +generic/395 +generic/396 +generic/397 +generic/398 +generic/400 +generic/402 +generic/404 +generic/406 +generic/407 +generic/408 +generic/412 +generic/413 +generic/414 +generic/417 +generic/419 +generic/420 +generic/421 +generic/422 +generic/424 +generic/425 +generic/427 +generic/439 +generic/440 +generic/446 +generic/449 +generic/450 +generic/451 +generic/453 +generic/454 +generic/456 +generic/458 +generic/462 +generic/463 +generic/465 +generic/466 +generic/468 +generic/469 +generic/470 +generic/471 +generic/474 +generic/485 +generic/487 +generic/488 +generic/491 +generic/492 +generic/499 +generic/501 +generic/503 +generic/505 +generic/506 +generic/507 +generic/508 +generic/511 +generic/513 +generic/514 +generic/515 +generic/516 +generic/517 +generic/518 +generic/519 +generic/520 +generic/528 +generic/530 +generic/536 +generic/537 +generic/538 +generic/539 +generic/540 +generic/541 +generic/542 +generic/543 +generic/544 +generic/545 +generic/546 +generic/548 +generic/549 +generic/550 +generic/552 +generic/553 +generic/555 +generic/556 +generic/566 +generic/567 +generic/572 +generic/573 +generic/574 +generic/575 +generic/576 +generic/577 +generic/578 +generic/580 +generic/581 +generic/582 +generic/583 +generic/584 +generic/586 +generic/587 +generic/588 +generic/591 +generic/592 +generic/593 +generic/594 +generic/595 +generic/596 +generic/597 +generic/598 +generic/599 +generic/600 +generic/601 +generic/602 +generic/603 +generic/605 +generic/606 +generic/607 +generic/608 +generic/609 +generic/610 +generic/612 +generic/613 +generic/621 +generic/623 +generic/624 +generic/625 +generic/626 +generic/628 +generic/629 +generic/630 +generic/635 +generic/644 +generic/645 +generic/646 +generic/647 +generic/651 +generic/652 +generic/653 +generic/654 +generic/655 +generic/657 +generic/658 +generic/659 +generic/660 +generic/661 +generic/662 +generic/663 +generic/664 +generic/665 +generic/666 +generic/667 +generic/668 +generic/669 +generic/673 +generic/674 +generic/675 +generic/677 +generic/678 +generic/679 +generic/680 +generic/681 +generic/682 +generic/683 +generic/684 +generic/685 +generic/686 +generic/687 +generic/688 +generic/689 +shared/002 +shared/032 +Passed all 512 tests diff --git a/tests/extra/xfstests/local.exclude b/tests/extra/xfstests/local.exclude new file mode 100644 index 00000000..fa274227 --- /dev/null +++ b/tests/extra/xfstests/local.exclude @@ -0,0 +1,44 @@ +generic/003 # missing atime update in buffered read +generic/075 # file content mismatch failures (fds, etc) +generic/103 # enospc causes trans commit failures +generic/108 # mount fails on failing device? +generic/112 # file content mismatch failures (fds, etc) +generic/213 # enospc causes trans commit failures +generic/318 # can't support user namespaces until v5.11 +generic/321 # requires selinux enabled for '+' in ls? +generic/338 # BUG_ON update inode error handling +generic/347 # _dmthin_mount doesn't work? +generic/356 # swap +generic/357 # swap +generic/409 # bind mounts not scripted yet +generic/410 # bind mounts not scripted yet +generic/411 # bind mounts not scripted yet +generic/423 # symlink inode size is strlen() + 1 on scoutfs +generic/430 # xfs_io copy_range missing in el7 +generic/431 # xfs_io copy_range missing in el7 +generic/432 # xfs_io copy_range missing in el7 +generic/433 # xfs_io copy_range missing in el7 +generic/434 # xfs_io copy_range missing in el7 +generic/441 # dm-mapper +generic/444 # el9's posix_acl_update_mode is buggy ? +generic/467 # open_by_handle ESTALE +generic/472 # swap +generic/484 # dm-mapper +generic/493 # swap +generic/494 # swap +generic/495 # swap +generic/496 # swap +generic/497 # swap +generic/532 # xfs_io statx attrib_mask missing in el7 +generic/554 # swap +generic/563 # cgroup+loopdev +generic/564 # xfs_io copy_range missing in el7 +generic/565 # xfs_io copy_range missing in el7 +generic/568 # falloc not resulting in block count increase +generic/569 # swap +generic/570 # swap +generic/620 # dm-hugedisk +generic/633 # id-mapped mounts missing in el7 +generic/636 # swap +generic/641 # swap +generic/643 # swap diff --git a/tests/funcs/exec.sh b/tests/funcs/exec.sh index 1e2d6f57..46949b5e 100644 --- a/tests/funcs/exec.sh +++ b/tests/funcs/exec.sh @@ -64,21 +64,27 @@ t_rc() } # -# redirect test output back to the output of the invoking script intead -# of the compared output. +# As run, stdout/err are redirected to a file that will be compared with +# the stored expected golden output of the test. This redirects +# stdout/err in the script to stdout of the invoking run-test. It's +# intended to give visible output of tests without being included in the +# golden output. # -t_restore_output() +# (see the goofy "exec" fd manipulation in the main run-tests as it runs +# each test) +# +t_stdout_invoked() { exec >&6 2>&1 } # -# redirect a command's output back to the compared output after the -# test has restored its output +# This undoes t_stdout_invokved, returning the test's stdout/err to the +# output file as it was when it was launched. # -t_compare_output() +t_stdout_compare() { - "$@" >&7 2>&1 + exec >&7 2>&1 } # diff --git a/tests/golden/xfstests b/tests/golden/xfstests index c4032ca9..e69de29b 100644 --- a/tests/golden/xfstests +++ b/tests/golden/xfstests @@ -1,882 +0,0 @@ -Ran: -generic/001 -generic/002 -generic/004 -generic/005 -generic/006 -generic/007 -generic/008 -generic/009 -generic/011 -generic/012 -generic/013 -generic/014 -generic/015 -generic/016 -generic/018 -generic/020 -generic/021 -generic/022 -generic/023 -generic/024 -generic/025 -generic/026 -generic/028 -generic/029 -generic/030 -generic/031 -generic/032 -generic/033 -generic/034 -generic/035 -generic/037 -generic/039 -generic/040 -generic/041 -generic/050 -generic/052 -generic/053 -generic/056 -generic/057 -generic/058 -generic/059 -generic/060 -generic/061 -generic/062 -generic/063 -generic/064 -generic/065 -generic/066 -generic/067 -generic/069 -generic/070 -generic/071 -generic/073 -generic/076 -generic/078 -generic/079 -generic/080 -generic/081 -generic/082 -generic/084 -generic/086 -generic/087 -generic/088 -generic/090 -generic/091 -generic/092 -generic/094 -generic/096 -generic/097 -generic/098 -generic/099 -generic/101 -generic/104 -generic/105 -generic/106 -generic/107 -generic/110 -generic/111 -generic/113 -generic/114 -generic/115 -generic/116 -generic/117 -generic/118 -generic/119 -generic/120 -generic/121 -generic/122 -generic/123 -generic/124 -generic/126 -generic/128 -generic/129 -generic/130 -generic/131 -generic/134 -generic/135 -generic/136 -generic/138 -generic/139 -generic/140 -generic/141 -generic/142 -generic/143 -generic/144 -generic/145 -generic/146 -generic/147 -generic/148 -generic/149 -generic/150 -generic/151 -generic/152 -generic/153 -generic/154 -generic/155 -generic/156 -generic/157 -generic/158 -generic/159 -generic/160 -generic/161 -generic/162 -generic/163 -generic/169 -generic/171 -generic/172 -generic/173 -generic/174 -generic/177 -generic/178 -generic/179 -generic/180 -generic/181 -generic/182 -generic/183 -generic/184 -generic/185 -generic/188 -generic/189 -generic/190 -generic/191 -generic/193 -generic/194 -generic/195 -generic/196 -generic/197 -generic/198 -generic/199 -generic/200 -generic/201 -generic/202 -generic/203 -generic/205 -generic/206 -generic/207 -generic/210 -generic/211 -generic/212 -generic/214 -generic/215 -generic/216 -generic/217 -generic/218 -generic/219 -generic/220 -generic/221 -generic/222 -generic/223 -generic/225 -generic/227 -generic/228 -generic/229 -generic/230 -generic/235 -generic/236 -generic/237 -generic/238 -generic/240 -generic/244 -generic/245 -generic/246 -generic/247 -generic/248 -generic/249 -generic/250 -generic/252 -generic/253 -generic/254 -generic/255 -generic/256 -generic/257 -generic/258 -generic/259 -generic/260 -generic/261 -generic/262 -generic/263 -generic/264 -generic/265 -generic/266 -generic/267 -generic/268 -generic/271 -generic/272 -generic/276 -generic/277 -generic/278 -generic/279 -generic/281 -generic/282 -generic/283 -generic/284 -generic/286 -generic/287 -generic/288 -generic/289 -generic/290 -generic/291 -generic/292 -generic/293 -generic/294 -generic/295 -generic/296 -generic/301 -generic/302 -generic/303 -generic/304 -generic/305 -generic/306 -generic/307 -generic/308 -generic/309 -generic/312 -generic/313 -generic/314 -generic/315 -generic/316 -generic/317 -generic/319 -generic/322 -generic/324 -generic/325 -generic/326 -generic/327 -generic/328 -generic/329 -generic/330 -generic/331 -generic/332 -generic/335 -generic/336 -generic/337 -generic/341 -generic/342 -generic/343 -generic/346 -generic/348 -generic/353 -generic/355 -generic/358 -generic/359 -generic/360 -generic/361 -generic/362 -generic/363 -generic/364 -generic/365 -generic/366 -generic/367 -generic/368 -generic/369 -generic/370 -generic/371 -generic/372 -generic/373 -generic/374 -generic/375 -generic/376 -generic/377 -generic/378 -generic/379 -generic/380 -generic/381 -generic/382 -generic/383 -generic/384 -generic/385 -generic/386 -generic/389 -generic/391 -generic/392 -generic/393 -generic/394 -generic/395 -generic/396 -generic/397 -generic/398 -generic/400 -generic/401 -generic/402 -generic/403 -generic/404 -generic/406 -generic/407 -generic/408 -generic/412 -generic/413 -generic/414 -generic/417 -generic/419 -generic/420 -generic/421 -generic/422 -generic/424 -generic/425 -generic/426 -generic/427 -generic/428 -generic/436 -generic/437 -generic/439 -generic/440 -generic/443 -generic/445 -generic/446 -generic/448 -generic/449 -generic/450 -generic/451 -generic/452 -generic/453 -generic/454 -generic/456 -generic/458 -generic/460 -generic/462 -generic/463 -generic/465 -generic/466 -generic/468 -generic/469 -generic/470 -generic/471 -generic/474 -generic/477 -generic/478 -generic/479 -generic/480 -generic/481 -generic/483 -generic/485 -generic/486 -generic/487 -generic/488 -generic/489 -generic/490 -generic/491 -generic/492 -generic/498 -generic/499 -generic/501 -generic/502 -generic/503 -generic/504 -generic/505 -generic/506 -generic/507 -generic/508 -generic/509 -generic/510 -generic/511 -generic/512 -generic/513 -generic/514 -generic/515 -generic/516 -generic/517 -generic/518 -generic/519 -generic/520 -generic/523 -generic/524 -generic/525 -generic/526 -generic/527 -generic/528 -generic/529 -generic/530 -generic/531 -generic/533 -generic/534 -generic/535 -generic/536 -generic/537 -generic/538 -generic/539 -generic/540 -generic/541 -generic/542 -generic/543 -generic/544 -generic/545 -generic/546 -generic/547 -generic/548 -generic/549 -generic/550 -generic/552 -generic/553 -generic/555 -generic/556 -generic/557 -generic/566 -generic/567 -generic/571 -generic/572 -generic/573 -generic/574 -generic/575 -generic/576 -generic/577 -generic/578 -generic/580 -generic/581 -generic/582 -generic/583 -generic/584 -generic/586 -generic/587 -generic/588 -generic/591 -generic/592 -generic/593 -generic/594 -generic/595 -generic/596 -generic/597 -generic/598 -generic/599 -generic/600 -generic/601 -generic/602 -generic/603 -generic/604 -generic/605 -generic/606 -generic/607 -generic/608 -generic/609 -generic/610 -generic/611 -generic/612 -generic/613 -generic/614 -generic/618 -generic/621 -generic/623 -generic/624 -generic/625 -generic/626 -generic/628 -generic/629 -generic/630 -generic/632 -generic/634 -generic/635 -generic/637 -generic/638 -generic/639 -generic/640 -generic/644 -generic/645 -generic/646 -generic/647 -generic/651 -generic/652 -generic/653 -generic/654 -generic/655 -generic/657 -generic/658 -generic/659 -generic/660 -generic/661 -generic/662 -generic/663 -generic/664 -generic/665 -generic/666 -generic/667 -generic/668 -generic/669 -generic/673 -generic/674 -generic/675 -generic/676 -generic/677 -generic/678 -generic/679 -generic/680 -generic/681 -generic/682 -generic/683 -generic/684 -generic/685 -generic/686 -generic/687 -generic/688 -generic/689 -shared/002 -shared/032 -Not -run: -generic/008 -generic/009 -generic/012 -generic/015 -generic/016 -generic/018 -generic/021 -generic/022 -generic/025 -generic/026 -generic/031 -generic/033 -generic/050 -generic/052 -generic/058 -generic/059 -generic/060 -generic/061 -generic/063 -generic/064 -generic/078 -generic/079 -generic/081 -generic/082 -generic/091 -generic/094 -generic/096 -generic/110 -generic/111 -generic/113 -generic/114 -generic/115 -generic/116 -generic/118 -generic/119 -generic/121 -generic/122 -generic/123 -generic/128 -generic/130 -generic/134 -generic/135 -generic/136 -generic/138 -generic/139 -generic/140 -generic/142 -generic/143 -generic/144 -generic/145 -generic/146 -generic/147 -generic/148 -generic/149 -generic/150 -generic/151 -generic/152 -generic/153 -generic/154 -generic/155 -generic/156 -generic/157 -generic/158 -generic/159 -generic/160 -generic/161 -generic/162 -generic/163 -generic/171 -generic/172 -generic/173 -generic/174 -generic/177 -generic/178 -generic/179 -generic/180 -generic/181 -generic/182 -generic/183 -generic/185 -generic/188 -generic/189 -generic/190 -generic/191 -generic/193 -generic/194 -generic/195 -generic/196 -generic/197 -generic/198 -generic/199 -generic/200 -generic/201 -generic/202 -generic/203 -generic/205 -generic/206 -generic/207 -generic/210 -generic/211 -generic/212 -generic/214 -generic/216 -generic/217 -generic/218 -generic/219 -generic/220 -generic/222 -generic/223 -generic/225 -generic/227 -generic/229 -generic/230 -generic/235 -generic/238 -generic/240 -generic/244 -generic/250 -generic/252 -generic/253 -generic/254 -generic/255 -generic/256 -generic/259 -generic/260 -generic/261 -generic/262 -generic/263 -generic/264 -generic/265 -generic/266 -generic/267 -generic/268 -generic/271 -generic/272 -generic/276 -generic/277 -generic/278 -generic/279 -generic/281 -generic/282 -generic/283 -generic/284 -generic/287 -generic/288 -generic/289 -generic/290 -generic/291 -generic/292 -generic/293 -generic/295 -generic/296 -generic/301 -generic/302 -generic/303 -generic/304 -generic/305 -generic/312 -generic/314 -generic/316 -generic/317 -generic/324 -generic/326 -generic/327 -generic/328 -generic/329 -generic/330 -generic/331 -generic/332 -generic/353 -generic/355 -generic/358 -generic/359 -generic/361 -generic/362 -generic/363 -generic/364 -generic/365 -generic/366 -generic/367 -generic/368 -generic/369 -generic/370 -generic/371 -generic/372 -generic/373 -generic/374 -generic/378 -generic/379 -generic/380 -generic/381 -generic/382 -generic/383 -generic/384 -generic/385 -generic/386 -generic/391 -generic/392 -generic/395 -generic/396 -generic/397 -generic/398 -generic/400 -generic/402 -generic/404 -generic/406 -generic/407 -generic/408 -generic/412 -generic/413 -generic/414 -generic/417 -generic/419 -generic/420 -generic/421 -generic/422 -generic/424 -generic/425 -generic/427 -generic/439 -generic/440 -generic/446 -generic/449 -generic/450 -generic/451 -generic/453 -generic/454 -generic/456 -generic/458 -generic/462 -generic/463 -generic/465 -generic/466 -generic/468 -generic/469 -generic/470 -generic/471 -generic/474 -generic/485 -generic/487 -generic/488 -generic/491 -generic/492 -generic/499 -generic/501 -generic/503 -generic/505 -generic/506 -generic/507 -generic/508 -generic/511 -generic/513 -generic/514 -generic/515 -generic/516 -generic/517 -generic/518 -generic/519 -generic/520 -generic/528 -generic/530 -generic/536 -generic/537 -generic/538 -generic/539 -generic/540 -generic/541 -generic/542 -generic/543 -generic/544 -generic/545 -generic/546 -generic/548 -generic/549 -generic/550 -generic/552 -generic/553 -generic/555 -generic/556 -generic/566 -generic/567 -generic/572 -generic/573 -generic/574 -generic/575 -generic/576 -generic/577 -generic/578 -generic/580 -generic/581 -generic/582 -generic/583 -generic/584 -generic/586 -generic/587 -generic/588 -generic/591 -generic/592 -generic/593 -generic/594 -generic/595 -generic/596 -generic/597 -generic/598 -generic/599 -generic/600 -generic/601 -generic/602 -generic/603 -generic/605 -generic/606 -generic/607 -generic/608 -generic/609 -generic/610 -generic/612 -generic/613 -generic/621 -generic/623 -generic/624 -generic/625 -generic/626 -generic/628 -generic/629 -generic/630 -generic/635 -generic/644 -generic/645 -generic/646 -generic/647 -generic/651 -generic/652 -generic/653 -generic/654 -generic/655 -generic/657 -generic/658 -generic/659 -generic/660 -generic/661 -generic/662 -generic/663 -generic/664 -generic/665 -generic/666 -generic/667 -generic/668 -generic/669 -generic/673 -generic/674 -generic/675 -generic/677 -generic/678 -generic/679 -generic/680 -generic/681 -generic/682 -generic/683 -generic/684 -generic/685 -generic/686 -generic/687 -generic/688 -generic/689 -shared/002 -shared/032 -Passed all 512 tests diff --git a/tests/run-tests.sh b/tests/run-tests.sh index f7e6d354..3a6f0377 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -613,6 +613,9 @@ for t in $tests; do # mark in dmesg as to what test we are running echo "run scoutfs test $test_name" > /dev/kmsg + # let the test get at its extra files + T_EXTRA="$T_TESTS/extra/$test_name" + for iter in $(seq 1 $T_LOOP_ITER); do # create a temporary dir and file path for the test diff --git a/tests/tests/xfstests.sh b/tests/tests/xfstests.sh index bd02a1a3..efc22cc6 100644 --- a/tests/tests/xfstests.sh +++ b/tests/tests/xfstests.sh @@ -63,73 +63,47 @@ export MOUNT_OPTIONS="-o quorum_slot_nr=0,metadev_path=$T_MB0" export TEST_FS_MOUNT_OPTS="-o quorum_slot_nr=0,metadev_path=$T_MB0" EOF -cat << EOF > local.exclude -generic/003 # missing atime update in buffered read -generic/075 # file content mismatch failures (fds, etc) -generic/103 # enospc causes trans commit failures -generic/108 # mount fails on failing device? -generic/112 # file content mismatch failures (fds, etc) -generic/213 # enospc causes trans commit failures -generic/318 # can't support user namespaces until v5.11 -generic/321 # requires selinux enabled for '+' in ls? -generic/338 # BUG_ON update inode error handling -generic/347 # _dmthin_mount doesn't work? -generic/356 # swap -generic/357 # swap -generic/409 # bind mounts not scripted yet -generic/410 # bind mounts not scripted yet -generic/411 # bind mounts not scripted yet -generic/423 # symlink inode size is strlen() + 1 on scoutfs -generic/430 # xfs_io copy_range missing in el7 -generic/431 # xfs_io copy_range missing in el7 -generic/432 # xfs_io copy_range missing in el7 -generic/433 # xfs_io copy_range missing in el7 -generic/434 # xfs_io copy_range missing in el7 -generic/441 # dm-mapper -generic/444 # el9's posix_acl_update_mode is buggy ? -generic/467 # open_by_handle ESTALE -generic/472 # swap -generic/484 # dm-mapper -generic/493 # swap -generic/494 # swap -generic/495 # swap -generic/496 # swap -generic/497 # swap -generic/532 # xfs_io statx attrib_mask missing in el7 -generic/554 # swap -generic/563 # cgroup+loopdev -generic/564 # xfs_io copy_range missing in el7 -generic/565 # xfs_io copy_range missing in el7 -generic/568 # falloc not resulting in block count increase -generic/569 # swap -generic/570 # swap -generic/620 # dm-hugedisk -generic/633 # id-mapped mounts missing in el7 -generic/636 # swap -generic/641 # swap -generic/643 # swap -EOF +cp "$T_EXTRA/local.exclude" local.exclude -t_restore_output +t_stdout_invoked echo " (showing output of xfstests)" args="-E local.exclude ${T_XFSTESTS_ARGS:--g quick}" ./check $args # the fs is unmounted when check finishes +t_stdout_compare + # -# ./check writes the results of the run to check.log. It lists -# the tests it ran, skipped, or failed. Then it writes a line saying -# everything passed or some failed. We scrape the most recent run and -# use it as the output to compare to make sure that we run the right -# tests and get the right results. +# ./check writes the results of the run to check.log. It lists the +# tests it ran, skipped, or failed. Then it writes a line saying +# everything passed or some failed. +# + +# +# If XFSTESTS_ARGS were specified then we just pass/fail to match the +# check run. +# +if [ -n "$T_XFSTESTS_ARGS" ]; then + if tail -1 results/check.log | grep -q "Failed"; then + t_fail + else + t_pass + fi +fi + +# +# Otherwise, typically, when there were no args then we scrape the most +# recent run and use it as the output to compare to make sure that we +# run the right tests and get the right results. # awk ' /^(Ran|Not run|Failures):.*/ { if (pf) { res="" pf="" - } res = res "\n" $0 + } + res = res "\n" $0 } /^(Passed|Failed).*tests$/ { pf=$0 @@ -139,10 +113,14 @@ awk ' }' < results/check.log > "$T_TMPDIR/results" # put a test per line so diff shows tests that differ -egrep "^(Ran|Not run|Failures):" "$T_TMPDIR/results" | \ - fmt -w 1 > "$T_TMPDIR/results.fmt" -egrep "^(Passed|Failed).*tests$" "$T_TMPDIR/results" >> "$T_TMPDIR/results.fmt" +grep -E "^(Ran|Not run|Failures):" "$T_TMPDIR/results" | fmt -w 1 > "$T_TMPDIR/results.fmt" +grep -E "^(Passed|Failed).*tests$" "$T_TMPDIR/results" >> "$T_TMPDIR/results.fmt" -t_compare_output cat "$T_TMPDIR/results.fmt" +diff -u "$T_EXTRA/expected-results" "$T_TMPDIR/results.fmt" > "$T_TMPDIR/results.diff" +if [ -s "$T_TMPDIR/results.diff" ]; then + echo "tests that were skipped/run differed from expected:" + cat "$T_TMPDIR/results.diff" + t_fail +fi t_pass From e182914e5180aea454c29c13663603e236419db0 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Sat, 8 Nov 2025 10:04:19 -0800 Subject: [PATCH 05/12] Fix double free of metadata blocks in log merging The log merging process is meant to provide parallelism across workers in mounts. The idea is that the server hands out a bunch of concurrent non-intersecting work that's based on the structure of the stable input fs_root btree. The nature of the parallel work (cow of the blocks that intersect a key range) means that the ranges of concurrently issued work can't overlap or the work will all cow the same input blocks, freeing that input stable block multiple times. We're seeing this in testing. Correctness was intended by having an advancing key that sweeps sorted ranges. Duplicate ranges would never be hit as the key advanced past each it visited. This was broken by the mapping of the fs item keys to log merge tree keys by clobbering the sk_zone key value. It effectively interleaves the ranges of each zone in the fs root (meta indexes, orphans, fs items). With just the right log merge conditions that involve logged items in the right places and partial completed work to insert remaining ranges behind the key, ranges can be stored at mapped keys that end up with ranges out of order. The server iterates over these and ends up issueing overlapping work, which results in duplicated frees of the input blocks. The fix, without changing the format of the stored log tree items, is to perform a full sweep of all the range items and determine the next item by looking at the full precision stored keys. This ensures that the processed ranges always advance and never overlap. Signed-off-by: Zach Brown --- kmod/src/server.c | 59 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index c8618273..b5022edb 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -994,10 +994,11 @@ static int for_each_rid_last_lt(struct super_block *sb, struct scoutfs_btree_roo } /* - * Log merge range items are stored at the starting fs key of the range. - * The only fs key field that doesn't hold information is the zone, so - * we use the zone to differentiate all types that we store in the log - * merge tree. + * Log merge range items are stored at the starting fs key of the range + * with the zone overwritten to indicate the log merge item type. This + * day0 mistake loses sorting information for items in the different + * zones in the fs root, so the range items aren't strictly sorted by + * the starting key of their range. */ static void init_log_merge_key(struct scoutfs_key *key, u8 zone, u64 first, u64 second) @@ -1029,6 +1030,51 @@ static int next_log_merge_item_key(struct super_block *sb, struct scoutfs_btree_ return ret; } +/* + * The range items aren't sorted by their range.start because + * _RANGE_ZONE clobbers the range's zone. We sweep all the items and + * find the range with the next least starting key that's greater than + * the caller's starting key. We have to be careful to iterate over the + * log_merge tree keys because the ranges can overlap as they're mapped + * to the log_merge keys by clobbering their zone. + */ +static int next_log_merge_range(struct super_block *sb, struct scoutfs_btree_root *root, + struct scoutfs_key *start, struct scoutfs_log_merge_range *rng) +{ + struct scoutfs_log_merge_range *next; + SCOUTFS_BTREE_ITEM_REF(iref); + struct scoutfs_key key; + int ret; + + key = *start; + key.sk_zone = SCOUTFS_LOG_MERGE_RANGE_ZONE; + scoutfs_key_set_ones(&rng->start); + + do { + ret = scoutfs_btree_next(sb, root, &key, &iref); + if (ret == 0) { + if (iref.key->sk_zone != SCOUTFS_LOG_MERGE_RANGE_ZONE) { + ret = -ENOENT; + } else if (iref.val_len != sizeof(struct scoutfs_log_merge_range)) { + ret = -EIO; + } else { + next = iref.val; + if (scoutfs_key_compare(&next->start, &rng->start) < 0 && + scoutfs_key_compare(&next->start, start) >= 0) + *rng = *next; + key = *iref.key; + scoutfs_key_inc(&key); + } + scoutfs_btree_put_iref(&iref); + } + } while (ret == 0); + + if (ret == -ENOENT && !scoutfs_key_is_ones(&rng->start)) + ret = 0; + + return ret; +} + static int next_log_merge_item(struct super_block *sb, struct scoutfs_btree_root *root, u8 zone, u64 first, u64 second, @@ -2720,10 +2766,7 @@ restart: /* find the next range, always checking for splicing */ for (;;) { - key = stat.next_range_key; - key.sk_zone = SCOUTFS_LOG_MERGE_RANGE_ZONE; - ret = next_log_merge_item_key(sb, &super->log_merge, SCOUTFS_LOG_MERGE_RANGE_ZONE, - &key, &rng, sizeof(rng)); + ret = next_log_merge_range(sb, &super->log_merge, &stat.next_range_key, &rng); if (ret < 0 && ret != -ENOENT) { err_str = "finding merge range item"; goto out; From 1ab798e7eb984197258856d1fcd96e4713743954 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Tue, 11 Nov 2025 10:33:16 -0800 Subject: [PATCH 06/12] Silence inconsistent srch on forced unmount Assembling a srch compaction operation creates an item and populates it with allocator state. It doesn't cleanly unwind the allocation and undo the compaction item change if allocation filling fails and issues a warning. This warning isn't needed if the error shows that we're in forced unmount. The inconsistent state won't be applied, it will be dropped on the floor as the mount is torn down. Signed-off-by: Zach Brown --- kmod/src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kmod/src/server.c b/kmod/src/server.c index b5022edb..4c6686b2 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -2140,7 +2140,7 @@ static int server_srch_get_compact(struct super_block *sb, apply: ret = server_apply_commit(sb, &hold, ret); - WARN_ON_ONCE(ret < 0 && ret != -ENOENT); /* XXX leaked busy item */ + WARN_ON_ONCE(ret < 0 && ret != -ENOENT && ret != -ENOLINK); /* XXX leaked busy item */ out: ret = scoutfs_net_response(sb, conn, cmd, id, ret, sc, sizeof(struct scoutfs_srch_compact)); From 90cb458cd5288ab65b4da0a494c0cf270a090acb Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Tue, 4 Nov 2025 15:23:52 -0800 Subject: [PATCH 07/12] Make mmap_stress not exceed a fixed amount of time. There's a scenarion where mmap_stress gets enough resources that twoe of the threads will starve the others, which then all take a very long time catching up committing changes. Because this test program didn't finish until all the threads had completed a fixed amount of work, essentially these threads all ended up tripping over eachother. In CI this would exceed 6h+, while originally I intended this to run in about 100s or so. Instead, cap the run time to ~30s by default. If threads exceed this time, they will immediately exit, which causes any clog in contention between the threads to drain relatively quickly. Signed-off-by: Auke Kok --- tests/src/mmap_stress.c | 23 ++++++++++++++++------- tests/tests/mmap.sh | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/src/mmap_stress.c b/tests/src/mmap_stress.c index 94a41484..c541bd6b 100644 --- a/tests/src/mmap_stress.c +++ b/tests/src/mmap_stress.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -29,7 +30,7 @@ #include static int size = 0; -static int count = 0; /* XXX make this duration instead */ +static int duration = 0; struct thread_info { int nr; @@ -41,6 +42,8 @@ static void *run_test_func(void *ptr) void *buf = NULL; char *addr = NULL; struct thread_info *tinfo = ptr; + uint64_t seconds = 0; + struct timespec ts; int c = 0; int fd; ssize_t read, written, ret; @@ -61,9 +64,15 @@ static void *run_test_func(void *ptr) usleep(100000); /* 0.1sec to allow all threads to start roughly at the same time */ + clock_gettime(CLOCK_REALTIME, &ts); /* record start time */ + seconds = ts.tv_sec + duration; + for (;;) { - if (++c > count) - break; + if (++c % 16 == 0) { + clock_gettime(CLOCK_REALTIME, &ts); + if (ts.tv_sec >= seconds) + break; + } switch (rand() % 4) { case 0: /* pread */ @@ -120,7 +129,7 @@ int main(int argc, char **argv) int i; if (argc != 8) { - fprintf(stderr, "%s requires 7 arguments - size count file1 file2 file3 file4 file5\n", argv[0]); + fprintf(stderr, "%s requires 7 arguments - size duration file1 file2 file3 file4 file5\n", argv[0]); exit(-1); } @@ -130,9 +139,9 @@ int main(int argc, char **argv) exit(-1); } - count = atoi(argv[2]); - if (count < 0) { - fprintf(stderr, "invalid count, must be greater than 0\n"); + duration = atoi(argv[2]); + if (duration < 0) { + fprintf(stderr, "invalid duration, must be greater than or equal to 0\n"); exit(-1); } diff --git a/tests/tests/mmap.sh b/tests/tests/mmap.sh index bf465ce9..4b14d516 100644 --- a/tests/tests/mmap.sh +++ b/tests/tests/mmap.sh @@ -5,7 +5,7 @@ t_require_commands mmap_stress mmap_validate scoutfs xfs_io echo "== mmap_stress" -mmap_stress 8192 2000 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort +mmap_stress 8192 30 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort echo "== basic mmap/read/write consistency checks" mmap_validate 256 1000 "$T_D0/mmap_val1" "$T_D1/mmap_val1" From ad078cd93c14d7eeab1a83ff08652cd6f84b2983 Mon Sep 17 00:00:00 2001 From: Auke Kok Date: Thu, 6 Nov 2025 11:56:33 -0800 Subject: [PATCH 08/12] Avoid lock stalling mmap_stress mmap_stress gets completely stalled in lock messaging and starving most of the mmap_stress threads, which causes it to delay and even time out in CI. Instead of spawning threads over all 5 test nodes, we reduce it to spawning over only 2 artificially. This still does a good number of operations on those node, and now the work is spread across the two nodes evenly. Additionaly, I've added a miniscule (10ms) delay in between operations that should hopefully be sufficient for other locking attempts to settle and allow the threads to better spread the work. This now shows that all the threads exit within < 0.25s on my test machine, which is a lot better than the 40s variation that I was seeing locally. Hopefully this fares better in CI. Signed-off-by: Auke Kok --- tests/src/mmap_stress.c | 2 ++ tests/tests/mmap.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/src/mmap_stress.c b/tests/src/mmap_stress.c index c541bd6b..4ec2220b 100644 --- a/tests/src/mmap_stress.c +++ b/tests/src/mmap_stress.c @@ -108,6 +108,8 @@ static void *run_test_func(void *ptr) memcpy(addr, buf, size); /* noerr */ break; } + + usleep(10000); } munmap(addr, size); diff --git a/tests/tests/mmap.sh b/tests/tests/mmap.sh index 4b14d516..8b617a36 100644 --- a/tests/tests/mmap.sh +++ b/tests/tests/mmap.sh @@ -5,7 +5,7 @@ t_require_commands mmap_stress mmap_validate scoutfs xfs_io echo "== mmap_stress" -mmap_stress 8192 30 "$T_D0/mmap_stress" "$T_D1/mmap_stress" "$T_D2/mmap_stress" "$T_D3/mmap_stress" "$T_D4/mmap_stress" | sed 's/:.*//g' | sort +mmap_stress 8192 30 "$T_D0/mmap_stress" "$T_D0/mmap_stress" "$T_D0/mmap_stress" "$T_D3/mmap_stress" "$T_D3/mmap_stress" | sed 's/:.*//g' | sort echo "== basic mmap/read/write consistency checks" mmap_validate 256 1000 "$T_D0/mmap_val1" "$T_D1/mmap_val1" From 92ac13287391d8041bbe9de5c45a00b69cb58f17 Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Nov 2025 14:40:22 -0800 Subject: [PATCH 09/12] Silence merge splice error when forcing Silence another error warning and assertion that's assuming that the result of the errors is going to be persistent. When we're forcing an unmount we've severed storage and networking. Signed-off-by: Zach Brown --- kmod/src/msg.h | 6 ++++++ kmod/src/server.c | 7 +++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/kmod/src/msg.h b/kmod/src/msg.h index e08682c8..54477825 100644 --- a/kmod/src/msg.h +++ b/kmod/src/msg.h @@ -35,6 +35,12 @@ do { \ } \ } while (0) \ +#define scoutfs_bug_on_err(sb, err, fmt, args...) \ +do { \ + __typeof__(err) _err = (err); \ + scoutfs_bug_on(sb, _err < 0 && _err != -ENOLINK, fmt, ##args); \ +} while (0) + /* * Each message is only generated once per volume. Remounting resets * the messages. diff --git a/kmod/src/server.c b/kmod/src/server.c index 4c6686b2..7ed04b52 100644 --- a/kmod/src/server.c +++ b/kmod/src/server.c @@ -2518,10 +2518,9 @@ out: } } - if (ret < 0) - scoutfs_err(sb, "server error %d splicing log merge completion: %s", ret, err_str); - - BUG_ON(ret); /* inconsistent */ + /* inconsistent */ + scoutfs_bug_on_err(sb, ret, + "server error %d splicing log merge completion: %s", ret, err_str); return ret ?: einprogress; } From 991e2cbdf8f8417a141004b0c2da01b294f14e2c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Fri, 31 Oct 2025 10:23:19 -0700 Subject: [PATCH 10/12] Ignore slow quorum hb transfers in tests We're getting test failures from messages that our guests can be unresponsive. They sure can be. We don't need to fail for this one specific case. Signed-off-by: Zach Brown --- tests/funcs/filter.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index b5a1a3cf..5e4675b5 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -166,6 +166,9 @@ t_filter_dmesg() # perf warning that it adjusted sample rate re="$re|perf: interrupt took too long.*lowering kernel.perf_event_max_sample_rate.*" + # some ci test guests are unresponsive + re="$re|longest quorum heartbeat .* delay" + egrep -v "($re)" | \ ignore_harmless_unwind_kasan_stack_oob } From fd80c17ab62951296fa463cc9b20598afe578c6c Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Wed, 12 Nov 2025 14:49:46 -0800 Subject: [PATCH 11/12] Filter out kernel message when guests are slow Ignore more kernel messages when debug guests are being slow. Signed-off-by: Zach Brown --- tests/funcs/filter.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/funcs/filter.sh b/tests/funcs/filter.sh index 5e4675b5..6e5fa2e2 100644 --- a/tests/funcs/filter.sh +++ b/tests/funcs/filter.sh @@ -121,6 +121,7 @@ t_filter_dmesg() # in debugging kernels we can slow things down a bit re="$re|hrtimer: interrupt took .*" + re="$re|clocksource: Long readout interval" # fencing tests force unmounts and trigger timeouts re="$re|scoutfs .* forcing unmount" From 8ddf9b8c8c118b9443ff32b2e15ffff2cd084f1f Mon Sep 17 00:00:00 2001 From: Zach Brown Date: Thu, 13 Nov 2025 12:15:43 -0800 Subject: [PATCH 12/12] Handle disappearing fencing requests and targets The userspace fencing process wasn't careful about handling underlying directories that disappear while it was working. On the server/fenced side, fencing requests can linger after they've been resolved by writing 1 to fenced or error. The script could come back around to see the directory before the server finally removes it, causing all later uses of the request dir to fail. We saw this in the logs as a bunch of cat errors for the various request files. On the local fence script side, all the mounts can be in the process of being unmounted so both the /sys/fs dirs and the mount it self can be removed while we're working. For both, when we're working with the /sys/fs files we read them without logging errors and then test that the dir still exists before using what we read. When fencing a mount, we stop if findmnt doesn't find the mount and then raise a umount error if the /sys/fs dir exists after umount fails. And while we're at it, we have each scripts logging append instead of truncating (if, say, it's a log file instead of an interactive tty). Signed-off-by: Zach Brown --- tests/fenced-local-force-unmount.sh | 38 ++++++++++++++--------------- utils/fenced/scoutfs-fenced | 34 ++++++++++++-------------- 2 files changed, 33 insertions(+), 39 deletions(-) diff --git a/tests/fenced-local-force-unmount.sh b/tests/fenced-local-force-unmount.sh index f5553be1..cf879b85 100755 --- a/tests/fenced-local-force-unmount.sh +++ b/tests/fenced-local-force-unmount.sh @@ -8,36 +8,34 @@ echo "$0 running rid '$SCOUTFS_FENCED_REQ_RID' ip '$SCOUTFS_FENCED_REQ_IP' args '$@'" -log() { - echo "$@" > /dev/stderr +echo_fail() { + echo "$@" >> /dev/stderr exit 1 } -echo_fail() { - echo "$@" > /dev/stderr - exit 1 +# silence error messages +quiet_cat() +{ + cat "$@" 2>/dev/null } rid="$SCOUTFS_FENCED_REQ_RID" +shopt -s nullglob for fs in /sys/fs/scoutfs/*; do - [ ! -d "$fs" ] && continue + fs_rid="$(quiet_cat $fs/rid)" + nr="$(quiet_cat $fs/data_device_maj_min)" + [ ! -d "$fs" -o "$fs_rid" != "$rid" ] && continue - fs_rid="$(cat $fs/rid)" || \ - echo_fail "failed to get rid in $fs" - if [ "$fs_rid" != "$rid" ]; then - continue - fi - - nr="$(cat $fs/data_device_maj_min)" || \ - echo_fail "failed to get data device major:minor in $fs" - - mnts=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \ + mnt=$(findmnt -l -n -t scoutfs -o TARGET -S $nr) || \ echo_fail "findmnt -t scoutfs -S $nr failed" - for mnt in $mnts; do - umount -f "$mnt" || \ - echo_fail "umout -f $mnt failed" - done + [ -z "$mnt" ] && continue + + if ! umount -qf "$mnt"; then + if [ -d "$fs" ]; then + echo_fail "umount -qf $mnt failed" + fi + fi done exit 0 diff --git a/utils/fenced/scoutfs-fenced b/utils/fenced/scoutfs-fenced index fa866e25..5070de4f 100755 --- a/utils/fenced/scoutfs-fenced +++ b/utils/fenced/scoutfs-fenced @@ -7,7 +7,7 @@ message_output() error_message() { - message_output "$@" >&2 + message_output "$@" >> /dev/stderr } error_exit() @@ -62,31 +62,27 @@ test -x "$SCOUTFS_FENCED_RUN" || \ # files disappear. # -# generate failure messages to stderr while still echoing 0 for the caller -careful_cat() +# silence error messages +quiet_cat() { - local path="$@" - - cat "$@" || echo 0 + cat "$@" 2>/dev/null } while sleep $SCOUTFS_FENCED_DELAY; do + shopt -s nullglob for fence in /sys/fs/scoutfs/*/fence/*; do - # catches unmatched regex when no dirs - if [ ! -d "$fence" ]; then - continue - fi - - # skip requests that have been handled - if [ "$(careful_cat $fence/fenced)" == 1 -o \ - "$(careful_cat $fence/error)" == 1 ]; then - continue - fi srv=$(basename $(dirname $(dirname $fence))) - rid="$(cat $fence/rid)" - ip="$(cat $fence/ipv4_addr)" - reason="$(cat $fence/reason)" + fenced="$(quiet_cat $fence/fenced)" + error="$(quiet_cat $fence/error)" + rid="$(quiet_cat $fence/rid)" + ip="$(quiet_cat $fence/ipv4_addr)" + reason="$(quiet_cat $fence/reason)" + + # request dirs can linger then disappear after fenced/error is set + if [ ! -d "$fence" -o "$fenced" == "1" -o "$error" == "1" ]; then + continue + fi log_message "server $srv fencing rid $rid at IP $ip for $reason"