ci: run each conformance test on its own (#10564)

* ci: run each conformance test on its own

Run as one batch with --no-abort, a test that fails part way leaves its
files behind and the next one fails creating them, so the report showed a
cascade of failures that were really one. The whole rdwr and flush group
passes when run alone, and was only ever collateral.

Each test now gets its own directory and its own invocation, which costs
a process start per test and makes the list mean what it says.

* ci: clear a case directory before reusing it

-Force creates the directory but leaves anything already in it, so a
leftover from an interrupted run would defeat the isolation this exists
to provide.

* ci: list the one real failure isolation exposed

With each test on its own, 31 of 32 pass. The exception is rdwr_mmap_test,
which compares mapped bytes against what was written and finds them
different — a genuine data mismatch that only appeared once the test could
run to completion instead of tripping over a previous one's leftovers.

test/winfsp-conformance/known_failures.txt

@@ -5,8 +5,10 @@
 # the run; a failure in anything NOT listed fails CI, which is what catches a
 # regression.
 #
-# Populated from the first real run: 20 of 50 passed. Extended attributes are
-# forwarded now, so that group runs rather than being excluded. Every entry below is a
+# Every test runs on its own in a clean directory, so an entry here is a real
+# defect rather than the wreckage of the test before it. The rdwr and flush
+# group is gone from this list for exactly that reason: it passes in isolation
+# and only failed as collateral. Every entry below is a
 # gap in the mount rather than a quirk of the suite, and the list is meant to
 # shrink. Keep a reason on each group — an entry with no reason cannot be told
 # apart from one nobody has looked at.
@@ -28,21 +30,13 @@ lock_*
 create_sd_test
 getsecurity_test
 
-# Cached and overlapped IO
-# ------------------------
-# The whole rdwr group fails together, so this is one defect rather than nine:
-# the mount does not yet satisfy what Windows expects of cached, write-through
-# and overlapped IO. The first thing worth fixing.
-rdwr_cached_test
-rdwr_cached_append_test
-rdwr_cached_overlapped_test
-rdwr_noncached_test
-rdwr_noncached_overlapped_test
-rdwr_writethru_test
-rdwr_writethru_append_test
-rdwr_writethru_overlapped_test
-rdwr_mixed_test
-flush_test
+# Memory-mapped IO
+# ----------------
+# rdwr-test.c:645 compares the mapped bytes against the pattern that was
+# written and finds them different. A real data mismatch, not a cascade: it
+# only surfaced once each test ran on its own and this one got far enough to
+# check. Worth its own investigation.
+rdwr_mmap_test
 
 # Delete semantics
 # ----------------

test/winfsp-conformance/run.ps1

@@ -62,19 +62,40 @@ Push-Location $workDir
 try {
     # --fuse-external: a third-party FUSE filesystem, not the bundled memfs.
     # --resilient:     tolerate operations this filesystem does not implement.
-    # --no-abort:      report every failure instead of stopping at the first.
-    $arguments = @('--fuse-external', '--resilient', '--no-abort')
-    foreach ($name in $excluded) {
-        $arguments += "-$name"
+    #
+    # One test per invocation, each in its own directory. Batched with
+    # --no-abort, a test that failed part way left its files behind and the
+    # next one failed creating them, reporting a cascade of failures that were
+    # really one. This costs a process start per test and makes the list mean
+    # what it says.
+    $base = @('--fuse-external', '--resilient')
+    $names = @(& $exe @base '--list' 2>&1 |
+        ForEach-Object { if ($_ -match '^([a-z_0-9]+)\s*$') { $Matches[1] } })
+    if ($names.Count -eq 0) { throw "could not list tests" }
+    Write-Host "listed $($names.Count) tests"
+
+    $failed = @()
+    $ran = 0
+    foreach ($name in $names) {
+        if ($excluded | Where-Object { $name -like $_ }) { continue }
+        $ran++
+        $caseDir = Join-Path $workDir $name
+        # -Force creates the directory but leaves anything already in it, and
+        # a leftover file is the very thing this isolation exists to avoid.
+        Remove-Item $caseDir -Recurse -Force -ErrorAction SilentlyContinue
+        New-Item -ItemType Directory -Force -Path $caseDir | Out-Null
+        Push-Location $caseDir
+        try {
+            $out = & $exe @base $name 2>&1
+            $out | ForEach-Object { Write-Host $_ }
+            if ($out -match '\s+KO\s*$' -or $LASTEXITCODE -ne 0) { $failed += $name }
+        } finally {
+            Pop-Location
+            Remove-Item $caseDir -Recurse -Force -ErrorAction SilentlyContinue
+        }
     }
-    Write-Host "running winfsp-tests with $($excluded.Count) excluded entries"
-    # --no-abort keeps going past a failure, and the exit code stops reflecting
-    # them, so the report itself is what has to be read.
-    $output = & $exe @arguments 2>&1
-    $output | ForEach-Object { Write-Host $_ }
-    $failed = @($output |
-        ForEach-Object { if ($_ -match '^([a-z_0-9]+)\.+\s+KO') { $Matches[1] } })
-    $code = if ($failed.Count -gt 0) { 1 } else { $LASTEXITCODE }
+    Write-Host "ran $ran tests, $($failed.Count) failed"
+    $code = if ($failed.Count -gt 0) { 1 } else { 0 }
 } finally {
     Pop-Location
     Remove-Item $workDir -Recurse -Force -ErrorAction SilentlyContinue