Speed up development and release candidate verification

dev/demo.sh

@@ -1,12 +1,28 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+dev_mode=0
+if [[ $# == 1 && $1 == --dev ]]; then
+    dev_mode=1
+elif [[ $# != 0 ]]; then
+    echo "Usage: $0 [--dev]" >&2
+    exit 2
+fi
+
 script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
 repo_root=$(cd -- "${script_dir}/.." && pwd)
 demo_root=${YATM_DEMO_ROOT:-"${TMPDIR:-/tmp}/yatm-demo"}
 listen=${YATM_DEMO_LISTEN:-127.0.0.1:18080}
+frontend_port=${YATM_DEMO_FRONTEND_PORT:-5173}
 case "$demo_root" in /*) ;; *) demo_root="$PWD/$demo_root" ;; esac
 
+if [[ $dev_mode == 1 ]]; then
+    if [[ ! $frontend_port =~ ^[1-9][0-9]{0,4}$ ]] || (( frontend_port > 65535 )); then
+        echo "Demo frontend port must be between 1 and 65535" >&2
+        exit 2
+    fi
+fi
+
 cd "${repo_root}"
 prepare_args=(-root "${demo_root}" -listen "${listen}")
 if [[ ${YATM_DEMO_RESET:-0} == 1 ]]; then
@@ -20,9 +36,45 @@ if [[ ${YATM_DEMO_IDENTICAL_FILES+x} ]]; then
 fi
 go run ./cmd/demo "${prepare_args[@]}"
 
-pnpm --dir frontend build
+if [[ $dev_mode == 0 ]]; then
+    pnpm --dir frontend build
+fi
 go build -o "${demo_root}/yatm-httpd" ./cmd/httpd
 
+if [[ $dev_mode == 1 ]]; then
+    backend_pid=
+    frontend_pid=
+    cleanup() {
+        trap - EXIT INT TERM
+        for pid in "$backend_pid" "$frontend_pid"; do
+            if [[ -n $pid ]]; then kill "$pid" 2>/dev/null || true; fi
+        done
+        for pid in "$backend_pid" "$frontend_pid"; do
+            if [[ -n $pid ]]; then wait "$pid" 2>/dev/null || true; fi
+        done
+    }
+    trap cleanup EXIT
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+
+    (cd "${demo_root}" && exec ./yatm-httpd -config ./config.yaml) &
+    backend_pid=$!
+    (cd "${repo_root}/frontend" && DEV_SERVICE_BASE="http://${listen}" exec node ./node_modules/vite/bin/vite.js \
+        --host localhost --port "$frontend_port" --strictPort) &
+    frontend_pid=$!
+
+    # Bash 3.2 has no wait -n. Stop both children when either one exits.
+    while kill -0 "$backend_pid" 2>/dev/null && kill -0 "$frontend_pid" 2>/dev/null; do
+        sleep 0.2
+    done
+    if kill -0 "$backend_pid" 2>/dev/null; then
+        wait "$frontend_pid"
+    else
+        wait "$backend_pid"
+    fi
+    exit
+fi
+
 rm -rf "${demo_root}/frontend"
 mkdir -p "${demo_root}/frontend"
 cp frontend/dist/index.html "${demo_root}/frontend/index.html"

dev/demo.test.mjs

@@ -0,0 +1,437 @@
+import assert from "node:assert/strict";
+import { spawn } from "node:child_process";
+import { once } from "node:events";
+import {
+  copyFile,
+  mkdir,
+  mkdtemp,
+  readFile,
+  realpath,
+  rm,
+  symlink,
+  writeFile,
+} from "node:fs/promises";
+import { createServer } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { setTimeout as delay } from "node:timers/promises";
+import test from "node:test";
+
+const frontend = fileURLToPath(new URL("../frontend/", import.meta.url));
+const record = `
+import { appendFileSync } from "node:fs";
+const args = process.argv.slice(2);
+const log = (kind) => appendFileSync(process.env.DEMO_TEST_LOG, JSON.stringify({
+  kind, args, cwd: process.cwd(), pid: process.pid, proxy: process.env.DEV_SERVICE_BASE,
+}) + "\\n");
+`;
+const backend = `#!/usr/bin/env node
+${record}
+import { createServer } from "node:http";
+log("backend");
+if (process.env.DEMO_TEST_BACKEND_EXIT) process.exit(Number(process.env.DEMO_TEST_BACKEND_EXIT));
+const address = new URL("http://" + process.env.YATM_DEMO_LISTEN);
+const server = createServer((request, response) => response.end("Demo " + request.url));
+server.on("error", (error) => { console.error(error.message); process.exit(23); });
+server.listen(Number(address.port), address.hostname, () => log("backend-ready"));
+process.on("SIGTERM", () => server.close(() => process.exit(0)));
+`;
+const go = `#!/usr/bin/env node
+${record}
+import { copyFileSync, mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+log("go");
+if (process.env.DEMO_TEST_GO_FAIL === args[0]) process.exit(31);
+if (args[0] === "run") {
+  const root = args[args.indexOf("-root") + 1];
+  mkdirSync(root, { recursive: true });
+  writeFileSync(join(root, "config.yaml"), "Demo fixture");
+} else if (args[0] === "build") {
+  const output = args[args.indexOf("-o") + 1];
+  mkdirSync(dirname(output), { recursive: true });
+  copyFileSync(process.env.DEMO_TEST_BACKEND, output);
+}
+`;
+const pnpm = `#!/usr/bin/env node
+${record}
+import { mkdirSync, writeFileSync } from "node:fs";
+log("pnpm");
+mkdirSync("frontend/dist/assets", { recursive: true });
+writeFileSync("frontend/dist/index.html", "production frontend");
+writeFileSync("frontend/dist/assets/app.js", "production assets");
+`;
+const vite = `${record}
+log("frontend");
+if (process.env.DEMO_TEST_FRONTEND_EXIT) process.exit(Number(process.env.DEMO_TEST_FRONTEND_EXIT));
+setInterval(() => {}, 1000);
+process.on("SIGTERM", () => process.exit(0));
+`;
+
+async function until(predicate, detail) {
+  const deadline = Date.now() + 15000;
+  while (Date.now() < deadline) {
+    if (await predicate()) return;
+    await delay(20);
+  }
+  assert.fail(`Timed out: ${detail}`);
+}
+
+async function listen(server, host = "127.0.0.1", port = 0) {
+  server.listen(port, host);
+  await once(server, "listening");
+  return server.address().port;
+}
+
+async function freePort() {
+  const server = createServer();
+  const port = await listen(server);
+  await new Promise((resolve) => server.close(resolve));
+  return port;
+}
+
+function alive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    if (error.code !== "ESRCH") throw error;
+    return false;
+  }
+}
+
+async function fixture(
+  t,
+  { realVite = false, env = {}, args = ["--dev"] } = {},
+) {
+  const root = await realpath(
+    await mkdtemp(join(tmpdir(), "yatm-demo-script-")),
+  );
+  const repo = join(root, "repo");
+  const caller = join(root, "caller directory");
+  const demoRoot = join(caller, "yatm-demo-review");
+  const logPath = join(root, "commands.jsonl");
+  const bin = join(root, "bin");
+  await Promise.all([
+    mkdir(join(repo, "dev"), { recursive: true }),
+    mkdir(join(repo, "frontend"), { recursive: true }),
+    mkdir(caller),
+    mkdir(bin),
+  ]);
+  await copyFile(
+    new URL("./demo.sh", import.meta.url),
+    join(repo, "dev/demo.sh"),
+  );
+  await writeFile(logPath, "");
+  await writeFile(join(bin, "backend"), backend, { mode: 0o755 });
+  await writeFile(join(bin, "go"), go, { mode: 0o755 });
+  await writeFile(join(bin, "pnpm"), pnpm, { mode: 0o755 });
+  if (realVite) {
+    await symlink(
+      join(frontend, "node_modules"),
+      join(repo, "frontend/node_modules"),
+    );
+    for (const file of ["vite.config.ts", "index.html", "package.json"]) {
+      await copyFile(join(frontend, file), join(repo, "frontend", file));
+    }
+  } else {
+    await mkdir(join(repo, "frontend/node_modules/vite/bin"), {
+      recursive: true,
+    });
+    await writeFile(join(repo, "frontend/package.json"), '{"type":"module"}');
+    await writeFile(join(repo, "frontend/node_modules/vite/bin/vite.js"), vite);
+  }
+  await mkdir(join(demoRoot, "frontend"), { recursive: true });
+  await writeFile(join(demoRoot, "frontend/reviewer.txt"), "retained frontend");
+
+  const environment = { ...process.env };
+  for (const key of Object.keys(environment)) {
+    if (key.startsWith("YATM_DEMO_") || key.startsWith("DEMO_TEST_"))
+      delete environment[key];
+  }
+  const backendPort = await freePort();
+  const child = spawn("bash", [join(repo, "dev/demo.sh"), ...args], {
+    cwd: caller,
+    detached: true,
+    env: {
+      ...environment,
+      PATH: `${bin}:${process.env.PATH}`,
+      YATM_DEMO_ROOT: "yatm-demo-review",
+      YATM_DEMO_LISTEN: `127.0.0.1:${backendPort}`,
+      DEMO_TEST_LOG: logPath,
+      DEMO_TEST_BACKEND: join(bin, "backend"),
+      ...env,
+    },
+  });
+  let output = "";
+  child.stdout.on("data", (data) => (output += data));
+  child.stderr.on("data", (data) => (output += data));
+  let exit;
+  child.on("exit", (code, signal) => (exit = { code, signal }));
+  t.after(async () => {
+    // The test owns this isolated process group, including cleanup after failed assertions.
+    if (alive(-child.pid)) process.kill(-child.pid, "SIGTERM");
+    await until(() => exit, output);
+    if (alive(-child.pid)) process.kill(-child.pid, "SIGKILL");
+    await rm(root, { force: true, recursive: true });
+  });
+  const commands = async () =>
+    (await readFile(logPath, "utf8"))
+      .trim()
+      .split("\n")
+      .filter(Boolean)
+      .map(JSON.parse);
+  return {
+    child,
+    commands,
+    repo,
+    demoRoot,
+    backendPort,
+    output: () => output,
+    async exited() {
+      await until(() => exit, output);
+      return exit;
+    },
+    async ready() {
+      await until(async () => {
+        const rows = await commands();
+        return (
+          rows.some((row) => row.kind === "backend-ready") &&
+          rows.some((row) => row.kind === "frontend")
+        );
+      }, "both Demo children started");
+    },
+  };
+}
+
+async function childrenStopped(demo) {
+  const children = (await demo.commands()).filter(
+    (row) => row.kind === "backend" || row.kind === "frontend",
+  );
+  assert.equal(children.length, 2, demo.output());
+  for (const child of children)
+    assert.equal(alive(child.pid), false, `${child.kind} must be reaped`);
+}
+
+test("production Demo builds and installs assets and forwards explicit fixture options", async (t) => {
+  const demo = await fixture(t, {
+    args: [],
+    env: {
+      YATM_DEMO_RESET: "1",
+      YATM_DEMO_VIDEO: "/tmp/clip with spaces.mp4",
+      YATM_DEMO_IDENTICAL_FILES: "10000",
+      DEMO_TEST_BACKEND_EXIT: "0",
+    },
+  });
+  assert.deepEqual(await demo.exited(), { code: 0, signal: null });
+  const rows = await demo.commands();
+  assert.deepEqual(
+    rows.map((row) => row.kind),
+    ["go", "pnpm", "go", "backend"],
+  );
+  assert.deepEqual(rows[0].args, [
+    "run",
+    "./cmd/demo",
+    "-root",
+    demo.demoRoot,
+    "-listen",
+    `127.0.0.1:${demo.backendPort}`,
+    "-reset",
+    "-video",
+    "/tmp/clip with spaces.mp4",
+    "-identical-files",
+    "10000",
+  ]);
+  assert.deepEqual(rows[1].args, ["--dir", "frontend", "build"]);
+  assert.deepEqual(rows[2].args, [
+    "build",
+    "-o",
+    join(demo.demoRoot, "yatm-httpd"),
+    "./cmd/httpd",
+  ]);
+  assert.equal(rows[0].cwd, demo.repo);
+  assert.equal(rows[3].cwd, demo.demoRoot);
+  assert.deepEqual(rows[3].args, ["-config", "./config.yaml"]);
+  assert.equal(
+    await readFile(join(demo.demoRoot, "frontend/index.html"), "utf8"),
+    "production frontend",
+  );
+  assert.equal(
+    await readFile(join(demo.demoRoot, "frontend/assets/app.js"), "utf8"),
+    "production assets",
+  );
+});
+
+for (const [signal, code] of [
+  ["SIGTERM", 143],
+  ["SIGINT", 130],
+]) {
+  test(`development Demo preserves fixtures and cleans both children on ${signal}`, async (t) => {
+    const demo = await fixture(t);
+    await demo.ready();
+    const rows = await demo.commands();
+    const frontend = rows.find((row) => row.kind === "frontend");
+    assert.deepEqual(
+      rows.filter((row) => row.kind === "go").map((row) => row.args),
+      [
+        [
+          "run",
+          "./cmd/demo",
+          "-root",
+          demo.demoRoot,
+          "-listen",
+          `127.0.0.1:${demo.backendPort}`,
+        ],
+        ["build", "-o", join(demo.demoRoot, "yatm-httpd"), "./cmd/httpd"],
+      ],
+    );
+    assert.equal(
+      rows.some((row) => row.kind === "pnpm"),
+      false,
+    );
+    assert.deepEqual(frontend.args, [
+      "--host",
+      "localhost",
+      "--port",
+      "5173",
+      "--strictPort",
+    ]);
+    assert.equal(frontend.proxy, `http://127.0.0.1:${demo.backendPort}`);
+    assert.equal(frontend.cwd, join(demo.repo, "frontend"));
+    assert.equal(
+      await readFile(join(demo.demoRoot, "frontend/reviewer.txt"), "utf8"),
+      "retained frontend",
+    );
+    demo.child.kill(signal);
+    assert.deepEqual(await demo.exited(), { code, signal: null });
+    await childrenStopped(demo);
+  });
+}
+
+for (const [kind, code] of [
+  ["BACKEND", 41],
+  ["FRONTEND", 42],
+]) {
+  test(`development Demo cleans its peer on early ${kind.toLowerCase()} failure`, async (t) => {
+    const demo = await fixture(t, {
+      env: { [`DEMO_TEST_${kind}_EXIT`]: String(code) },
+    });
+    assert.deepEqual(await demo.exited(), { code, signal: null });
+    // A peer may be stopped before its Node entrypoint records startup.
+    for (const child of (await demo.commands()).filter(
+      (row) => row.kind === "backend" || row.kind === "frontend",
+    )) {
+      assert.equal(alive(child.pid), false);
+    }
+    assert.equal(
+      alive(-demo.child.pid),
+      false,
+      "no child process remains in the Demo group",
+    );
+  });
+}
+
+for (const stage of ["run", "build"]) {
+  test(`development Demo starts no runtime children after failed Go ${stage}`, async (t) => {
+    const demo = await fixture(t, { env: { DEMO_TEST_GO_FAIL: stage } });
+    assert.deepEqual(await demo.exited(), { code: 31, signal: null });
+    assert.equal(
+      (await demo.commands()).every((row) => row.kind === "go"),
+      true,
+    );
+  });
+}
+
+test("development Demo forwards reset only when explicitly requested", async (t) => {
+  const demo = await fixture(t, {
+    env: { YATM_DEMO_RESET: "1", YATM_DEMO_FRONTEND_PORT: "5199" },
+  });
+  await demo.ready();
+  const rows = await demo.commands();
+  assert.equal(rows[0].args.at(-1), "-reset");
+  assert.deepEqual(rows.find((row) => row.kind === "frontend").args, [
+    "--host",
+    "localhost",
+    "--port",
+    "5199",
+    "--strictPort",
+  ]);
+  demo.child.kill("SIGTERM");
+  await demo.exited();
+  await childrenStopped(demo);
+});
+
+test("invalid options and ports fail before fixture preparation", async (t) => {
+  for (const options of [
+    { args: ["--unknown"] },
+    { args: ["--dev", "extra"] },
+    ...["0", "65536", "invalid"].map((port) => ({
+      env: { YATM_DEMO_FRONTEND_PORT: port },
+    })),
+  ]) {
+    const demo = await fixture(t, options);
+    assert.deepEqual(await demo.exited(), { code: 2, signal: null });
+    assert.deepEqual(await demo.commands(), []);
+  }
+});
+
+test("real Vite serves HMR and proxies both Demo backend routes on a custom port", async (t) => {
+  const port = await freePort();
+  const demo = await fixture(t, {
+    realVite: true,
+    env: { YATM_DEMO_FRONTEND_PORT: String(port) },
+  });
+  let html;
+  await until(async () => {
+    try {
+      const response = await fetch(`http://localhost:${port}/`);
+      html = await response.text();
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }, "Vite frontend started");
+  assert.match(html, /window\.apiBase = "\/services"/);
+  assert.match(html, /\/@vite\/client/);
+  assert.equal(
+    (await fetch(`http://localhost:${port}/@vite/client`)).status,
+    200,
+  );
+  for (const route of ["/services/probe", "/files/probe"]) {
+    assert.equal(
+      await (await fetch(`http://localhost:${port}${route}`)).text(),
+      `Demo ${route}`,
+    );
+  }
+  demo.child.kill("SIGTERM");
+  assert.deepEqual(await demo.exited(), { code: 143, signal: null });
+  await assert.rejects(fetch(`http://localhost:${port}/`));
+  await assert.rejects(fetch(`http://127.0.0.1:${demo.backendPort}/`));
+});
+
+test("an occupied frontend port fails instead of moving and stops the backend", async (t) => {
+  const blocker = createServer();
+  const port = await listen(blocker, "localhost");
+  t.after(() => new Promise((resolve) => blocker.close(resolve)));
+  const demo = await fixture(t, {
+    realVite: true,
+    env: { YATM_DEMO_FRONTEND_PORT: String(port) },
+  });
+  assert.deepEqual(await demo.exited(), { code: 1, signal: null });
+  assert.match(demo.output(), new RegExp(`Port ${port} is already in use`));
+  const backend = (await demo.commands()).find((row) => row.kind === "backend");
+  assert.equal(alive(backend.pid), false);
+  await assert.rejects(fetch(`http://127.0.0.1:${demo.backendPort}/`));
+});
+
+test("an occupied backend port fails and stops the frontend", async (t) => {
+  const blocker = createServer();
+  const port = await listen(blocker);
+  t.after(() => new Promise((resolve) => blocker.close(resolve)));
+  const demo = await fixture(t, {
+    env: { YATM_DEMO_LISTEN: `127.0.0.1:${port}` },
+  });
+  assert.deepEqual(await demo.exited(), { code: 23, signal: null });
+  assert.match(demo.output(), /EADDRINUSE/);
+  assert.equal(alive(-demo.child.pid), false);
+});

dev/performance/collect.mjs

@@ -7,11 +7,12 @@ import { resolveGo } from "./toolchain.mjs";
 import { digest, readSamples, validateInventory, validateSamples } from "./evidence.mjs";
 
 export function options(args) {
-  const result = { tier: "fast", cpu: 2, benchtime: "1s", cgo: "1" };
+  const result = { tier: "fast", cpu: 2, benchtime: "1s", cgo: "1", packages: [] };
   for (let index = 0; index < args.length; index++) {
     const key = args[index].replace(/^--/, "");
     if (key === "idle") { result.idle = true; continue; }
-    if (!["baseline", "candidate", "harness", "out", "tier", "cpu", "benchtime", "cgo", "environment", "manifest"].includes(key) || !args[index + 1] || args[index + 1].startsWith("--")) throw new Error(`Unknown or incomplete option ${args[index]}`);
+    if (!["baseline", "candidate", "harness", "out", "tier", "cpu", "benchtime", "cgo", "environment", "manifest", "package"].includes(key) || !args[index + 1] || args[index + 1].startsWith("--")) throw new Error(`Unknown or incomplete option ${args[index]}`);
+    if (key === "package") { result.packages.push(args[++index]); continue; }
     result[key] = args[++index];
   }
   for (const key of ["baseline", "candidate", "out", "environment"]) if (!result[key]) throw new Error(`--${key} is required`);
@@ -84,8 +85,15 @@ function idle() {
 
 export function collect(config, run = command, checkIdle = idle) {
   const manifestPath = resolve(config.manifest ?? join(dirname(fileURLToPath(import.meta.url)), "suites.json"));
-  const tier = validateTier(JSON.parse(readFileSync(manifestPath, "utf8")).tiers[config.tier]);
-  if (config.tier === "release" && !tier.structural?.length) throw new Error("Release tier requires explicit 1m structural checks");
+  const manifest = readFileSync(manifestPath);
+  const fullTier = validateTier(JSON.parse(manifest).tiers[config.tier]);
+  if (config.tier === "release" && !fullTier.structural?.length) throw new Error("Release tier requires explicit 1m structural checks");
+  const requested = [...new Set(config.packages ?? [])];
+  for (const pkg of requested) if (!fullTier.suites.some((suite) => suite.package === pkg)) throw new Error(`Package is not in the ${config.tier} tier: ${pkg}`);
+  const suites = fullTier.suites.filter((suite) => !requested.length || requested.includes(suite.package));
+  const packages = suites.map((suite) => suite.package);
+  const tier = { ...fullTier, suites, harness: fullTier.harness.filter((name) => packages.some((pkg) => name.startsWith(localPath(pkg) + "/"))),
+    structural: (fullTier.structural ?? []).filter((check) => packages.includes(check.package)) };
   const output = physicalPath(config.out), harnessRoot = realpathSync(config.harness ?? config.candidate);
   if (existsSync(output)) throw new Error("Output must be a new directory; existing evidence is never overwritten");
   for (const root of [config.baseline, config.candidate, harnessRoot]) {
@@ -95,7 +103,12 @@ export function collect(config, run = command, checkIdle = idle) {
   checkIdle();
   mkdirSync(output, { recursive: true });
   const workspace = mkdtempSync(join(dirname(output), ".yatm-performance-"));
-  const record = { format: 1, complete: false, tier: config.tier, idleConfirmed: true, startedAt: new Date().toISOString(), sources: {}, runs: [], structural: [] };
+  const record = { format: 1, complete: false, tier: config.tier, idleConfirmed: true, startedAt: new Date().toISOString(), sources: {}, runs: [], structural: [],
+    fullReleaseComparison: false, scope: { requestedPackages: requested.length ? requested : packages, selectedPackages: packages, executedPackages: [],
+      skippedPackages: fullTier.suites.map((suite) => suite.package).filter((pkg) => !packages.includes(pkg)),
+      structural: tier.structural.map(({ package: pkg, tests }) => ({ package: pkg, tests })),
+      skippedStructural: (fullTier.structural ?? []).filter((check) => !packages.includes(check.package)).map(({ package: pkg, tests }) => ({ package: pkg, tests })) } };
+  record.scope.fullTier = record.scope.skippedPackages.length === 0;
   const save = () => writeFileSync(join(output, "pair.json"), JSON.stringify(record, null, 2) + "\n");
   try {
     // One environment and fixture filesystem serve both source revisions.
@@ -106,6 +119,10 @@ export function collect(config, run = command, checkIdle = idle) {
     mkdirSync(env.TMPDIR);
     const selectedGo = resolveGo(`go${toolchain}`, harnessRoot, env, run);
     env = selectedGo.env;
+    const fixtureEnvs = Object.fromEntries(["baseline", "candidate"].map((side) => {
+      const TMPDIR = join(env.TMPDIR, side); mkdirSync(TMPDIR);
+      return [side, { ...env, TMPDIR }];
+    }));
     const go = selectedGo.info;
     const fs = statfsSync(workspace);
     record.environment = { label: config.environment, go, compiler: config.cgo === "1" ? run(go.CC, ["--version"], { env }).split("\n")[0] : null,
@@ -122,7 +139,7 @@ export function collect(config, run = command, checkIdle = idle) {
       if (!lstatSync(path).isFile()) throw new Error(`Harness paths must name files: ${name}`);
       return { name, bytes: readFileSync(path) };
     });
-    record.collector = { sha256: digest(readFileSync(fileURLToPath(import.meta.url))), manifestSHA256: digest(readFileSync(manifestPath)) };
+    record.collector = { sha256: digest(readFileSync(fileURLToPath(import.meta.url))), manifestSHA256: digest(manifest) };
     record.harness = { commit: run("git", ["-C", harnessRoot, "rev-parse", "HEAD"]).trim(), files: harness.map(({ name, bytes }) => [name, digest(bytes)]), sha256: digest(JSON.stringify(harness.map(({ name, bytes }) => [name, digest(bytes)]))) };
     for (const side of ["baseline", "candidate"]) {
       for (const suite of tier.suites) {
@@ -134,15 +151,17 @@ export function collect(config, run = command, checkIdle = idle) {
         writeFileSync(join(workspace, side, name), bytes);
       }
       writeFileSync(join(output, `${side}.bench`), "");
-      for (const [index, suite] of record.suites.entries()) run(selectedGo.file, ["test", "-c", "-p=1", "-o", join(workspace, `${side}-${index}.test`), suite.package], { cwd: join(workspace, side), env });
+      for (const [index, suite] of record.suites.entries()) run(selectedGo.file, ["test", "-c", "-p=1", "-o", join(workspace, `${side}-${index}.test`), suite.package], { cwd: join(workspace, side), env: fixtureEnvs[side] });
     }
     save();
 
     // Run each standard Go benchmark suite once per source. Go calibrates its own loop.
     for (const [index, suite] of record.suites.entries()) {
       for (const side of ["baseline", "candidate"]) {
+        if (!record.scope.executedPackages.includes(suite.package)) record.scope.executedPackages.push(suite.package);
+        save();
         const args = ["-test.run=^$", `-test.bench=${suite.bench}`, "-test.benchmem", `-test.benchtime=${suite.benchtime}`, "-test.count=1", `-test.cpu=${config.cpu}`, "-test.parallel=1", "-test.timeout=30m"];
-        const stdout = run(join(workspace, `${side}-${index}.test`), args, { cwd: join(workspace, side, localPath(suite.package)), env });
+        const stdout = run(join(workspace, `${side}-${index}.test`), args, { cwd: join(workspace, side, localPath(suite.package)), env: fixtureEnvs[side] });
         const text = `pkg: ${suite.importPath}\n${stdout}\n`;
         appendFileSync(join(output, `${side}.bench`), text);
         if (!/^PASS\s*$/m.test(stdout)) throw new Error("Benchmark suite did not complete");
@@ -156,7 +175,7 @@ export function collect(config, run = command, checkIdle = idle) {
       const result = { ...check, passed: false, outputs: {} }; record.structural.push(result);
       // Structural correctness is checked once on the candidate, separately from timing.
       const binary = join(workspace, `candidate-${suiteIndex}.test`);
-      const settings = { cwd: join(workspace, "candidate", localPath(check.package)), env: { ...env, ...check.env } };
+      const settings = { cwd: join(workspace, "candidate", localPath(check.package)), env: { ...fixtureEnvs.candidate, ...check.env } };
       const listed = run(binary, [`-test.list=${check.run}`], settings).trim().split(/\r?\n/).sort();
       if (JSON.stringify(listed) !== JSON.stringify([...check.tests].sort())) throw new Error("Required candidate structural tests are absent");
       const stdout = run(binary, [`-test.run=${check.run}`, "-test.count=1", "-test.v", "-test.timeout=60m"], settings);
@@ -167,7 +186,8 @@ export function collect(config, run = command, checkIdle = idle) {
     }
     validateSamples(readSamples(readFileSync(join(output, "baseline.bench"), "utf8")), readSamples(readFileSync(join(output, "candidate.bench"), "utf8")), record.suites);
     record.outputs = Object.fromEntries(["baseline", "candidate"].map((side) => [side, digest(readFileSync(join(output, `${side}.bench`)))]));
-    record.complete = true; record.finishedAt = new Date().toISOString(); save();
+    record.complete = true; record.fullReleaseComparison = config.tier === "release" && record.scope.fullTier;
+    record.finishedAt = new Date().toISOString(); save();
     return record;
   } catch (error) {
     record.error = error.message; save();

dev/performance/collect.test.mjs

@@ -6,7 +6,7 @@ import { basename, join } from "node:path";
 import test from "node:test";
 import { fileURLToPath } from "node:url";
 import { collect, options, validateTier } from "./collect.mjs";
-import { readSamples, validateInventory, validatePair, validateSamples } from "./evidence.mjs";
+import { digest, readSamples, validateInventory, validatePair, validateSamples } from "./evidence.mjs";
 import { comparePair } from "../check-performance.mjs";
 
 function fixture(t) {
@@ -359,3 +359,127 @@ test("snapshot rejects source symlinks including dangling links", async (t) => {
     assert.equal(JSON.parse(readFileSync(join(f.config.out, "pair.json"))).complete, false);
   });
 });
+
+function packageFixture(t) {
+  const f = fixture(t);
+  const extra = "internal/other/performance_test.go";
+  f.tier.harness.push(extra);
+  f.tier.suites.push({ package: "./internal/other", bench: "^BenchmarkOther$", benchmarks: ["BenchmarkOther"], benchtime: "1x" });
+  f.tier.structural = f.tier.suites.map((suite, index) => ({ scope: "candidate", package: suite.package,
+    run: `^TestRows${index}$`, tests: [`TestRows${index}`] }));
+  f.config.tier = "release";
+  writeFileSync(f.config.manifest, JSON.stringify({ tiers: { release: f.tier } }));
+  for (const side of ["baseline", "candidate"]) {
+    mkdirSync(join(f.root, side, "internal/other"), { recursive: true });
+    writeFileSync(join(f.root, side, extra), `other harness from ${side}\n`);
+  }
+  const original = f.run;
+  f.run = (file, args, settings = {}) => {
+    if (file === "git" && args.includes("ls-files")) return original(file, args, settings) + extra + "\0";
+    if (basename(file) === "go" && args[0] === "test") {
+      f.calls.push({ file, args, settings });
+      const pkg = args.at(-1).replace(/^\.\//, "");
+      const expected = pkg.endsWith("other") ? "other harness from candidate\n" : "common harness from candidate\n";
+      assert.equal(readFileSync(join(settings.cwd, pkg, "performance_test.go"), "utf8"), expected);
+      assert.equal(f.calls.some((call) => call.file.endsWith(".test")), false);
+      return "";
+    }
+    if (file.endsWith(".test")) {
+      f.calls.push({ file, args, settings });
+      const list = /^-test.list=\^(TestRows\d+)\$$/.exec(args[0]);
+      if (list) return list[1] + "\n";
+      const structural = /^-test.run=\^(TestRows\d+)\$$/.exec(args[0]);
+      if (structural) return `--- PASS: ${structural[1]} (0.00s)\nPASS\n`;
+      const benchmark = basename(settings.cwd) === "other" ? "BenchmarkOther" : "BenchmarkList";
+      return `goos: linux\ngoarch: amd64\ncpu: synthetic test CPU\n${benchmark}-2 1 100 ns/op 0 B/op 0 allocs/op\nPASS\n`;
+    }
+    return original(file, args, settings);
+  };
+  return f;
+}
+
+test("package selection filters an existing tier, harness and applicable structural checks", (t) => {
+  const f = packageFixture(t);
+  f.config.packages = ["./internal/other", "./internal/other"];
+  const record = collect(f.config, (file, args, settings) => {
+    if (args.includes("-test.run=^$")) {
+      const state = join(settings.env.TMPDIR, "mutable-fixture");
+      assert.equal(existsSync(state), false, "the baseline fixture cannot leak into the candidate");
+      writeFileSync(state, "owned by this source");
+    }
+    return f.run(file, args, settings);
+  }, () => {});
+  assert.deepEqual(record.scope, { requestedPackages: ["./internal/other"], selectedPackages: ["./internal/other"],
+    executedPackages: ["./internal/other"], skippedPackages: ["./internal/example"], fullTier: false,
+    structural: [{ package: "./internal/other", tests: ["TestRows1"] }],
+    skippedStructural: [{ package: "./internal/example", tests: ["TestRows0"] }] });
+  assert.equal(record.fullReleaseComparison, false, "a completed subset cannot claim release comparison");
+  assert.deepEqual(record.harness.files.map(([name]) => name), ["internal/other/performance_test.go"]);
+  assert.deepEqual(record.runs.map(({ side, package: pkg }) => [side, pkg]), [["baseline", "./internal/other"], ["candidate", "./internal/other"]]);
+  const benchmarks = f.calls.filter((call) => call.args.includes("-test.run=^$"));
+  assert.equal(benchmarks.length, 2);
+  assert.deepEqual(benchmarks[0].args, benchmarks[1].args);
+  assert.ok(benchmarks[0].args.includes("-test.count=1"));
+  assert.ok(benchmarks[0].args.includes("-test.benchtime=1x"));
+  assert.notEqual(benchmarks[0].settings.env.TMPDIR, benchmarks[1].settings.env.TMPDIR);
+  assert.equal(validatePair(f.config.out).fullReleaseComparison, false);
+  assert.equal(comparePair(f.config.out).flagged, false);
+
+  for (const change of [
+    (value) => { value.fullReleaseComparison = true; },
+    (value) => { value.scope.fullTier = true; },
+    (value) => { value.scope.executedPackages = []; },
+    (value) => { value.structural = []; },
+    (value) => { delete value.scope; },
+  ]) {
+    const invalid = structuredClone(record); change(invalid);
+    writeFileSync(join(f.config.out, "pair.json"), JSON.stringify(invalid));
+    assert.throws(() => validatePair(f.config.out), /scope|structural/);
+  }
+});
+
+test("explicit package requests retain manifest order and complete default release scope", (t) => {
+  const f = packageFixture(t);
+  f.config.packages = ["./internal/other", "./internal/example"];
+  const record = collect(f.config, f.run, () => {});
+  assert.deepEqual(record.scope.requestedPackages, f.config.packages);
+  assert.deepEqual(record.scope.executedPackages, ["./internal/example", "./internal/other"]);
+  assert.equal(record.fullReleaseComparison, true);
+  assert.equal(validatePair(f.config.out).scope.fullTier, true);
+  const compile = f.calls.filter((call) => call.args.includes("-c"));
+  const benchmarks = f.calls.filter((call) => call.args.includes("-test.run=^$"));
+  assert.equal(compile.length, 4);
+  assert.equal(benchmarks.length, 4);
+  for (const call of benchmarks) {
+    const side = basename(call.file).split("-")[0], pkg = "./internal/" + basename(call.settings.cwd);
+    assert.equal(call.settings.env.TMPDIR, compile.find((item) => basename(item.settings.cwd) === side && item.args.at(-1) === pkg).settings.env.TMPDIR);
+  }
+});
+
+test("unknown or incomplete package requests fail before source or fixture setup", (t) => {
+  const f = packageFixture(t);
+  assert.throws(() => collect({ ...f.config, packages: ["./internal/absent"] }, f.run, () => {}), /not in the release tier/);
+  assert.equal(f.calls.length, 0);
+  assert.equal(existsSync(f.config.out), false);
+  const args = ["--baseline", "a", "--candidate", "b", "--out", "c", "--environment", "d", "--idle"];
+  assert.deepEqual(options([...args, "--package", "./internal/apis", "--package", "./internal/executor/scan"]).packages,
+    ["./internal/apis", "./internal/executor/scan"]);
+  assert.throws(() => options([...args, "--package"]), /incomplete option --package/);
+});
+
+test("collection records attempted package scope on failure and hashes the consumed immutable manifest", (t) => {
+  const f = packageFixture(t);
+  f.config.packages = ["./internal/other"];
+  const manifest = readFileSync(f.config.manifest);
+  assert.throws(() => collect(f.config, (file, args, settings) => {
+    if (basename(file) === "go" && args[0] === "env") writeFileSync(f.config.manifest, "changed after selection");
+    if (file.endsWith("baseline-0.test")) throw new Error("selected benchmark failed");
+    return f.run(file, args, settings);
+  }, () => {}), /selected benchmark failed/);
+  const record = JSON.parse(readFileSync(join(f.config.out, "pair.json")));
+  assert.equal(record.collector.manifestSHA256, digest(manifest));
+  assert.deepEqual(record.scope.executedPackages, ["./internal/other"]);
+  assert.equal(record.complete, false);
+  assert.equal(record.fullReleaseComparison, false);
+  assert.equal(readdirSync(f.root).some((name) => name.startsWith(".yatm-performance-")), false);
+});

dev/performance/evidence.mjs

@@ -63,6 +63,17 @@ export function validateSamples(base, candidate, suites) {
 export function validatePair(directory) {
   const record = JSON.parse(readFileSync(resolve(directory, "pair.json"), "utf8"));
   if (record.format !== 1 || record.complete !== true || !record.idleConfirmed || !record.environment?.label || !record.environment?.go?.GOVERSION || !record.environment?.host?.platform || !record.harness?.sha256 || !record.suites?.length) throw new Error("Missing completed comparable collection metadata");
+  if (record.fullReleaseComparison !== undefined && !record.scope) throw new Error("Missing recorded package scope");
+  if (record.scope) {
+    const packages = record.suites.map((suite) => suite.package), scope = record.scope;
+    if (JSON.stringify(scope.selectedPackages) !== JSON.stringify(packages) || JSON.stringify(scope.executedPackages) !== JSON.stringify(packages)
+      || !scope.requestedPackages?.length || JSON.stringify([...scope.requestedPackages].sort()) !== JSON.stringify([...packages].sort())
+      || !Array.isArray(scope.skippedPackages) || scope.skippedPackages.some((pkg) => packages.includes(pkg))
+      || scope.fullTier !== (scope.skippedPackages.length === 0)
+      || record.fullReleaseComparison !== (record.tier === "release" && scope.fullTier)) throw new Error("Recorded package scope differs from executed comparison");
+    const checks = (record.structural ?? []).map(({ package: pkg, tests }) => ({ package: pkg, tests }));
+    if (JSON.stringify(scope.structural) !== JSON.stringify(checks)) throw new Error("Required selected structural checks did not finish");
+  }
   for (const side of ["baseline", "candidate"]) {
     if (!/^[a-f0-9]{40,64}$/.test(record.sources?.[side]?.commit ?? "") || !/^[a-f0-9]{64}$/.test(record.sources[side].sha256 ?? "")) throw new Error(`Missing recorded ${side} source identity`);
     const contents = readFileSync(resolve(directory, `${side}.bench`));
@@ -77,6 +88,6 @@ export function validatePair(directory) {
     if (check.scope !== "candidate" || JSON.stringify(Object.keys(check.outputs)) !== '["candidate"]') throw new Error("Structural evidence must explicitly cover only candidate correctness");
     if (digest(readFileSync(resolve(directory, check.outputs.candidate.file))) !== check.outputs.candidate.sha256) throw new Error("Structural evidence changed");
   }
-  if (record.tier === "release" && !record.structural?.length) throw new Error("Release evidence requires the 1m structural checks");
+  if (record.tier === "release" && record.scope?.fullTier !== false && !record.structural?.length) throw new Error("Release evidence requires the 1m structural checks");
   return record;
 }