test/e2e: exercise the stale-preview refusal and the real GC sweep

Three scripts, split by what they cost to run. batch07-stale-preview.mjs stages a preview and waits out the 30-minute maxPreviewAgeForDelete constant. batch07-stale-click.mjs is the resumable half: it re-renders whatever preview the hold already holds and clicks delete on it. GET /admin/api/gc/status re-renders lastPreview WITHOUT touching lastPreviewAt, so showing an old preview does not reset its age — which is what makes a failed run cost seconds instead of another 31 minutes. Result against the dev hold: "preview is 34m0s old (limit 30m0s) — run Scan again before deleting" rendered through the progress-to-error fragment chain, with all 387 records still there afterwards. That chain is the point; the refusal logic itself already has a Go test, but a refusal that renders as nothing is indistinguishable from "there was nothing to delete". batch07-sweep.mjs then runs the destructive path for real: 387 records deleted of 387 staged, orphaned count to zero, referenced blobs unchanged at 15. Safe only against the dev hold on Storj; production is Bunny + UpCloud and is not reachable from here. page.on('dialog') did not reliably intercept hx-confirm on this page, and an unaccepted native dialog blocks every later evaluate() and innerText(), so the script hangs rather than fails — the worst failure mode for an unattended check. Both scripts now strip the hx-confirm attribute before clicking. The confirm is not what is under test. Two findings worth carrying, neither introduced by this range: * deleteOrphanedBlobs is still unexercised. The bucket holds 19 objects, of which 8 are past the 7-day blob grace, and none are unreferenced — so there is nothing for it to collect. More pushes cannot help: fresh blobs are inside the grace window by definition. * Storage accounting is derived from layer records, so this sweep moved the dashboard from 1.3 GB to 1.1 KB while the bucket held 147 MB throughout. It was overstating by ~9x before (records for blobs held by another hold) and understates now (referenced blobs with no layer records). Quotas and billing read the same number. Belongs to batch 12. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-02 00:06:58 +00:00 · 2026-08-25 16:34:25 -05:00
parent 0071528b8f
commit 5f74299bd7
3 changed files with 356 additions and 0 deletions
@@ -0,0 +1,101 @@
+// batch07-stale-click.mjs — click delete on an already-stale preview.
+//
+// Companion to batch07-stale-preview.mjs, which stages a preview and waits out
+// maxPreviewAgeForDelete. This half assumes the wait is already done: it
+// re-renders whatever preview the hold is holding and clicks delete on it.
+//
+// GET /admin/api/gc/status re-renders lastPreview WITHOUT touching
+// lastPreviewAt, so displaying an old preview does not reset its age. That is
+// what makes this resumable after a failed run, instead of costing another 31
+// minutes.
+//
+// Two things learned the hard way, both encoded below:
+//
+//   * page.on('dialog') did not reliably intercept the hx-confirm on this page.
+//     The unaccepted native dialog then blocks every subsequent evaluate() and
+//     innerText(), so the script hangs rather than failing. The confirm is not
+//     what is under test, so the attribute is stripped before clicking and no
+//     dialog is ever raised.
+//
+//   * The delete button is not the first "Delete" on the page when a Reconcile
+//     button is also rendered, so match the full label.
+//
+//   node test/e2e/batch07-stale-click.mjs
+import { open, adminPage, reporter } from './lib.mjs';
+
+const ctx = await open();
+const page = await adminPage(ctx);
+const r = reporter();
+const results = page.locator('#gc-results');
+
+await page.click('a[href="/admin#storage"]');
+await page.waitForTimeout(1500);
+
+// How old does the hold think its preview is? Rendered on the tab itself.
+const age = await page.locator('text=/Last scan:/').first().innerText().catch(() => '(none)');
+console.log(`Hold reports: ${age}`);
+
+// --- Re-render the staged preview without restaging it ---------------------
+await page.evaluate(() =>
+  window.htmx.ajax('GET', '/admin/api/gc/status', { target: '#gc-results', swap: 'innerHTML' }),
+);
+
+let staged = '';
+const renderDeadline = Date.now() + 30_000;
+while (Date.now() < renderDeadline) {
+  staged = await results.innerText().catch(() => '');
+  if (staged.includes('Referenced Blobs')) break;
+  await page.waitForTimeout(1000);
+}
+r.record('staged preview re-rendered without restaging', staged.includes('Referenced Blobs'));
+
+const m = staged.match(/Delete (\d+) Orphaned Records/);
+const before = m ? Number(m[1]) : null;
+r.record('delete control present', before !== null, `${before} orphaned records`);
+if (before === null) {
+  r.summarize('batch 07 — stale-preview refusal');
+  await ctx.close();
+  process.exit(1);
+}
+
+// --- Click, with the confirm removed --------------------------------------
+const btn = page.locator(`button:has-text("Delete ${before} Orphaned Records")`).first();
+await btn.evaluate((el) => el.removeAttribute('hx-confirm'));
+await btn.click();
+
+let out = '';
+const deadline = Date.now() + 120_000;
+while (Date.now() < deadline) {
+  out = await results.innerText().catch(() => '');
+  if (/preview is .* old \(limit/.test(out)) break;
+  if (/Records Deleted/.test(out)) break;
+  await page.waitForTimeout(2000);
+}
+
+console.log('\n#gc-results after the click:\n' + out.slice(0, 500));
+
+r.record('stale preview is refused, visibly',
+  /preview is .* old \(limit .*\)/.test(out),
+  out ? out.split('\n').filter(Boolean)[0] : 'nothing rendered');
+
+r.record('refusal names the remedy',
+  out.includes('run Scan again'),
+  'the operator has to be told what to do next');
+
+// --- Nothing was deleted ---------------------------------------------------
+await page.locator('button:has-text("Scan for Orphans")').first().click();
+let after = '';
+const rescanDeadline = Date.now() + 180_000;
+while (Date.now() < rescanDeadline) {
+  after = await results.innerText().catch(() => '');
+  if (after.includes('Referenced Blobs')) break;
+  await page.waitForTimeout(2000);
+}
+const m2 = after.match(/Delete (\d+) Orphaned Records/);
+r.record('no records were deleted by the refused click',
+  m2 !== null && Number(m2[1]) === before,
+  `${before} before, ${m2 ? m2[1] : 'unknown'} after`);
+
+const failed = r.summarize('batch 07 — stale-preview refusal');
+await ctx.close();
+process.exit(failed ? 1 : 0);
@@ -0,0 +1,117 @@
+// batch07-stale-preview.mjs — the stale-preview refusal, end to end.
+//
+// pkg/hold/gc covers the refusal itself (TestDeleteOrphanedRecords_StalePreviewIsRefused).
+// What no Go test can cover is whether the refusal ever reaches a human: the
+// delete handler returns gc_progress.html, which polls /admin/api/gc/status,
+// which renders the error through gc_error.html. A break anywhere in that chain
+// turns a refusal into a silent no-op, and a silent no-op reads exactly like
+// "nothing needed deleting".
+//
+// maxPreviewAgeForDelete is a package constant at 30 minutes, so this takes its
+// own preview and then waits it out. Roughly 32 minutes. It deliberately does
+// NOT reuse an existing preview: taking its own is the only way to know the age
+// of the thing it is about to click on.
+//
+// The click is destructive if the refusal is broken. That is the point, and the
+// blast radius is bounded: the records staged here were confirmed orphaned
+// against their owners' PDSes by batch07-gc.mjs.
+//
+// Do not touch a tracked .go/.html/.css/.js file while this runs. Air rebuilds
+// the hold and a rebuild takes the in-memory admin session with it.
+//
+//   node test/e2e/batch07-stale-preview.mjs
+import { open, adminPage, reporter } from './lib.mjs';
+
+const STALE_WAIT_MS = 31.5 * 60 * 1000;
+
+const ctx = await open();
+const page = await adminPage(ctx);
+const r = reporter();
+const results = page.locator('#gc-results');
+
+const waitForPreview = async (timeoutMs = 180_000) => {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const text = await results.innerText().catch(() => '');
+    if (text.includes('Referenced Blobs')) return text;
+    await page.waitForTimeout(2000);
+  }
+  return null;
+};
+
+const orphanCount = (text) => {
+  const m = text.match(/Delete (\d+) Orphaned Records/);
+  return m ? Number(m[1]) : null;
+};
+
+// --- Stage a preview and start the clock -----------------------------------
+await page.click('a[href="/admin#storage"]');
+await page.waitForTimeout(1500);
+await page.locator('button:has-text("Scan for Orphans")').first().click();
+
+const staged = await waitForPreview();
+r.record('preview staged', staged !== null);
+if (!staged) {
+  await ctx.close();
+  process.exit(1);
+}
+const previewAt = Date.now();
+const before = orphanCount(staged);
+r.record('preview offers a delete control', before !== null && before > 0, `${before} orphaned records`);
+console.log(`Preview staged at ${new Date(previewAt).toLocaleTimeString()} with ${before} orphaned records.`);
+console.log(`Waiting ${(STALE_WAIT_MS / 60000).toFixed(1)} minutes for it to go stale...`);
+
+// --- Wait it out -----------------------------------------------------------
+// Idle only. Navigating here would reload the fragment and lose the button,
+// and re-scanning would reset the very age under test.
+while (Date.now() - previewAt < STALE_WAIT_MS) {
+  await page.waitForTimeout(30_000);
+  const left = Math.ceil((STALE_WAIT_MS - (Date.now() - previewAt)) / 60000);
+  if (left % 5 === 0) console.log(`  ${left} minutes left`);
+}
+
+// --- Click delete on a preview that is now too old -------------------------
+// The button carries hx-confirm, which is a native confirm() dialog. Without a
+// handler Playwright dismisses it and the request is never sent, which would
+// look like a passing refusal while proving nothing.
+let dialogSeen = false;
+page.on('dialog', async (d) => {
+  dialogSeen = true;
+  await d.accept();
+});
+
+await page.locator('button:has-text("Delete")').first().click();
+await page.waitForTimeout(1000);
+r.record('hx-confirm dialog was raised and accepted', dialogSeen);
+
+// --- The refusal has to become visible -------------------------------------
+let refusal = '';
+const deadline = Date.now() + 90_000;
+while (Date.now() < deadline) {
+  const text = await results.innerText().catch(() => '');
+  if (/preview is .* old \(limit/.test(text)) { refusal = text; break; }
+  if (text.includes('Deleted') || text.includes('Records Deleted')) { refusal = text; break; }
+  await page.waitForTimeout(2000);
+}
+
+console.log('\n#gc-results after the click:\n' + refusal.slice(0, 600));
+
+r.record('stale preview is refused, visibly',
+  /preview is .* old \(limit .*\)/.test(refusal),
+  refusal ? refusal.split('\n')[0] : 'nothing rendered');
+
+r.record('refusal names the remedy',
+  refusal.includes('run Scan again'),
+  'the message should tell the operator what to do next');
+
+// --- And nothing was actually deleted --------------------------------------
+await page.locator('button:has-text("Scan for Orphans")').first().click();
+const after = await waitForPreview();
+const afterCount = after ? orphanCount(after) : null;
+r.record('no records were deleted by the refused click',
+  afterCount === before,
+  `${before} before, ${afterCount} after`);
+
+const failed = r.summarize('batch 07 — stale-preview refusal');
+await ctx.close();
+process.exit(failed ? 1 : 0);
@@ -0,0 +1,138 @@
+// batch07-sweep.mjs — run the destructive half of GC for real.
+//
+// Everything else in batch 07 proves GC decides correctly. Nothing proves it
+// can carry a decision out: deleteOrphanedRecords against a real CAR store, and
+// deleteOrphanedBlobs against a real S3 listing, are untested by any Go test and
+// unreachable from the in-process harness.
+//
+// Safe to run ONLY against the dev hold on Storj (gateway.storjshare.io, bucket
+// "atcr"). Production is Bunny + UpCloud and is not reachable from here. Confirm
+// before running:
+//
+//   docker compose exec atcr-hold env | grep -E '^(S3_|AWS_)'
+//
+// Sequence: scan, delete records, re-scan, delete blobs if any appeared,
+// re-scan. Each step reports what changed, because the interesting result is
+// not "it worked" but what the second scan reveals once the records are gone.
+//
+//   node test/e2e/batch07-sweep.mjs
+import { open, adminPage, reporter } from './lib.mjs';
+
+const ctx = await open();
+const page = await adminPage(ctx);
+const r = reporter();
+const results = page.locator('#gc-results');
+
+// hx-confirm raises a native confirm() that page.on('dialog') did not reliably
+// intercept here; an unaccepted dialog then blocks every later evaluate() and
+// the script hangs instead of failing. The confirm is not under test, so it is
+// removed before each destructive click.
+const clickDelete = async (label) => {
+  const btn = page.locator(`button:has-text("${label}")`).first();
+  await btn.evaluate((el) => el.removeAttribute('hx-confirm'));
+  await btn.click();
+};
+
+const waitFor = async (needle, timeoutMs = 300_000) => {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    const text = await results.innerText().catch(() => '');
+    if (text.includes(needle)) return text;
+    if (/preview is .* old \(limit/.test(text)) return text; // stale refusal
+    await page.waitForTimeout(2000);
+  }
+  return null;
+};
+
+const statsOf = async () =>
+  page.evaluate(() => {
+    const out = {};
+    for (const s of document.querySelectorAll('.stat-title')) {
+      const v = s.parentElement?.querySelector('.stat-value');
+      if (v) out[s.textContent.trim()] = v.textContent.trim().split('—')[0].trim();
+    }
+    return out;
+  });
+
+const scan = async (label) => {
+  await page.locator('button:has-text("Scan for Orphans")').first().click();
+  const text = await waitFor('Referenced Blobs');
+  if (!text) return null;
+  const s = await statsOf();
+  console.log(`\n[${label}] records=${s['Orphaned Records']} blobs=${s['Orphaned Blobs']} ` +
+              `missing=${s['Missing Records']} referenced=${s['Referenced Blobs']}`);
+  return s;
+};
+
+await page.click('a[href="/admin#storage"]');
+await page.waitForTimeout(1500);
+
+// --- 1. Scan ---------------------------------------------------------------
+const before = await scan('scan 1');
+r.record('initial scan completed', before !== null);
+if (!before) { await ctx.close(); process.exit(1); }
+
+const recordsToDelete = Number(before['Orphaned Records']);
+r.record('there is something to sweep', recordsToDelete > 0, `${recordsToDelete} orphaned records`);
+
+// --- 2. Delete records -----------------------------------------------------
+// Immediately after the scan, so the preview-age guard cannot fire.
+if (recordsToDelete > 0) {
+  await clickDelete(`Delete ${recordsToDelete} Orphaned Records`);
+  const out = await waitFor('Records Deleted');
+  console.log('\nAfter delete-records:\n' + (out ?? '<nothing>').slice(0, 400));
+
+  r.record('delete-records was not refused as stale',
+    out !== null && !/preview is .* old \(limit/.test(out));
+
+  const after = await statsOf();
+  const deleted = Number(after['Records Deleted']);
+  r.record('every staged record was deleted',
+    deleted === recordsToDelete,
+    `${deleted} deleted of ${recordsToDelete} staged`);
+}
+
+// --- 3. Re-scan ------------------------------------------------------------
+// The question this answers: with the layer records gone, do their blobs now
+// fall out of the referenced set and surface as orphaned? If the bytes were
+// never in this bucket, nothing appears and the count stays flat.
+const mid = await scan('scan 2');
+r.record('re-scan completed', mid !== null);
+if (mid) {
+  r.record('orphaned records are cleared',
+    Number(mid['Orphaned Records']) === 0,
+    `${mid['Orphaned Records']} remain`);
+}
+
+// --- 4. Delete blobs, if any surfaced --------------------------------------
+const blobsToDelete = mid ? Number(mid['Orphaned Blobs']) : 0;
+if (blobsToDelete > 0) {
+  console.log(`\n${blobsToDelete} orphaned blobs surfaced; deleting from S3.`);
+  await clickDelete(`Delete ${blobsToDelete} Orphaned Blobs`);
+  const out = await waitFor('Blobs Deleted');
+  console.log('\nAfter delete-blobs:\n' + (out ?? '<nothing>').slice(0, 400));
+
+  const after = await statsOf();
+  r.record('blobs were deleted from S3',
+    Number(after['Blobs Deleted']) === blobsToDelete,
+    `${after['Blobs Deleted']} of ${blobsToDelete}, ${after['Space Reclaimed'] ?? '?'} reclaimed`);
+
+  const end = await scan('scan 3');
+  if (end) {
+    r.record('orphaned blobs are cleared',
+      Number(end['Orphaned Blobs']) === 0,
+      `${end['Orphaned Blobs']} remain`);
+    r.record('referenced blobs survived the sweep',
+      Number(end['Referenced Blobs']) === Number(before['Referenced Blobs']),
+      `${before['Referenced Blobs']} before, ${end['Referenced Blobs']} after`);
+  }
+} else {
+  console.log('\nNo orphaned blobs surfaced after the records were deleted.');
+  r.record('referenced blobs survived the sweep',
+    mid !== null && Number(mid['Referenced Blobs']) === Number(before['Referenced Blobs']),
+    `${before['Referenced Blobs']} before, ${mid?.['Referenced Blobs']} after`);
+}
+
+const failed = r.summarize('batch 07 — destructive sweep');
+await ctx.close();
+process.exit(failed ? 1 : 0);