diff --git a/test/e2e/batch07-stale-click.mjs b/test/e2e/batch07-stale-click.mjs new file mode 100644 index 0000000..3e86960 --- /dev/null +++ b/test/e2e/batch07-stale-click.mjs @@ -0,0 +1,101 @@ +// batch07-stale-click.mjs — click delete on an already-stale preview. +// +// Companion to batch07-stale-preview.mjs, which stages a preview and waits out +// maxPreviewAgeForDelete. This half assumes the wait is already done: it +// re-renders whatever preview the hold is holding and clicks delete on it. +// +// GET /admin/api/gc/status re-renders lastPreview WITHOUT touching +// lastPreviewAt, so displaying an old preview does not reset its age. That is +// what makes this resumable after a failed run, instead of costing another 31 +// minutes. +// +// Two things learned the hard way, both encoded below: +// +// * page.on('dialog') did not reliably intercept the hx-confirm on this page. +// The unaccepted native dialog then blocks every subsequent evaluate() and +// innerText(), so the script hangs rather than failing. The confirm is not +// what is under test, so the attribute is stripped before clicking and no +// dialog is ever raised. +// +// * The delete button is not the first "Delete" on the page when a Reconcile +// button is also rendered, so match the full label. +// +// node test/e2e/batch07-stale-click.mjs +import { open, adminPage, reporter } from './lib.mjs'; + +const ctx = await open(); +const page = await adminPage(ctx); +const r = reporter(); +const results = page.locator('#gc-results'); + +await page.click('a[href="/admin#storage"]'); +await page.waitForTimeout(1500); + +// How old does the hold think its preview is? Rendered on the tab itself. +const age = await page.locator('text=/Last scan:/').first().innerText().catch(() => '(none)'); +console.log(`Hold reports: ${age}`); + +// --- Re-render the staged preview without restaging it --------------------- +await page.evaluate(() => + window.htmx.ajax('GET', '/admin/api/gc/status', { target: '#gc-results', swap: 'innerHTML' }), +); + +let staged = ''; +const renderDeadline = Date.now() + 30_000; +while (Date.now() < renderDeadline) { + staged = await results.innerText().catch(() => ''); + if (staged.includes('Referenced Blobs')) break; + await page.waitForTimeout(1000); +} +r.record('staged preview re-rendered without restaging', staged.includes('Referenced Blobs')); + +const m = staged.match(/Delete (\d+) Orphaned Records/); +const before = m ? Number(m[1]) : null; +r.record('delete control present', before !== null, `${before} orphaned records`); +if (before === null) { + r.summarize('batch 07 — stale-preview refusal'); + await ctx.close(); + process.exit(1); +} + +// --- Click, with the confirm removed -------------------------------------- +const btn = page.locator(`button:has-text("Delete ${before} Orphaned Records")`).first(); +await btn.evaluate((el) => el.removeAttribute('hx-confirm')); +await btn.click(); + +let out = ''; +const deadline = Date.now() + 120_000; +while (Date.now() < deadline) { + out = await results.innerText().catch(() => ''); + if (/preview is .* old \(limit/.test(out)) break; + if (/Records Deleted/.test(out)) break; + await page.waitForTimeout(2000); +} + +console.log('\n#gc-results after the click:\n' + out.slice(0, 500)); + +r.record('stale preview is refused, visibly', + /preview is .* old \(limit .*\)/.test(out), + out ? out.split('\n').filter(Boolean)[0] : 'nothing rendered'); + +r.record('refusal names the remedy', + out.includes('run Scan again'), + 'the operator has to be told what to do next'); + +// --- Nothing was deleted --------------------------------------------------- +await page.locator('button:has-text("Scan for Orphans")').first().click(); +let after = ''; +const rescanDeadline = Date.now() + 180_000; +while (Date.now() < rescanDeadline) { + after = await results.innerText().catch(() => ''); + if (after.includes('Referenced Blobs')) break; + await page.waitForTimeout(2000); +} +const m2 = after.match(/Delete (\d+) Orphaned Records/); +r.record('no records were deleted by the refused click', + m2 !== null && Number(m2[1]) === before, + `${before} before, ${m2 ? m2[1] : 'unknown'} after`); + +const failed = r.summarize('batch 07 — stale-preview refusal'); +await ctx.close(); +process.exit(failed ? 1 : 0); diff --git a/test/e2e/batch07-stale-preview.mjs b/test/e2e/batch07-stale-preview.mjs new file mode 100644 index 0000000..1dfd0ac --- /dev/null +++ b/test/e2e/batch07-stale-preview.mjs @@ -0,0 +1,117 @@ +// batch07-stale-preview.mjs — the stale-preview refusal, end to end. +// +// pkg/hold/gc covers the refusal itself (TestDeleteOrphanedRecords_StalePreviewIsRefused). +// What no Go test can cover is whether the refusal ever reaches a human: the +// delete handler returns gc_progress.html, which polls /admin/api/gc/status, +// which renders the error through gc_error.html. A break anywhere in that chain +// turns a refusal into a silent no-op, and a silent no-op reads exactly like +// "nothing needed deleting". +// +// maxPreviewAgeForDelete is a package constant at 30 minutes, so this takes its +// own preview and then waits it out. Roughly 32 minutes. It deliberately does +// NOT reuse an existing preview: taking its own is the only way to know the age +// of the thing it is about to click on. +// +// The click is destructive if the refusal is broken. That is the point, and the +// blast radius is bounded: the records staged here were confirmed orphaned +// against their owners' PDSes by batch07-gc.mjs. +// +// Do not touch a tracked .go/.html/.css/.js file while this runs. Air rebuilds +// the hold and a rebuild takes the in-memory admin session with it. +// +// node test/e2e/batch07-stale-preview.mjs +import { open, adminPage, reporter } from './lib.mjs'; + +const STALE_WAIT_MS = 31.5 * 60 * 1000; + +const ctx = await open(); +const page = await adminPage(ctx); +const r = reporter(); +const results = page.locator('#gc-results'); + +const waitForPreview = async (timeoutMs = 180_000) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const text = await results.innerText().catch(() => ''); + if (text.includes('Referenced Blobs')) return text; + await page.waitForTimeout(2000); + } + return null; +}; + +const orphanCount = (text) => { + const m = text.match(/Delete (\d+) Orphaned Records/); + return m ? Number(m[1]) : null; +}; + +// --- Stage a preview and start the clock ----------------------------------- +await page.click('a[href="/admin#storage"]'); +await page.waitForTimeout(1500); +await page.locator('button:has-text("Scan for Orphans")').first().click(); + +const staged = await waitForPreview(); +r.record('preview staged', staged !== null); +if (!staged) { + await ctx.close(); + process.exit(1); +} +const previewAt = Date.now(); +const before = orphanCount(staged); +r.record('preview offers a delete control', before !== null && before > 0, `${before} orphaned records`); +console.log(`Preview staged at ${new Date(previewAt).toLocaleTimeString()} with ${before} orphaned records.`); +console.log(`Waiting ${(STALE_WAIT_MS / 60000).toFixed(1)} minutes for it to go stale...`); + +// --- Wait it out ----------------------------------------------------------- +// Idle only. Navigating here would reload the fragment and lose the button, +// and re-scanning would reset the very age under test. +while (Date.now() - previewAt < STALE_WAIT_MS) { + await page.waitForTimeout(30_000); + const left = Math.ceil((STALE_WAIT_MS - (Date.now() - previewAt)) / 60000); + if (left % 5 === 0) console.log(` ${left} minutes left`); +} + +// --- Click delete on a preview that is now too old ------------------------- +// The button carries hx-confirm, which is a native confirm() dialog. Without a +// handler Playwright dismisses it and the request is never sent, which would +// look like a passing refusal while proving nothing. +let dialogSeen = false; +page.on('dialog', async (d) => { + dialogSeen = true; + await d.accept(); +}); + +await page.locator('button:has-text("Delete")').first().click(); +await page.waitForTimeout(1000); +r.record('hx-confirm dialog was raised and accepted', dialogSeen); + +// --- The refusal has to become visible ------------------------------------- +let refusal = ''; +const deadline = Date.now() + 90_000; +while (Date.now() < deadline) { + const text = await results.innerText().catch(() => ''); + if (/preview is .* old \(limit/.test(text)) { refusal = text; break; } + if (text.includes('Deleted') || text.includes('Records Deleted')) { refusal = text; break; } + await page.waitForTimeout(2000); +} + +console.log('\n#gc-results after the click:\n' + refusal.slice(0, 600)); + +r.record('stale preview is refused, visibly', + /preview is .* old \(limit .*\)/.test(refusal), + refusal ? refusal.split('\n')[0] : 'nothing rendered'); + +r.record('refusal names the remedy', + refusal.includes('run Scan again'), + 'the message should tell the operator what to do next'); + +// --- And nothing was actually deleted -------------------------------------- +await page.locator('button:has-text("Scan for Orphans")').first().click(); +const after = await waitForPreview(); +const afterCount = after ? orphanCount(after) : null; +r.record('no records were deleted by the refused click', + afterCount === before, + `${before} before, ${afterCount} after`); + +const failed = r.summarize('batch 07 — stale-preview refusal'); +await ctx.close(); +process.exit(failed ? 1 : 0); diff --git a/test/e2e/batch07-sweep.mjs b/test/e2e/batch07-sweep.mjs new file mode 100644 index 0000000..2fa1913 --- /dev/null +++ b/test/e2e/batch07-sweep.mjs @@ -0,0 +1,138 @@ +// batch07-sweep.mjs — run the destructive half of GC for real. +// +// Everything else in batch 07 proves GC decides correctly. Nothing proves it +// can carry a decision out: deleteOrphanedRecords against a real CAR store, and +// deleteOrphanedBlobs against a real S3 listing, are untested by any Go test and +// unreachable from the in-process harness. +// +// Safe to run ONLY against the dev hold on Storj (gateway.storjshare.io, bucket +// "atcr"). Production is Bunny + UpCloud and is not reachable from here. Confirm +// before running: +// +// docker compose exec atcr-hold env | grep -E '^(S3_|AWS_)' +// +// Sequence: scan, delete records, re-scan, delete blobs if any appeared, +// re-scan. Each step reports what changed, because the interesting result is +// not "it worked" but what the second scan reveals once the records are gone. +// +// node test/e2e/batch07-sweep.mjs +import { open, adminPage, reporter } from './lib.mjs'; + +const ctx = await open(); +const page = await adminPage(ctx); +const r = reporter(); +const results = page.locator('#gc-results'); + +// hx-confirm raises a native confirm() that page.on('dialog') did not reliably +// intercept here; an unaccepted dialog then blocks every later evaluate() and +// the script hangs instead of failing. The confirm is not under test, so it is +// removed before each destructive click. +const clickDelete = async (label) => { + const btn = page.locator(`button:has-text("${label}")`).first(); + await btn.evaluate((el) => el.removeAttribute('hx-confirm')); + await btn.click(); +}; + +const waitFor = async (needle, timeoutMs = 300_000) => { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const text = await results.innerText().catch(() => ''); + if (text.includes(needle)) return text; + if (/preview is .* old \(limit/.test(text)) return text; // stale refusal + await page.waitForTimeout(2000); + } + return null; +}; + +const statsOf = async () => + page.evaluate(() => { + const out = {}; + for (const s of document.querySelectorAll('.stat-title')) { + const v = s.parentElement?.querySelector('.stat-value'); + if (v) out[s.textContent.trim()] = v.textContent.trim().split('—')[0].trim(); + } + return out; + }); + +const scan = async (label) => { + await page.locator('button:has-text("Scan for Orphans")').first().click(); + const text = await waitFor('Referenced Blobs'); + if (!text) return null; + const s = await statsOf(); + console.log(`\n[${label}] records=${s['Orphaned Records']} blobs=${s['Orphaned Blobs']} ` + + `missing=${s['Missing Records']} referenced=${s['Referenced Blobs']}`); + return s; +}; + +await page.click('a[href="/admin#storage"]'); +await page.waitForTimeout(1500); + +// --- 1. Scan --------------------------------------------------------------- +const before = await scan('scan 1'); +r.record('initial scan completed', before !== null); +if (!before) { await ctx.close(); process.exit(1); } + +const recordsToDelete = Number(before['Orphaned Records']); +r.record('there is something to sweep', recordsToDelete > 0, `${recordsToDelete} orphaned records`); + +// --- 2. Delete records ----------------------------------------------------- +// Immediately after the scan, so the preview-age guard cannot fire. +if (recordsToDelete > 0) { + await clickDelete(`Delete ${recordsToDelete} Orphaned Records`); + const out = await waitFor('Records Deleted'); + console.log('\nAfter delete-records:\n' + (out ?? '').slice(0, 400)); + + r.record('delete-records was not refused as stale', + out !== null && !/preview is .* old \(limit/.test(out)); + + const after = await statsOf(); + const deleted = Number(after['Records Deleted']); + r.record('every staged record was deleted', + deleted === recordsToDelete, + `${deleted} deleted of ${recordsToDelete} staged`); +} + +// --- 3. Re-scan ------------------------------------------------------------ +// The question this answers: with the layer records gone, do their blobs now +// fall out of the referenced set and surface as orphaned? If the bytes were +// never in this bucket, nothing appears and the count stays flat. +const mid = await scan('scan 2'); +r.record('re-scan completed', mid !== null); +if (mid) { + r.record('orphaned records are cleared', + Number(mid['Orphaned Records']) === 0, + `${mid['Orphaned Records']} remain`); +} + +// --- 4. Delete blobs, if any surfaced -------------------------------------- +const blobsToDelete = mid ? Number(mid['Orphaned Blobs']) : 0; +if (blobsToDelete > 0) { + console.log(`\n${blobsToDelete} orphaned blobs surfaced; deleting from S3.`); + await clickDelete(`Delete ${blobsToDelete} Orphaned Blobs`); + const out = await waitFor('Blobs Deleted'); + console.log('\nAfter delete-blobs:\n' + (out ?? '').slice(0, 400)); + + const after = await statsOf(); + r.record('blobs were deleted from S3', + Number(after['Blobs Deleted']) === blobsToDelete, + `${after['Blobs Deleted']} of ${blobsToDelete}, ${after['Space Reclaimed'] ?? '?'} reclaimed`); + + const end = await scan('scan 3'); + if (end) { + r.record('orphaned blobs are cleared', + Number(end['Orphaned Blobs']) === 0, + `${end['Orphaned Blobs']} remain`); + r.record('referenced blobs survived the sweep', + Number(end['Referenced Blobs']) === Number(before['Referenced Blobs']), + `${before['Referenced Blobs']} before, ${end['Referenced Blobs']} after`); + } +} else { + console.log('\nNo orphaned blobs surfaced after the records were deleted.'); + r.record('referenced blobs survived the sweep', + mid !== null && Number(mid['Referenced Blobs']) === Number(before['Referenced Blobs']), + `${before['Referenced Blobs']} before, ${mid?.['Referenced Blobs']} after`); +} + +const failed = r.summarize('batch 07 — destructive sweep'); +await ctx.close(); +process.exit(failed ? 1 : 0);