at-container-registry/test/e2e/batch07-sweep.mjs

// batch07-sweep.mjs — run the destructive half of GC for real.
//
// Everything else in batch 07 proves GC decides correctly. Nothing proves it
// can carry a decision out: deleteOrphanedRecords against a real CAR store, and
// deleteOrphanedBlobs against a real S3 listing, are untested by any Go test and
// unreachable from the in-process harness.
//
// Safe to run ONLY against the dev hold on Storj (gateway.storjshare.io, bucket
// "atcr"). Production is Bunny + UpCloud and is not reachable from here. Confirm
// before running:
//
//   docker compose exec atcr-hold env | grep -E '^(S3_|AWS_)'
//
// Sequence: scan, delete records, re-scan, delete blobs if any appeared,
// re-scan. Each step reports what changed, because the interesting result is
// not "it worked" but what the second scan reveals once the records are gone.
//
//   node test/e2e/batch07-sweep.mjs
import { open, adminPage, reporter } from './lib.mjs';

const ctx = await open();
const page = await adminPage(ctx);
const r = reporter();
const results = page.locator('#gc-results');

// hx-confirm raises a native confirm() that page.on('dialog') did not reliably
// intercept here; an unaccepted dialog then blocks every later evaluate() and
// the script hangs instead of failing. The confirm is not under test, so it is
// removed before each destructive click.
const clickDelete = async (label) => {
  const btn = page.locator(`button:has-text("${label}")`).first();
  await btn.evaluate((el) => el.removeAttribute('hx-confirm'));
  await btn.click();
};

const waitFor = async (needle, timeoutMs = 300_000) => {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const text = await results.innerText().catch(() => '');
    if (text.includes(needle)) return text;
    if (/preview is .* old \(limit/.test(text)) return text; // stale refusal
    await page.waitForTimeout(2000);
  }
  return null;
};

const statsOf = async () =>
  page.evaluate(() => {
    const out = {};
    for (const s of document.querySelectorAll('.stat-title')) {
      const v = s.parentElement?.querySelector('.stat-value');
      if (v) out[s.textContent.trim()] = v.textContent.trim().split('—')[0].trim();
    }
    return out;
  });

const scan = async (label) => {
  await page.locator('button:has-text("Scan for Orphans")').first().click();
  const text = await waitFor('Referenced Blobs');
  if (!text) return null;
  const s = await statsOf();
  console.log(`\n[${label}] records=${s['Orphaned Records']} blobs=${s['Orphaned Blobs']} ` +
              `missing=${s['Missing Records']} referenced=${s['Referenced Blobs']}`);
  return s;
};

await page.click('a[href="/admin#storage"]');
await page.waitForTimeout(1500);

// --- 1. Scan ---------------------------------------------------------------
const before = await scan('scan 1');
r.record('initial scan completed', before !== null);
if (!before) { await ctx.close(); process.exit(1); }

const recordsToDelete = Number(before['Orphaned Records']);
r.record('there is something to sweep', recordsToDelete > 0, `${recordsToDelete} orphaned records`);

// --- 2. Delete records -----------------------------------------------------
// Immediately after the scan, so the preview-age guard cannot fire.
if (recordsToDelete > 0) {
  await clickDelete(`Delete ${recordsToDelete} Orphaned Records`);
  const out = await waitFor('Records Deleted');
  console.log('\nAfter delete-records:\n' + (out ?? '<nothing>').slice(0, 400));

  r.record('delete-records was not refused as stale',
    out !== null && !/preview is .* old \(limit/.test(out));

  const after = await statsOf();
  const deleted = Number(after['Records Deleted']);
  r.record('every staged record was deleted',
    deleted === recordsToDelete,
    `${deleted} deleted of ${recordsToDelete} staged`);
}

// --- 3. Re-scan ------------------------------------------------------------
// The question this answers: with the layer records gone, do their blobs now
// fall out of the referenced set and surface as orphaned? If the bytes were
// never in this bucket, nothing appears and the count stays flat.
const mid = await scan('scan 2');
r.record('re-scan completed', mid !== null);
if (mid) {
  r.record('orphaned records are cleared',
    Number(mid['Orphaned Records']) === 0,
    `${mid['Orphaned Records']} remain`);
}

// --- 4. Delete blobs, if any surfaced --------------------------------------
const blobsToDelete = mid ? Number(mid['Orphaned Blobs']) : 0;
if (blobsToDelete > 0) {
  console.log(`\n${blobsToDelete} orphaned blobs surfaced; deleting from S3.`);
  await clickDelete(`Delete ${blobsToDelete} Orphaned Blobs`);
  const out = await waitFor('Blobs Deleted');
  console.log('\nAfter delete-blobs:\n' + (out ?? '<nothing>').slice(0, 400));

  const after = await statsOf();
  r.record('blobs were deleted from S3',
    Number(after['Blobs Deleted']) === blobsToDelete,
    `${after['Blobs Deleted']} of ${blobsToDelete}, ${after['Space Reclaimed'] ?? '?'} reclaimed`);

  const end = await scan('scan 3');
  if (end) {
    r.record('orphaned blobs are cleared',
      Number(end['Orphaned Blobs']) === 0,
      `${end['Orphaned Blobs']} remain`);
    r.record('referenced blobs survived the sweep',
      Number(end['Referenced Blobs']) === Number(before['Referenced Blobs']),
      `${before['Referenced Blobs']} before, ${end['Referenced Blobs']} after`);
  }
} else {
  console.log('\nNo orphaned blobs surfaced after the records were deleted.');
  r.record('referenced blobs survived the sweep',
    mid !== null && Number(mid['Referenced Blobs']) === Number(before['Referenced Blobs']),
    `${before['Referenced Blobs']} before, ${mid?.['Referenced Blobs']} after`);
}

const failed = r.summarize('batch 07 — destructive sweep');
await ctx.close();
process.exit(failed ? 1 : 0);