at-container-registry/scanner/internal/e2e/bench_test.go

package e2e

// Performance and resource scenarios for the scan pipeline.
//
// Nothing in here runs during an ordinary `go test ./...`: every Test is gated
// on ATCR_SCANNER_PERF=1 and every Benchmark needs -bench, and both skip when
// the image fixture they need is absent. Fixtures are the gitignored OCI
// layouts under ../mockhold/testdata/blobs; see fixtureLayout for how to pull
// the ones these scenarios use.
//
// # Why several scenarios drive a replica of the pipeline rather than the
// # WorkerPool
//
// worker.go used to dereference result.Summary.Total on every successful scan
// while Summary is only populated when cfg.Vuln.Enabled. The harness disables
// Grype (enabling it would download a multi-hundred-MB database), so the first
// successful scan through the real WorkerPool panicked and took the whole test
// binary with it. That is fixed; the scenarios below were written under it and
// have not been re-cut, so they still avoid successful scans through the pool.
//
// The consequence for measurement was concrete: no scenario that drives the
// real worker could observe more than one successful scan, so throughput,
// sustained-memory and concurrency numbers cannot come from that path. Those
// scenarios instead run pipelineOnce, which performs the same four steps
// against the same libraries and the same mock hold: presign + download, OCI
// layout assembly, stereoscope load/extract, Syft catalog, SPDX encode. It is
// a replica, not the production function — buildOCILayout and generateSBOM are
// unexported in package scan and this file may not add a seam to them — so
// treat its absolute numbers as "what the work costs", and the WorkerPool
// scenarios below as "what the worker adds on top".
//
// The scenarios that do drive the real WorkerPool (queue burst, cooldown
// cadence, reconnect leaks) are all built from jobs that terminate in an error
// or a skip, which is the only way to run many jobs through the real worker
// today.

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io/fs"
	"math"
	"net/http"
	"net/http/httptest"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"runtime/debug"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	scanner "atcr.io/scanner"
	"atcr.io/scanner/internal/client"
	"atcr.io/scanner/internal/mockhold"
	"atcr.io/scanner/internal/scan"

	"github.com/anchore/stereoscope/pkg/file"
	"github.com/anchore/stereoscope/pkg/image/oci"
	"github.com/anchore/syft/syft"
	"github.com/anchore/syft/syft/format"
	"github.com/anchore/syft/syft/format/spdxjson"
	"github.com/anchore/syft/syft/source/stereoscopesource"
)

// perfEnv gates every Test in this file. Benchmarks are gated by -bench.
const perfEnv = "ATCR_SCANNER_PERF"

func requirePerf(t *testing.T) {
	t.Helper()
	if os.Getenv(perfEnv) != "1" {
		t.Skipf("set %s=1 to run performance scenarios", perfEnv)
	}
}

// ---------------------------------------------------------------------------
// Fixtures
// ---------------------------------------------------------------------------

// perfFixtures are the image layouts the scaling scenarios sweep over, in
// increasing cost. Each is an OCI layout under ../mockhold/testdata/blobs,
// which is what `skopeo copy docker://<ref> oci:<dir>:img` writes:
//
//	cd scanner/internal/mockhold/testdata/blobs
//	skopeo copy docker://docker.io/library/alpine:3.20      oci:perf-alpine:img
//	skopeo copy docker://docker.io/library/python:3.12-slim oci:perf-python:img
//	skopeo copy docker://docker.io/library/node:22          oci:perf-node:img
//
// hsm-secrets-operator comes from testdata/fetch-blobs.sh and is the one
// fixture whose descriptors are also in corpus.json. The others are pulled
// straight from Docker Hub because the corpus's larger images live on a
// private hold: the scan job for them is reconstructed from the layout itself
// by jobFromLayout, so no corpus entry is needed.
//
// perf-loom19 is optional: a 19-layer, ~37 MB corpus image that would separate
// layer count from total bytes on real data. Pulling it with the authfile
// dance fetch-blobs.sh performs (corpus digest
// sha256:30d7f33c7f15ff3c6a1e4302575dcebfe31f3e2403486a2902a6bf30d44c7cdd)
// failed here with "blob unknown to registry" — the hold no longer has one of
// its layers — so the layer-count sweep in TestPerfSyntheticScaling covers
// that axis synthetically instead. The name is kept so the fixture drops in if
// a pullable equivalent turns up.
var perfFixtures = []string{
	"hsm-secrets-operator",
	"perf-alpine",
	"perf-python",
	"perf-loom19",
	"perf-node",
}

func fixtureDir(name string) string {
	return filepath.Join("..", "mockhold", "testdata", "blobs", name)
}

func hasFixture(name string) bool {
	_, err := os.Stat(filepath.Join(fixtureDir(name), "oci-layout"))
	return err == nil
}

// availableFixtures returns the subset of perfFixtures present on disk.
func availableFixtures(tb testing.TB) []string {
	tb.Helper()
	var out []string
	for _, f := range perfFixtures {
		if hasFixture(f) {
			out = append(out, f)
		}
	}
	if len(out) == 0 {
		tb.Skip("no image fixtures present; see the perfFixtures comment for the skopeo commands")
	}
	return out
}

// jobFromLayout reconstructs the scan job for the single image in an OCI
// layout: the descriptors the hold would have sent, read back out of the
// bytes skopeo wrote. This is what lets any pulled image act as a fixture
// without a matching corpus record.
func jobFromLayout(tb testing.TB, dir string) *scanner.ScanJob {
	tb.Helper()

	var index struct {
		Manifests []struct {
			MediaType string `json:"mediaType"`
			Digest    string `json:"digest"`
		} `json:"manifests"`
	}
	readJSON(tb, filepath.Join(dir, "index.json"), &index)
	if len(index.Manifests) == 0 {
		tb.Fatalf("%s: index.json declares no manifests", dir)
	}

	digest := index.Manifests[0].Digest
	var manifest struct {
		MediaType string                   `json:"mediaType"`
		Config    scanner.BlobDescriptor   `json:"config"`
		Layers    []scanner.BlobDescriptor `json:"layers"`
		Manifests []struct {
			Digest string `json:"digest"`
		} `json:"manifests"`
	}
	readJSON(tb, blobPath(dir, digest), &manifest)

	// A layout pulled without --all still occasionally carries an index at the
	// top: follow one hop into the first child manifest.
	if manifest.Config.Digest == "" && len(manifest.Manifests) > 0 {
		readJSON(tb, blobPath(dir, manifest.Manifests[0].Digest), &manifest)
	}
	if manifest.Config.Digest == "" {
		tb.Fatalf("%s: no image manifest found in layout", dir)
	}

	return &scanner.ScanJob{
		ManifestDigest: digest,
		Repository:     filepath.Base(dir),
		Tag:            "img",
		Tier:           "deckhand",
		Config:         manifest.Config,
		Layers:         manifest.Layers,
	}
}

func blobPath(dir, digest string) string {
	return filepath.Join(dir, "blobs", "sha256", mockhold.DigestHex(digest))
}

func readJSON(tb testing.TB, path string, v any) {
	tb.Helper()
	data, err := os.ReadFile(path)
	if err != nil {
		tb.Fatalf("read %s: %v", path, err)
	}
	if err := json.Unmarshal(data, v); err != nil {
		tb.Fatalf("parse %s: %v", path, err)
	}
}

// compressedBytes is the size the scanner's own MaxImageSize check would see.
func compressedBytes(job *scanner.ScanJob) int64 {
	total := job.Config.Size
	for _, l := range job.Layers {
		total += l.Size
	}
	return total
}

// ---------------------------------------------------------------------------
// Pipeline replica
// ---------------------------------------------------------------------------

// stageTimes is one pass of the pipeline, broken into the four phases that
// production runs back to back inside processJob.
type stageTimes struct {
	Download time.Duration // presign + fetch every blob, write the layout
	Load     time.Duration // stereoscope: read the layout, extract layers
	Catalog  time.Duration // syft.CreateSBOM
	Encode   time.Duration // SPDX JSON encode
	Total    time.Duration

	Packages int
	SBOMSize int
	TmpPeak  int64 // peak bytes under TMPDIR while this pass ran
}

func (s stageTimes) String() string {
	return fmt.Sprintf("total=%s download=%s load=%s catalog=%s encode=%s packages=%d sbom=%dKiB tmpPeak=%.1fMiB",
		round(s.Total), round(s.Download), round(s.Load), round(s.Catalog), round(s.Encode),
		s.Packages, s.SBOMSize/1024, float64(s.TmpPeak)/(1<<20))
}

func round(d time.Duration) time.Duration { return d.Round(time.Millisecond) }

// pipelineOnce runs one scan the way processJob does and reports what each
// phase cost. See the file comment for why this is a replica.
//
// tmpDir stands in for cfg.Vuln.TmpDir: the layout is assembled there and
// TMPDIR points at it, so stereoscope's extraction lands there too and a
// single directory walk measures the whole disk footprint of a scan.
//
// It returns an error rather than calling Fatalf so the concurrency scenario
// can run it from several goroutines at once.
func pipelineOnce(job *scanner.ScanJob, tmpDir string) (st stageTimes, err error) {
	watcher := watchDir(tmpDir, 20*time.Millisecond)
	defer func() { st.TmpPeak = watcher.stop() }()

	start := time.Now()

	layoutDir, cleanup, aerr := assembleLayout(job, tmpDir)
	if aerr != nil {
		return st, fmt.Errorf("assemble layout: %w", aerr)
	}
	defer cleanup()
	st.Download = time.Since(start)

	ctx := context.Background()
	t0 := time.Now()
	tmpGen := file.NewTempDirGenerator("syft-scan")
	defer tmpGen.Cleanup()

	img, err := oci.NewDirectoryProvider(tmpGen, layoutDir).Provide(ctx)
	if err != nil {
		return st, fmt.Errorf("provide image: %w", err)
	}
	if err := img.Read(); err != nil {
		img.Cleanup()
		return st, fmt.Errorf("read image: %w", err)
	}
	src := stereoscopesource.New(img, stereoscopesource.ImageConfig{Reference: layoutDir})
	defer src.Close()
	st.Load = time.Since(t0)

	t0 = time.Now()
	sbomResult, err := syft.CreateSBOM(ctx, src, nil)
	if err != nil {
		return st, fmt.Errorf("create sbom: %w", err)
	}
	st.Catalog = time.Since(t0)
	st.Packages = sbomResult.Artifacts.Packages.PackageCount()

	t0 = time.Now()
	encoder, err := spdxjson.NewFormatEncoderWithConfig(spdxjson.DefaultEncoderConfig())
	if err != nil {
		return st, fmt.Errorf("encoder: %w", err)
	}
	sbomJSON, err := format.Encode(*sbomResult, encoder)
	if err != nil {
		return st, fmt.Errorf("encode sbom: %w", err)
	}
	_ = sha256.Sum256(sbomJSON)
	st.Encode = time.Since(t0)
	st.SBOMSize = len(sbomJSON)

	st.Total = time.Since(start)
	return st, nil
}

// mustPipeline runs pipelineOnce and fails the test on error.
func mustPipeline(tb testing.TB, job *scanner.ScanJob, tmpDir string) stageTimes {
	tb.Helper()
	st, err := pipelineOnce(job, tmpDir)
	if err != nil {
		tb.Fatalf("pipeline: %v", err)
	}
	return st
}

// assembleLayout mirrors buildOCILayout: download the config and every tar
// layer through the hold's presign indirection, then write the manifest,
// index.json and oci-layout beside them.
func assembleLayout(job *scanner.ScanJob, tmpDir string) (string, func(), error) {
	scanDir, err := os.MkdirTemp(tmpDir, "bench-scan-*")
	if err != nil {
		return "", nil, err
	}
	cleanup := func() { os.RemoveAll(scanDir) }

	blobsDir := filepath.Join(scanDir, "blobs", "sha256")
	if err := os.MkdirAll(blobsDir, 0o755); err != nil {
		cleanup()
		return "", nil, err
	}

	// Mirrors buildOCILayout's boundary too: parse the digest once, then let
	// the same value name the blob on the wire and the file on disk.
	fetch := func(digest string) error {
		parsed, err := scanner.ParseDigest(digest)
		if err != nil {
			return err
		}
		url, err := client.GetBlobPresignedURL(context.Background(), job.HoldEndpoint, job.HoldDID, parsed, "")
		if err != nil {
			return err
		}
		_, err = client.DownloadBlob(context.Background(), url, filepath.Join(blobsDir, parsed.Hex), client.BlobExpectation{
			Digest:   parsed,
			MaxBytes: -1,
		})
		return err
	}

	if err := fetch(job.Config.Digest); err != nil {
		cleanup()
		return "", nil, fmt.Errorf("config blob: %w", err)
	}

	type desc struct {
		MediaType string `json:"mediaType"`
		Digest    string `json:"digest"`
		Size      int64  `json:"size"`
	}
	layers := make([]desc, 0, len(job.Layers))
	for i, l := range job.Layers {
		if l.Digest == "" || (l.MediaType != "" && !strings.Contains(l.MediaType, "tar")) {
			continue
		}
		if err := fetch(l.Digest); err != nil {
			cleanup()
			return "", nil, fmt.Errorf("layer %d: %w", i, err)
		}
		mt := l.MediaType
		if mt == "" {
			mt = "application/vnd.oci.image.layer.v1.tar+gzip"
		}
		layers = append(layers, desc{MediaType: mt, Digest: l.Digest, Size: l.Size})
	}

	cfgType := job.Config.MediaType
	if cfgType == "" {
		cfgType = "application/vnd.oci.image.config.v1+json"
	}
	manifestJSON, _ := json.Marshal(map[string]any{
		"schemaVersion": 2,
		"mediaType":     "application/vnd.oci.image.manifest.v1+json",
		"config":        desc{MediaType: cfgType, Digest: job.Config.Digest, Size: job.Config.Size},
		"layers":        layers,
	})
	sum := sha256.Sum256(manifestJSON)
	if err := os.WriteFile(filepath.Join(blobsDir, fmt.Sprintf("%x", sum)), manifestJSON, 0o644); err != nil {
		cleanup()
		return "", nil, err
	}
	indexJSON, _ := json.Marshal(map[string]any{
		"schemaVersion": 2,
		"manifests": []desc{{
			MediaType: "application/vnd.oci.image.manifest.v1+json",
			Digest:    fmt.Sprintf("sha256:%x", sum),
			Size:      int64(len(manifestJSON)),
		}},
	})
	if err := os.WriteFile(filepath.Join(scanDir, "index.json"), indexJSON, 0o644); err != nil {
		cleanup()
		return "", nil, err
	}
	if err := os.WriteFile(filepath.Join(scanDir, "oci-layout"),
		[]byte(`{"imageLayoutVersion":"1.0.0"}`), 0o644); err != nil {
		cleanup()
		return "", nil, err
	}
	return scanDir, cleanup, nil
}

// perfHold starts a mock hold serving a fixture layout and points the job at
// it, and redirects TMPDIR at a scratch directory the way WorkerPool.Start
// redirects it at cfg.Vuln.TmpDir. It returns that directory.
func perfHold(tb testing.TB, source mockhold.BlobSource, job *scanner.ScanJob) string {
	tb.Helper()
	hold := mockhold.New(source)
	tb.Cleanup(hold.Close)
	job.HoldEndpoint = hold.URL()

	tmp := tb.TempDir()
	setEnv(tb, "TMPDIR", tmp)
	return tmp
}

// setEnv is t.Setenv, spelled out because testing.TB does not carry it and
// several of these scenarios are benchmarks.
func setEnv(tb testing.TB, key, value string) {
	prev, had := os.LookupEnv(key)
	os.Setenv(key, value)
	tb.Cleanup(func() {
		if had {
			os.Setenv(key, prev)
			return
		}
		os.Unsetenv(key)
	})
}

// ---------------------------------------------------------------------------
// Process-level sampling
// ---------------------------------------------------------------------------

// procSample is one observation of the whole process, which is what matters
// here: Go's heap accounting misses the mmap'd regions stereoscope and
// SQLite bring in, and RSS is what the container's memory limit counts.
type procSample struct {
	At         time.Time
	RSS        int64 // bytes, /proc/self/statm
	HeapAlloc  int64
	HeapSys    int64
	Goroutines int
	FDs        int
}

type monitor struct {
	stop     chan struct{}
	done     chan struct{}
	mu       sync.Mutex
	samples  []procSample
	interval time.Duration
}

func startMonitor(interval time.Duration) *monitor {
	m := &monitor{stop: make(chan struct{}), done: make(chan struct{}), interval: interval}
	go func() {
		defer close(m.done)
		tick := time.NewTicker(interval)
		defer tick.Stop()
		m.record()
		for {
			select {
			case <-m.stop:
				m.record()
				return
			case <-tick.C:
				m.record()
			}
		}
	}()
	return m
}

func (m *monitor) record() {
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	s := procSample{
		At:         time.Now(),
		RSS:        rssBytes(),
		HeapAlloc:  int64(ms.HeapAlloc),
		HeapSys:    int64(ms.HeapSys),
		Goroutines: runtime.NumGoroutine(),
		FDs:        fdCount(),
	}
	m.mu.Lock()
	m.samples = append(m.samples, s)
	m.mu.Unlock()
}

func (m *monitor) finish() []procSample {
	close(m.stop)
	<-m.done
	m.mu.Lock()
	defer m.mu.Unlock()
	return append([]procSample(nil), m.samples...)
}

func peakRSS(samples []procSample) int64 {
	var max int64
	for _, s := range samples {
		if s.RSS > max {
			max = s.RSS
		}
	}
	return max
}

func peakHeap(samples []procSample) int64 {
	var max int64
	for _, s := range samples {
		if s.HeapAlloc > max {
			max = s.HeapAlloc
		}
	}
	return max
}

// settle forces the flattest heap this process can be talked into, so a
// measurement is not reading the previous scenario's garbage. It is what makes
// the peak numbers below comparable across iterations of a sweep.
func settle() {
	runtime.GC()
	runtime.GC()
	time.Sleep(100 * time.Millisecond)
}

// growth reports the peak RSS and heap of a run measured from its own first
// sample, which is the part attributable to the work rather than to whatever
// the process was already holding.
func growth(samples []procSample) (rss, heap int64) {
	if len(samples) == 0 {
		return 0, 0
	}
	return peakRSS(samples) - samples[0].RSS, peakHeap(samples) - samples[0].HeapAlloc
}

// rssBytes reads resident set size from /proc/self/statm. Returns 0 where
// procfs is unavailable, which is the honest answer rather than a guess.
func rssBytes() int64 {
	data, err := os.ReadFile("/proc/self/statm")
	if err != nil {
		return 0
	}
	fields := strings.Fields(string(data))
	if len(fields) < 2 {
		return 0
	}
	pages, err := strconv.ParseInt(fields[1], 10, 64)
	if err != nil {
		return 0
	}
	return pages * int64(os.Getpagesize())
}

// vmHWM is the kernel's own peak-RSS watermark for the process, which no
// sampling interval can miss. It never decreases, so it is only meaningful as
// "the highest this process ever reached", not per-scenario.
func vmHWM() int64 {
	f, err := os.Open("/proc/self/status")
	if err != nil {
		return 0
	}
	defer f.Close()
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		if !strings.HasPrefix(sc.Text(), "VmHWM:") {
			continue
		}
		fields := strings.Fields(sc.Text())
		if len(fields) < 2 {
			return 0
		}
		kb, err := strconv.ParseInt(fields[1], 10, 64)
		if err != nil {
			return 0
		}
		return kb * 1024
	}
	return 0
}

func fdCount() int {
	entries, err := os.ReadDir("/proc/self/fd")
	if err != nil {
		return -1
	}
	return len(entries)
}

// dirBytes sums the apparent size of every regular file under root.
func dirBytes(root string) int64 {
	var total int64
	_ = filepath.WalkDir(root, func(_ string, d fs.DirEntry, err error) error {
		if err != nil || d.IsDir() {
			return nil //nolint:nilerr // a file vanishing mid-walk is expected
		}
		if info, err := d.Info(); err == nil {
			total += info.Size()
		}
		return nil
	})
	return total
}

type dirWatcher struct {
	stopCh chan struct{}
	done   chan struct{}
	peak   int64
}

// watchDir samples the on-disk footprint of a tree until stopped, and reports
// the high-water mark. Sampling can undershoot a short spike; the ratios it is
// used for are large enough that this does not change the conclusion.
func watchDir(root string, interval time.Duration) *dirWatcher {
	w := &dirWatcher{stopCh: make(chan struct{}), done: make(chan struct{})}
	go func() {
		defer close(w.done)
		tick := time.NewTicker(interval)
		defer tick.Stop()
		for {
			select {
			case <-w.stopCh:
				return
			case <-tick.C:
				if n := dirBytes(root); n > w.peak {
					w.peak = n
				}
			}
		}
	}()
	return w
}

func (w *dirWatcher) stop() int64 {
	close(w.stopCh)
	<-w.done
	return w.peak
}

// ---------------------------------------------------------------------------
// 1. Per-stage cost on real images
// ---------------------------------------------------------------------------

// TestPerfStageBreakdown times each phase of the pipeline for every fixture
// present, which is the only way to see which one actually dominates. It runs
// each image three times and reports every pass rather than an average: on a
// shared machine the spread between passes is the honest error bar, and the
// first pass also carries the page-cache cost of reading the fixture off disk.
func TestPerfStageBreakdown(t *testing.T) {
	requirePerf(t)

	for _, name := range availableFixtures(t) {
		t.Run(name, func(t *testing.T) {
			dir := fixtureDir(name)
			job := jobFromLayout(t, dir)
			tmp := perfHold(t, mockhold.NewOCILayout(dir), job)

			t.Logf("%s: %d layers, %.1f MiB compressed", name,
				len(job.Layers), float64(compressedBytes(job))/(1<<20))

			for i := 0; i < 3; i++ {
				settle()
				m := startMonitor(25 * time.Millisecond)
				st := mustPipeline(t, job, tmp)
				samples := m.finish()
				dRSS, dHeap := growth(samples)
				t.Logf("pass %d: %s peakRSS=%.0fMiB(+%.0f) peakHeap=%.0fMiB(+%.0f)",
					i, st, float64(peakRSS(samples))/(1<<20), float64(dRSS)/(1<<20),
					float64(peakHeap(samples))/(1<<20), float64(dHeap)/(1<<20))
			}
		})
	}
}

// BenchmarkPipeline is the same work under the benchmark harness, for when a
// stable per-op number matters more than the stage split.
func BenchmarkPipeline(b *testing.B) {
	for _, name := range availableFixtures(b) {
		b.Run(name, func(b *testing.B) {
			dir := fixtureDir(name)
			job := jobFromLayout(b, dir)
			tmp := perfHold(b, mockhold.NewOCILayout(dir), job)

			b.ReportAllocs()
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				if _, err := pipelineOnce(job, tmp); err != nil {
					b.Fatal(err)
				}
			}
			b.StopTimer()
			// RSS at the end of the run, not a peak: a benchmark loop offers no
			// quiet moment to sample one, and the peak is what
			// TestPerfStageBreakdown reports.
			b.ReportMetric(float64(rssBytes())/(1<<20), "endRSS_MiB")
		})
	}
}

// ---------------------------------------------------------------------------
// 2. Sustained load, and whether the GC-plus-cooldown pause earns its keep
// ---------------------------------------------------------------------------

// cooldownMode selects what TestPerfSustainedLoad does between jobs.
//
// The two modes must be compared across processes, not within one: whichever
// runs second inherits the other's already-grown heap and page-cache state, so
// a single process cannot answer "does memory return to baseline" for both.
// Run it twice, once per mode:
//
//	ATCR_SCANNER_PERF=1 ATCR_SCANNER_PERF_COOLDOWN=prod go test ./internal/e2e/ -run TestPerfSustainedLoad -v -timeout 30m
//	ATCR_SCANNER_PERF=1 ATCR_SCANNER_PERF_COOLDOWN=none go test ./internal/e2e/ -run TestPerfSustainedLoad -v -timeout 30m
const cooldownEnv = "ATCR_SCANNER_PERF_COOLDOWN"

// TestPerfSustainedLoad runs the same image repeatedly and records RSS and
// heap after every job, with and without the production `runtime.GC()` plus
// ten second pause. It answers two separate questions that are easy to
// conflate: whether memory ratchets across jobs, and whether the pause is what
// stops it.
func TestPerfSustainedLoad(t *testing.T) {
	requirePerf(t)

	mode := os.Getenv(cooldownEnv)
	if mode == "" {
		mode = "prod"
	}
	if mode != "prod" && mode != "none" {
		t.Fatalf("%s must be prod or none, got %q", cooldownEnv, mode)
	}

	name := heaviestFixture(t)
	dir := fixtureDir(name)
	job := jobFromLayout(t, dir)
	tmp := perfHold(t, mockhold.NewOCILayout(dir), job)

	const jobs = 6
	m := startMonitor(25 * time.Millisecond)
	baselineRSS := rssBytes()
	start := time.Now()
	var work time.Duration

	t.Logf("fixture=%s mode=%s jobs=%d baselineRSS=%.0fMiB", name, mode, jobs,
		float64(baselineRSS)/(1<<20))

	for i := 0; i < jobs; i++ {
		t0 := time.Now()
		st := mustPipeline(t, job, tmp)
		work += time.Since(t0)

		var before runtime.MemStats
		runtime.ReadMemStats(&before)
		rssBefore := rssBytes()

		if mode == "prod" {
			// Exactly what worker.go does between jobs.
			runtime.GC()
			time.Sleep(10 * time.Second)
		}

		var after runtime.MemStats
		runtime.ReadMemStats(&after)
		t.Logf("job %d: %s | RSS %.0f→%.0fMiB heap %.0f→%.0fMiB goroutines=%d fds=%d",
			i, round(st.Total),
			float64(rssBefore)/(1<<20), float64(rssBytes())/(1<<20),
			float64(before.HeapAlloc)/(1<<20), float64(after.HeapAlloc)/(1<<20),
			runtime.NumGoroutine(), fdCount())
	}

	wall := time.Since(start)
	samples := m.finish()
	t.Logf("RESULT mode=%s jobs=%d wall=%s work=%s throughput=%.2f jobs/min peakRSS=%.0fMiB peakHeap=%.0fMiB endRSS=%.0fMiB vmHWM=%.0fMiB",
		mode, jobs, round(wall), round(work), float64(jobs)/wall.Minutes(),
		float64(peakRSS(samples))/(1<<20), float64(peakHeap(samples))/(1<<20),
		float64(rssBytes())/(1<<20), float64(vmHWM())/(1<<20))
}

// heaviestFixture picks the largest fixture present, since memory behaviour
// only shows up on an image big enough to allocate.
func heaviestFixture(tb testing.TB) string {
	tb.Helper()
	available := availableFixtures(tb)
	best, bestSize := "", int64(-1)
	for _, name := range available {
		if n := dirBytes(fixtureDir(name)); n > bestSize {
			best, bestSize = name, n
		}
	}
	return best
}

// ---------------------------------------------------------------------------
// 3. Scaling with layer count and with bytes
// ---------------------------------------------------------------------------

// syntheticImage builds an image out of nothing: layers layers, each holding
// files of fileSize bytes, with content chosen by fill.
//
// Deterministic pseudo-random content ("random") is close to incompressible,
// so compressed and extracted sizes stay within a few percent of each other
// and the sweep measures bytes rather than the gzip ratio. Zero content
// ("zeros") is the opposite extreme, and is what the disk-amplification
// scenario uses.
func syntheticImage(tb testing.TB, layers, filesPerLayer int, fileSize int64, fill string) (*scanner.ScanJob, *mockhold.Memory) {
	tb.Helper()

	mem := mockhold.NewMemory()
	job := &scanner.ScanJob{
		ManifestDigest: "sha256:" + strings.Repeat("0", 64),
		Repository:     "synthetic",
		Tag:            "img",
		Tier:           "deckhand",
	}

	var diffIDs []string
	for i := 0; i < layers; i++ {
		raw := buildTar(tb, i, filesPerLayer, fileSize, fill)
		diffIDs = append(diffIDs, fmt.Sprintf("sha256:%x", sha256.Sum256(raw)))

		var gz bytes.Buffer
		zw, err := gzip.NewWriterLevel(&gz, gzip.BestSpeed)
		if err != nil {
			tb.Fatalf("gzip writer: %v", err)
		}
		if _, err := zw.Write(raw); err != nil {
			tb.Fatalf("gzip write: %v", err)
		}
		if err := zw.Close(); err != nil {
			tb.Fatalf("gzip close: %v", err)
		}

		data := gz.Bytes()
		digest := fmt.Sprintf("sha256:%x", sha256.Sum256(data))
		mem.Add(digest, append([]byte(nil), data...))
		job.Layers = append(job.Layers, scanner.BlobDescriptor{
			Digest:    digest,
			Size:      int64(len(data)),
			MediaType: "application/vnd.oci.image.layer.v1.tar+gzip",
		})
	}

	cfg, err := json.Marshal(map[string]any{
		"architecture": "amd64",
		"os":           "linux",
		"config":       map[string]any{},
		"rootfs":       map[string]any{"type": "layers", "diff_ids": diffIDs},
	})
	if err != nil {
		tb.Fatalf("marshal config: %v", err)
	}
	cfgDigest := fmt.Sprintf("sha256:%x", sha256.Sum256(cfg))
	mem.Add(cfgDigest, cfg)
	job.Config = scanner.BlobDescriptor{
		Digest:    cfgDigest,
		Size:      int64(len(cfg)),
		MediaType: "application/vnd.oci.image.config.v1+json",
	}

	return job, mem
}

func buildTar(tb testing.TB, layer, files int, size int64, fill string) []byte {
	tb.Helper()
	var buf bytes.Buffer
	tw := tar.NewWriter(&buf)
	for f := 0; f < files; f++ {
		content := fillBytes(size, fill, int64(layer*1000+f))
		hdr := &tar.Header{
			Name:     fmt.Sprintf("layer%02d/file%04d.bin", layer, f),
			Mode:     0o644,
			Size:     int64(len(content)),
			Typeflag: tar.TypeReg,
			ModTime:  time.Unix(0, 0),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			tb.Fatalf("tar header: %v", err)
		}
		if _, err := tw.Write(content); err != nil {
			tb.Fatalf("tar write: %v", err)
		}
	}
	if err := tw.Close(); err != nil {
		tb.Fatalf("tar close: %v", err)
	}
	return buf.Bytes()
}

// fillBytes generates content without importing math/rand: a xorshift over a
// seed is enough to defeat gzip while staying reproducible.
func fillBytes(n int64, fill string, seed int64) []byte {
	out := make([]byte, n)
	if fill == "zeros" {
		return out
	}
	x := uint64(seed*2654435761 + 12345)
	for i := range out {
		x ^= x << 13
		x ^= x >> 7
		x ^= x << 17
		out[i] = byte(x)
	}
	return out
}

// TestPerfSyntheticScaling sweeps layer count at fixed total bytes, then total
// bytes at fixed layer count. Splitting them is the point: the two are
// confounded in any real corpus, where more layers usually also means more
// bytes.
func TestPerfSyntheticScaling(t *testing.T) {
	requirePerf(t)

	const totalMiB = 64

	t.Run("layers_at_fixed_bytes", func(t *testing.T) {
		for _, layers := range []int{1, 2, 4, 8, 16, 32} {
			perLayer := int64(totalMiB<<20) / int64(layers)
			job, mem := syntheticImage(t, layers, 4, perLayer/4, "random")
			tmp := perfHold(t, mem, job)

			settle()
			m := startMonitor(25 * time.Millisecond)
			st := mustPipeline(t, job, tmp)
			samples := m.finish()
			dRSS, dHeap := growth(samples)
			t.Logf("layers=%2d compressed=%.1fMiB %s ΔRSS=%.0fMiB ΔHeap=%.0fMiB peakRSS=%.0fMiB",
				layers, float64(compressedBytes(job))/(1<<20), st,
				float64(dRSS)/(1<<20), float64(dHeap)/(1<<20), float64(peakRSS(samples))/(1<<20))
		}
	})

	t.Run("bytes_at_fixed_layers", func(t *testing.T) {
		for _, mib := range []int64{8, 32, 128, 512} {
			job, mem := syntheticImage(t, 4, 4, (mib<<20)/16, "random")
			tmp := perfHold(t, mem, job)

			settle()
			m := startMonitor(25 * time.Millisecond)
			st := mustPipeline(t, job, tmp)
			samples := m.finish()
			dRSS, dHeap := growth(samples)
			t.Logf("bytes=%4dMiB compressed=%.1fMiB %s ΔRSS=%.0fMiB ΔHeap=%.0fMiB peakRSS=%.0fMiB",
				mib, float64(compressedBytes(job))/(1<<20), st,
				float64(dRSS)/(1<<20), float64(dHeap)/(1<<20), float64(peakRSS(samples))/(1<<20))
		}
	})

	t.Run("files_at_fixed_bytes", func(t *testing.T) {
		// Entry count, not byte count, is what a filesystem catalog walks.
		for _, files := range []int{16, 256, 4096} {
			job, mem := syntheticImage(t, 4, files, (16<<20)/int64(files), "random")
			tmp := perfHold(t, mem, job)

			settle()
			m := startMonitor(25 * time.Millisecond)
			st := mustPipeline(t, job, tmp)
			samples := m.finish()
			dRSS, dHeap := growth(samples)
			t.Logf("filesPerLayer=%5d (%d total) %s ΔRSS=%.0fMiB ΔHeap=%.0fMiB peakRSS=%.0fMiB",
				files, files*4, st,
				float64(dRSS)/(1<<20), float64(dHeap)/(1<<20), float64(peakRSS(samples))/(1<<20))
		}
	})
}

// ---------------------------------------------------------------------------
// 4. Disk: what MaxImageSize does not bound
// ---------------------------------------------------------------------------

// TestPerfDiskAmplification measures the gap between the compressed bytes the
// MaxImageSize guard checks and the bytes a scan actually puts on disk.
// A layer of zeros is the extreme case, but the direction is the same for any
// real layer, and the guard sees only the compressed figure.
func TestPerfDiskAmplification(t *testing.T) {
	requirePerf(t)

	for _, mib := range []int64{64, 512} {
		job, mem := syntheticImage(t, 1, 8, (mib<<20)/8, "zeros")
		tmp := perfHold(t, mem, job)

		compressed := compressedBytes(job)
		st := mustPipeline(t, job, tmp)
		leftover := dirBytes(tmp)

		t.Logf("uncompressed=%dMiB compressed=%.2fMiB ratio=%.0fx peakTmp=%.1fMiB leftoverAfterScan=%dB %s",
			mib, float64(compressed)/(1<<20), float64(mib<<20)/float64(compressed),
			float64(st.TmpPeak)/(1<<20), leftover, st)
	}
}

// TestPerfTempCleanup checks that nothing accumulates under the scan tmp dir
// across a run of jobs, which is the failure mode a long-lived scanner would
// hit long before it hit a memory limit.
func TestPerfTempCleanup(t *testing.T) {
	requirePerf(t)

	name := availableFixtures(t)[0]
	dir := fixtureDir(name)
	job := jobFromLayout(t, dir)
	tmp := perfHold(t, mockhold.NewOCILayout(dir), job)

	for i := 0; i < 5; i++ {
		mustPipeline(t, job, tmp)
		entries, err := os.ReadDir(tmp)
		if err != nil {
			t.Fatalf("read tmp: %v", err)
		}
		t.Logf("after job %d: %d entries, %d bytes under %s", i, len(entries), dirBytes(tmp), tmp)
		if len(entries) != 0 {
			names := make([]string, 0, len(entries))
			for _, e := range entries {
				names = append(names, e.Name())
			}
			t.Errorf("job %d left %d entries behind: %v", i, len(entries), names)
		}
	}
}

// ---------------------------------------------------------------------------
// 5. Concurrency
// ---------------------------------------------------------------------------

// TestPerfConcurrency runs the same image through 1, 2 and 4 concurrent
// pipelines and reports throughput and peak RSS for each.
//
// This measures the work, not the worker pool: the pool cannot be driven to
// completion on a successful scan (see the file comment), so what it shows is
// the ceiling raising scanner.workers could reach if the pool itself adds no
// contention of its own. The vulnerability-database lock, which the pool does
// add once Grype is enabled, is modelled separately below.
func TestPerfConcurrency(t *testing.T) {
	requirePerf(t)

	name := heaviestFixture(t)
	dir := fixtureDir(name)
	tmpl := jobFromLayout(t, dir)
	tmp := perfHold(t, mockhold.NewOCILayout(dir), tmpl)

	const perWorker = 2
	for _, workers := range []int{1, 2, 4} {
		settle()
		m := startMonitor(25 * time.Millisecond)
		start := time.Now()

		var wg sync.WaitGroup
		errs := make(chan error, workers*perWorker)
		for w := 0; w < workers; w++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				job := *tmpl // each pipeline gets its own job value
				for i := 0; i < perWorker; i++ {
					if _, err := pipelineOnce(&job, tmp); err != nil {
						errs <- err
						return
					}
				}
			}()
		}
		wg.Wait()
		close(errs)
		for err := range errs {
			t.Fatalf("workers=%d: %v", workers, err)
		}

		wall := time.Since(start)
		samples := m.finish()
		total := workers * perWorker
		dRSS, dHeap := growth(samples)
		t.Logf("workers=%d scans=%d wall=%s perScan=%s throughput=%.2f scans/min peakRSS=%.0fMiB ΔRSS=%.0fMiB ΔHeap=%.0fMiB",
			workers, total, round(wall), round(wall/time.Duration(total)),
			float64(total)/wall.Minutes(), float64(peakRSS(samples))/(1<<20),
			float64(dRSS)/(1<<20), float64(dHeap)/(1<<20))

		runtime.GC()
	}
}

// TestPerfVulnDBLockModel measures what the RWMutex discipline in grype.go
// costs when a reload lands during steady-state scanning.
//
// It is a model, not the real code: loadVulnDB is unexported in package scan
// and cannot be stubbed from here, and running the real thing would download
// the database this file is forbidden to fetch. What the model reproduces is
// exactly the structure at grype.go:105 and grype.go:187 — every scan holds
// the read lock for the whole of FindMatches, a reload holds the write lock
// for the whole of the download — and what it measures is the stall that
// structure imposes on workers that are not reloading anything.
func TestPerfVulnDBLockModel(t *testing.T) {
	requirePerf(t)

	const (
		workers    = 4
		scanWork   = 50 * time.Millisecond
		reloadWork = 2 * time.Second // stands in for a database download
	)

	var lock sync.RWMutex
	var stalls sync.Map // worker id -> longest wait for the read lock

	stop := make(chan struct{})
	var wg sync.WaitGroup

	for w := 0; w < workers; w++ {
		wg.Add(1)
		go func(id int) {
			defer wg.Done()
			var worst time.Duration
			for {
				select {
				case <-stop:
					stalls.Store(id, worst)
					return
				default:
				}
				t0 := time.Now()
				lock.RLock()
				if wait := time.Since(t0); wait > worst {
					worst = wait
				}
				time.Sleep(scanWork)
				lock.RUnlock()
			}
		}(w)
	}

	time.Sleep(500 * time.Millisecond) // steady state
	before := time.Now()
	lock.Lock()
	acquired := time.Since(before)
	time.Sleep(reloadWork)
	lock.Unlock()
	time.Sleep(500 * time.Millisecond)

	close(stop)
	wg.Wait()

	var worst time.Duration
	stalls.Range(func(_, v any) bool {
		if d := v.(time.Duration); d > worst {
			worst = d
		}
		return true
	})

	t.Logf("model: %d workers, %s per scan, %s reload — writer waited %s to acquire, "+
		"worst reader stall %s (%.1fx the reload)",
		workers, scanWork, reloadWork, round(acquired), round(worst),
		float64(worst)/float64(reloadWork))
	t.Logf("in production the reload is a database download of several hundred MB, " +
		"so the reader stall scales with that download, not with the model's 2s")
}

// ---------------------------------------------------------------------------
// 6. The real worker pool: cadence, queue burst, reconnect
// ---------------------------------------------------------------------------

// failFastJob is a job the pipeline rejects before it opens a socket: no
// config digest, no layers, so buildOCILayout fails on the first check. It is
// the cheapest way to push many jobs through the real worker without paying
// for a real image per job (see the file comment).
func failFastJob(repo string) *scanner.ScanJob {
	return &scanner.ScanJob{
		ManifestDigest: "sha256:" + strings.Repeat("1", 64),
		Repository:     repo,
		Tag:            "latest",
		Tier:           "deckhand",
	}
}

// TestPerfWorkerCadence measures the real inter-job gap at the production
// JobCooldown. The jobs themselves do no work, so what is left is exactly the
// per-job overhead the worker adds: the GC call plus the sleep.
func TestPerfWorkerCadence(t *testing.T) {
	requirePerf(t)

	h := Start(t, mockhold.NewMemory())
	// Restore the production value the harness shortens. Start already
	// registered a cleanup that puts back whatever it found, and cleanups run
	// last-in-first-out, so this needs no undo of its own.
	scan.JobCooldown = 10 * time.Second

	const jobs = 4
	start := time.Now()
	var seqs []int64
	for i := 0; i < jobs; i++ {
		seq, err := h.Hold.SendJob(failFastJob("cadence"))
		if err != nil {
			t.Fatalf("send job %d: %v", i, err)
		}
		seqs = append(seqs, seq)
	}

	var prev time.Time
	for i, seq := range seqs {
		msg := h.AwaitTerminal(t, seq, 2*time.Minute)
		gap := time.Duration(0)
		if i > 0 {
			gap = msg.At.Sub(prev)
		}
		prev = msg.At
		t.Logf("job %d terminal at +%s (gap %s) type=%s", i,
			round(msg.At.Sub(start)), round(gap), msg.Type)
	}

	wall := time.Since(start)
	t.Logf("RESULT %d no-op jobs took %s at the production cooldown: %s per job, "+
		"%.1f jobs/min ceiling for a single worker independent of scan cost",
		jobs, round(wall), round(wall/jobs), float64(jobs)/wall.Minutes())
}

// TestPerfQueueBurst fills the queue to its configured depth and measures how
// long the last job in it waits, which is the number that has to be compared
// against the hold's ten minute processing timeout: the scanner acks a job the
// moment it arrives (client/hold.go:159), which moves the hold's row to
// 'processing' without moving assigned_at, and reDispatchTimedOut fails any
// processing row older than ten minutes
// (pkg/hold/pds/scan_broadcaster.go:833). Queue time is spent against that
// deadline.
//
// The first job is pointed at a server that never answers, which parks the
// single worker while the burst arrives. Every burst job then fails instantly,
// so what the last one's latency measures is queue position plus cooldown and
// nothing else — a floor for the real thing, where each job ahead of it also
// costs a scan.
func TestPerfQueueBurst(t *testing.T) {
	requirePerf(t)

	release := make(chan struct{})
	var releaseOnce sync.Once
	stall := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		<-release
		http.Error(w, "gone", http.StatusNotFound)
	}))
	defer stall.Close()
	defer releaseOnce.Do(func() { close(release) })

	h := Start(t, mockhold.NewMemory())
	depth := h.Cfg.Scanner.QueueSize

	// Park the worker on a presign request that never returns.
	parked := failFastJob("parked")
	parked.Config = scanner.BlobDescriptor{
		Digest:    "sha256:" + strings.Repeat("2", 64),
		Size:      10,
		MediaType: "application/vnd.oci.image.config.v1+json",
	}
	parked.HoldEndpoint = stall.URL
	if _, err := h.Hold.SendJob(parked); err != nil {
		t.Fatalf("send parking job: %v", err)
	}
	deadline := time.Now().Add(10 * time.Second)
	for len(h.Hold.BlobRequests()) == 0 && h.Queue.Len() == 0 && time.Now().Before(deadline) {
		time.Sleep(10 * time.Millisecond)
	}
	time.Sleep(200 * time.Millisecond)

	// Exactly enough to fill the queue, then a few more that cannot fit.
	const overflow = 10
	sent := make([]int64, 0, depth+overflow)
	sentAt := time.Now()
	for i := 0; i < depth+overflow; i++ {
		seq, err := h.Hold.SendJob(failFastJob(fmt.Sprintf("burst-%03d", i)))
		if err != nil {
			t.Fatalf("send burst job %d: %v", i, err)
		}
		sent = append(sent, seq)
	}
	dispatchTook := time.Since(sentAt)

	// Wait for the client's read loop to drain the socket, then look at how
	// deep the queue actually got.
	maxDepth := 0
	deadline = time.Now().Add(10 * time.Second)
	for time.Now().Before(deadline) {
		if n := h.Queue.Len(); n > maxDepth {
			maxDepth = n
		}
		if maxDepth >= depth {
			break
		}
		time.Sleep(5 * time.Millisecond)
	}
	t.Logf("dispatched %d jobs in %s; queue reached %d of %d",
		len(sent), round(dispatchTook), maxDepth, depth)

	// The read loop enqueues in arrival order with the worker parked, so the
	// first `depth` jobs are the accepted ones and the rest are rejected.
	lastAccepted := sent[depth-1]
	releaseOnce.Do(func() { close(release) })

	last := h.AwaitTerminal(t, lastAccepted, 5*time.Minute)
	first := h.AwaitTerminal(t, sent[0], time.Minute)

	var full int
	for _, m := range h.Hold.Transcript() {
		if m.Type == "error" && strings.Contains(m.Error, "queue full") {
			full++
		}
	}

	drain := last.At.Sub(first.At)
	perJob := drain / time.Duration(depth-1)
	t.Logf("RESULT queueDepth=%d rejectedAsQueueFull=%d firstTerminal=+%s lastTerminal=+%s drain=%s perJob=%s (cooldown=%s)",
		depth, full, round(first.At.Sub(sentAt)), round(last.At.Sub(sentAt)),
		round(drain), round(perJob), scan.JobCooldown)

	overhead := perJob - scan.JobCooldown
	t.Logf("EXTRAPOLATION per-job overhead outside the cooldown is %s; at the production "+
		"10s cooldown a full %d-deep queue of no-op jobs drains in %s, and a queue of "+
		"16s node:22 scans in %s — against the hold's 10m processing timeout, which is "+
		"exceeded by queue position %d and %d respectively",
		round(overhead), depth,
		round(time.Duration(depth)*(overhead+10*time.Second)),
		round(time.Duration(depth)*(overhead+10*time.Second+16*time.Second)),
		int(10*time.Minute/(overhead+10*time.Second)),
		int(10*time.Minute/(overhead+26*time.Second)))
}

// TestPerfConnectionChurn watches goroutines and file descriptors across
// reconnects. The client redials on a fixed five second backoff, so each cycle
// costs that much; the count is kept low deliberately.
func TestPerfConnectionChurn(t *testing.T) {
	requirePerf(t)

	h := Start(t, mockhold.NewMemory())

	settle := func() {
		runtime.GC()
		time.Sleep(200 * time.Millisecond)
	}
	settle()
	baseGoroutines, baseFDs := runtime.NumGoroutine(), fdCount()
	t.Logf("baseline: goroutines=%d fds=%d", baseGoroutines, baseFDs)

	const cycles = 4
	for i := 0; i < cycles; i++ {
		mode := mockhold.DropAbrupt
		if i%2 == 1 {
			mode = mockhold.DropClean
		}
		h.Hold.DropConnections(mode)

		deadline := time.Now().Add(30 * time.Second)
		want := len(h.Hold.Dials()) + 1
		for len(h.Hold.Dials()) < want && time.Now().Before(deadline) {
			time.Sleep(100 * time.Millisecond)
		}
		if len(h.Hold.Dials()) < want {
			t.Fatalf("cycle %d: scanner never redialled", i)
		}
		if _, err := h.Hold.SendJob(failFastJob("churn")); err != nil {
			t.Fatalf("cycle %d: send job: %v", i, err)
		}
		settle()
		t.Logf("cycle %d (%v): goroutines=%d fds=%d", i, mode, runtime.NumGoroutine(), fdCount())
	}

	settle()
	t.Logf("RESULT after %d reconnects: goroutines %d→%d fds %d→%d",
		cycles, baseGoroutines, runtime.NumGoroutine(), baseFDs, fdCount())
}

// TestPerfJobChurn pushes many no-op jobs through the real worker and watches
// for goroutine or descriptor growth. It cannot cover the scanning path, which
// is where a leak would most plausibly live; that limitation is the BLOCKER's,
// not the scenario's.
func TestPerfJobChurn(t *testing.T) {
	requirePerf(t)

	h := Start(t, mockhold.NewMemory())
	runtime.GC()
	time.Sleep(200 * time.Millisecond)
	baseGoroutines, baseFDs := runtime.NumGoroutine(), fdCount()
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	baseHeap := ms.HeapAlloc

	const jobs = 200
	var lastSeq int64
	for i := 0; i < jobs; i++ {
		seq, err := h.Hold.SendJob(failFastJob("churn"))
		if err != nil {
			t.Fatalf("send job %d: %v", i, err)
		}
		lastSeq = seq
		if i%20 == 0 {
			time.Sleep(50 * time.Millisecond) // stay inside the 100-deep queue
		}
	}
	h.AwaitTerminal(t, lastSeq, 3*time.Minute)

	runtime.GC()
	time.Sleep(200 * time.Millisecond)
	runtime.ReadMemStats(&ms)
	t.Logf("RESULT after %d jobs: goroutines %d→%d fds %d→%d heap %.1f→%.1fMiB",
		jobs, baseGoroutines, runtime.NumGoroutine(), baseFDs, fdCount(),
		float64(baseHeap)/(1<<20), float64(ms.HeapAlloc)/(1<<20))
}

// ---------------------------------------------------------------------------
// 7. The real worker on a real image, in a subprocess
// ---------------------------------------------------------------------------

// TestPerfRealWorkerScan measures one successful scan through the actual
// WorkerPool in a child process. It was written when the worker panicked on
// result.Summary.Total the instant a scan succeeded, so the child died with a
// SIGSEGV that would have taken the whole test binary with it; the child is
// kept because it also isolates the measurement from this process.
//
// The measurement is the child's own log line "Scan pipeline completed
// duration=...", produced by production code, so it can be compared directly
// against the replica's number to see what the worker adds.
func TestPerfRealWorkerScan(t *testing.T) {
	requirePerf(t)

	name := heaviestFixture(t)
	cmd := exec.Command(os.Args[0],
		"-test.run", "^TestPerfRealWorkerChild$",
		"-test.v",
		"-test.timeout", "10m")
	cmd.Env = append(os.Environ(),
		perfEnv+"=1",
		"ATCR_SCANNER_PERF_CHILD="+name)

	start := time.Now()
	out, err := cmd.CombinedOutput()
	wall := time.Since(start)

	var duration, panicked string
	for _, line := range strings.Split(string(out), "\n") {
		if strings.Contains(line, "Scan pipeline completed") {
			duration = line
		}
		if strings.HasPrefix(line, "panic:") {
			panicked = line
		}
	}

	t.Logf("child fixture=%s wall=%s exit=%v", name, round(wall), err)
	if duration != "" {
		t.Logf("production log line: %s", strings.TrimSpace(duration))
	}
	if panicked != "" {
		t.Errorf("child panicked: %s\noutput tail:\n%s",
			strings.TrimSpace(panicked), tail(string(out), 20))
	}
}

// TestPerfRealWorkerChild is the child half of TestPerfRealWorkerScan. It is
// inert unless ATCR_SCANNER_PERF_CHILD names a fixture.
func TestPerfRealWorkerChild(t *testing.T) {
	name := os.Getenv("ATCR_SCANNER_PERF_CHILD")
	if name == "" {
		t.Skip("child process scenario; driven by TestPerfRealWorkerScan")
	}
	dir := fixtureDir(name)
	if !hasFixture(name) {
		t.Skipf("fixture %q absent", name)
	}

	job := jobFromLayout(t, dir)
	h := Start(t, mockhold.NewOCILayout(dir))
	seq, err := h.Hold.SendJob(job)
	if err != nil {
		t.Fatalf("send job: %v", err)
	}
	msg := h.AwaitTerminal(t, seq, 5*time.Minute)
	t.Logf("terminal=%s", msg.Type)
}

func tail(s string, n int) string {
	lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
	if len(lines) > n {
		lines = lines[len(lines)-n:]
	}
	return strings.Join(lines, "\n")
}

// ---------------------------------------------------------------------------
// 8. The production soft memory limit
// ---------------------------------------------------------------------------

// TestPerfMemoryLimit runs the same scan under the soft memory limit the
// scanner sets on itself at cmd/scanner/main.go:46 and again with the limit
// effectively removed, and reports what the limit costs.
//
// GOMEMLIMIT is a soft limit: the runtime does not fail an allocation that
// crosses it, it runs the collector harder. When a scan's live heap is at or
// above the limit there is nothing left to collect, so the collector runs
// continuously against a heap it cannot shrink. GCCPUFraction is the number
// that shows this happening.
//
// The limit is process-wide, so scanner.workers > 1 divides it between
// concurrent scans rather than multiplying it.
func TestPerfMemoryLimit(t *testing.T) {
	requirePerf(t)

	name := heaviestFixture(t)
	dir := fixtureDir(name)
	job := jobFromLayout(t, dir)
	tmp := perfHold(t, mockhold.NewOCILayout(dir), job)

	const prodLimit = 512 * 1024 * 1024 // cmd/scanner/main.go:46

	run := func(label string, limit int64) {
		prev := debug.SetMemoryLimit(limit)
		defer debug.SetMemoryLimit(prev)

		settle()
		var before runtime.MemStats
		runtime.ReadMemStats(&before)

		m := startMonitor(25 * time.Millisecond)
		st := mustPipeline(t, job, tmp)
		samples := m.finish()

		var after runtime.MemStats
		runtime.ReadMemStats(&after)
		dRSS, dHeap := growth(samples)
		t.Logf("%s (limit=%s): %s peakRSS=%.0fMiB ΔRSS=%.0fMiB ΔHeap=%.0fMiB gcCycles=%d gcPause=%s gcCPU=%.1f%%",
			label, limitLabel(limit), st,
			float64(peakRSS(samples))/(1<<20), float64(dRSS)/(1<<20), float64(dHeap)/(1<<20),
			after.NumGC-before.NumGC,
			round(time.Duration(after.PauseTotalNs-before.PauseTotalNs)),
			after.GCCPUFraction*100)
	}

	// The same again with two scans in flight, because the limit is
	// process-wide: scanner.workers > 1 divides it rather than multiplying it,
	// and the production template sets workers: 2.
	runConcurrent := func(label string, limit int64, n int) {
		prev := debug.SetMemoryLimit(limit)
		defer debug.SetMemoryLimit(prev)

		settle()
		var before runtime.MemStats
		runtime.ReadMemStats(&before)

		m := startMonitor(25 * time.Millisecond)
		start := time.Now()
		var wg sync.WaitGroup
		errs := make(chan error, n)
		for i := 0; i < n; i++ {
			wg.Add(1)
			go func() {
				defer wg.Done()
				j := *job
				if _, err := pipelineOnce(&j, tmp); err != nil {
					errs <- err
				}
			}()
		}
		wg.Wait()
		close(errs)
		for err := range errs {
			t.Fatalf("%s: %v", label, err)
		}
		wall := time.Since(start)
		samples := m.finish()

		var after runtime.MemStats
		runtime.ReadMemStats(&after)
		dRSS, dHeap := growth(samples)
		t.Logf("%s x%d (limit=%s): wall=%s perScan=%s peakRSS=%.0fMiB ΔRSS=%.0fMiB ΔHeap=%.0fMiB gcCycles=%d gcPause=%s gcCPU=%.1f%%",
			label, n, limitLabel(limit), round(wall), round(wall/time.Duration(n)),
			float64(peakRSS(samples))/(1<<20), float64(dRSS)/(1<<20), float64(dHeap)/(1<<20),
			after.NumGC-before.NumGC,
			round(time.Duration(after.PauseTotalNs-before.PauseTotalNs)),
			after.GCCPUFraction*100)
	}

	t.Logf("fixture=%s %d layers %.0fMiB compressed", name, len(job.Layers),
		float64(compressedBytes(job))/(1<<20))
	run("unlimited", math.MaxInt64)
	run("production", prodLimit)
	run("unlimited-again", math.MaxInt64) // ordering control
	runConcurrent("unlimited", math.MaxInt64, 2)
	runConcurrent("production", prodLimit, 2)
	runConcurrent("unlimited", math.MaxInt64, 4)
	runConcurrent("production", prodLimit, 4)
}

func limitLabel(limit int64) string {
	if limit == math.MaxInt64 {
		return "off"
	}
	return fmt.Sprintf("%dMiB", limit/(1<<20))
}