at-container-registry/scanner/internal/scan/vulndb_refresh_test.go

package scan

import (
	"context"
	"errors"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"

	"atcr.io/scanner/internal/client"
	"atcr.io/scanner/internal/config"
	"atcr.io/scanner/internal/mockhold"
	"atcr.io/scanner/internal/queue"
	"github.com/anchore/grype/grype"
	v6 "github.com/anchore/grype/grype/db/v6"
	v6dist "github.com/anchore/grype/grype/db/v6/distribution"
	v6inst "github.com/anchore/grype/grype/db/v6/installation"
	"github.com/anchore/grype/grype/vulnerability"
)

// This file covers the production complaint "grype is out of date, please
// update the database": the vulnerability database stops refreshing and scans
// start failing, with no way back to health short of a restart.
//
// grype_test.go pins the ordinary lifecycle of loadVulnDatabase. What follows
// pins the failure lifecycle — cold start, upstream outage, an on-disk database
// that is corrupt rather than merely old, the periodic reload, and the ceiling
// on how old a database may get before the scanner refuses to vouch for its
// results.
//
// Every test that reaches a real Grype code path does so without network
// access. The only calls into grype.LoadVulnerabilityDB use update=false, so
// no listing is fetched and no archive is downloaded; the on-disk database is
// synthesised locally with the exported v6 low-level writer.
//
// These tests share the package-level vulnDB globals with grype_test.go and so
// must not be run in parallel with anything.

// slowLoader installs a loader that blocks for d before returning. It is how
// the tests below stand in for a real refresh, which in production is a 30s
// listing check plus an up-to-300s archive download plus hydration, all of it
// inside loadVulnDatabase's exclusive lock.
func slowLoader(t *testing.T, d time.Duration, built time.Time, err error) (*fakeProvider, *int) {
	t.Helper()
	loaded := &fakeProvider{name: "slow"}
	calls := 0
	var mu sync.Mutex
	loadVulnDB = func(v6dist.Config, v6inst.Config, bool) (vulnerability.Provider, *vulnerability.ProviderStatus, error) {
		mu.Lock()
		calls++
		mu.Unlock()
		time.Sleep(d)
		if err != nil {
			return nil, nil, err
		}
		return loaded, &vulnerability.ProviderStatus{Built: built}, nil
	}
	return loaded, &calls
}

// scriptedLoader installs a loader that returns the given responses in order,
// repeating the last one once the script runs out. It is how the tests below
// express "the upstream fails and then recovers" without a network.
type loaderResponse struct {
	built time.Time
	err   error
}

func scriptedLoader(t *testing.T, responses ...loaderResponse) (*fakeProvider, *int) {
	t.Helper()
	loaded := &fakeProvider{name: "scripted"}
	calls := 0
	var mu sync.Mutex
	loadVulnDB = func(v6dist.Config, v6inst.Config, bool) (vulnerability.Provider, *vulnerability.ProviderStatus, error) {
		mu.Lock()
		i := calls
		calls++
		mu.Unlock()
		if i >= len(responses) {
			i = len(responses) - 1
		}
		r := responses[i]
		if r.err != nil {
			return nil, nil, r.err
		}
		return loaded, &vulnerability.ProviderStatus{Built: r.built}, nil
	}
	return loaded, &calls
}

// --- the database lifecycle under failure ---

// TestColdStart_ThrottlesRepeatedAttempts. With no provider in hand the
// retry backoff must still apply, or every queued scan runs a complete download
// attempt (30s listing check plus up to a 300s archive fetch) under the
// exclusive lock while the hold's stale loop keeps re-queueing the failures.
//
// The throttled calls must still report an error — the scan genuinely cannot
// run — but they must not repeat the download.
func TestColdStart_ThrottlesRepeatedAttempts(t *testing.T) {
	resetVulnDBState(t)

	_, calls := stubLoader(t, time.Time{}, errors.New("upstream unreachable"))

	const attempts = 5
	for i := 0; i < attempts; i++ {
		got, err := loadVulnDatabase(context.Background(), t.TempDir())
		if err == nil {
			t.Fatalf("attempt %d: a cold start with no database returned success", i)
		}
		if got != nil {
			t.Fatalf("attempt %d: a provider was returned alongside the error", i)
		}
	}

	if *calls != 1 {
		t.Errorf("loader called %d times across %d cold-start scans, want 1: the retry "+
			"backoff must apply with no provider in hand, not only with one", *calls, attempts)
	}
}

// TestColdStart_ConcurrentWorkersShareOneAttempt is the pool-level consequence
// of B1. Four workers arriving together must produce one download attempt, not
// four serialized ones, and must finish in about one attempt's time.
func TestColdStart_ConcurrentWorkersShareOneAttempt(t *testing.T) {
	resetVulnDBState(t)

	const workers = 4
	const loadDelay = 40 * time.Millisecond
	_, calls := slowLoader(t, loadDelay, time.Time{}, errors.New("upstream unreachable"))

	start := time.Now()
	var wg sync.WaitGroup
	for i := 0; i < workers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, _ = loadVulnDatabase(context.Background(), t.TempDir())
		}()
	}
	wg.Wait()
	elapsed := time.Since(start)

	if *calls != 1 {
		t.Errorf("loader called %d times for %d concurrent workers, want 1", *calls, workers)
	}
	if max := time.Duration(workers-1) * loadDelay; elapsed >= max {
		t.Errorf("elapsed %s for %d workers at %s each: the attempts are still being "+
			"serialized through the exclusive lock", elapsed, workers, loadDelay)
	}
}

// TestColdStart_RecoversWhenUpstreamReturns is the other half of the backoff:
// it has to let go. A scanner that failed its cold start must load the database
// on its own once the upstream comes back, with no process restart.
func TestColdStart_RecoversWhenUpstreamReturns(t *testing.T) {
	resetVulnDBState(t)

	freshBuild := time.Now().Add(-1 * time.Hour)
	loaded, calls := scriptedLoader(t,
		loaderResponse{err: errors.New("upstream unreachable")},
		loaderResponse{built: freshBuild},
	)

	if _, err := loadVulnDatabase(context.Background(), t.TempDir()); err == nil {
		t.Fatal("the first cold-start attempt was expected to fail")
	}

	// Age out the cold-start backoff, standing in for the passage of time.
	vulnDBLock.Lock()
	vulnDBAttempt = time.Now().Add(-vulnDBColdRetryBackoff - time.Minute)
	vulnDBLock.Unlock()

	got, err := loadVulnDatabase(context.Background(), t.TempDir())
	if err != nil {
		t.Fatalf("the scanner did not recover once the upstream returned: %v", err)
	}
	if got != loaded {
		t.Error("the newly loaded provider was not adopted")
	}
	if *calls != 2 {
		t.Errorf("loader called %d times, want 2", *calls)
	}
	if !vulnDBBuilt.Equal(freshBuild) {
		t.Errorf("vulnDBBuilt = %v, want %v", vulnDBBuilt, freshBuild)
	}
}

// TestPeriodicReload_FiresOnSchedule. The "close and reopen the DB every N
// scans to flush SQLite's page cache and mmap region" mitigation has to
// actually run; for most of this file's history it never did. The counter therefore has to live on the path scans take — the
// read-lock fast path — because a fresh database never reaches the write lock.
func TestPeriodicReload_FiresOnSchedule(t *testing.T) {
	resetVulnDBState(t)

	cached := &fakeProvider{name: "cached"}
	vulnDB = cached
	vulnDBBuilt = time.Now().Add(-1 * time.Hour) // fresh

	loaded, calls := stubLoader(t, time.Now().Add(-1*time.Hour), nil)

	const scans = 200
	for i := 0; i < scans; i++ {
		got, err := loadVulnDatabase(context.Background(), t.TempDir())
		if err != nil {
			t.Fatalf("scan %d: %v", i, err)
		}
		if got == nil {
			t.Fatalf("scan %d returned no provider", i)
		}
	}

	if want := scans / vulnDBReloadEvery; *calls != want {
		t.Errorf("loader called %d times in %d fresh scans, want %d (one every %d)",
			*calls, scans, want, vulnDBReloadEvery)
	}
	if !cached.closed {
		t.Error("the replaced provider was not closed: every periodic reload would leak its predecessor")
	}
	if vulnDB != loaded {
		t.Error("the reloaded provider was not adopted")
	}
	if n := vulnDBScans.Load(); n != scans {
		t.Errorf("vulnDBScans = %d after %d scans, want %d: the counter is not on the path scans take", n, scans, scans)
	}
}

// TestPeriodicReload_FailureKeepsTheWorkingProvider is the interaction that
// makes the reload dangerous to enable on its own. It used to null the provider
// before loading its replacement, so one transient failure would drop a scanner
// that was working perfectly into the no-provider state above.
//
// Load first, swap second: a failed periodic reload must be a no-op.
func TestPeriodicReload_FailureKeepsTheWorkingProvider(t *testing.T) {
	resetVulnDBState(t)

	cached := &fakeProvider{name: "cached"}
	vulnDB = cached
	vulnDBBuilt = time.Now().Add(-1 * time.Hour) // fresh

	_, calls := stubLoader(t, time.Time{}, errors.New("upstream blip"))

	for i := 0; i < vulnDBReloadEvery+5; i++ {
		got, err := loadVulnDatabase(context.Background(), t.TempDir())
		if err != nil {
			t.Fatalf("scan %d: a failed periodic reload must not fail the scan: %v", i, err)
		}
		if got != cached {
			t.Fatalf("scan %d: the working provider was dropped for a failed reload", i)
		}
	}

	if cached.closed {
		t.Error("the working provider was closed before its replacement was in hand")
	}
	if *calls != 1 {
		t.Errorf("loader called %d times, want 1", *calls)
	}
}

// TestStaleFallback_RefusesPastTheCeiling. Serving a slightly old database
// beats refusing to scan, which is why the fallback exists. Serving one of
// unbounded age while reporting success is a different thing: a scanner whose
// egress is blocked would publish confident "0 critical" verdicts for months.
//
// Grype's own MaxAllowedBuiltAge is the natural line, since that is the
// guarantee grypeDBConfig already asks for.
func TestStaleFallback_RefusesPastTheCeiling(t *testing.T) {
	resetVulnDBState(t)

	ancient := &fakeProvider{name: "ancient"}
	vulnDB = ancient
	vulnDBBuilt = time.Now().Add(-120 * 24 * time.Hour) // four months old

	_, calls := stubLoader(t, time.Time{}, errors.New("upstream unreachable"))

	got, err := loadVulnDatabase(context.Background(), t.TempDir())
	if err == nil {
		t.Fatal("a database built 120 days ago was served with a nil error")
	}
	if got != nil {
		t.Error("a provider past the ceiling was returned alongside the error")
	}
	if !strings.Contains(err.Error(), "too old") {
		t.Errorf("error %q does not say the database is too old to trust", err)
	}
	if *calls != 1 {
		t.Errorf("loader called %d times, want 1", *calls)
	}

	// And the refusal is throttled like any other failure, rather than running
	// a download per scan.
	if _, err := loadVulnDatabase(context.Background(), t.TempDir()); err == nil {
		t.Fatal("the second scan was served from the same ancient database")
	}
	if *calls != 1 {
		t.Errorf("loader called %d times across two scans, want 1", *calls)
	}
}

// TestStaleFallback_ServesInsideTheCeiling pins the other side of the trade: a
// database that is stale but still inside Grype's max allowed age keeps
// scanning through an upstream outage.
func TestStaleFallback_ServesInsideTheCeiling(t *testing.T) {
	resetVulnDBState(t)

	old := &fakeProvider{name: "old"}
	vulnDB = old
	vulnDBBuilt = time.Now().Add(-10 * 24 * time.Hour) // stale, inside the 14-day ceiling

	_, calls := stubLoader(t, time.Time{}, errors.New("upstream unreachable"))

	got, err := loadVulnDatabase(context.Background(), t.TempDir())
	if err != nil {
		t.Fatalf("a stale but usable database must keep scanning: %v", err)
	}
	if got != old {
		t.Error("the previously loaded provider was not served")
	}
	if old.closed {
		t.Error("the provider still in use was closed")
	}
	if *calls != 1 {
		t.Errorf("loader called %d times, want 1", *calls)
	}
}

// TestCorruptDB_IsDeletedAndRetriedOnce covers the database that is current but
// whose import metadata was lost — an interrupted activate, a truncated write,
// a restored volume snapshot. curator.Update only installs something strictly
// newer, and a checksum failure leaves the on-disk description in place, so
// such a database never re-downloads: it fails every scan forever and nothing
// deletes it. Delete it and retry once, which is what `grype db delete && grype
// db update` does for CLI users.
func TestCorruptDB_IsDeletedAndRetriedOnce(t *testing.T) {
	resetVulnDBState(t)

	root := t.TempDir()
	marker := filepath.Join(root, "6", "vulnerability.db")
	if err := os.MkdirAll(filepath.Dir(marker), 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}
	if err := os.WriteFile(marker, []byte("corrupt"), 0o644); err != nil {
		t.Fatalf("write: %v", err)
	}

	freshBuild := time.Now().Add(-1 * time.Hour)
	loaded, calls := scriptedLoader(t,
		loaderResponse{err: errors.New("no import metadata file at: " + filepath.Join(root, "6", "import.json"))},
		loaderResponse{built: freshBuild},
	)

	got, err := loadVulnDatabase(context.Background(), root)
	if err != nil {
		t.Fatalf("a corrupt database must be deleted and re-downloaded, not served as a failure: %v", err)
	}
	if got != loaded {
		t.Error("the re-downloaded provider was not adopted")
	}
	if *calls != 2 {
		t.Fatalf("loader called %d times, want 2 (the load and one retry after the delete)", *calls)
	}
	if _, err := os.Stat(marker); !os.IsNotExist(err) {
		t.Errorf("the corrupt database file still exists at %s (stat err %v)", marker, err)
	}
}

// TestCorruptDB_RetriesOnlyOncePerAttempt keeps the self-heal from becoming its
// own storm: if the retry fails too, the attempt ends there and the backoff
// takes over.
func TestCorruptDB_RetriesOnlyOncePerAttempt(t *testing.T) {
	resetVulnDBState(t)

	root := t.TempDir()
	_, calls := stubLoader(t, time.Time{}, errors.New("no import metadata file at: "+root+"/6/import.json"))

	if _, err := loadVulnDatabase(context.Background(), root); err == nil {
		t.Fatal("a database that stays corrupt must still fail the scan")
	}
	if *calls != 2 {
		t.Errorf("loader called %d times, want 2: the delete-and-retry must happen once per attempt", *calls)
	}

	// And the next scan is throttled rather than repeating the delete-and-retry.
	if _, err := loadVulnDatabase(context.Background(), root); err == nil {
		t.Fatal("the second scan unexpectedly succeeded")
	}
	if *calls != 2 {
		t.Errorf("loader called %d times across two scans, want 2", *calls)
	}
}

// TestLoadVulnDatabase_HonoursContextCancellation. A shutdown, or a hold that
// has abandoned the job, must not start a fresh database download that then
// holds the exclusive lock until Grype's own timeouts expire — up to 300s for
// the archive alone, unbounded for hydration.
func TestLoadVulnDatabase_HonoursContextCancellation(t *testing.T) {
	resetVulnDBState(t)

	_, calls := stubLoader(t, time.Now(), nil)

	ctx, cancel := context.WithCancel(context.Background())
	cancel()

	if _, err := loadVulnDatabase(ctx, t.TempDir()); !errors.Is(err, context.Canceled) {
		t.Errorf("loadVulnDatabase err = %v, want context.Canceled", err)
	}
	if *calls != 0 {
		t.Errorf("loader called %d times with an already-cancelled context, want 0", *calls)
	}

	// A cancelled context must not take a usable database away from a caller
	// that needs no load at all.
	cached := &fakeProvider{name: "cached"}
	vulnDBLock.Lock()
	vulnDB = cached
	vulnDBBuilt = time.Now().Add(-1 * time.Hour)
	vulnDBLock.Unlock()

	got, err := loadVulnDatabase(ctx, t.TempDir())
	if err != nil {
		t.Fatalf("a cached provider needs no load and must not be refused: %v", err)
	}
	if got != cached {
		t.Error("the cached provider was not served")
	}
}

// TestRefresh_BlocksAllInFlightScans quantifies the third finding: a refresh
// holds the write lock for the entire download, and scanVulnerabilities holds
// the read lock across FindMatches. A scan that arrives during a refresh
// therefore waits out the whole download before it can even begin matching.
//
// The wait is measured on vulnDBLock directly rather than through
// scanVulnerabilities, which would need a real SBOM and a real provider. It is
// the same lock and the same acquisition scanVulnerabilities performs at
// grype.go:105.
func TestRefresh_BlocksAllInFlightScans(t *testing.T) {
	resetVulnDBState(t)

	old := &fakeProvider{name: "old"}
	vulnDB = old
	vulnDBBuilt = time.Now().Add(-10 * 24 * time.Hour)                // stale, so a reload is due
	vulnDBAttempt = time.Now().Add(-vulnDBRetryBackoff - time.Minute) // backoff expired

	const loadDelay = 150 * time.Millisecond
	_, calls := slowLoader(t, loadDelay, time.Now(), nil)

	refreshDone := make(chan struct{})
	go func() {
		defer close(refreshDone)
		if _, err := loadVulnDatabase(context.Background(), t.TempDir()); err != nil {
			t.Errorf("refresh: %v", err)
		}
	}()

	// Give the refresher time to take the write lock before the scan tries for
	// the read lock. Any interleaving still produces a valid measurement; this
	// only makes the intended one likely.
	time.Sleep(20 * time.Millisecond)

	start := time.Now()
	vulnDBLock.RLock()
	blocked := time.Since(start)
	vulnDBLock.RUnlock()

	<-refreshDone

	if *calls != 1 {
		t.Fatalf("loader called %d times, want 1", *calls)
	}
	if blocked < loadDelay/2 {
		t.Skipf("scan acquired the read lock in %s: the refresher had not taken the "+
			"write lock yet, so this run measured nothing", blocked)
	}
	t.Logf("a scan arriving during a refresh waited %s for the read lock; in production "+
		"the same wait is the full listing check plus archive download plus hydration, "+
		"which grype bounds only by its 300s UpdateTimeout", blocked)
}

// --- what Grype itself does with the configuration this package passes it ---

// writeSyntheticDB creates a minimal but genuine Grype v6 database on disk with
// the given build timestamp, at the layout grypeDBConfig expects
// (<root>/<ModelVersion>/vulnerability.db) and returns that directory.
//
// No import.json is written. That file is produced by the curator's activate
// step, and its absence is deliberate: writing a valid one needs grype's
// xxhash digest helper (and a new direct module dependency), and the tests
// that call into the curator here are about Status(), which reports the age
// failure whether or not the checksum check also fires.
func writeSyntheticDB(t *testing.T, root string, built time.Time) string {
	t.Helper()

	dir := filepath.Join(root, strconv.Itoa(v6.ModelVersion))
	if err := os.MkdirAll(dir, 0o755); err != nil {
		t.Fatalf("mkdir: %v", err)
	}

	gdb, err := v6.NewLowLevelDB(filepath.Join(dir, v6.VulnerabilityDBFileName), true, true, false)
	if err != nil {
		t.Fatalf("create db: %v", err)
	}
	ts := built.UTC().Round(time.Second)
	err = gdb.Create(&v6.DBMetadata{
		BuildTimestamp: &ts,
		Model:          v6.ModelVersion,
		Revision:       v6.Revision,
		Addition:       v6.Addition,
	}).Error
	if err != nil {
		t.Fatalf("write metadata: %v", err)
	}
	sqlDB, err := gdb.DB()
	if err != nil {
		t.Fatalf("unwrap db: %v", err)
	}
	if err := sqlDB.Close(); err != nil {
		t.Fatalf("close db: %v", err)
	}
	return dir
}

// TestGrype_ExpiredOnDiskDBIsAHardError is the production error message,
// reproduced with no network.
//
// grypeDBConfig sets ValidateAge with a 14-day MaxAllowedBuiltAge. Once the
// on-disk database crosses that line, curator.Status() fails, and
// grype.LoadVulnerabilityDB returns that failure instead of a provider — after
// the update attempt, not before it. So an expired database plus an upstream
// that cannot be reached is a hard load failure, not a degraded scan.
//
// The call here uses update=false purely to keep the test off the network.
// Production calls with update=true, and on the path that matters (the update
// fails, or finds nothing to install) reaches the identical Status() check.
func TestGrype_ExpiredOnDiskDBIsAHardError(t *testing.T) {
	root := t.TempDir()
	writeSyntheticDB(t, root, time.Now().Add(-20*24*time.Hour))

	distCfg, instCfg := grypeDBConfig(root)
	provider, _, err := grype.LoadVulnerabilityDB(distCfg, instCfg, false)
	if err == nil {
		if provider != nil {
			provider.Close()
		}
		t.Fatal("a database built 20 days ago loaded successfully; MaxAllowedBuiltAge is not being enforced")
	}
	if provider != nil {
		t.Error("a provider was returned alongside the error")
	}

	// This is Grype's own wording, from curator.validateAge
	// (grype/db/v6/installation/curator.go:613). It is what reaches the hold,
	// wrapped twice by grype.go, and what users report as "grype is out of
	// date, please update the database".
	if !strings.Contains(err.Error(), "the vulnerability database was built") ||
		!strings.Contains(err.Error(), "max allowed age") {
		t.Errorf("unexpected error text %q; the age check may have moved", err)
	}
	t.Logf("grype: %v", err)
}

// TestGrype_FreshDBWithoutImportMetadataIsAlsoAHardError is a second, quieter
// way into the same symptom, and unlike the age cliff it does not self-heal.
//
// grypeDBConfig also sets ValidateChecksum. Status() joins the checksum failure
// into the same error, so a database with a missing or corrupt import.json
// fails to load even when its build timestamp is minutes old.
//
// The asymmetry is what makes this dangerous. curator.Update() nils out the
// current description when validateAge fails, and isSupersededBy(nil, ...) is
// unconditionally true, so an expired database is always re-downloaded. A
// checksum failure leaves the description in place, so a download only happens
// if the upstream has something strictly newer. A database that is current but
// whose import metadata was lost (an interrupted activate, a truncated write, a
// restored volume snapshot) therefore fails every scan and no update fixes it.
func TestGrype_FreshDBWithoutImportMetadataIsAlsoAHardError(t *testing.T) {
	root := t.TempDir()
	writeSyntheticDB(t, root, time.Now().Add(-1*time.Hour))

	distCfg, instCfg := grypeDBConfig(root)
	provider, _, err := grype.LoadVulnerabilityDB(distCfg, instCfg, false)
	if err == nil {
		if provider != nil {
			provider.Close()
		}
		t.Fatal("a database with no import metadata loaded successfully; ValidateChecksum is not being enforced")
	}
	if !strings.Contains(err.Error(), "import metadata") {
		t.Errorf("unexpected error text %q; expected the checksum/import-metadata failure", err)
	}
	t.Logf("grype: %v", err)
}

// TestGrype_InMemoryProviderDoesNotRevalidateAge checks the claim grype.go
// leans on at line 210: "the in-memory provider doesn't re-validate build age
// on queries, so a stale-but-loaded DB still scans fine".
//
// The claim holds, and this is the mechanism the serve-the-old-DB fallback
// depends on. v6.NewVulnerabilityProvider closes over a Reader and an
// architecture-alias map and nothing else: no Config, no MaxAllowedBuiltAge, no
// build timestamp. Age is checked by the curator when opening the file on disk,
// never by the provider that results. Which is also why the fallback has no
// ceiling: there is nothing in the object that could impose one.
//
// The provider is built the way LoadVulnerabilityDB builds it, but without the
// curator, so no age or checksum gate runs at all.
func TestGrype_InMemoryProviderDoesNotRevalidateAge(t *testing.T) {
	root := t.TempDir()
	// A year old: far past MaxAllowedBuiltAge, which the curator would refuse
	// and the provider cannot see.
	built := time.Now().Add(-365 * 24 * time.Hour)
	dir := writeSyntheticDB(t, root, built)

	rdr, err := v6.NewReader(v6.Config{DBDirPath: dir})
	if err != nil {
		t.Fatalf("open reader: %v", err)
	}
	defer rdr.Close()

	meta, err := rdr.GetDBMetadata()
	if err != nil {
		t.Fatalf("read metadata: %v", err)
	}
	if age := time.Since(*meta.BuildTimestamp); age < 300*24*time.Hour {
		t.Fatalf("test setup drifted: build age is %s", age)
	}

	provider := v6.NewVulnerabilityProvider(rdr)

	// A query against a year-old database. It answers, rather than reporting
	// that the database is out of date, and would answer identically at any
	// age. That is the whole content of the comment at grype.go:210.
	vulns, err := provider.FindVulnerabilities()
	if err != nil {
		t.Fatalf("query: %v", err)
	}
	if len(vulns) != 0 {
		t.Fatalf("synthetic database returned %d vulnerabilities", len(vulns))
	}
}

// --- the symptom, through the real worker pool ---

// TestWorkerPool_EveryScanFailsWhileTheDBIsUnloadable drives the whole scan
// pipeline against a mock hold with the database loader failing, which is what
// production looks like during the outage. It is here rather than in
// internal/e2e because loadVulnDB is unexported and the harness cannot stub it.
//
// Two things are asserted that the unit tests cannot: the failure reaches the
// hold as an "error" message (which the hold's stale-scan loop retries, unlike
// a "skipped"), and the cold-start backoff holds across jobs arriving from the
// hold — so the hold's retry loop no longer drives one full download attempt
// per queued scan for as long as the upstream is down.
func TestWorkerPool_EveryScanFailsWhileTheDBIsUnloadable(t *testing.T) {
	fixture := filepath.Join("..", "mockhold", "testdata", "blobs", "hsm-secrets-operator")
	if _, err := os.Stat(filepath.Join(fixture, "oci-layout")); err != nil {
		t.Skipf("fixture not present; run scanner/internal/mockhold/testdata/fetch-blobs.sh")
	}

	resetVulnDBState(t)
	_, calls := stubLoader(t, time.Time{}, errors.New("upstream unreachable"))

	const secret = "test-scanner-secret"
	hold := mockhold.New(mockhold.NewOCILayout(fixture), mockhold.WithSecret(secret))
	t.Cleanup(hold.Close)

	cfg := config.DefaultConfig()
	cfg.Hold.URL = hold.URL()
	cfg.Hold.Secret = secret
	cfg.Scanner.Workers = 1
	cfg.Vuln.Enabled = true // the point of the test
	cfg.Vuln.DBPath = t.TempDir()
	cfg.Vuln.TmpDir = t.TempDir()

	// WorkerPool.Start exports TMPDIR process-wide and never restores it.
	origTmpDir, hadTmpDir := os.LookupEnv("TMPDIR")
	t.Cleanup(func() {
		if hadTmpDir {
			os.Setenv("TMPDIR", origTmpDir)
			return
		}
		os.Unsetenv("TMPDIR")
	})

	restoreCooldown := JobCooldown
	JobCooldown = 10 * time.Millisecond

	q := queue.NewJobQueue(cfg.Scanner.QueueSize)
	c := client.NewHoldClient(cfg.Hold.URL, cfg.Hold.Secret, q)
	pool := NewWorkerPool(cfg, q, c)

	ctx, cancel := context.WithCancel(context.Background())
	pool.Start(ctx)
	go c.Connect()
	t.Cleanup(func() {
		cancel()
		c.Close()
		q.Close()
		// Wait for the workers before restoring JobCooldown. The worker loop
		// reads it on every iteration, so restoring it while a worker is still
		// running is a genuine data race that -race reports. (The e2e harness
		// restores it without waiting, and has the same latent race.)
		pool.Wait()
		JobCooldown = restoreCooldown
	})

	if err := hold.WaitForScanner(10 * time.Second); err != nil {
		t.Fatalf("scanner never connected: %v", err)
	}

	const target = "sha256:1cfa4e2b09e127b9c4ed43578d3f3c18e7d44ea47b9ea98475c0cbe9086525f8"
	all, err := mockhold.Corpus()
	if err != nil {
		t.Fatalf("load corpus: %v", err)
	}
	var manifest mockhold.Manifest
	for _, m := range all {
		if m.Digest == target {
			manifest = m
			break
		}
	}
	if manifest.Digest == "" {
		t.Fatalf("digest %s not in corpus", target)
	}

	// The startup goroutine in WorkerPool.Start also calls the loader once.
	// Take a baseline after connecting so the per-job count is unambiguous.
	baseline := *calls

	const jobs = 2
	for i := 0; i < jobs; i++ {
		seq, err := hold.SendJob(manifest.Job())
		if err != nil {
			t.Fatalf("send job %d: %v", i, err)
		}
		msg, err := hold.WaitForMessage(func(m mockhold.Message) bool {
			return m.Seq == seq && (m.Type == "result" || m.Type == "error" || m.Type == "skipped")
		}, 3*time.Minute)
		if err != nil {
			t.Fatalf("no terminal message for seq %d: %v", seq, err)
		}
		if msg.Type != "error" {
			t.Fatalf("job %d: want error while the vulnerability DB is unloadable, got %s", i, msg.Type)
		}
		if !strings.Contains(msg.Error, "failed to load vulnerability database") {
			t.Errorf("job %d: error %q does not name the database failure", i, msg.Error)
		}
		t.Logf("job %d: %q", i, msg.Error)
	}

	// The jobs still fail — there is genuinely nothing to scan against — but
	// they fail cheaply. Both arrive inside vulnDBColdRetryBackoff, so at most
	// one of them reaches the loader.
	if extra := *calls - baseline; extra > 1 {
		t.Errorf("loader ran %d times across %d jobs, want at most 1: the cold-start "+
			"backoff must throttle the hold's retry loop", extra, jobs)
	}
}