Files
at-container-registry/scanner/internal/scan/deadline_test.go
T
Evan JarrettandClaude Opus 5 22058cc5f4 scanner: bound a scan job, and lose the race to the hold on purpose
Nothing limited how long one job could take. The worker's context was the
process's, buildOCILayout took none, and blob downloads used a package-level
client whose five-minute timeout is per request with no context, so a 19-layer
image had a hundred-minute worst case on downloads alone and cancellation could
not touch it. At the default single worker, one wedged job stopped that scanner
entirely.

scanner.job_timeout, default 8m, against the hold's 10m scanning timeout. Both
clocks start at the same instant: the worker sends "started" on dequeue and
derives the job context on the next line, so the scanner loses by two minutes,
which is enough for its terminal message to cross the socket and be recorded.
If the hold wins instead it re-dispatches while this scanner is still working,
which is duplicate work recorded under a generic reason. A scanner cannot read
the hold's config, so the relation is a mirrored constant used only for a
boot-time warning, and the same warning fires if the deadline is disabled.

What is actually bounded, since a deadline the code cannot honour is worse than
none: presign, download, stereoscope's Provide, Syft's CreateSBOM, and Grype,
which does have FindMatchesContext even though FindMatches does not.
stereoscope's img.Read takes no context and is 81% of a scan, so it is checked
either side rather than interrupted. Abandoning it on a goroutine would trade a
bounded overrun for one writing gigabytes into a directory the caller has
already deleted. max_image_size remains the real bound on that stage.

A timeout reports error, not skipped. It describes this host at this moment, a
contended CPU or a slow bucket, not the image, and skips are never retried, so
one bad afternoon would retire an image permanently with nothing in the record
to say why. Retry cost is bounded on the other side by max_image_size and by
the stale-scan schedule. The classification asks the job context rather than
the error, because several stages replace the cause and the uninterruptible one
knows nothing about the deadline, and a job that finishes after an overrun
still reports its real result rather than throwing away completed work.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
2026-09-05 16:16:16 -05:00

151 lines
5.0 KiB
Go

package scan
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
scanner "atcr.io/scanner"
"atcr.io/scanner/internal/config"
)
// These are the unit-level assertions behind the per-job deadline: how the
// worker derives a job's context, how the shipped default relates to the
// hold's own budget, and that the scan directory is removed when the deadline
// fires mid-download. The end-to-end shape lives in internal/e2e/deadline_test.go.
func poolWithJobTimeout(d time.Duration) *WorkerPool {
cfg := config.DefaultConfig()
cfg.Scanner.JobTimeout = d
return &WorkerPool{cfg: cfg}
}
// TestJobContextCarriesTheConfiguredDeadline pins the knob to the context the
// job actually runs under. Everything downstream (the HTTP requests, Syft's
// cataloging, Grype's matcher) is bounded by this context and by nothing else.
func TestJobContextCarriesTheConfiguredDeadline(t *testing.T) {
wp := poolWithJobTimeout(5 * time.Second)
ctx, cancel := wp.jobContext(context.Background())
defer cancel()
deadline, ok := ctx.Deadline()
if !ok {
t.Fatal("job context carries no deadline, so nothing downstream can be bounded")
}
if left := time.Until(deadline); left <= 0 || left > 5*time.Second {
t.Errorf("job deadline is %s away, want (0, 5s]", left)
}
}
// TestZeroJobTimeoutDisablesTheDeadline keeps the escape hatch honest. An
// operator scanning very large images on a hold with a longer budget can turn
// the deadline off, and must still get a context that shutdown can cancel.
func TestZeroJobTimeoutDisablesTheDeadline(t *testing.T) {
wp := poolWithJobTimeout(0)
parent, cancelParent := context.WithCancel(context.Background())
ctx, cancel := wp.jobContext(parent)
defer cancel()
if _, ok := ctx.Deadline(); ok {
t.Error("job_timeout=0 still produced a deadline")
}
cancelParent()
select {
case <-ctx.Done():
case <-time.After(time.Second):
t.Error("job context does not follow the pool context, so shutdown cannot interrupt a job")
}
}
// TestDefaultJobTimeoutFiresBeforeTheHoldGivesUp is the ordering constraint,
// asserted against the constant the scanner keeps for the hold's budget.
//
// If the hold's deadline lands first it marks the row failed and re-dispatches
// it while this worker is still scanning: two scanners on one image, and the
// recorded reason is the hold's generic timeout rather than anything the
// scanner observed. The scanner has to lose the race on purpose.
func TestDefaultJobTimeoutFiresBeforeTheHoldGivesUp(t *testing.T) {
got := config.DefaultConfig().Scanner.JobTimeout
if got <= 0 {
t.Fatal("the shipped default disables the deadline")
}
if got >= holdScanningTimeout {
t.Fatalf("default job_timeout %s is not below the hold's %s scanning budget", got, holdScanningTimeout)
}
if margin := holdScanningTimeout - got; margin < time.Minute {
t.Errorf("only %s of margin for the terminal message to reach the hold", margin)
}
}
// TestBuildOCILayoutCleansUpWhenTheDeadlineFires is the temp-directory
// assertion on the timeout path specifically.
//
// buildOCILayout creates scan-* before its first download and removes it on
// every failure it returns. A deadline that fires mid-transfer is a new way to
// leave that function, and the 3.8x extraction amplification measured for a
// node:22 image is what a leak costs: a scanner that times out repeatedly
// fills its volume.
func TestBuildOCILayoutCleansUpWhenTheDeadlineFires(t *testing.T) {
stalled := make(chan struct{})
defer close(stalled)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-stalled:
}
}))
defer srv.Close()
tmpDir := t.TempDir()
job := &scanner.ScanJob{
ManifestDigest: "sha256:" + strings.Repeat("a", 64),
Repository: "deadline/cleanup",
HoldDID: "did:web:hold.example",
HoldEndpoint: srv.URL,
Config: scanner.BlobDescriptor{
Digest: "sha256:" + strings.Repeat("c", 64),
Size: 100,
MediaType: "application/vnd.oci.image.config.v1+json",
},
}
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()
started := time.Now()
_, _, err := buildOCILayout(ctx, job, tmpDir, "", 0)
if err == nil {
t.Fatal("buildOCILayout succeeded against a server that never answers")
}
if elapsed := time.Since(started); elapsed > 5*time.Second {
t.Errorf("buildOCILayout took %s to notice a 200ms deadline", elapsed)
}
if !errors.Is(err, context.DeadlineExceeded) {
t.Errorf("error does not carry the deadline: %v", err)
}
// A timeout must not be reported as a skip. The hold never re-offers a
// skipped record, so a slow afternoon would retire the image for good.
var skip *SkipError
if errors.As(err, &skip) {
t.Errorf("a timed-out download was classified as a permanent skip: %v", err)
}
entries, err := os.ReadDir(tmpDir)
if err != nil {
t.Fatalf("read tmp dir: %v", err)
}
for _, e := range entries {
t.Errorf("scan directory left behind after the deadline fired: %s", e.Name())
}
}