mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-25 19:54:15 +00:00
Nothing limited how long one job could take. The worker's context was the process's, buildOCILayout took none, and blob downloads used a package-level client whose five-minute timeout is per request with no context, so a 19-layer image had a hundred-minute worst case on downloads alone and cancellation could not touch it. At the default single worker, one wedged job stopped that scanner entirely. scanner.job_timeout, default 8m, against the hold's 10m scanning timeout. Both clocks start at the same instant: the worker sends "started" on dequeue and derives the job context on the next line, so the scanner loses by two minutes, which is enough for its terminal message to cross the socket and be recorded. If the hold wins instead it re-dispatches while this scanner is still working, which is duplicate work recorded under a generic reason. A scanner cannot read the hold's config, so the relation is a mirrored constant used only for a boot-time warning, and the same warning fires if the deadline is disabled. What is actually bounded, since a deadline the code cannot honour is worse than none: presign, download, stereoscope's Provide, Syft's CreateSBOM, and Grype, which does have FindMatchesContext even though FindMatches does not. stereoscope's img.Read takes no context and is 81% of a scan, so it is checked either side rather than interrupted. Abandoning it on a goroutine would trade a bounded overrun for one writing gigabytes into a directory the caller has already deleted. max_image_size remains the real bound on that stage. A timeout reports error, not skipped. It describes this host at this moment, a contended CPU or a slow bucket, not the image, and skips are never retried, so one bad afternoon would retire an image permanently with nothing in the record to say why. Retry cost is bounded on the other side by max_image_size and by the stale-scan schedule. The classification asks the job context rather than the error, because several stages replace the cause and the uninterruptible one knows nothing about the deadline, and a job that finishes after an overrun still reports its real result rather than throwing away completed work. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01U1Km3N3uUmeGaj7VbaM8PF
151 lines
5.0 KiB
Go
151 lines
5.0 KiB
Go
package scan
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
scanner "atcr.io/scanner"
|
|
"atcr.io/scanner/internal/config"
|
|
)
|
|
|
|
// These are the unit-level assertions behind the per-job deadline: how the
|
|
// worker derives a job's context, how the shipped default relates to the
|
|
// hold's own budget, and that the scan directory is removed when the deadline
|
|
// fires mid-download. The end-to-end shape lives in internal/e2e/deadline_test.go.
|
|
|
|
func poolWithJobTimeout(d time.Duration) *WorkerPool {
|
|
cfg := config.DefaultConfig()
|
|
cfg.Scanner.JobTimeout = d
|
|
return &WorkerPool{cfg: cfg}
|
|
}
|
|
|
|
// TestJobContextCarriesTheConfiguredDeadline pins the knob to the context the
|
|
// job actually runs under. Everything downstream (the HTTP requests, Syft's
|
|
// cataloging, Grype's matcher) is bounded by this context and by nothing else.
|
|
func TestJobContextCarriesTheConfiguredDeadline(t *testing.T) {
|
|
wp := poolWithJobTimeout(5 * time.Second)
|
|
|
|
ctx, cancel := wp.jobContext(context.Background())
|
|
defer cancel()
|
|
|
|
deadline, ok := ctx.Deadline()
|
|
if !ok {
|
|
t.Fatal("job context carries no deadline, so nothing downstream can be bounded")
|
|
}
|
|
if left := time.Until(deadline); left <= 0 || left > 5*time.Second {
|
|
t.Errorf("job deadline is %s away, want (0, 5s]", left)
|
|
}
|
|
}
|
|
|
|
// TestZeroJobTimeoutDisablesTheDeadline keeps the escape hatch honest. An
|
|
// operator scanning very large images on a hold with a longer budget can turn
|
|
// the deadline off, and must still get a context that shutdown can cancel.
|
|
func TestZeroJobTimeoutDisablesTheDeadline(t *testing.T) {
|
|
wp := poolWithJobTimeout(0)
|
|
|
|
parent, cancelParent := context.WithCancel(context.Background())
|
|
ctx, cancel := wp.jobContext(parent)
|
|
defer cancel()
|
|
|
|
if _, ok := ctx.Deadline(); ok {
|
|
t.Error("job_timeout=0 still produced a deadline")
|
|
}
|
|
|
|
cancelParent()
|
|
select {
|
|
case <-ctx.Done():
|
|
case <-time.After(time.Second):
|
|
t.Error("job context does not follow the pool context, so shutdown cannot interrupt a job")
|
|
}
|
|
}
|
|
|
|
// TestDefaultJobTimeoutFiresBeforeTheHoldGivesUp is the ordering constraint,
|
|
// asserted against the constant the scanner keeps for the hold's budget.
|
|
//
|
|
// If the hold's deadline lands first it marks the row failed and re-dispatches
|
|
// it while this worker is still scanning: two scanners on one image, and the
|
|
// recorded reason is the hold's generic timeout rather than anything the
|
|
// scanner observed. The scanner has to lose the race on purpose.
|
|
func TestDefaultJobTimeoutFiresBeforeTheHoldGivesUp(t *testing.T) {
|
|
got := config.DefaultConfig().Scanner.JobTimeout
|
|
if got <= 0 {
|
|
t.Fatal("the shipped default disables the deadline")
|
|
}
|
|
if got >= holdScanningTimeout {
|
|
t.Fatalf("default job_timeout %s is not below the hold's %s scanning budget", got, holdScanningTimeout)
|
|
}
|
|
if margin := holdScanningTimeout - got; margin < time.Minute {
|
|
t.Errorf("only %s of margin for the terminal message to reach the hold", margin)
|
|
}
|
|
}
|
|
|
|
// TestBuildOCILayoutCleansUpWhenTheDeadlineFires is the temp-directory
|
|
// assertion on the timeout path specifically.
|
|
//
|
|
// buildOCILayout creates scan-* before its first download and removes it on
|
|
// every failure it returns. A deadline that fires mid-transfer is a new way to
|
|
// leave that function, and the 3.8x extraction amplification measured for a
|
|
// node:22 image is what a leak costs: a scanner that times out repeatedly
|
|
// fills its volume.
|
|
func TestBuildOCILayoutCleansUpWhenTheDeadlineFires(t *testing.T) {
|
|
stalled := make(chan struct{})
|
|
defer close(stalled)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
select {
|
|
case <-r.Context().Done():
|
|
case <-stalled:
|
|
}
|
|
}))
|
|
defer srv.Close()
|
|
|
|
tmpDir := t.TempDir()
|
|
job := &scanner.ScanJob{
|
|
ManifestDigest: "sha256:" + strings.Repeat("a", 64),
|
|
Repository: "deadline/cleanup",
|
|
HoldDID: "did:web:hold.example",
|
|
HoldEndpoint: srv.URL,
|
|
Config: scanner.BlobDescriptor{
|
|
Digest: "sha256:" + strings.Repeat("c", 64),
|
|
Size: 100,
|
|
MediaType: "application/vnd.oci.image.config.v1+json",
|
|
},
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
|
defer cancel()
|
|
|
|
started := time.Now()
|
|
_, _, err := buildOCILayout(ctx, job, tmpDir, "", 0)
|
|
if err == nil {
|
|
t.Fatal("buildOCILayout succeeded against a server that never answers")
|
|
}
|
|
if elapsed := time.Since(started); elapsed > 5*time.Second {
|
|
t.Errorf("buildOCILayout took %s to notice a 200ms deadline", elapsed)
|
|
}
|
|
if !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Errorf("error does not carry the deadline: %v", err)
|
|
}
|
|
|
|
// A timeout must not be reported as a skip. The hold never re-offers a
|
|
// skipped record, so a slow afternoon would retire the image for good.
|
|
var skip *SkipError
|
|
if errors.As(err, &skip) {
|
|
t.Errorf("a timed-out download was classified as a permanent skip: %v", err)
|
|
}
|
|
|
|
entries, err := os.ReadDir(tmpDir)
|
|
if err != nil {
|
|
t.Fatalf("read tmp dir: %v", err)
|
|
}
|
|
for _, e := range entries {
|
|
t.Errorf("scan directory left behind after the deadline fired: %s", e.Name())
|
|
}
|
|
}
|