package e2e import ( "encoding/json" "net/http" "net/http/httptest" "strings" "testing" "time" scanner "atcr.io/scanner" "atcr.io/scanner/internal/client" "atcr.io/scanner/internal/config" "atcr.io/scanner/internal/mockhold" "atcr.io/scanner/internal/queue" ) // --- helpers --------------------------------------------------------------- // silenceWindow is how long a test waits before concluding the scanner sent // nothing back. It is short on purpose: the failure mode being pinned is // "nothing, ever", and the hold's own ackTimeout is five minutes, so anything // the scanner has not said within a second here it will not say at all. const silenceWindow = 1 * time.Second // synthJob builds a descriptor-only container image job. The digests point at // nothing, which is fine for every scenario that either refuses the job before // downloading or deliberately stalls the download. func synthJob(repo string) *scanner.ScanJob { return &scanner.ScanJob{ ManifestDigest: "sha256:" + strings.Repeat("a", 64), Repository: repo, Tag: "latest", Tier: "deckhand", HoldDID: "did:web:hold.example", Config: scanner.BlobDescriptor{ Digest: "sha256:" + strings.Repeat("c", 64), Size: 100, MediaType: "application/vnd.oci.image.config.v1+json", }, Layers: []scanner.BlobDescriptor{{ Digest: "sha256:" + strings.Repeat("1", 64), Size: 200, MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", }}, } } // helmJob returns a corpus Helm chart, the cheapest job that reaches a // terminal message: the scanner refuses it on config media type alone, before // any tmp dir or download work, so it is a reliable liveness probe. func helmJob(t *testing.T) *scanner.ScanJob { t.Helper() charts, err := mockhold.CorpusByShape(mockhold.ShapeHelm) if err != nil { t.Fatalf("load corpus: %v", err) } if len(charts) == 0 { t.Fatal("corpus contains no helm manifests") } return charts[0].Job() } // rawJob renders a job frame from a map, so a test can omit a field, or give // it a shape the scanner cannot decode. func rawJob(t *testing.T, fields map[string]any) []byte { t.Helper() data, err := json.Marshal(fields) if err != nil { t.Fatalf("marshal raw job: %v", err) } return data } // expectSilence fails if any message matching match arrives within the window. func expectSilence(t *testing.T, h *Harness, match func(mockhold.Message) bool, window time.Duration, what string) { t.Helper() msg, err := h.Hold.WaitForMessage(match, window) if err == nil { t.Fatalf("%s: expected no reply, got %s for seq %d", what, msg.Type, msg.Seq) } } // forSeq matches any message about one job. func forSeq(seq int64) func(mockhold.Message) bool { return func(m mockhold.Message) bool { return m.Seq == seq } } // assertAlive proves the WebSocket survived whatever the previous step did to // it, by pushing a job the scanner is guaranteed to answer. func assertAlive(t *testing.T, h *Harness) { t.Helper() seq, err := h.Hold.SendJob(helmJob(t)) if err != nil { t.Fatalf("connection did not survive: %v", err) } if msg := h.AwaitTerminal(t, seq, 30*time.Second); msg.Type != "skipped" { t.Fatalf("liveness probe: want skipped, got %s", msg.Type) } } // --- 1. frames the scanner cannot parse ------------------------------------- // TestUnparseableFramesAreAnsweredWithSkipped is the central protocol finding, // and the shape of a nine-day outage. // // connectOnce decodes three things — the frame, then the config sub-document, // then the layers sub-document — and every failure branch used to be // slog.Error followed by continue, sending nothing back at all. The ack was // sent only after both sub-document unmarshals, so a job whose config or // layers did not decode was never even acknowledged. // // The hold has already written status='assigned' for that seq before it wrote // the frame. Its only escape was the five-minute ackTimeout in // reDispatchTimedOut, after which the row was re-offered — to the same // scanner, which dropped it again for exactly the same reason, because a // decoding disagreement is permanent. And because hasActiveJobs counts // 'assigned' rows and dispatchLoop admits one proactive candidate at a time // behind waitForCapacity, one such row meant no proactive scan was ever // dispatched again, deployment-wide. // // The scanner now answers "skipped" for any frame that carries a usable seq. // Skipped is the correct verdict rather than "error": the hold retries // failures on the rescan interval, and no retry of an undecodable frame can // ever succeed, whereas handleSkipped writes a terminal record and releases // the row and the in-flight digest for good. The hold-side half is // TestScanSkipped_RetiresAnUndecodableFrame in // pkg/hold/pds/scan_broadcaster_stuck_test.go. func TestUnparseableFramesAreAnsweredWithSkipped(t *testing.T) { h := Start(t, mockhold.NewMemory()) base := func(seq int64) map[string]any { return map[string]any{ "type": "job", "seq": seq, "manifestDigest": "sha256:" + strings.Repeat("a", 64), "repository": "probe/unparseable", "tag": "latest", "userDid": "did:plc:probe", "holdDid": "did:web:hold.example", "holdEndpoint": h.Hold.URL(), "tier": "deckhand", "config": map[string]any{"digest": "sha256:" + strings.Repeat("c", 64), "size": 1, "mediaType": "application/vnd.oci.image.config.v1+json"}, "layers": []any{}, } } cases := []struct { name string frame func(seq int64) []byte }{ {"config is a string", func(seq int64) []byte { f := base(seq) f["config"] = "not-an-object" return rawJob(t, f) }}, {"config field absent", func(seq int64) []byte { f := base(seq) delete(f, "config") return rawJob(t, f) }}, {"layers is an object", func(seq int64) []byte { f := base(seq) f["layers"] = map[string]any{"oops": 1} return rawJob(t, f) }}, {"layers field absent", func(seq int64) []byte { f := base(seq) delete(f, "layers") return rawJob(t, f) }}, // encoding/json records the first type error and keeps decoding the // rest of the object, so a frame the top-level unmarshal rejects can // still yield the seq that addresses the hold's row. That row must be // answered too, which is why the reply is keyed on the seq rather than // on which of the three decodes failed. {"frame has a type-mismatched field", func(seq int64) []byte { f := base(seq) f["tier"] = 12345 return rawJob(t, f) }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { seq := h.Hold.NextSeq() if err := h.Hold.SendRaw(tc.frame(seq)); err != nil { t.Fatalf("send raw frame: %v", err) } msg, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool { return m.Seq == seq && m.Type != "ack" }, 10*time.Second) if err != nil { t.Fatalf("%s: no reply for seq %d; the hold's row stays "+ "assigned and is re-offered to this same scanner forever. "+ "transcript: %s", tc.name, seq, describe(h.Hold.Transcript())) } if msg.Type != "skipped" { t.Fatalf("%s: reply for seq %d was %s (%s%s), want skipped: an "+ "undecodable frame is a permanent condition and the hold "+ "retries anything it records as a failure", tc.name, seq, msg.Type, msg.Error, msg.Reason) } if msg.Reason == "" { t.Errorf("%s: skipped with no reason; the hold stores it on the "+ "scan record and it is all a user ever sees", tc.name) } }) } // The connection survives all of it, which is what made the retry loop // infinite rather than self-limiting. assertAlive(t, h) } // TestFramesWithNoUsableSeqAreDroppedSilently is the deliberate exception. // // A frame that does not decode far enough to yield a seq addresses no job: // there is no row to retire and no seq to put in a reply, so logging is the // only thing left. This is safe in a way the config/layers case never was — // the hold writes status='assigned' keyed by seq before it sends, so a frame // whose seq never made it onto the wire cannot be the frame that stranded a // row. The connection must survive, since the hold will keep using it. func TestFramesWithNoUsableSeqAreDroppedSilently(t *testing.T) { h := Start(t, mockhold.NewMemory()) for _, tc := range []struct { name string frame []byte }{ {"frame is not JSON", []byte("{ this is not json")}, {"frame is a JSON array", []byte(`[1,2,3]`)}, {"frame carries seq 0", rawJob(t, map[string]any{"type": "job", "seq": 0, "config": "nope"})}, } { t.Run(tc.name, func(t *testing.T) { before := len(h.Hold.Transcript()) if err := h.Hold.SendRaw(tc.frame); err != nil { t.Fatalf("send raw frame: %v", err) } expectSilence(t, h, func(m mockhold.Message) bool { return m.Seq == 0 }, silenceWindow, tc.name) if got := len(h.Hold.Transcript()); got != before { t.Errorf("%s: scanner sent %d messages for an unaddressable frame", tc.name, got-before) } }) } assertAlive(t, h) } // TestNullConfigIsAckedThenFailsRetryably covers the shape that *does* decode: // a JSON null unmarshals into a zero BlobDescriptor without error, so the job // is acked and enters the pipeline, then dies in buildOCILayout on the empty // config digest. That is reported as "error", which the hold treats as // transient and retries on the rescan interval forever, even though no retry // can ever succeed: nothing about a null config will change. func TestNullConfigIsAckedThenFailsRetryably(t *testing.T) { h := Start(t, mockhold.NewMemory()) seq := h.Hold.NextSeq() frame := rawJob(t, map[string]any{ "type": "job", "seq": seq, "manifestDigest": "sha256:" + strings.Repeat("b", 64), "repository": "probe/null-config", "tag": "latest", "userDid": "did:plc:probe", "holdDid": "did:web:hold.example", "holdEndpoint": h.Hold.URL(), "tier": "deckhand", "config": nil, "layers": []any{map[string]any{ "digest": "sha256:" + strings.Repeat("1", 64), "size": 10, "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", }}, }) if err := h.Hold.SendRaw(frame); err != nil { t.Fatalf("send raw frame: %v", err) } if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool { return m.Seq == seq && m.Type == "ack" }, 10*time.Second); err != nil { t.Fatalf("null config was not acked: %v", err) } msg := h.AwaitTerminal(t, seq, 30*time.Second) if msg.Type != "error" { t.Fatalf("want the current retryable-error behaviour, got %s (%s%s)", msg.Type, msg.Error, msg.Reason) } if !strings.Contains(msg.Error, "empty digest") { t.Errorf("unexpected error text %q", msg.Error) } t.Logf("null config is a permanent condition reported as a retryable error: %q", msg.Error) } // TestUnknownMessageTypeIsIgnored pins the benign half: a frame the scanner // does not recognise is logged and skipped, the connection survives, and // nothing is sent back. Harmless today because the hold only ever sends // "job", but it means any future message type is silently swallowed by an // older scanner rather than refused. func TestUnknownMessageTypeIsIgnored(t *testing.T) { h := Start(t, mockhold.NewMemory()) seq := h.Hold.NextSeq() if err := h.Hold.SendRaw(rawJob(t, map[string]any{"type": "cancel", "seq": seq})); err != nil { t.Fatalf("send raw frame: %v", err) } expectSilence(t, h, forSeq(seq), silenceWindow, "unknown message type") assertAlive(t, h) } // --- 2. duplicate delivery -------------------------------------------------- // TestDuplicateSeqIsProcessedTwice shows the scanner has no idea it has seen a // job before. Nothing dedupes on seq or on manifest digest: the job is acked // twice, queued twice, and scanned twice. // // On the hold side the second ack is a no-op (handleAck's UPDATE is guarded by // status='assigned', which the first ack already cleared) but the second // terminal message is not: handleSkipped/handleResult/handleError re-run // unconditionally, writing a second scan record to the PDS for the same // manifest. For a real image this is also a full second download and Syft run. func TestDuplicateSeqIsProcessedTwice(t *testing.T) { h := Start(t, mockhold.NewMemory()) job := helmJob(t) job.Seq = h.Hold.NextSeq() for i := 0; i < 2; i++ { dup := *job if _, err := h.Hold.SendJob(&dup); err != nil { t.Fatalf("send job %d: %v", i, err) } } deadline := time.Now().Add(30 * time.Second) var acks, terminals int for time.Now().Before(deadline) { acks, terminals = 0, 0 for _, m := range h.Hold.Transcript() { if m.Seq != job.Seq { continue } switch m.Type { case "ack": acks++ case "result", "error", "skipped": terminals++ } } if acks >= 2 && terminals >= 2 { break } time.Sleep(20 * time.Millisecond) } if acks != 2 || terminals != 2 { t.Fatalf("duplicate seq: got %d acks and %d terminal messages, want 2 and 2", acks, terminals) } t.Logf("seq %d was acked %d times and answered %d times: no dedup anywhere in the scanner", job.Seq, acks, terminals) } // --- 3. queue capacity ------------------------------------------------------ // TestQueueFullIsReportedAsRetryableError pins what happens past the queue's // high-water mark: the job is acked (hold: assigned -> processing) and then // immediately answered with "error: scanner queue full", which the hold // records as a *failure*. Failures are retryable, so the same job comes back // on the rescan interval and will overflow again for as long as the backlog // persists. A capacity signal is being reported through the channel reserved // for scan outcomes, and it lands in the user's scan history as a failed scan. func TestQueueFullIsReportedAsRetryableError(t *testing.T) { const queueSize = 2 h := Start(t, mockhold.NewMemory(), WithWorkers(0), // nothing drains the queue, so the Nth job is deterministic func(c *config.Config) { c.Scanner.QueueSize = queueSize }) var seqs []int64 for i := 0; i < queueSize+2; i++ { seq, err := h.Hold.SendJob(synthJob("probe/overflow")) if err != nil { t.Fatalf("send job %d: %v", i, err) } seqs = append(seqs, seq) } for _, seq := range seqs[queueSize:] { msg := h.AwaitTerminal(t, seq, 10*time.Second) if msg.Type != "error" || !strings.Contains(msg.Error, "queue full") { t.Fatalf("seq %d: want a queue-full error, got %s (%s%s)", seq, msg.Type, msg.Error, msg.Reason) } } // And the overflowed jobs were acked first, so the hold saw them go // assigned -> processing -> failed for a condition that never involved the // job at all. for _, seq := range seqs[queueSize:] { if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool { return m.Seq == seq && m.Type == "ack" }, time.Second); err != nil { t.Errorf("seq %d was rejected without ever being acked", seq) } } t.Log("queue overflow is reported as a retryable per-job failure, not as backpressure") } // TestZeroQueueSizeRejectsEveryJob is the configuration corner of the same // path. scanner.queue_size = 0 passes validation, and NewJobQueue(0) then // refuses every Enqueue, so a scanner that looks healthy (connected, health // endpoint green, workers idle) fails 100% of jobs with "scanner queue full" // and the hold retries all of them forever. func TestZeroQueueSizeRejectsEveryJob(t *testing.T) { h := Start(t, mockhold.NewMemory(), func(c *config.Config) { c.Scanner.QueueSize = 0 }) seq, err := h.Hold.SendJob(helmJob(t)) if err != nil { t.Fatalf("send job: %v", err) } msg := h.AwaitTerminal(t, seq, 10*time.Second) if msg.Type != "error" || !strings.Contains(msg.Error, "queue full") { t.Fatalf("queue_size=0: want a queue-full error, got %s (%s%s)", msg.Type, msg.Error, msg.Reason) } } // --- 4. worker configuration ------------------------------------------------ // TestZeroWorkersAcksAndStrands: scanner.workers = 0 passes validation and // starts a pool with no workers at all. The client still acks everything it // receives, so the hold moves each job to 'processing' and then waits out the // ten-minute processing timeout in reDispatchTimedOut before failing it. The // scanner logs "Scanner worker pool started workers=0" once at boot and // nothing else; there is no health signal that distinguishes this from idle. func TestZeroWorkersAcksAndStrands(t *testing.T) { h := Start(t, mockhold.NewMemory(), WithWorkers(0)) seq, err := h.Hold.SendJob(helmJob(t)) if err != nil { t.Fatalf("send job: %v", err) } if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool { return m.Seq == seq && m.Type == "ack" }, 10*time.Second); err != nil { t.Fatalf("job was not even acked: %v", err) } expectSilence(t, h, func(m mockhold.Message) bool { return m.Seq == seq && (m.Type == "result" || m.Type == "error" || m.Type == "skipped") }, 2*time.Second, "workers=0") if n := h.Queue.Len(); n != 1 { t.Errorf("queue holds %d jobs, want 1 (acked and stranded)", n) } } // TestEmptyTmpDirFailsEveryJob: vuln.tmp_dir = "" is accepted by config // loading, skips the TMPDIR export in WorkerPool.Start, and then fails every // single job in processJob's ensureDir, because os.MkdirAll("") is an error. // The failure is retryable, so every job in the deployment loops forever. func TestEmptyTmpDirFailsEveryJob(t *testing.T) { h := Start(t, mockhold.NewMemory(), func(c *config.Config) { c.Vuln.TmpDir = "" }) seq, err := h.Hold.SendJob(synthJob("probe/no-tmpdir")) if err != nil { t.Fatalf("send job: %v", err) } msg := h.AwaitTerminal(t, seq, 30*time.Second) if msg.Type != "error" || !strings.Contains(msg.Error, "tmp dir") { t.Fatalf("empty tmp_dir: want a tmp dir error, got %s (%s%s)", msg.Type, msg.Error, msg.Reason) } if n := len(h.Hold.BlobRequests()); n != 0 { t.Errorf("failed before download but still fetched %d blobs", n) } t.Logf("every job fails with %q, retryably", msg.Error) } // --- 5. priority ------------------------------------------------------------ // gate is an HTTP stand-in for a hold whose getBlob hangs. A job pointed at it // occupies a worker for exactly as long as the test wants, which is how the // backlog scenarios below build a queue without needing real image bytes. type gate struct { srv *httptest.Server entered chan struct{} release chan struct{} } func newGate(t *testing.T) *gate { t.Helper() g := &gate{entered: make(chan struct{}, 8), release: make(chan struct{})} g.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case g.entered <- struct{}{}: default: } <-g.release http.Error(w, "gate released", http.StatusNotFound) })) t.Cleanup(g.srv.Close) return g } func (g *gate) waitEntered(t *testing.T, timeout time.Duration) { t.Helper() select { case <-g.entered: case <-time.After(timeout): t.Fatal("worker never reached the stalled blob fetch") } } func (g *gate) open() { close(g.release) } // TestHighTierJumpsQueuedBacklog confirms the priority heap does what it // claims across the real client/queue/worker path: while one worker is busy, // a later owner-tier job overtakes an earlier deckhand-tier one. // // It also shows the limit of that guarantee. Priority is consulted only at // Dequeue, so a high-tier job that arrives while the single worker is inside // a scan waits for that scan to finish plus the full JobCooldown. With the // production 10s cooldown and multi-minute scans, "priority" means position in // a queue, not preemption, and a saturated scanner starves the low tier // entirely: every owner job admitted during a scan is dequeued before any // deckhand job, no matter how long the deckhand job has waited. func TestHighTierJumpsQueuedBacklog(t *testing.T) { h := Start(t, mockhold.NewMemory()) g := newGate(t) // Occupy the single worker with a job whose blob fetch never returns. blocker := synthJob("probe/blocker") blocker.HoldEndpoint = g.srv.URL blockerSeq, err := h.Hold.SendJob(blocker) if err != nil { t.Fatalf("send blocker: %v", err) } g.waitEntered(t, 15*time.Second) // Queue a deckhand job first, then an owner job. low := helmJob(t) low.Tier = "deckhand" lowSeq, err := h.Hold.SendJob(low) if err != nil { t.Fatalf("send low: %v", err) } high := helmJob(t) high.Tier = "owner" highSeq, err := h.Hold.SendJob(high) if err != nil { t.Fatalf("send high: %v", err) } // Both must be in the queue before the worker is freed, or the test would // be measuring arrival order rather than priority. for _, seq := range []int64{lowSeq, highSeq} { if _, err := h.Hold.WaitForMessage(func(m mockhold.Message) bool { return m.Seq == seq && m.Type == "ack" }, 10*time.Second); err != nil { t.Fatalf("seq %d never acked: %v", seq, err) } } if n := h.Queue.Len(); n != 2 { t.Fatalf("queue depth %d, want 2 backlogged jobs", n) } g.open() h.AwaitTerminal(t, blockerSeq, 30*time.Second) highMsg := h.AwaitTerminal(t, highSeq, 30*time.Second) lowMsg := h.AwaitTerminal(t, lowSeq, 30*time.Second) if !highMsg.At.Before(lowMsg.At) { t.Errorf("owner-tier job finished at %s, deckhand at %s: priority did not apply", highMsg.At, lowMsg.At) } } // --- 6. shutdown ------------------------------------------------------------ // TestQueueCloseDrainsRatherThanCancels documents queue.Close semantics, which // are not what "close" suggests. Dequeue returns nil only when the queue is // closed *and* empty, so a shutdown with a backlog hands every remaining job // to a worker rather than dropping it. Combined with HoldClient.Close having // already severed the socket, whatever those jobs produce is written into a // dead connection and lost, while the hold sits on them until the ten-minute // processing timeout. func TestQueueCloseDrainsRatherThanCancels(t *testing.T) { q := queue.NewJobQueue(10) for i := 0; i < 3; i++ { if !q.Enqueue(&scanner.ScanJob{Seq: int64(i + 1), Tier: "deckhand"}) { t.Fatalf("enqueue %d refused", i) } } q.Close() var drained []int64 for { job := q.Dequeue() if job == nil { break } drained = append(drained, job.Seq) } if len(drained) != 3 { t.Fatalf("Close() discarded the backlog: drained %v, want 3 jobs", drained) } t.Logf("Close() left %d jobs to be dequeued and scanned after shutdown began", len(drained)) } // Shutdown and in-flight downloads used to be pinned here, by // TestShutdownDoesNotInterruptInFlightDownload: the blob fetches built their // requests with http.NewRequest and no context, so cancelling the pool context // changed nothing and WorkerPool.Wait blocked until the HTTP client's own // five-minute timeout expired. That is fixed, and the opposite is now asserted // by TestShutdownAbortsAnInFlightDownload in deadline_test.go, which also // checks the server sees the request hang up rather than the worker merely // walking away from it. // TestHoldClientCloseIsNotIdempotent pins a sharp edge rather than a live bug: // HoldClient.Close closes c.done unconditionally, so a second call panics the // process with "close of closed channel". cmd/scanner calls it exactly once // today, which is the only reason this is not already an incident, and there // is no guard if a future shutdown path (a health-check restart, a reconnect // supervisor) calls it again. func TestHoldClientCloseIsNotIdempotent(t *testing.T) { c := client.NewHoldClient("ws://127.0.0.1:1", "secret", queue.NewJobQueue(1)) c.Close() defer func() { if recover() == nil { t.Fatal("Close is idempotent now; this finding is fixed and the test should be inverted") } }() c.Close() t.Fatal("unreachable") }