hold/scanner: stop one undispatchable job freezing all scanning

Vulnerability scanning produced nothing across the whole deployment for
nine days, from 2026-08-25 01:20:48 until a scanner restart on 2026-09-03.
The scanner was connected and idle, the hold's discovery pass kept
reporting unscannedFound=15 every four hours, and no scan_jobs row was
created in that entire window.

hasActiveJobs counted pending, assigned and processing rows globally with
no age bound, and waitForCapacity spins while it is true. dispatchLoop
calls it before popping any candidate, so a single pending row that never
reached a terminal state reported "busy" forever: discovery kept pushing
candidates into unscannedQueue and nothing ever popped them. That is why
the symptom was an empty queue rather than a growing one.

Nothing papered over it because push-triggered enqueue only fires for
owner or a tier with scan_on_push, which in production means pro alone.
All 210 manifests pushed to this hold in that window came from free,
supporter, or accounts with no crew row, so the frozen proactive loop was
the only source of jobs.

Nor could it recover on its own. Only Enqueue and drainPendingJobs
dispatch a pending row, and drainPendingJobs runs only when a scanner
newly connects; reDispatchTimedOut considered assigned rows only. The
hold had been up since Aug 14 and the scanner since Aug 21 on the same
websocket, so the drain path had not run since the row appeared.

So bound the capacity gate to pending rows younger than pendingStaleAfter,
give reDispatchTimedOut a pending reclaim, and check RowsAffected on the
assign UPDATE now that two dispatchers can race for a row. waitForCapacity
warns and names the blocking jobs after ten minutes without capacity,
because the failure mode above was completely silent.

Two adjacent fixes for the same outage. The scanner never called
InitLogger, so log_level and log_shipper were dead config and an idle
scanner was mute, which is what made nine days invisible. And skipReason
now also skips a job whose layers contain nothing tar-shaped: the job that
wedged this queue was an in-toto attestation whose config mediaType is an
ordinary image config, so the existing config-type check missed it and
buildOCILayout would have handed Syft an empty image.

The regression tests were verified against the old logic first: three of
them fail on it and pass on the fix.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DPWkeCKcbtGoyXyyeMhSps

scanner/cmd/scanner/main.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"atcr.io/pkg/logging"
 	"atcr.io/scanner/internal/client"
 	"atcr.io/scanner/internal/config"
 	"atcr.io/scanner/internal/queue"
@@ -50,6 +51,20 @@ Environment variables always override file values (SCANNER_ prefix).`,
 			return fmt.Errorf("failed to load config: %w", err)
 		}
 
+		// Without this the scanner runs on slog's default handler: log_level and
+		// log_shipper are silently ignored, nothing is shipped anywhere, and an
+		// idle scanner is indistinguishable from a wedged one.
+		logging.InitLoggerWithShipper(cfg.LogLevel, logging.ShipperConfig{
+			Backend:       cfg.LogShipper.Backend,
+			URL:           cfg.LogShipper.URL,
+			BatchSize:     cfg.LogShipper.BatchSize,
+			FlushInterval: cfg.LogShipper.FlushInterval,
+			Service:       "scanner",
+			Username:      cfg.LogShipper.Username,
+			Password:      cfg.LogShipper.Password,
+		})
+		defer logging.Shutdown()
+
 		slog.Info("Starting ATCR scanner",
 			"hold_url", cfg.Hold.URL,
 			"workers", cfg.Scanner.Workers,

scanner/internal/scan/grype.go

@@ -125,11 +125,11 @@ func scanVulnerabilities(ctx context.Context, s *sbom.SBOM, vulnDBPath string) (
 		"low", summary.Low,
 		"total", summary.Total)
 
-	report := map[string]interface{}{
+	report := map[string]any{
 		"matches": allMatches.Sorted(),
 		"source":  s.Source,
 		"distro":  s.Artifacts.LinuxDistribution,
-		"descriptor": map[string]interface{}{
+		"descriptor": map[string]any{
 			"name":    "grype",
 			"version": "v0.107.1",
 		},

scanner/internal/scan/worker.go

@@ -9,6 +9,7 @@ import (
 	"log/slog"
 	"os"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -151,6 +152,37 @@ var unscannableConfigTypes = map[string]bool{
 	"application/vnd.dsse.envelope.v1+json":    true, // DSSE envelopes (SLSA)
 }
 
+// skipReason reports why a job cannot be scanned, or "" when it can be.
+func skipReason(job *scanner.ScanJob) string {
+	if unscannableConfigTypes[job.Config.MediaType] {
+		return fmt.Sprintf("unscannable artifact type %s", job.Config.MediaType)
+	}
+
+	// A buildx attestation manifest carries an ordinary image config with a
+	// single in-toto or DSSE payload as its layer, so the config media type
+	// alone does not identify it. buildOCILayout drops every non-tar layer,
+	// which would hand Syft an image with nothing in it.
+	if len(job.Layers) > 0 && !hasScannableLayer(job.Layers) {
+		return fmt.Sprintf("no scannable layers (%s)", job.Layers[0].MediaType)
+	}
+
+	return ""
+}
+
+// hasScannableLayer mirrors the layer filter in buildOCILayout: anything that
+// is not a tar of some flavour is not something Syft can read.
+func hasScannableLayer(layers []scanner.BlobDescriptor) bool {
+	for _, layer := range layers {
+		if layer.Digest == "" {
+			continue
+		}
+		if layer.MediaType == "" || strings.Contains(layer.MediaType, "tar") {
+			return true
+		}
+	}
+	return false
+}
+
 func (wp *WorkerPool) processJob(ctx context.Context, job *scanner.ScanJob) (*scanner.ScanResult, error) {
 	startTime := time.Now()
 
@@ -158,8 +190,8 @@ func (wp *WorkerPool) processJob(ctx context.Context, job *scanner.ScanJob) (*sc
 	// Returning *SkipError tells the worker dispatch loop to send a "skipped"
 	// message rather than an "error" — the hold marks these records as
 	// permanently skipped and won't retry them on the rescan interval.
-	if unscannableConfigTypes[job.Config.MediaType] {
-		return nil, &SkipError{Reason: fmt.Sprintf("unscannable artifact type %s", job.Config.MediaType)}
+	if reason := skipReason(job); reason != "" {
+		return nil, &SkipError{Reason: reason}
 	}
 
 	// Ensure tmp dir exists

scanner/internal/scan/worker_skip_test.go

@@ -0,0 +1,83 @@
+package scan
+
+import (
+	"testing"
+
+	scanner "atcr.io/scanner"
+)
+
+// TestSkipReason covers the artifact shapes the scanner must refuse before it
+// spends a download on them. The attestation case is the one that reached
+// production: an in-toto SLSA provenance manifest carries an ordinary image
+// config, so the config media type check alone waved it through and Syft was
+// handed an OCI layout with no layers in it.
+func TestSkipReason(t *testing.T) {
+	tests := []struct {
+		name       string
+		configType string
+		layerTypes []string
+		wantSkip   bool
+	}{
+		{
+			name:       "container image",
+			configType: "application/vnd.oci.image.config.v1+json",
+			layerTypes: []string{"application/vnd.oci.image.layer.v1.tar+gzip"},
+		},
+		{
+			name:       "docker image",
+			configType: "application/vnd.docker.container.image.v1+json",
+			layerTypes: []string{"application/vnd.docker.image.rootfs.diff.tar.gzip"},
+		},
+		{
+			name:       "layer media type absent",
+			configType: "application/vnd.oci.image.config.v1+json",
+			layerTypes: []string{""},
+		},
+		{
+			name:       "helm chart",
+			configType: "application/vnd.cncf.helm.config.v1+json",
+			layerTypes: []string{"application/vnd.cncf.helm.chart.content.v1.tar+gzip"},
+			wantSkip:   true,
+		},
+		{
+			name:       "in-toto attestation with an image config",
+			configType: "application/vnd.oci.image.config.v1+json",
+			layerTypes: []string{"application/vnd.in-toto+json"},
+			wantSkip:   true,
+		},
+		{
+			name:       "dsse envelope layer",
+			configType: "application/vnd.oci.image.config.v1+json",
+			layerTypes: []string{"application/vnd.dsse.envelope.v1+json"},
+			wantSkip:   true,
+		},
+		{
+			name:       "mixed layers keep the scannable one",
+			configType: "application/vnd.oci.image.config.v1+json",
+			layerTypes: []string{"application/vnd.in-toto+json", "application/vnd.oci.image.layer.v1.tar"},
+		},
+		{
+			name:       "no layers at all is left to the pipeline",
+			configType: "application/vnd.oci.image.config.v1+json",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			job := &scanner.ScanJob{
+				Config: scanner.BlobDescriptor{MediaType: tt.configType, Digest: "sha256:config"},
+			}
+			for i, mt := range tt.layerTypes {
+				job.Layers = append(job.Layers, scanner.BlobDescriptor{
+					MediaType: mt,
+					Digest:    "sha256:layer" + string(rune('a'+i)),
+				})
+			}
+
+			reason := skipReason(job)
+			if got := reason != ""; got != tt.wantSkip {
+				t.Errorf("skipReason = %q, wantSkip=%v", reason, tt.wantSkip)
+			}
+		})
+	}
+}

scanner/types.go

@@ -6,16 +6,16 @@ import "encoding/json"
 
 // ScanJob represents a vulnerability scanning job received from the hold service
 type ScanJob struct {
-	Seq            int64           `json:"seq"`
-	ManifestDigest string          `json:"manifestDigest"`
-	Repository     string          `json:"repository"`
-	Tag            string          `json:"tag"`
-	UserDID        string          `json:"userDid"`
-	UserHandle     string          `json:"userHandle"`
-	HoldDID        string          `json:"holdDid"`
-	HoldEndpoint   string          `json:"holdEndpoint"`
-	Tier           string          `json:"tier"`
-	Config         BlobDescriptor  `json:"config"`
+	Seq            int64            `json:"seq"`
+	ManifestDigest string           `json:"manifestDigest"`
+	Repository     string           `json:"repository"`
+	Tag            string           `json:"tag"`
+	UserDID        string           `json:"userDid"`
+	UserHandle     string           `json:"userHandle"`
+	HoldDID        string           `json:"holdDid"`
+	HoldEndpoint   string           `json:"holdEndpoint"`
+	Tier           string           `json:"tier"`
+	Config         BlobDescriptor   `json:"config"`
 	Layers         []BlobDescriptor `json:"layers"`
 }