diff --git a/deploy/upcloud/configs/scanner.yaml.tmpl b/deploy/upcloud/configs/scanner.yaml.tmpl index 3c2b2f1..dd349a3 100644 --- a/deploy/upcloud/configs/scanner.yaml.tmpl +++ b/deploy/upcloud/configs/scanner.yaml.tmpl @@ -13,9 +13,27 @@ hold: url: "ws://localhost:8080" secret: "{{.ScannerSecret}}" scanner: - workers: 2 + # One worker, deliberately. Two reasons, and the first one may go away: + # - The hold's proactive dispatch loop gates on waitForCapacity() and + # hands out one job at a time hold-wide, so a second worker only ever + # receives work when two pushes coincide. + # - Peak RSS is per concurrent scan. Two concurrent scans of a + # node:22-class image measured 687 MiB with the 512 MiB GOMEMLIMIT in + # force and 1357 MiB without it. On a host this size that is the + # difference between working and OOM-killing the hold. + # Raise this only together with MemoryMax in the unit file and the memory + # available on the host. + workers: 1 queue_size: 100 vuln: enabled: true db_path: "{{.BasePath}}/scanner/vulndb" + # Must be on disk, not tmpfs. Layers are extracted here and amplify + # roughly 3.8x over their compressed size (measured: node:22, 389 MiB -> + # 1493 MiB), so a tmpfs would spend host memory to hold them. tmp_dir: "{{.BasePath}}/scanner/tmp" + # 512 MiB compressed. The shipped default is 2 GiB, which no small host + # can survive: peak RSS tracks image size, and 389 MiB compressed already + # reached 561 MiB RSS with the memory limit applied. Images above this are + # rejected before any blob is downloaded. + max_image_size: 536870912 diff --git a/deploy/upcloud/systemd/scanner.service.tmpl b/deploy/upcloud/systemd/scanner.service.tmpl index cd8de5d..ef1c742 100644 --- a/deploy/upcloud/systemd/scanner.service.tmpl +++ b/deploy/upcloud/systemd/scanner.service.tmpl @@ -11,6 +11,39 @@ ExecStart={{.BinaryPath}} serve --config {{.ConfigPath}} Restart=on-failure RestartSec=10 +# Memory containment. The scanner sets a 512 MiB Go soft limit itself +# (GOMEMLIMIT, see cmd/scanner/main.go), but that is soft: the runtime +# collects harder to respect it and never fails an allocation to honour it, so +# a large enough image walks straight through it. Without a cgroup cap the +# kernel OOM killer chooses its own victim, and on a shared host the other +# large process is the hold, meaning the scanner's overshoot kills the service +# it reports to. These make the scanner the one that dies. +# +# MemoryHigh throttles and reclaims; MemoryMax kills. Sized from measurement: +# one scan of a node:22-class image peaks at 561 MiB RSS with the Go limit in +# force, so MemoryHigh sits above that and MemoryMax leaves headroom for the +# hold and the OS on a 1 GiB host. Raise both together with scanner.workers. +MemoryHigh=640M +MemoryMax=768M + +# Scheduling priority. The scanner is the lowest-value process on a shared +# host: a scan finishing a minute later costs nothing, a hold that cannot +# answer a pull costs a user. Syft's extraction saturates CPU and writes +# several times the compressed image size to disk, so left at the default +# weight it competes evenly with the services that matter. +# +# Weights apply only under contention: the scanner still uses the whole box +# when nothing else wants it, which is what makes this better than the fixed +# inter-job sleep in worker.go (that yields on a timer whether or not anyone +# needs the CPU, and yields nothing while a scan is actually running). +# Everything else runs at the default weight of 100. +# +# IOWeight needs the io controller with a scheduler that honours it (BFQ, or +# io.cost configured); where it is unsupported systemd ignores it silently and +# CPUWeight still applies. +CPUWeight=20 +IOWeight=20 + ReadWritePaths={{.DataDir}} ProtectSystem=strict ProtectHome=yes