deploy: give the scanner its own server, and stop the tool from undoing production fixes on provision

The scanner shared the hold's 1 GB host and thrashed it twice: 11 hours on
2026-09-12 and again on the 13th (644 MiB resident plus 1.9 GB of swap, 504
on every repo page). It is memory-bound, not CPU-bound, so it now gets a
dedicated STARTER-2xCPU-4GB server: own state entry, own plan flag (pinned
name, shape match if UpCloud renames the tier again, picker last), own
cloud-init, firewall, `update scanner`, `ssh scanner`, status, backup and
teardown. Its config reaches the hold over the private network and its unit
sets MemorySwapMax=0 so an overshoot is an OOM kill and a restart, not a
wedged host. The hold's cloud-init and update paths no longer carry it.

Three defects the first provision run exposed, all fixed here:

- Frontend HTTP/2 defaulted to on and was reconciled onto the LB every run.
  Re-enabling it on the 12th stranded the appview<->hold connections for
  25 minutes. Default is now off and reconciled off, with a guard test.
- The TLS step requested Let's Encrypt bundles for every registry domain,
  re-adding the .cr ones that were removed when those moved behind Bunny.
  It now skips any domain whose DNS does not resolve to the LB.
- Each prompt built its own bufio.Scanner on stdin, so the first swallowed
  every piped answer and the second read EOF and took the default, which
  re-ran cloud-init on the production hold. One shared reader, and no answer
  now means skip.

Also: STARTER- plans take standard storage (maxiops fails with TIER_INVALID),
and the cloud-init wait polls for up to 20 minutes instead of one SSH call
capped at five, which a first boot with npm exceeds.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Hho5da4daoCoPBJ9tCrL7s
This commit is contained in:
Evan Jarrett
2026-09-12 21:49:26 -05:00
co-authored by Claude Fable 5.1
parent ba1f5eab40
commit 83092d9aee
12 changed files with 691 additions and 292 deletions
+7 -7
View File
@@ -91,8 +91,9 @@ func preDeployBackup(ip string, s backupSpec) (string, error) {
// backupSpecFor lists what a deploy of target replaces on its server: the
// service binary, its config and unit, plus the companion service that ships
// with it (scanner with hold, labeler with appview). The appview's SQLite
// database is snapshotted too because migrations run on the next boot.
// with it (labeler with appview). The scanner is its own target on its own
// server. The appview's SQLite database is snapshotted too because migrations
// run on the next boot.
func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec {
s := backupSpec{
BaseDir: naming.BasePath(),
@@ -115,11 +116,10 @@ func backupSpecFor(target string, naming Naming, state *InfraState, now time.Tim
s.Binaries = []string{bin + naming.Hold()}
s.Configs = []string{naming.HoldConfigPath()}
s.Units = []string{naming.Hold()}
if state.ScannerEnabled {
s.Binaries = append(s.Binaries, bin+naming.Scanner())
s.Configs = append(s.Configs, naming.ScannerConfigPath())
s.Units = append(s.Units, naming.Scanner())
}
case "scanner":
s.Binaries = []string{bin + naming.Scanner()}
s.Configs = []string{naming.ScannerConfigPath()}
s.Units = []string{naming.Scanner()}
}
return s
}
+10 -2
View File
@@ -30,8 +30,16 @@ func TestBackupSpecForAppviewIncludesDatabaseAndLabeler(t *testing.T) {
if h.SQLiteDB != "" {
t.Fatalf("hold backup must not snapshot a database, got %q", h.SQLiteDB)
}
if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold,/opt/seamark/bin/seamark-scanner" {
t.Fatalf("hold Binaries = %v", h.Binaries)
if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold" {
t.Fatalf("hold Binaries = %v (the scanner has its own server and its own backup)", h.Binaries)
}
sc := backupSpecFor("scanner", naming, state, now, 5)
if strings.Join(sc.Binaries, ",") != "/opt/seamark/bin/seamark-scanner" {
t.Fatalf("scanner Binaries = %v", sc.Binaries)
}
if strings.Join(sc.Configs, ",") != "/etc/seamark/scanner.yaml" || strings.Join(sc.Units, ",") != "seamark-scanner" {
t.Fatalf("scanner Configs = %v Units = %v", sc.Configs, sc.Units)
}
}
+85 -57
View File
@@ -57,6 +57,9 @@ type ConfigValues struct {
// Scanner (auto-generated shared secret)
ScannerSecret string // hex-encoded 32-byte secret; empty disables scanning
// ScannerHoldURL is the hold's WebSocket address as seen from the scanner
// server, e.g. "ws://10.0.1.3:8080" over the private network.
ScannerHoldURL string
}
// renderConfig executes a Go template with the given values.
@@ -95,15 +98,49 @@ func renderServiceUnit(tmplStr string, p serviceUnitParams) (string, error) {
}
// scannerServiceUnitParams holds values for rendering the scanner systemd unit.
// Extends the standard fields with HoldServiceName for the After= dependency.
// Extends the standard fields with the memory limits, which are sized to the
// scanner's own server (see scannerPlanSpec) rather than shared with a hold.
type scannerServiceUnitParams struct {
DisplayName string // e.g. "Seamark"
User string // e.g. "seamark"
BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner"
ConfigPath string // e.g. "/etc/seamark/scanner.yaml"
DataDir string // e.g. "/var/lib/seamark"
ServiceName string // e.g. "seamark-scanner"
HoldServiceName string // e.g. "seamark-hold" (After= dependency)
DisplayName string // e.g. "Seamark"
User string // e.g. "seamark"
BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner"
ConfigPath string // e.g. "/etc/seamark/scanner.yaml"
DataDir string // e.g. "/var/lib/seamark"
ServiceName string // e.g. "seamark-scanner"
GoMemLimit string // GOMEMLIMIT, e.g. "2560MiB"
MemoryHigh string // systemd MemoryHigh, e.g. "3G"
MemoryMax string // systemd MemoryMax, e.g. "3500M"
}
// scannerMemoryLimits sizes the unit's memory knobs from the plan's RAM. The
// OS and page cache keep roughly the last 500 MiB; MemoryMax sits just under
// that line, MemoryHigh 500 MiB below it so reclaim starts before the kill,
// and the Go soft limit another 500 MiB below so the collector gets the first
// try. On the 4 GB scanner plan that is 2560MiB / 3G / 3500M.
func scannerMemoryLimits(memoryMB int) (goMemLimit, memoryHigh, memoryMax string) {
maxMB := memoryMB - 596
highMB := maxMB - 428
goMB := highMB - 512
if goMB < 512 {
goMB = 512
}
return fmt.Sprintf("%dMiB", goMB), fmt.Sprintf("%dM", highMB), fmt.Sprintf("%dM", maxMB)
}
// scannerUnitParams builds the scanner unit parameters for a deployment.
func scannerUnitParams(naming Naming) scannerServiceUnitParams {
goLimit, high, maxMem := scannerMemoryLimits(scannerPlanSpec.MemoryMB)
return scannerServiceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
ConfigPath: naming.ScannerConfigPath(),
DataDir: naming.BasePath(),
ServiceName: naming.Scanner(),
GoMemLimit: goLimit,
MemoryHigh: high,
MemoryMax: maxMem,
}
}
func renderScannerServiceUnit(p scannerServiceUnitParams) (string, error) {
@@ -242,10 +279,10 @@ echo "=== Labeler setup complete ==="
return script + labelerPhase, nil
}
// generateHoldCloudInit generates the cloud-init user-data script for the hold server.
// When withScanner is true, a second phase is appended that creates scanner data
// directories and installs a scanner systemd service. Binaries are deployed separately via SCP.
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner bool) (string, error) {
// generateHoldCloudInit generates the cloud-init user-data script for the hold
// server. Binaries are deployed separately via SCP. The scanner has its own
// server (generateScannerCloudInit) and no longer rides along here.
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) {
naming := cfg.Naming()
configYAML, err := renderConfig(holdConfigTmpl, vals)
@@ -265,7 +302,7 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo
return "", fmt.Errorf("hold service unit: %w", err)
}
script, err := generateCloudInit(cloudInitParams{
return generateCloudInit(cloudInitParams{
BinaryName: naming.Hold(),
ServiceUnit: serviceUnit,
ConfigYAML: configYAML,
@@ -278,69 +315,60 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo
LogFile: naming.LogFile(),
DisplayName: naming.DisplayName(),
})
if err != nil {
return "", err
}
}
if !withScanner {
return script, nil
// generateScannerCloudInit generates the cloud-init user-data script for the
// scanner's own server. It is the standard base script (packages, swap,
// service user, config, unit) plus the vulnerability database and extraction
// directories the scanner needs. The binary is deployed separately via SCP.
// vals.ScannerHoldURL must already point at the hold's private address.
func generateScannerCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) {
if vals.ScannerHoldURL == "" {
return "", fmt.Errorf("scanner cloud-init: hold URL is empty (hold server must exist first)")
}
naming := cfg.Naming()
// Render scanner config YAML
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
configYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return "", fmt.Errorf("scanner config: %w", err)
}
// Append scanner setup phase (no build — binary deployed via SCP)
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
ConfigPath: naming.ScannerConfigPath(),
DataDir: naming.BasePath(),
ServiceName: naming.Scanner(),
HoldServiceName: naming.Hold(),
})
serviceUnit, err := renderScannerServiceUnit(scannerUnitParams(naming))
if err != nil {
return "", fmt.Errorf("scanner service unit: %w", err)
}
// Escape single quotes for heredoc embedding
scannerUnit = strings.ReplaceAll(scannerUnit, "'", "'\\''")
scannerConfigYAML = strings.ReplaceAll(scannerConfigYAML, "'", "'\\''")
script, err := generateCloudInit(cloudInitParams{
BinaryName: naming.Scanner(),
ServiceUnit: serviceUnit,
ConfigYAML: configYAML,
ConfigPath: naming.ScannerConfigPath(),
ServiceName: naming.Scanner(),
DataDir: naming.BasePath(),
InstallDir: naming.InstallDir(),
SystemUser: naming.SystemUser(),
ConfigDir: naming.ConfigDir(),
LogFile: naming.LogFile(),
DisplayName: naming.DisplayName(),
})
if err != nil {
return "", err
}
scannerPhase := fmt.Sprintf(`
# === Scanner Setup ===
return script + scannerDirsScript(naming), nil
}
# Scanner data dirs
// scannerDirsScript creates the scanner's data directories. Shared between
// cloud-init (first boot) and update (a server whose directories were removed).
func scannerDirsScript(naming Naming) string {
return fmt.Sprintf(`
# Scanner data dirs (vulnerability database, layer extraction scratch)
mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s
# Scanner config
cat > %s << 'CFGEOF'
%s
CFGEOF
# Scanner systemd service
cat > /etc/systemd/system/%s.service << 'SVCEOF'
%s
SVCEOF
systemctl daemon-reload
systemctl enable %s
echo "=== Scanner setup complete ==="
`,
naming.ScannerDataDir(), naming.ScannerDataDir(),
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir(),
naming.ScannerConfigPath(),
scannerConfigYAML,
naming.Scanner(),
scannerUnit,
naming.Scanner(),
)
return script + scannerPhase, nil
}
type cloudInitParams struct {
+99 -1
View File
@@ -4,9 +4,11 @@ import (
"context"
"fmt"
"os"
"sort"
"strings"
"time"
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/client"
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service"
"go.yaml.in/yaml/v3"
@@ -22,6 +24,7 @@ const (
type InfraConfig struct {
Zone string
Plan string
ScannerPlan string // plan for the scanner server; resolved by resolveScannerPlan when empty
SSHPublicKey string
S3SecretKey string
@@ -39,7 +42,7 @@ func (c *InfraConfig) Naming() Naming {
return Naming{ClientName: c.ClientName}
}
func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
func loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
sshKey, err := readSSHPublicKey(sshKeyPath)
if err != nil {
return nil, err
@@ -53,6 +56,7 @@ func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
return &InfraConfig{
Zone: zone,
Plan: plan,
ScannerPlan: scannerPlan,
SSHPublicKey: sshKey,
S3SecretKey: s3Secret,
ClientName: clientName,
@@ -128,6 +132,100 @@ func resolveInteractive(ctx context.Context, svc *service.Service, cfg *InfraCon
return nil
}
// scannerPlanSpec is the shape of the scanner's own server. The scanner is
// memory-bound, not CPU-bound: Grype keeps a large working set of its 2 GB
// vulnerability database resident (400-600 MiB idle) and Syft catalogs an
// extracted image in memory on top of that. On the 1 GB hold host the two
// together peaked at 644 MiB resident plus 1.9 GB of swap and thrashed the
// box for eleven hours (2026-09-12). 4 GB fits the worst scan seen so far with
// headroom; 30 GB of disk holds the database plus the ~3.8x extraction
// amplification of a 1 GiB image.
type planSpec struct {
Cores int
MemoryMB int
DiskGB int
}
var scannerPlanSpec = planSpec{Cores: 2, MemoryMB: 4096, DiskGB: 30}
// defaultScannerPlan is the UpCloud plan of that shape at the time of
// writing (the Starter tier, which UpCloud called Developer / DEV- until
// 2026). resolveScannerPlan checks it still exists and falls back to a shape
// match if UpCloud renames the tier again.
const defaultScannerPlan = "STARTER-2xCPU-4GB"
// isStarterPlan reports whether a plan is in UpCloud's entry tier, which only
// accepts "standard" storage: creating one with maxiops fails with
// TIER_INVALID. The tier was named DEV- before it became STARTER-.
func isStarterPlan(name string) bool {
upper := strings.ToUpper(name)
return strings.HasPrefix(upper, "STARTER-") || strings.HasPrefix(upper, "DEV-")
}
// hasPlan reports whether name is one of the plans UpCloud offers.
func hasPlan(plans []upcloud.Plan, name string) bool {
for _, p := range plans {
if p.Name == name {
return true
}
}
return false
}
// matchPlan returns the plan names whose shape equals spec. Starter-tier
// plans sort first because they are the cheapest way to buy that shape.
func matchPlan(plans []upcloud.Plan, spec planSpec) []string {
var names []string
for _, p := range plans {
if p.GPUAmount > 0 {
continue
}
if p.CoreNumber == spec.Cores && p.MemoryAmount == spec.MemoryMB && p.StorageSize == spec.DiskGB {
names = append(names, p.Name)
}
}
sort.SliceStable(names, func(i, j int) bool {
di, dj := isStarterPlan(names[i]), isStarterPlan(names[j])
if di != dj {
return di
}
return names[i] < names[j]
})
return names
}
// resolveScannerPlan fills cfg.ScannerPlan: the --scanner-plan flag wins,
// then defaultScannerPlan if UpCloud still offers it, then the first plan
// matching scannerPlanSpec, then the interactive picker.
func resolveScannerPlan(ctx context.Context, svc *service.Service, cfg *InfraConfig) error {
if cfg.ScannerPlan != "" {
return nil
}
resp, err := svc.GetPlans(ctx)
if err != nil {
return fmt.Errorf("fetch plans: %w", err)
}
if hasPlan(resp.Plans, defaultScannerPlan) {
cfg.ScannerPlan = defaultScannerPlan
return nil
}
fmt.Printf("Plan %s is no longer offered; matching by shape.\n", defaultScannerPlan)
if names := matchPlan(resp.Plans, scannerPlanSpec); len(names) > 0 {
cfg.ScannerPlan = names[0]
fmt.Printf("Scanner plan: %s (%d CPU, %d GB RAM, %d GB disk)\n", names[0],
scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB)
return nil
}
fmt.Printf("No plan offers %d CPU / %d GB RAM / %d GB disk; pick one for the scanner.\n",
scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB)
p, err := pickPlan(ctx, svc)
if err != nil {
return fmt.Errorf("scanner plan picker: %w", err)
}
cfg.ScannerPlan = p
return nil
}
// newService creates an UpCloud API client. If token is non-empty it's used
// directly; otherwise credentials are read from UPCLOUD_TOKEN env var.
func newService(token string) (*service.Service, error) {
+20 -19
View File
@@ -10,19 +10,18 @@ log_shipper:
server:
addr: :9090
hold:
url: "ws://localhost:8080"
# The hold's private-network address. The scanner runs on its own server
# and reaches the hold over the UpCloud SDN, never through the load
# balancer (the LB frontend has a 10s client timeout that would cut the
# scan WebSocket).
url: "{{.ScannerHoldURL}}"
secret: "{{.ScannerSecret}}"
scanner:
# One worker, deliberately. Two reasons, and the first one may go away:
# - The hold's proactive dispatch loop gates on waitForCapacity() and
# hands out one job at a time hold-wide, so a second worker only ever
# receives work when two pushes coincide.
# - Peak RSS is per concurrent scan. Two concurrent scans of a
# node:22-class image measured 687 MiB with the 512 MiB GOMEMLIMIT in
# force and 1357 MiB without it. On a host this size that is the
# difference between working and OOM-killing the hold.
# Raise this only together with MemoryMax in the unit file and the memory
# available on the host.
# One worker, deliberately: the hold's proactive dispatch loop gates on
# waitForCapacity() and hands out one job at a time hold-wide, so a second
# worker only ever receives work when two pushes coincide. Peak RSS is per
# concurrent scan (two node:22-class scans measured 1357 MiB with no Go
# limit), so raise this only together with MemoryMax in the unit file.
workers: 1
queue_size: 100
# Must stay below the hold's 10m scanning timeout, which it measures from
@@ -42,14 +41,16 @@ vuln:
# roughly 3.8x over their compressed size (measured: node:22, 389 MiB ->
# 1493 MiB), so a tmpfs would spend host memory to hold them.
tmp_dir: "{{.BasePath}}/scanner/tmp"
# 512 MiB compressed. The shipped default is 2 GiB, which no small host
# can survive: peak RSS tracks image size, and 389 MiB compressed already
# reached 561 MiB RSS with the memory limit applied. Images above this are
# rejected before any blob is downloaded.
max_image_size: 536870912
# 1 GiB compressed. Peak RSS tracks image size (389 MiB compressed reached
# 561 MiB RSS with a 512 MiB Go limit) and extraction writes ~3.8x the
# compressed size to tmp_dir, so this bounds both memory and disk on the
# 4 GB / 30 GB scanner server. Images above it are rejected before any
# blob is downloaded. The shipped default is 2 GiB.
max_image_size: 1073741824
# Reclaim scan directories left behind by a scanner that was killed
# mid-scan (a restart or a deploy). Those never run their own cleanup, and
# on this host they reached 8.8 GB on a 20 GB disk before scans started
# failing with "no space left on device". Above the 8m job timeout, so a
# scan in flight elsewhere in the same directory is never touched.
# on the old shared host they reached 8.8 GB on a 20 GB disk before scans
# started failing with "no space left on device". Above the 8m job
# timeout, so a scan in flight elsewhere in the same directory is never
# touched.
sweep_max_age: 1h
+225 -54
View File
@@ -8,6 +8,7 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
"net"
"os"
"path/filepath"
"strings"
@@ -26,28 +27,30 @@ var provisionCmd = &cobra.Command{
token, _ := cmd.Root().PersistentFlags().GetString("token")
zone, _ := cmd.Flags().GetString("zone")
plan, _ := cmd.Flags().GetString("plan")
scannerPlan, _ := cmd.Flags().GetString("scanner-plan")
sshKey, _ := cmd.Flags().GetString("ssh-key")
s3Secret, _ := cmd.Flags().GetString("s3-secret")
withScanner, _ := cmd.Flags().GetBool("with-scanner")
withLabeler, _ := cmd.Flags().GetBool("with-labeler")
withBilling, _ := cmd.Flags().GetBool("with-billing")
return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner, withLabeler, withBilling)
return cmdProvision(token, zone, plan, scannerPlan, sshKey, s3Secret, withScanner, withLabeler, withBilling)
},
}
func init() {
provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)")
provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)")
provisionCmd.Flags().String("plan", "", "Server plan for appview and hold (interactive picker if omitted)")
provisionCmd.Flags().String("scanner-plan", "", fmt.Sprintf("Server plan for the scanner's own server (default %s, or whichever plan offers %d CPU / %d GB RAM / %d GB disk if that name is gone)", defaultScannerPlan, scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB))
provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required when creating new servers)")
provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold")
provisionCmd.Flags().Bool("with-scanner", false, "Deploy the vulnerability scanner on its own server")
provisionCmd.Flags().Bool("with-labeler", false, "Deploy content moderation labeler alongside appview")
provisionCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default.")
rootCmd.AddCommand(provisionCmd)
}
func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error {
cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret)
func cmdProvision(token, zone, plan, scannerPlan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error {
cfg, err := loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret)
if err != nil {
return err
}
@@ -81,6 +84,16 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
}
}
// The scanner gets its own server, on its own plan. Resolve it only when
// that server still has to be created; an existing one keeps its plan.
scannerEnabled := withScanner || state.ScannerEnabled
needsScannerServer := scannerEnabled && state.Scanner.UUID == ""
if needsScannerServer {
if err := resolveScannerPlan(ctx, svc, cfg); err != nil {
return err
}
}
if state.Zone == "" {
state.Zone = cfg.Zone
}
@@ -111,6 +124,9 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
if needsServers {
fmt.Printf("Server plan: %s\n", cfg.Plan)
}
if needsScannerServer {
fmt.Printf("Scanner server plan: %s\n", cfg.ScannerPlan)
}
fmt.Println()
// S3 secret key — from flag for existing storage, from API for new
@@ -239,7 +255,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
if err != nil {
return err
}
appview, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
appview, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
if err != nil {
return fmt.Errorf("create appview: %w", err)
}
@@ -253,7 +269,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
holdCreated := false
if state.Hold.UUID != "" {
fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
holdScript, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
holdScript, err := generateHoldCloudInit(cfg, vals)
if err != nil {
return err
}
@@ -267,22 +283,13 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML, true); err != nil {
return fmt.Errorf("hold config sync: %w", err)
}
if state.ScannerEnabled {
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return fmt.Errorf("render scanner config: %w", err)
}
if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil {
return fmt.Errorf("scanner config sync: %w", err)
}
}
} else {
fmt.Println("Creating hold server...")
holdUserData, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
holdUserData, err := generateHoldCloudInit(cfg, vals)
if err != nil {
return err
}
hold, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
hold, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
if err != nil {
return fmt.Errorf("create hold: %w", err)
}
@@ -292,15 +299,63 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
fmt.Printf(" Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP)
}
// 4b. Scanner server. Its own box: the scanner is memory-bound (Grype's
// database working set plus Syft's in-memory catalog) and sharing a 1 GB
// host with the hold swapped that host unresponsive for eleven hours on
// 2026-09-12. It reaches the hold over the private network, so the hold's
// private IP has to be known before its config can be rendered.
scannerCreated := false
if scannerEnabled {
vals.ScannerHoldURL = scannerHoldURL(state.Hold.PrivateIP)
if state.Scanner.UUID != "" {
fmt.Printf("Scanner: %s (exists)\n", state.Scanner.UUID)
scannerScript, err := generateScannerCloudInit(cfg, vals)
if err != nil {
return err
}
if err := syncCloudInit("scanner", state.Scanner.PublicIP, scannerScript); err != nil {
return err
}
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return fmt.Errorf("render scanner config: %w", err)
}
if err := syncConfigKeys("scanner", state.Scanner.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil {
return fmt.Errorf("scanner config sync: %w", err)
}
} else {
fmt.Println("Creating scanner server...")
scannerUserData, err := generateScannerCloudInit(cfg, vals)
if err != nil {
return err
}
scanner, err := createServer(ctx, svc, cfg, cfg.ScannerPlan, templateUUID, state.Network.UUID, naming.Scanner(), scannerUserData)
if err != nil {
return fmt.Errorf("create scanner: %w", err)
}
state.Scanner = *scanner
_ = saveState(state)
scannerCreated = true
fmt.Printf(" Scanner: %s (public: %s, private: %s)\n", scanner.UUID, scanner.PublicIP, scanner.PrivateIP)
}
}
// 5. Firewall rules (idempotent — replaces all rules)
fmt.Println("Configuring firewall rules...")
for _, s := range []struct {
firewalled := []struct {
name string
uuid string
}{
{"appview", state.Appview.UUID},
{"hold", state.Hold.UUID},
} {
}
if state.Scanner.UUID != "" {
firewalled = append(firewalled, struct {
name string
uuid string
}{"scanner", state.Scanner.UUID})
}
for _, s := range firewalled {
if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil {
return fmt.Errorf("firewall %s: %w", s.name, err)
}
@@ -372,7 +427,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
}
// 7. Build locally and deploy binaries to new servers
if appviewCreated || holdCreated {
if appviewCreated || holdCreated || scannerCreated {
rootDir := projectRoot()
if err := runMakeBuildTrixie(rootDir, withBilling); err != nil {
@@ -393,6 +448,11 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
return err
}
}
if scannerCreated {
if err := waitForSetup(state.Scanner.PublicIP, "scanner"); err != nil {
return err
}
}
fmt.Println("\nDeploying binaries...")
if appviewCreated {
@@ -415,12 +475,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil {
return fmt.Errorf("upload hold: %w", err)
}
if state.ScannerEnabled {
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
if err := scpFile(scannerLocal, state.Hold.PublicIP, scannerRemote); err != nil {
return fmt.Errorf("upload scanner: %w", err)
}
}
if scannerCreated {
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
if err := scpFile(scannerLocal, state.Scanner.PublicIP, scannerRemote); err != nil {
return fmt.Errorf("upload scanner: %w", err)
}
}
}
@@ -459,9 +519,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
fmt.Println("SSH access:")
fmt.Printf(" ssh root@%s # appview\n", state.Appview.PublicIP)
fmt.Printf(" ssh root@%s # hold\n", state.Hold.PublicIP)
if state.Scanner.UUID != "" {
fmt.Printf(" ssh root@%s # scanner\n", state.Scanner.PublicIP)
}
fmt.Println()
fmt.Println("Next steps:")
if appviewCreated || holdCreated {
if appviewCreated || holdCreated || scannerCreated {
fmt.Println(" 1. Edit configs if needed, then start services:")
} else {
fmt.Println(" 1. Start services:")
@@ -588,12 +651,15 @@ func objectStorageRegion(zone string) string {
}
}
func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, plan, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
if cfg.SSHPublicKey == "" {
return nil, fmt.Errorf("creating server %s requires --ssh-key (path to SSH public key file)", title)
}
if plan == "" {
return nil, fmt.Errorf("creating server %s: no plan selected", title)
}
storageTier := "maxiops"
if strings.HasPrefix(strings.ToUpper(cfg.Plan), "DEV-") {
if isStarterPlan(plan) {
storageTier = "standard"
}
@@ -602,7 +668,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t
plans, err := svc.GetPlans(ctx)
if err == nil {
for _, p := range plans.Plans {
if p.Name == cfg.Plan {
if p.Name == plan {
diskSize = p.StorageSize
break
}
@@ -613,7 +679,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t
Zone: cfg.Zone,
Title: title,
Hostname: title,
Plan: cfg.Plan,
Plan: plan,
Metadata: upcloud.True,
UserData: userData,
Firewall: "on",
@@ -737,17 +803,26 @@ const lbClientTimeout = 10
// httpFrontendProperties are the https frontend's properties.
//
// HTTP/2 is the one that matters for the browser: over HTTP/1.1 a browser
// opens at most ~6 connections per origin, so a page that fans out many small
// requests (the admin crew tab issues one per member) queues them six at a
// time and blocks every other request to the same host behind them. h2
// multiplexes them over a single connection and the queue disappears.
// HTTP/2 stays OFF on this frontend, and ensureLBHTTP2 turns it off on an LB
// that has it on. It was on until 2026-09-12, for the browser's sake (h2
// multiplexes the admin crew tab's fan-out over one connection instead of
// six). Two things changed the answer:
//
// Clients that need an HTTP/1.1 upgrade are unaffected: h2 is negotiated per
// connection via ALPN, so a WebSocket client simply selects http/1.1.
// - Re-enabling it that day caused a 25-minute /auth/token outage. The
// appview and the hold call each other through this LB on long-lived Go
// HTTP clients; the frontend reconfigure left those connections open at
// TCP level but dead, Go's HTTP/2 transport multiplexed every call onto
// them, and no transport here sets ReadIdleTimeout to notice. Only
// restarting both services recovered. Over HTTP/1.1 a dead connection
// fails one request and the pool opens another.
// - The 64 KB HTTP/2 stream window capped pushes at ~2.5 MB/s through the
// LB; median push fell from 16.5 s to 3.1 s when it went off.
//
// The registry and web hostnames are fronted by Bunny now, so the browser
// case this bought is served by the CDN's own h2 edge rather than by the LB.
func httpFrontendProperties() *upcloud.LoadBalancerFrontendProperties {
return &upcloud.LoadBalancerFrontendProperties{
HTTP2Enabled: new(true),
HTTP2Enabled: new(false),
TimeoutClient: lbClientTimeout,
}
}
@@ -974,13 +1049,28 @@ func labelerFrontendRule(labelerDomain string) request.LoadBalancerFrontendRule
}
// ensureLBCertificates reconciles TLS certificate bundles on the load balancer.
// It skips domains that already have a TLS config attached and creates missing ones.
// It skips domains that already have a TLS config attached and creates missing
// ones, but only for domains whose DNS resolves to this LB. A dynamic
// (Let's Encrypt) bundle is validated over HTTP at the domain, so a domain
// fronted by a CDN cannot be issued here anyway; requesting it only leaves a
// bundle stuck in validation. The .cr registry domains moved behind Bunny on
// 2026-09-12 and their bundles were removed by hand; on 2026-09-13 this
// function put them straight back. Hence the check.
func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error {
lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID})
if err != nil {
return fmt.Errorf("get load balancer: %w", err)
}
lbAddrs := map[string]bool{}
for _, n := range lb.Networks {
if n.Type == upcloud.LoadBalancerNetworkTypePublic && n.DNSName != "" {
for _, a := range lookupHost(n.DNSName) {
lbAddrs[a] = true
}
}
}
// Build set of existing TLS config names on the "https" frontend
existing := make(map[string]bool)
for _, fe := range lb.Frontends {
@@ -997,6 +1087,10 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri
fmt.Printf(" TLS certificate: %s (exists)\n", domain)
continue
}
if !pointsAt(domain, lbAddrs) {
fmt.Printf(" TLS certificate: %s skipped (DNS does not point at this LB; fronted elsewhere or not yet delegated)\n", domain)
continue
}
bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{
Type: upcloud.LoadBalancerCertificateBundleTypeDynamic,
@@ -1025,11 +1119,32 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri
return nil
}
// lookupHost resolves name to its IPv4/IPv6 addresses; failures resolve to none.
func lookupHost(name string) []string {
addrs, err := net.LookupHost(name)
if err != nil {
return nil
}
return addrs
}
// pointsAt reports whether any address domain resolves to is one of lbAddrs.
// An empty lbAddrs (the LB's DNS name did not resolve) matches nothing, so
// no bundle is requested when the precondition cannot be checked.
func pointsAt(domain string, lbAddrs map[string]bool) bool {
for _, a := range lookupHost(domain) {
if lbAddrs[a] {
return true
}
}
return false
}
// ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule.
// This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers,
// overwriting any pre-existing values (prevents spoofing).
// ensureLBHTTP2 reconciles HTTP/2 and the client timeout onto an LB that
// already exists.
// ensureLBHTTP2 reconciles the HTTP/2 flag (off, see httpFrontendProperties)
// and the client timeout onto an LB that already exists.
//
// createLoadBalancer only runs when there is no LB yet, so without this the
// properties above would reach a fresh deployment and never an existing one.
@@ -1045,10 +1160,11 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err
}
want := httpFrontendProperties()
wantHTTP2 := want.HTTP2Enabled != nil && *want.HTTP2Enabled
haveHTTP2 := fe.Properties != nil && fe.Properties.HTTP2Enabled != nil && *fe.Properties.HTTP2Enabled
haveTimeout := fe.Properties != nil && fe.Properties.TimeoutClient == want.TimeoutClient
if haveHTTP2 && haveTimeout {
fmt.Println(" Frontend HTTP/2 + timeout: already set")
if haveHTTP2 == wantHTTP2 && haveTimeout {
fmt.Printf(" Frontend HTTP/2 %s + timeout_client=%ds: already set\n", onOff(wantHTTP2), want.TimeoutClient)
} else {
if _, err := svc.ModifyLoadBalancerFrontend(ctx, &request.ModifyLoadBalancerFrontendRequest{
ServiceUUID: lbUUID,
@@ -1059,12 +1175,19 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err
}); err != nil {
return fmt.Errorf("modify https frontend: %w", err)
}
fmt.Printf(" Frontend HTTP/2: enabled (timeout_client=%ds)\n", want.TimeoutClient)
fmt.Printf(" Frontend HTTP/2: %s (timeout_client=%ds)\n", onOff(wantHTTP2), want.TimeoutClient)
}
return nil
}
func onOff(b bool) string {
if b {
return "on"
}
return "off"
}
func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error {
rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
ServiceUUID: lbUUID,
@@ -1449,9 +1572,16 @@ func syncCloudInit(name, ip, localScript string) error {
}
fmt.Printf(" Re-run cloud-init on %s? [Y/n] ", name)
scanner := bufio.NewScanner(os.Stdin)
scanner.Scan()
answer := strings.TrimSpace(strings.ToLower(scanner.Text()))
answer, ok := readLine()
if !ok {
// No answer at all (stdin closed, or a non-interactive run) must not
// re-run setup on a production server. Only a typed yes does.
fmt.Printf(" no answer, skipped\n")
if err := writeRemoteCloudInit(ip, localScript); err != nil {
fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
}
return nil
}
if answer != "" && answer != "y" && answer != "yes" {
fmt.Printf(" Skipped\n")
// Still update the remote reference so next provision sees an accurate diff
@@ -1479,6 +1609,33 @@ func syncCloudInit(name, ip, localScript string) error {
return nil
}
// scannerHoldURL is the hold's WebSocket address from the scanner server: the
// hold's private IP over the SDN, never the load balancer (whose 10s client
// timeout would cut the long-lived scan socket).
func scannerHoldURL(holdPrivateIP string) string {
if holdPrivateIP == "" {
return ""
}
return "ws://" + holdPrivateIP + ":8080"
}
// stdin is shared by every prompt in the tool. Each prompt used to wrap
// os.Stdin in its own bufio.Scanner, and the first one swallowed everything
// that was already buffered (all of a piped "n\nn\n"), so the second prompt
// read EOF, took the default, and re-ran cloud-init on the production hold
// (2026-09-13). One reader keeps the unread lines for the next prompt.
var stdin = bufio.NewReader(os.Stdin)
// readLine returns the next line typed on stdin, lower-cased and trimmed, and
// false when stdin has nothing more to give.
func readLine() (string, bool) {
line, err := stdin.ReadString('\n')
if err != nil && line == "" {
return "", false
}
return strings.TrimSpace(strings.ToLower(line)), true
}
// generateScannerSecret generates a random 32-byte hex-encoded shared secret
// for authenticating scanner-to-hold WebSocket connections.
func generateScannerSecret() (string, error) {
@@ -1515,11 +1672,25 @@ func waitForSetup(ip, name string) error {
time.Sleep(10 * time.Second)
}
// Poll rather than `cloud-init status --wait`: runSSH caps one command at
// five minutes, and a fresh server's first boot (apt upgrade plus git,
// gcc, nodejs, npm) takes longer than that. The 2026-09-13 scanner
// provision timed out here with the server healthy and half installed.
fmt.Printf(" %s: waiting for cloud-init...\n", name)
_, err := runSSH(ip, "cloud-init status --wait 2>/dev/null || true", false)
if err != nil {
return fmt.Errorf("cloud-init wait on %s: %w", name, err)
deadline := time.Now().Add(20 * time.Minute)
for {
out, err := runSSH(ip, "cloud-init status 2>/dev/null | head -1", false)
status := strings.TrimSpace(out)
switch {
case err == nil && strings.HasPrefix(status, "status: done"):
fmt.Printf(" %s: ready\n", name)
return nil
case err == nil && strings.HasPrefix(status, "status: error"):
return fmt.Errorf("cloud-init on %s finished with errors; see /var/log/cloud-init-output.log on the server", name)
}
if time.Now().After(deadline) {
return fmt.Errorf("cloud-init on %s not done after 20 minutes (last status %q)", name, status)
}
time.Sleep(15 * time.Second)
}
fmt.Printf(" %s: ready\n", name)
return nil
}
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"strings"
"testing"
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
)
// The scanner's server is chosen by shape, not by name, because UpCloud
// renames tiers. Developer plans win a tie: same shape, cheapest tier.
func TestMatchPlanPrefersDeveloperTier(t *testing.T) {
plans := []upcloud.Plan{
{Name: "2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 80},
{Name: "GPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30, GPUAmount: 1},
{Name: "HICPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30},
{Name: "STARTER-2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30},
{Name: "STARTER-1xCPU-1GB", CoreNumber: 1, MemoryAmount: 1024, StorageSize: 10},
}
got := matchPlan(plans, scannerPlanSpec)
if strings.Join(got, ",") != "STARTER-2xCPU-4GB,HICPU-2xCPU-4GB-30GB" {
t.Fatalf("matchPlan = %v", got)
}
// Both the current and the pre-2026 tier names take standard storage;
// asking for maxiops on them fails with TIER_INVALID (seen 2026-09-13).
for _, n := range []string{"STARTER-2xCPU-4GB", "DEV-2xCPU-4GB-30GB", "dev-1xcpu-1gb-10gb"} {
if !isStarterPlan(n) {
t.Errorf("%s should be a starter-tier plan", n)
}
}
if isStarterPlan("2xCPU-4GB") || isStarterPlan("HICPU-2xCPU-4GB-30GB") {
t.Error("general purpose and high-CPU plans are not starter tier")
}
if len(matchPlan(plans, planSpec{Cores: 8, MemoryMB: 65536, DiskGB: 1000})) != 0 {
t.Fatal("a shape nobody sells must match nothing, so the picker takes over")
}
if !hasPlan(plans, defaultScannerPlan) || hasPlan(plans, "STARTER-2xCPU-4GB-80GB") {
t.Fatal("hasPlan must match the exact plan name")
}
// The pinned default must be the shape the spec describes, or the two
// paths in resolveScannerPlan would buy different servers.
if got := matchPlan(plans, scannerPlanSpec); got[0] != defaultScannerPlan {
t.Fatalf("defaultScannerPlan %s is not the DEV plan of shape %+v (matcher chose %s)", defaultScannerPlan, scannerPlanSpec, got[0])
}
}
// Limits step down from the plan's RAM: MemoryMax leaves the OS ~600 MiB,
// MemoryHigh sits below that, and the Go soft limit below both so the
// collector runs before the kernel reclaims or kills.
func TestScannerMemoryLimits(t *testing.T) {
goLimit, high, maxMem := scannerMemoryLimits(4096)
if goLimit != "2560MiB" || high != "3072M" || maxMem != "3500M" {
t.Fatalf("4 GB plan: got %s / %s / %s", goLimit, high, maxMem)
}
goLimit, _, _ = scannerMemoryLimits(1024)
if goLimit != "512MiB" {
t.Fatalf("the Go limit must never drop below the scanner's own 512 MiB default, got %s", goLimit)
}
}
func TestScannerCloudInitTargetsHoldPrivateIP(t *testing.T) {
cfg := &InfraConfig{ClientName: "seamark"}
vals := &ConfigValues{
BasePath: "/var/lib/seamark",
ScannerSecret: "abc",
ScannerHoldURL: scannerHoldURL("10.0.1.3"),
}
script, err := generateScannerCloudInit(cfg, vals)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
`url: "ws://10.0.1.3:8080"`,
"mkdir -p /var/lib/seamark/scanner/vulndb /var/lib/seamark/scanner/tmp",
"Environment=GOMEMLIMIT=2560MiB",
"MemorySwapMax=0",
"systemctl enable seamark-scanner",
} {
if !strings.Contains(script, want) {
t.Errorf("scanner cloud-init missing %q", want)
}
}
if strings.Contains(script, "seamark-hold.service") {
t.Error("a standalone scanner must not order itself after a hold unit that is not on its host")
}
if _, err := generateScannerCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark"}); err == nil {
t.Error("rendering a scanner config with no hold address must fail, not ship ws://:8080")
}
if scannerHoldURL("") != "" {
t.Error("no hold IP must render as no URL so the guard above fires")
}
}
// The hold's cloud-init no longer carries the scanner: its unit, config and
// directories belong to the scanner server.
func TestHoldCloudInitHasNoScanner(t *testing.T) {
cfg := &InfraConfig{ClientName: "seamark"}
script, err := generateHoldCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark", HoldDomain: "x.cove.seamark.dev"})
if err != nil {
t.Fatal(err)
}
if strings.Contains(script, "seamark-scanner") || strings.Contains(script, "vulndb") {
t.Error("hold cloud-init still installs the scanner")
}
}
// Re-enabling frontend HTTP/2 on 2026-09-12 stranded the appview<->hold
// connections through the LB for 25 minutes. Every provision run reconciles
// this property, so the default is the outage switch.
func TestLBFrontendHTTP2StaysOff(t *testing.T) {
p := httpFrontendProperties()
if p.HTTP2Enabled == nil || *p.HTTP2Enabled {
t.Fatal("LB frontend HTTP/2 must be off (see httpFrontendProperties)")
}
if p.TimeoutClient != lbClientTimeout {
t.Fatalf("timeout_client = %d, want %d", p.TimeoutClient, lbClientTimeout)
}
}
// A domain fronted by a CDN never resolves to the LB, so the tool must not
// request a certificate for it (it re-added the .cr bundles on 2026-09-13).
func TestPointsAtRequiresOverlap(t *testing.T) {
lb := map[string]bool{"203.0.113.10": true}
if pointsAt("localhost", lb) {
t.Fatal("localhost does not resolve to the LB")
}
if pointsAt("localhost", map[string]bool{}) {
t.Fatal("an unresolved LB must match nothing")
}
if !pointsAt("localhost", map[string]bool{"127.0.0.1": true, "::1": true}) {
t.Fatal("a domain resolving to an LB address must match")
}
}
+1
View File
@@ -16,6 +16,7 @@ type InfraState struct {
Network StateRef `json:"network"`
Appview ServerState `json:"appview"`
Hold ServerState `json:"hold"`
Scanner ServerState `json:"scanner,omitempty"`
LB StateRef `json:"loadbalancer"`
ObjectStorage ObjectStorageState `json:"object_storage"`
ScannerEnabled bool `json:"scanner_enabled,omitempty"`
+11 -27
View File
@@ -43,7 +43,7 @@ func cmdStatus(token string) error {
fmt.Printf("Zone: %s\n\n", state.Zone)
// Server status
for _, s := range []struct {
servers := []struct {
name string
ss ServerState
serviceName string
@@ -51,7 +51,16 @@ func cmdStatus(token string) error {
}{
{"Appview", state.Appview, naming.Appview(), "http://localhost:5000/health"},
{"Hold", state.Hold, naming.Hold(), "http://localhost:8080/xrpc/_health"},
} {
}
if state.ScannerEnabled || state.Scanner.UUID != "" {
servers = append(servers, struct {
name string
ss ServerState
serviceName string
healthURL string
}{"Scanner", state.Scanner, naming.Scanner(), "http://localhost:9090/healthz"})
}
for _, s := range servers {
fmt.Printf("%-8s UUID: %s\n", s.name, s.ss.UUID)
fmt.Printf(" Public: %s\n", s.ss.PublicIP)
fmt.Printf(" Private: %s\n", s.ss.PrivateIP)
@@ -90,31 +99,6 @@ func cmdStatus(token string) error {
fmt.Println()
}
// Scanner status (runs on hold server)
if state.ScannerEnabled {
fmt.Printf("Scanner (on hold server)\n")
if state.Hold.PublicIP != "" {
output, err := runSSH(state.Hold.PublicIP, fmt.Sprintf(
"systemctl is-active %s 2>/dev/null || echo 'inactive'; curl -sf http://localhost:9090/healthz > /dev/null 2>&1 && echo 'health:ok' || echo 'health:fail'",
naming.Scanner(),
), false)
if err != nil {
fmt.Printf(" Service: unreachable\n")
} else {
lines := strings.SplitSeq(strings.TrimSpace(output), "\n")
for line := range lines {
line = strings.TrimSpace(line)
if line == "active" || line == "inactive" {
fmt.Printf(" Service: %s\n", line)
} else if after, ok := strings.CutPrefix(line, "health:"); ok {
fmt.Printf(" Health: %s\n", after)
}
}
}
}
fmt.Println()
}
// LB status
if state.LB.UUID != "" {
fmt.Printf("Load Balancer: %s\n", state.LB.UUID)
+19 -31
View File
@@ -1,6 +1,6 @@
[Unit]
Description={{.DisplayName}} Scanner (Vulnerability Scanning)
After=network-online.target {{.HoldServiceName}}.service
After=network-online.target
Wants=network-online.target
[Service]
@@ -11,38 +11,26 @@ ExecStart={{.BinaryPath}} serve --config {{.ConfigPath}}
Restart=on-failure
RestartSec=10
# Memory containment. The scanner sets a 512 MiB Go soft limit itself
# (GOMEMLIMIT, see cmd/scanner/main.go), but that is soft: the runtime
# collects harder to respect it and never fails an allocation to honour it, so
# a large enough image walks straight through it. Without a cgroup cap the
# kernel OOM killer chooses its own victim, and on a shared host the other
# large process is the hold, meaning the scanner's overshoot kills the service
# it reports to. These make the scanner the one that dies.
# The scanner runs on its own server, so it is the only large process on the
# host and the limits below are sized to the box rather than to a neighbour.
#
# MemoryHigh throttles and reclaims; MemoryMax kills. Sized from measurement:
# one scan of a node:22-class image peaks at 561 MiB RSS with the Go limit in
# force, so MemoryHigh sits above that and MemoryMax leaves headroom for the
# hold and the OS on a 1 GiB host. Raise both together with scanner.workers.
MemoryHigh=640M
MemoryMax=768M
# GOMEMLIMIT is the Go soft limit: the runtime collects harder as the heap
# approaches it and never fails an allocation to honour it, so a large enough
# image walks straight through. It sits below MemoryHigh so the GC gets a
# chance before the kernel starts reclaiming. Without the variable the scanner
# defaults itself to 512 MiB (cmd/scanner/main.go), which on a 4 GB host only
# wastes CPU on collection.
Environment=GOMEMLIMIT={{.GoMemLimit}}
# Scheduling priority. The scanner is the lowest-value process on a shared
# host: a scan finishing a minute later costs nothing, a hold that cannot
# answer a pull costs a user. Syft's extraction saturates CPU and writes
# several times the compressed image size to disk, so left at the default
# weight it competes evenly with the services that matter.
#
# Weights apply only under contention: the scanner still uses the whole box
# when nothing else wants it, which is what makes this better than the fixed
# inter-job sleep in worker.go (that yields on a timer whether or not anyone
# needs the CPU, and yields nothing while a scan is actually running).
# Everything else runs at the default weight of 100.
#
# IOWeight needs the io controller with a scheduler that honours it (BFQ, or
# io.cost configured); where it is unsupported systemd ignores it silently and
# CPUWeight still applies.
CPUWeight=20
IOWeight=20
# MemoryHigh throttles and reclaims; MemoryMax kills. MemorySwapMax=0 is the
# one that matters: with swap available a scanner over its cap is not killed,
# it is paged out, and the host thrashes instead of failing. That is what took
# the shared hold host down for eleven hours on 2026-09-12 (644 MiB resident,
# 1.9 GB swapped). Refusing swap turns that into an OOM kill of one job and a
# clean restart ten seconds later.
MemoryHigh={{.MemoryHigh}}
MemoryMax={{.MemoryMax}}
MemorySwapMax=0
ReadWritePaths={{.DataDir}}
ProtectSystem=strict
+5 -6
View File
@@ -1,11 +1,8 @@
package main
import (
"bufio"
"context"
"fmt"
"os"
"strings"
"time"
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request"
@@ -39,14 +36,15 @@ func cmdTeardown(token string) error {
fmt.Printf(" Zone: %s\n", state.Zone)
fmt.Printf(" Appview: %s (%s)\n", state.Appview.UUID, state.Appview.PublicIP)
fmt.Printf(" Hold: %s (%s)\n", state.Hold.UUID, state.Hold.PublicIP)
if state.Scanner.UUID != "" {
fmt.Printf(" Scanner: %s (%s)\n", state.Scanner.UUID, state.Scanner.PublicIP)
}
fmt.Printf(" Network: %s\n", state.Network.UUID)
fmt.Printf(" LB: %s\n", state.LB.UUID)
fmt.Println()
fmt.Print("Type 'yes' to confirm: ")
scanner := bufio.NewScanner(os.Stdin)
scanner.Scan()
if strings.TrimSpace(scanner.Text()) != "yes" {
if answer, _ := readLine(); answer != "yes" {
fmt.Println("Aborted.")
return nil
}
@@ -76,6 +74,7 @@ func cmdTeardown(token string) error {
}{
{"appview", state.Appview.UUID},
{"hold", state.Hold.UUID},
{"scanner", state.Scanner.UUID},
} {
if s.uuid == "" {
continue
+75 -88
View File
@@ -18,7 +18,7 @@ var updateCmd = &cobra.Command{
Use: "update [target]",
Short: "Deploy updates to servers",
Args: cobra.MaximumNArgs(1),
ValidArgs: []string{"all", "appview", "hold"},
ValidArgs: []string{"all", "appview", "hold", "scanner"},
RunE: func(cmd *cobra.Command, args []string) error {
target := "all"
if len(args) > 0 {
@@ -37,7 +37,7 @@ var sshCmd = &cobra.Command{
Use: "ssh <target>",
Short: "SSH into a server",
Args: cobra.ExactArgs(1),
ValidArgs: []string{"appview", "hold"},
ValidArgs: []string{"appview", "hold", "scanner"},
RunE: func(cmd *cobra.Command, args []string) error {
return cmdSSH(args[0])
},
@@ -45,7 +45,7 @@ var sshCmd = &cobra.Command{
func init() {
updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.")
updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
updateCmd.Flags().Bool("with-scanner", false, "Enable the vulnerability scanner (its server is created by `provision --with-scanner`)")
updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview")
updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into <data dir>/predeploy-<stamp>/ before replacing them")
updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup")
@@ -68,7 +68,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
naming := state.Naming()
rootDir := projectRoot()
// Enable scanner retroactively via --with-scanner on update
// Enable scanner retroactively via --with-scanner on update. The secret
// lands in the hold's config on this run; the scanner server itself is
// created by provision, which is the only command that creates servers.
if withScanner && !state.ScannerEnabled {
state.ScannerEnabled = true
if state.ScannerSecret == "" {
@@ -80,6 +82,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
fmt.Printf("Generated scanner shared secret\n")
}
_ = saveState(state)
if state.Scanner.UUID == "" {
fmt.Println("Scanner enabled. Run `provision --with-scanner` to create its server, then `update scanner` deploys to it.")
}
}
// Enable labeler retroactively via --with-labeler on update
@@ -90,38 +95,64 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
vals := configValsFromState(state)
// renderUnit is per target because the scanner unit carries memory limits
// the generic serviceUnitParams does not know about.
standardUnit := func(tmpl, binaryName, configPath, serviceName string) func() (string, error) {
return func() (string, error) {
return renderServiceUnit(tmpl, serviceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + binaryName,
ConfigPath: configPath,
DataDir: naming.BasePath(),
ServiceName: serviceName,
})
}
}
targets := map[string]struct {
ip string
binaryName string
buildCmd string
localBinary string
serviceName string
healthURL string
configTmpl string
configPath string
unitTmpl string
renderUnit func() (string, error)
setup string // extra remote shell run before restart; empty for none
}{
"appview": {
ip: state.Appview.PublicIP,
binaryName: naming.Appview(),
buildCmd: "appview",
localBinary: "atcr-appview",
serviceName: naming.Appview(),
healthURL: "http://localhost:5000/health",
configTmpl: appviewConfigTmpl,
configPath: naming.AppviewConfigPath(),
unitTmpl: appviewServiceTmpl,
renderUnit: standardUnit(appviewServiceTmpl, naming.Appview(), naming.AppviewConfigPath(), naming.Appview()),
},
"hold": {
ip: state.Hold.PublicIP,
binaryName: naming.Hold(),
buildCmd: "hold",
localBinary: "atcr-hold",
serviceName: naming.Hold(),
healthURL: "http://localhost:8080/xrpc/_health",
configTmpl: holdConfigTmpl,
configPath: naming.HoldConfigPath(),
unitTmpl: holdServiceTmpl,
renderUnit: standardUnit(holdServiceTmpl, naming.Hold(), naming.HoldConfigPath(), naming.Hold()),
},
"scanner": {
ip: state.Scanner.PublicIP,
binaryName: naming.Scanner(),
localBinary: "atcr-scanner",
serviceName: naming.Scanner(),
healthURL: "http://localhost:9090/healthz",
configTmpl: scannerConfigTmpl,
configPath: naming.ScannerConfigPath(),
renderUnit: func() (string, error) {
return renderScannerServiceUnit(scannerUnitParams(naming))
},
setup: scannerDirsScript(naming),
},
}
@@ -129,10 +160,18 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
switch target {
case "all":
toUpdate = []string{"appview", "hold"}
if state.Scanner.UUID != "" {
toUpdate = append(toUpdate, "scanner")
}
case "appview", "hold":
toUpdate = []string{target}
case "scanner":
if state.Scanner.UUID == "" {
return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one")
}
toUpdate = []string{target}
default:
return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target)
return fmt.Errorf("unknown target: %s (use: all, appview, hold, scanner)", target)
}
// Build all binaries via `make build-trixie` so output links against
@@ -182,14 +221,7 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
}
// Sync systemd service unit
renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName,
ConfigPath: t.configPath,
DataDir: naming.BasePath(),
ServiceName: t.serviceName,
})
renderedUnit, err := t.renderUnit()
if err != nil {
return fmt.Errorf("render %s service unit: %w", name, err)
}
@@ -210,58 +242,10 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
daemonReload = "systemctl daemon-reload"
}
// Scanner additions for hold server
scannerRestart := ""
scannerHealthCheck := ""
if name == "hold" && state.ScannerEnabled {
// Sync scanner config keys
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
if err != nil {
return fmt.Errorf("render scanner config: %w", err)
if t.setup != "" {
if _, err := runSSH(t.ip, t.setup, false); err != nil {
return fmt.Errorf("%s setup: %w", name, err)
}
if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML, false); err != nil {
return fmt.Errorf("scanner config sync: %w", err)
}
// Sync scanner service unit
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
DisplayName: naming.DisplayName(),
User: naming.SystemUser(),
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
ConfigPath: naming.ScannerConfigPath(),
DataDir: naming.BasePath(),
ServiceName: naming.Scanner(),
HoldServiceName: naming.Hold(),
})
if err != nil {
return fmt.Errorf("render scanner service unit: %w", err)
}
scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
if err != nil {
return fmt.Errorf("scanner service unit sync: %w", err)
}
if scannerUnitChanged {
daemonReload = "systemctl daemon-reload"
}
// Upload scanner binary
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil {
return fmt.Errorf("upload scanner: %w", err)
}
// Ensure scanner data dirs exist on server
scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s`,
naming.ScannerDataDir(), naming.ScannerDataDir(),
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())
if _, err := runSSH(t.ip, scannerSetup, false); err != nil {
return fmt.Errorf("scanner dir setup: %w", err)
}
scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
scannerHealthCheck = renderHealthProbe("scanner", "http://localhost:9090/healthz", healthProbeTries)
}
// Labeler additions for appview server
@@ -324,9 +308,9 @@ chown -R %s:%s %s`,
// status and only shows up as a health URL that never answers.
restartScript := fmt.Sprintf(`set -euo pipefail
%s
systemctl restart %s%s%s
%s%s%s`, daemonReload, t.serviceName, scannerRestart, labelerRestart,
renderHealthProbe(name, t.healthURL, healthProbeTries), scannerHealthCheck, labelerHealthCheck)
systemctl restart %s%s
%s%s`, daemonReload, t.serviceName, labelerRestart,
renderHealthProbe(name, t.healthURL, healthProbeTries), labelerHealthCheck)
output, err := runSSH(t.ip, restartScript, true)
if err != nil {
@@ -340,9 +324,6 @@ systemctl restart %s%s%s
results := parseProbes(output)
reportProbe(results, name, t.ip, t.serviceName)
if name == "hold" && state.ScannerEnabled {
reportProbe(results, "scanner", t.ip, naming.Scanner())
}
if name == "appview" && state.LabelerEnabled {
reportProbe(results, "labeler", t.ip, naming.Labeler())
}
@@ -361,17 +342,18 @@ func configValsFromState(state *InfraState) *ConfigValues {
labelerDomain := "labeler." + baseDomain
return &ConfigValues{
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: "", // not persisted in state; existing value on server is preserved
Zone: state.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
LabelerDomain: labelerDomain,
BasePath: naming.BasePath(),
ScannerSecret: state.ScannerSecret,
S3Endpoint: state.ObjectStorage.Endpoint,
S3Region: state.ObjectStorage.Region,
S3Bucket: state.ObjectStorage.Bucket,
S3AccessKey: state.ObjectStorage.AccessKeyID,
S3SecretKey: "", // not persisted in state; existing value on server is preserved
Zone: state.Zone,
HoldDomain: holdDomain,
HoldDid: "did:web:" + holdDomain,
LabelerDomain: labelerDomain,
BasePath: naming.BasePath(),
ScannerSecret: state.ScannerSecret,
ScannerHoldURL: scannerHoldURL(state.Hold.PrivateIP),
}
}
@@ -420,8 +402,13 @@ func cmdSSH(target string) error {
ip = state.Appview.PublicIP
case "hold":
ip = state.Hold.PublicIP
case "scanner":
ip = state.Scanner.PublicIP
if ip == "" {
return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one")
}
default:
return fmt.Errorf("unknown target: %s (use: appview, hold)", target)
return fmt.Errorf("unknown target: %s (use: appview, hold, scanner)", target)
}
fmt.Printf("Connecting to %s (%s)...\n", target, ip)