diff --git a/deploy/upcloud/backup.go b/deploy/upcloud/backup.go index 4ba1ff5..7c02fa4 100644 --- a/deploy/upcloud/backup.go +++ b/deploy/upcloud/backup.go @@ -91,8 +91,9 @@ func preDeployBackup(ip string, s backupSpec) (string, error) { // backupSpecFor lists what a deploy of target replaces on its server: the // service binary, its config and unit, plus the companion service that ships -// with it (scanner with hold, labeler with appview). The appview's SQLite -// database is snapshotted too because migrations run on the next boot. +// with it (labeler with appview). The scanner is its own target on its own +// server. The appview's SQLite database is snapshotted too because migrations +// run on the next boot. func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec { s := backupSpec{ BaseDir: naming.BasePath(), @@ -115,11 +116,10 @@ func backupSpecFor(target string, naming Naming, state *InfraState, now time.Tim s.Binaries = []string{bin + naming.Hold()} s.Configs = []string{naming.HoldConfigPath()} s.Units = []string{naming.Hold()} - if state.ScannerEnabled { - s.Binaries = append(s.Binaries, bin+naming.Scanner()) - s.Configs = append(s.Configs, naming.ScannerConfigPath()) - s.Units = append(s.Units, naming.Scanner()) - } + case "scanner": + s.Binaries = []string{bin + naming.Scanner()} + s.Configs = []string{naming.ScannerConfigPath()} + s.Units = []string{naming.Scanner()} } return s } diff --git a/deploy/upcloud/backup_test.go b/deploy/upcloud/backup_test.go index 3cadf6d..67d2b23 100644 --- a/deploy/upcloud/backup_test.go +++ b/deploy/upcloud/backup_test.go @@ -30,8 +30,16 @@ func TestBackupSpecForAppviewIncludesDatabaseAndLabeler(t *testing.T) { if h.SQLiteDB != "" { t.Fatalf("hold backup must not snapshot a database, got %q", h.SQLiteDB) } - if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold,/opt/seamark/bin/seamark-scanner" { - t.Fatalf("hold Binaries = %v", h.Binaries) + if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold" { + t.Fatalf("hold Binaries = %v (the scanner has its own server and its own backup)", h.Binaries) + } + + sc := backupSpecFor("scanner", naming, state, now, 5) + if strings.Join(sc.Binaries, ",") != "/opt/seamark/bin/seamark-scanner" { + t.Fatalf("scanner Binaries = %v", sc.Binaries) + } + if strings.Join(sc.Configs, ",") != "/etc/seamark/scanner.yaml" || strings.Join(sc.Units, ",") != "seamark-scanner" { + t.Fatalf("scanner Configs = %v Units = %v", sc.Configs, sc.Units) } } diff --git a/deploy/upcloud/cloudinit.go b/deploy/upcloud/cloudinit.go index b0e2a7b..75ba9ec 100644 --- a/deploy/upcloud/cloudinit.go +++ b/deploy/upcloud/cloudinit.go @@ -57,6 +57,9 @@ type ConfigValues struct { // Scanner (auto-generated shared secret) ScannerSecret string // hex-encoded 32-byte secret; empty disables scanning + // ScannerHoldURL is the hold's WebSocket address as seen from the scanner + // server, e.g. "ws://10.0.1.3:8080" over the private network. + ScannerHoldURL string } // renderConfig executes a Go template with the given values. @@ -95,15 +98,49 @@ func renderServiceUnit(tmplStr string, p serviceUnitParams) (string, error) { } // scannerServiceUnitParams holds values for rendering the scanner systemd unit. -// Extends the standard fields with HoldServiceName for the After= dependency. +// Extends the standard fields with the memory limits, which are sized to the +// scanner's own server (see scannerPlanSpec) rather than shared with a hold. type scannerServiceUnitParams struct { - DisplayName string // e.g. "Seamark" - User string // e.g. "seamark" - BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner" - ConfigPath string // e.g. "/etc/seamark/scanner.yaml" - DataDir string // e.g. "/var/lib/seamark" - ServiceName string // e.g. "seamark-scanner" - HoldServiceName string // e.g. "seamark-hold" (After= dependency) + DisplayName string // e.g. "Seamark" + User string // e.g. "seamark" + BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner" + ConfigPath string // e.g. "/etc/seamark/scanner.yaml" + DataDir string // e.g. "/var/lib/seamark" + ServiceName string // e.g. "seamark-scanner" + GoMemLimit string // GOMEMLIMIT, e.g. "2560MiB" + MemoryHigh string // systemd MemoryHigh, e.g. "3G" + MemoryMax string // systemd MemoryMax, e.g. "3500M" +} + +// scannerMemoryLimits sizes the unit's memory knobs from the plan's RAM. The +// OS and page cache keep roughly the last 500 MiB; MemoryMax sits just under +// that line, MemoryHigh 500 MiB below it so reclaim starts before the kill, +// and the Go soft limit another 500 MiB below so the collector gets the first +// try. On the 4 GB scanner plan that is 2560MiB / 3G / 3500M. +func scannerMemoryLimits(memoryMB int) (goMemLimit, memoryHigh, memoryMax string) { + maxMB := memoryMB - 596 + highMB := maxMB - 428 + goMB := highMB - 512 + if goMB < 512 { + goMB = 512 + } + return fmt.Sprintf("%dMiB", goMB), fmt.Sprintf("%dM", highMB), fmt.Sprintf("%dM", maxMB) +} + +// scannerUnitParams builds the scanner unit parameters for a deployment. +func scannerUnitParams(naming Naming) scannerServiceUnitParams { + goLimit, high, maxMem := scannerMemoryLimits(scannerPlanSpec.MemoryMB) + return scannerServiceUnitParams{ + DisplayName: naming.DisplayName(), + User: naming.SystemUser(), + BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(), + ConfigPath: naming.ScannerConfigPath(), + DataDir: naming.BasePath(), + ServiceName: naming.Scanner(), + GoMemLimit: goLimit, + MemoryHigh: high, + MemoryMax: maxMem, + } } func renderScannerServiceUnit(p scannerServiceUnitParams) (string, error) { @@ -242,10 +279,10 @@ echo "=== Labeler setup complete ===" return script + labelerPhase, nil } -// generateHoldCloudInit generates the cloud-init user-data script for the hold server. -// When withScanner is true, a second phase is appended that creates scanner data -// directories and installs a scanner systemd service. Binaries are deployed separately via SCP. -func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner bool) (string, error) { +// generateHoldCloudInit generates the cloud-init user-data script for the hold +// server. Binaries are deployed separately via SCP. The scanner has its own +// server (generateScannerCloudInit) and no longer rides along here. +func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) { naming := cfg.Naming() configYAML, err := renderConfig(holdConfigTmpl, vals) @@ -265,7 +302,7 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo return "", fmt.Errorf("hold service unit: %w", err) } - script, err := generateCloudInit(cloudInitParams{ + return generateCloudInit(cloudInitParams{ BinaryName: naming.Hold(), ServiceUnit: serviceUnit, ConfigYAML: configYAML, @@ -278,69 +315,60 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo LogFile: naming.LogFile(), DisplayName: naming.DisplayName(), }) - if err != nil { - return "", err - } +} - if !withScanner { - return script, nil +// generateScannerCloudInit generates the cloud-init user-data script for the +// scanner's own server. It is the standard base script (packages, swap, +// service user, config, unit) plus the vulnerability database and extraction +// directories the scanner needs. The binary is deployed separately via SCP. +// vals.ScannerHoldURL must already point at the hold's private address. +func generateScannerCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) { + if vals.ScannerHoldURL == "" { + return "", fmt.Errorf("scanner cloud-init: hold URL is empty (hold server must exist first)") } + naming := cfg.Naming() - // Render scanner config YAML - scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) + configYAML, err := renderConfig(scannerConfigTmpl, vals) if err != nil { return "", fmt.Errorf("scanner config: %w", err) } - // Append scanner setup phase (no build — binary deployed via SCP) - scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{ - DisplayName: naming.DisplayName(), - User: naming.SystemUser(), - BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(), - ConfigPath: naming.ScannerConfigPath(), - DataDir: naming.BasePath(), - ServiceName: naming.Scanner(), - HoldServiceName: naming.Hold(), - }) + serviceUnit, err := renderScannerServiceUnit(scannerUnitParams(naming)) if err != nil { return "", fmt.Errorf("scanner service unit: %w", err) } - // Escape single quotes for heredoc embedding - scannerUnit = strings.ReplaceAll(scannerUnit, "'", "'\\''") - scannerConfigYAML = strings.ReplaceAll(scannerConfigYAML, "'", "'\\''") + script, err := generateCloudInit(cloudInitParams{ + BinaryName: naming.Scanner(), + ServiceUnit: serviceUnit, + ConfigYAML: configYAML, + ConfigPath: naming.ScannerConfigPath(), + ServiceName: naming.Scanner(), + DataDir: naming.BasePath(), + InstallDir: naming.InstallDir(), + SystemUser: naming.SystemUser(), + ConfigDir: naming.ConfigDir(), + LogFile: naming.LogFile(), + DisplayName: naming.DisplayName(), + }) + if err != nil { + return "", err + } - scannerPhase := fmt.Sprintf(` -# === Scanner Setup === + return script + scannerDirsScript(naming), nil +} -# Scanner data dirs +// scannerDirsScript creates the scanner's data directories. Shared between +// cloud-init (first boot) and update (a server whose directories were removed). +func scannerDirsScript(naming Naming) string { + return fmt.Sprintf(` +# Scanner data dirs (vulnerability database, layer extraction scratch) mkdir -p %s/vulndb %s/tmp chown -R %s:%s %s - -# Scanner config -cat > %s << 'CFGEOF' -%s -CFGEOF - -# Scanner systemd service -cat > /etc/systemd/system/%s.service << 'SVCEOF' -%s -SVCEOF -systemctl daemon-reload -systemctl enable %s - -echo "=== Scanner setup complete ===" `, naming.ScannerDataDir(), naming.ScannerDataDir(), naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir(), - naming.ScannerConfigPath(), - scannerConfigYAML, - naming.Scanner(), - scannerUnit, - naming.Scanner(), ) - - return script + scannerPhase, nil } type cloudInitParams struct { diff --git a/deploy/upcloud/config.go b/deploy/upcloud/config.go index f01a347..7169a3d 100644 --- a/deploy/upcloud/config.go +++ b/deploy/upcloud/config.go @@ -4,9 +4,11 @@ import ( "context" "fmt" "os" + "sort" "strings" "time" + "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud" "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/client" "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service" "go.yaml.in/yaml/v3" @@ -22,6 +24,7 @@ const ( type InfraConfig struct { Zone string Plan string + ScannerPlan string // plan for the scanner server; resolved by resolveScannerPlan when empty SSHPublicKey string S3SecretKey string @@ -39,7 +42,7 @@ func (c *InfraConfig) Naming() Naming { return Naming{ClientName: c.ClientName} } -func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) { +func loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret string) (*InfraConfig, error) { sshKey, err := readSSHPublicKey(sshKeyPath) if err != nil { return nil, err @@ -53,6 +56,7 @@ func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) { return &InfraConfig{ Zone: zone, Plan: plan, + ScannerPlan: scannerPlan, SSHPublicKey: sshKey, S3SecretKey: s3Secret, ClientName: clientName, @@ -128,6 +132,100 @@ func resolveInteractive(ctx context.Context, svc *service.Service, cfg *InfraCon return nil } +// scannerPlanSpec is the shape of the scanner's own server. The scanner is +// memory-bound, not CPU-bound: Grype keeps a large working set of its 2 GB +// vulnerability database resident (400-600 MiB idle) and Syft catalogs an +// extracted image in memory on top of that. On the 1 GB hold host the two +// together peaked at 644 MiB resident plus 1.9 GB of swap and thrashed the +// box for eleven hours (2026-09-12). 4 GB fits the worst scan seen so far with +// headroom; 30 GB of disk holds the database plus the ~3.8x extraction +// amplification of a 1 GiB image. +type planSpec struct { + Cores int + MemoryMB int + DiskGB int +} + +var scannerPlanSpec = planSpec{Cores: 2, MemoryMB: 4096, DiskGB: 30} + +// defaultScannerPlan is the UpCloud plan of that shape at the time of +// writing (the Starter tier, which UpCloud called Developer / DEV- until +// 2026). resolveScannerPlan checks it still exists and falls back to a shape +// match if UpCloud renames the tier again. +const defaultScannerPlan = "STARTER-2xCPU-4GB" + +// isStarterPlan reports whether a plan is in UpCloud's entry tier, which only +// accepts "standard" storage: creating one with maxiops fails with +// TIER_INVALID. The tier was named DEV- before it became STARTER-. +func isStarterPlan(name string) bool { + upper := strings.ToUpper(name) + return strings.HasPrefix(upper, "STARTER-") || strings.HasPrefix(upper, "DEV-") +} + +// hasPlan reports whether name is one of the plans UpCloud offers. +func hasPlan(plans []upcloud.Plan, name string) bool { + for _, p := range plans { + if p.Name == name { + return true + } + } + return false +} + +// matchPlan returns the plan names whose shape equals spec. Starter-tier +// plans sort first because they are the cheapest way to buy that shape. +func matchPlan(plans []upcloud.Plan, spec planSpec) []string { + var names []string + for _, p := range plans { + if p.GPUAmount > 0 { + continue + } + if p.CoreNumber == spec.Cores && p.MemoryAmount == spec.MemoryMB && p.StorageSize == spec.DiskGB { + names = append(names, p.Name) + } + } + sort.SliceStable(names, func(i, j int) bool { + di, dj := isStarterPlan(names[i]), isStarterPlan(names[j]) + if di != dj { + return di + } + return names[i] < names[j] + }) + return names +} + +// resolveScannerPlan fills cfg.ScannerPlan: the --scanner-plan flag wins, +// then defaultScannerPlan if UpCloud still offers it, then the first plan +// matching scannerPlanSpec, then the interactive picker. +func resolveScannerPlan(ctx context.Context, svc *service.Service, cfg *InfraConfig) error { + if cfg.ScannerPlan != "" { + return nil + } + resp, err := svc.GetPlans(ctx) + if err != nil { + return fmt.Errorf("fetch plans: %w", err) + } + if hasPlan(resp.Plans, defaultScannerPlan) { + cfg.ScannerPlan = defaultScannerPlan + return nil + } + fmt.Printf("Plan %s is no longer offered; matching by shape.\n", defaultScannerPlan) + if names := matchPlan(resp.Plans, scannerPlanSpec); len(names) > 0 { + cfg.ScannerPlan = names[0] + fmt.Printf("Scanner plan: %s (%d CPU, %d GB RAM, %d GB disk)\n", names[0], + scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB) + return nil + } + fmt.Printf("No plan offers %d CPU / %d GB RAM / %d GB disk; pick one for the scanner.\n", + scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB) + p, err := pickPlan(ctx, svc) + if err != nil { + return fmt.Errorf("scanner plan picker: %w", err) + } + cfg.ScannerPlan = p + return nil +} + // newService creates an UpCloud API client. If token is non-empty it's used // directly; otherwise credentials are read from UPCLOUD_TOKEN env var. func newService(token string) (*service.Service, error) { diff --git a/deploy/upcloud/configs/scanner.yaml.tmpl b/deploy/upcloud/configs/scanner.yaml.tmpl index eca1469..ebf349a 100644 --- a/deploy/upcloud/configs/scanner.yaml.tmpl +++ b/deploy/upcloud/configs/scanner.yaml.tmpl @@ -10,19 +10,18 @@ log_shipper: server: addr: :9090 hold: - url: "ws://localhost:8080" + # The hold's private-network address. The scanner runs on its own server + # and reaches the hold over the UpCloud SDN, never through the load + # balancer (the LB frontend has a 10s client timeout that would cut the + # scan WebSocket). + url: "{{.ScannerHoldURL}}" secret: "{{.ScannerSecret}}" scanner: - # One worker, deliberately. Two reasons, and the first one may go away: - # - The hold's proactive dispatch loop gates on waitForCapacity() and - # hands out one job at a time hold-wide, so a second worker only ever - # receives work when two pushes coincide. - # - Peak RSS is per concurrent scan. Two concurrent scans of a - # node:22-class image measured 687 MiB with the 512 MiB GOMEMLIMIT in - # force and 1357 MiB without it. On a host this size that is the - # difference between working and OOM-killing the hold. - # Raise this only together with MemoryMax in the unit file and the memory - # available on the host. + # One worker, deliberately: the hold's proactive dispatch loop gates on + # waitForCapacity() and hands out one job at a time hold-wide, so a second + # worker only ever receives work when two pushes coincide. Peak RSS is per + # concurrent scan (two node:22-class scans measured 1357 MiB with no Go + # limit), so raise this only together with MemoryMax in the unit file. workers: 1 queue_size: 100 # Must stay below the hold's 10m scanning timeout, which it measures from @@ -42,14 +41,16 @@ vuln: # roughly 3.8x over their compressed size (measured: node:22, 389 MiB -> # 1493 MiB), so a tmpfs would spend host memory to hold them. tmp_dir: "{{.BasePath}}/scanner/tmp" - # 512 MiB compressed. The shipped default is 2 GiB, which no small host - # can survive: peak RSS tracks image size, and 389 MiB compressed already - # reached 561 MiB RSS with the memory limit applied. Images above this are - # rejected before any blob is downloaded. - max_image_size: 536870912 + # 1 GiB compressed. Peak RSS tracks image size (389 MiB compressed reached + # 561 MiB RSS with a 512 MiB Go limit) and extraction writes ~3.8x the + # compressed size to tmp_dir, so this bounds both memory and disk on the + # 4 GB / 30 GB scanner server. Images above it are rejected before any + # blob is downloaded. The shipped default is 2 GiB. + max_image_size: 1073741824 # Reclaim scan directories left behind by a scanner that was killed # mid-scan (a restart or a deploy). Those never run their own cleanup, and - # on this host they reached 8.8 GB on a 20 GB disk before scans started - # failing with "no space left on device". Above the 8m job timeout, so a - # scan in flight elsewhere in the same directory is never touched. + # on the old shared host they reached 8.8 GB on a 20 GB disk before scans + # started failing with "no space left on device". Above the 8m job + # timeout, so a scan in flight elsewhere in the same directory is never + # touched. sweep_max_age: 1h diff --git a/deploy/upcloud/provision.go b/deploy/upcloud/provision.go index 3661830..8f42fd1 100644 --- a/deploy/upcloud/provision.go +++ b/deploy/upcloud/provision.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "net" "os" "path/filepath" "strings" @@ -26,28 +27,30 @@ var provisionCmd = &cobra.Command{ token, _ := cmd.Root().PersistentFlags().GetString("token") zone, _ := cmd.Flags().GetString("zone") plan, _ := cmd.Flags().GetString("plan") + scannerPlan, _ := cmd.Flags().GetString("scanner-plan") sshKey, _ := cmd.Flags().GetString("ssh-key") s3Secret, _ := cmd.Flags().GetString("s3-secret") withScanner, _ := cmd.Flags().GetBool("with-scanner") withLabeler, _ := cmd.Flags().GetBool("with-labeler") withBilling, _ := cmd.Flags().GetBool("with-billing") - return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner, withLabeler, withBilling) + return cmdProvision(token, zone, plan, scannerPlan, sshKey, s3Secret, withScanner, withLabeler, withBilling) }, } func init() { provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)") - provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)") + provisionCmd.Flags().String("plan", "", "Server plan for appview and hold (interactive picker if omitted)") + provisionCmd.Flags().String("scanner-plan", "", fmt.Sprintf("Server plan for the scanner's own server (default %s, or whichever plan offers %d CPU / %d GB RAM / %d GB disk if that name is gone)", defaultScannerPlan, scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB)) provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required when creating new servers)") provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)") - provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold") + provisionCmd.Flags().Bool("with-scanner", false, "Deploy the vulnerability scanner on its own server") provisionCmd.Flags().Bool("with-labeler", false, "Deploy content moderation labeler alongside appview") provisionCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default.") rootCmd.AddCommand(provisionCmd) } -func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error { - cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret) +func cmdProvision(token, zone, plan, scannerPlan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error { + cfg, err := loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret) if err != nil { return err } @@ -81,6 +84,16 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w } } + // The scanner gets its own server, on its own plan. Resolve it only when + // that server still has to be created; an existing one keeps its plan. + scannerEnabled := withScanner || state.ScannerEnabled + needsScannerServer := scannerEnabled && state.Scanner.UUID == "" + if needsScannerServer { + if err := resolveScannerPlan(ctx, svc, cfg); err != nil { + return err + } + } + if state.Zone == "" { state.Zone = cfg.Zone } @@ -111,6 +124,9 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w if needsServers { fmt.Printf("Server plan: %s\n", cfg.Plan) } + if needsScannerServer { + fmt.Printf("Scanner server plan: %s\n", cfg.ScannerPlan) + } fmt.Println() // S3 secret key — from flag for existing storage, from API for new @@ -239,7 +255,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w if err != nil { return err } - appview, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData) + appview, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData) if err != nil { return fmt.Errorf("create appview: %w", err) } @@ -253,7 +269,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w holdCreated := false if state.Hold.UUID != "" { fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID) - holdScript, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled) + holdScript, err := generateHoldCloudInit(cfg, vals) if err != nil { return err } @@ -267,22 +283,13 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML, true); err != nil { return fmt.Errorf("hold config sync: %w", err) } - if state.ScannerEnabled { - scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) - if err != nil { - return fmt.Errorf("render scanner config: %w", err) - } - if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil { - return fmt.Errorf("scanner config sync: %w", err) - } - } } else { fmt.Println("Creating hold server...") - holdUserData, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled) + holdUserData, err := generateHoldCloudInit(cfg, vals) if err != nil { return err } - hold, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Hold(), holdUserData) + hold, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Hold(), holdUserData) if err != nil { return fmt.Errorf("create hold: %w", err) } @@ -292,15 +299,63 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w fmt.Printf(" Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP) } + // 4b. Scanner server. Its own box: the scanner is memory-bound (Grype's + // database working set plus Syft's in-memory catalog) and sharing a 1 GB + // host with the hold swapped that host unresponsive for eleven hours on + // 2026-09-12. It reaches the hold over the private network, so the hold's + // private IP has to be known before its config can be rendered. + scannerCreated := false + if scannerEnabled { + vals.ScannerHoldURL = scannerHoldURL(state.Hold.PrivateIP) + if state.Scanner.UUID != "" { + fmt.Printf("Scanner: %s (exists)\n", state.Scanner.UUID) + scannerScript, err := generateScannerCloudInit(cfg, vals) + if err != nil { + return err + } + if err := syncCloudInit("scanner", state.Scanner.PublicIP, scannerScript); err != nil { + return err + } + scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) + if err != nil { + return fmt.Errorf("render scanner config: %w", err) + } + if err := syncConfigKeys("scanner", state.Scanner.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil { + return fmt.Errorf("scanner config sync: %w", err) + } + } else { + fmt.Println("Creating scanner server...") + scannerUserData, err := generateScannerCloudInit(cfg, vals) + if err != nil { + return err + } + scanner, err := createServer(ctx, svc, cfg, cfg.ScannerPlan, templateUUID, state.Network.UUID, naming.Scanner(), scannerUserData) + if err != nil { + return fmt.Errorf("create scanner: %w", err) + } + state.Scanner = *scanner + _ = saveState(state) + scannerCreated = true + fmt.Printf(" Scanner: %s (public: %s, private: %s)\n", scanner.UUID, scanner.PublicIP, scanner.PrivateIP) + } + } + // 5. Firewall rules (idempotent — replaces all rules) fmt.Println("Configuring firewall rules...") - for _, s := range []struct { + firewalled := []struct { name string uuid string }{ {"appview", state.Appview.UUID}, {"hold", state.Hold.UUID}, - } { + } + if state.Scanner.UUID != "" { + firewalled = append(firewalled, struct { + name string + uuid string + }{"scanner", state.Scanner.UUID}) + } + for _, s := range firewalled { if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil { return fmt.Errorf("firewall %s: %w", s.name, err) } @@ -372,7 +427,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w } // 7. Build locally and deploy binaries to new servers - if appviewCreated || holdCreated { + if appviewCreated || holdCreated || scannerCreated { rootDir := projectRoot() if err := runMakeBuildTrixie(rootDir, withBilling); err != nil { @@ -393,6 +448,11 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w return err } } + if scannerCreated { + if err := waitForSetup(state.Scanner.PublicIP, "scanner"); err != nil { + return err + } + } fmt.Println("\nDeploying binaries...") if appviewCreated { @@ -415,12 +475,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil { return fmt.Errorf("upload hold: %w", err) } - if state.ScannerEnabled { - scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner") - scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner() - if err := scpFile(scannerLocal, state.Hold.PublicIP, scannerRemote); err != nil { - return fmt.Errorf("upload scanner: %w", err) - } + } + if scannerCreated { + scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner") + scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner() + if err := scpFile(scannerLocal, state.Scanner.PublicIP, scannerRemote); err != nil { + return fmt.Errorf("upload scanner: %w", err) } } } @@ -459,9 +519,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w fmt.Println("SSH access:") fmt.Printf(" ssh root@%s # appview\n", state.Appview.PublicIP) fmt.Printf(" ssh root@%s # hold\n", state.Hold.PublicIP) + if state.Scanner.UUID != "" { + fmt.Printf(" ssh root@%s # scanner\n", state.Scanner.PublicIP) + } fmt.Println() fmt.Println("Next steps:") - if appviewCreated || holdCreated { + if appviewCreated || holdCreated || scannerCreated { fmt.Println(" 1. Edit configs if needed, then start services:") } else { fmt.Println(" 1. Start services:") @@ -588,12 +651,15 @@ func objectStorageRegion(zone string) string { } } -func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, templateUUID, networkUUID, title, userData string) (*ServerState, error) { +func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, plan, templateUUID, networkUUID, title, userData string) (*ServerState, error) { if cfg.SSHPublicKey == "" { return nil, fmt.Errorf("creating server %s requires --ssh-key (path to SSH public key file)", title) } + if plan == "" { + return nil, fmt.Errorf("creating server %s: no plan selected", title) + } storageTier := "maxiops" - if strings.HasPrefix(strings.ToUpper(cfg.Plan), "DEV-") { + if isStarterPlan(plan) { storageTier = "standard" } @@ -602,7 +668,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t plans, err := svc.GetPlans(ctx) if err == nil { for _, p := range plans.Plans { - if p.Name == cfg.Plan { + if p.Name == plan { diskSize = p.StorageSize break } @@ -613,7 +679,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t Zone: cfg.Zone, Title: title, Hostname: title, - Plan: cfg.Plan, + Plan: plan, Metadata: upcloud.True, UserData: userData, Firewall: "on", @@ -737,17 +803,26 @@ const lbClientTimeout = 10 // httpFrontendProperties are the https frontend's properties. // -// HTTP/2 is the one that matters for the browser: over HTTP/1.1 a browser -// opens at most ~6 connections per origin, so a page that fans out many small -// requests (the admin crew tab issues one per member) queues them six at a -// time and blocks every other request to the same host behind them. h2 -// multiplexes them over a single connection and the queue disappears. +// HTTP/2 stays OFF on this frontend, and ensureLBHTTP2 turns it off on an LB +// that has it on. It was on until 2026-09-12, for the browser's sake (h2 +// multiplexes the admin crew tab's fan-out over one connection instead of +// six). Two things changed the answer: // -// Clients that need an HTTP/1.1 upgrade are unaffected: h2 is negotiated per -// connection via ALPN, so a WebSocket client simply selects http/1.1. +// - Re-enabling it that day caused a 25-minute /auth/token outage. The +// appview and the hold call each other through this LB on long-lived Go +// HTTP clients; the frontend reconfigure left those connections open at +// TCP level but dead, Go's HTTP/2 transport multiplexed every call onto +// them, and no transport here sets ReadIdleTimeout to notice. Only +// restarting both services recovered. Over HTTP/1.1 a dead connection +// fails one request and the pool opens another. +// - The 64 KB HTTP/2 stream window capped pushes at ~2.5 MB/s through the +// LB; median push fell from 16.5 s to 3.1 s when it went off. +// +// The registry and web hostnames are fronted by Bunny now, so the browser +// case this bought is served by the CDN's own h2 edge rather than by the LB. func httpFrontendProperties() *upcloud.LoadBalancerFrontendProperties { return &upcloud.LoadBalancerFrontendProperties{ - HTTP2Enabled: new(true), + HTTP2Enabled: new(false), TimeoutClient: lbClientTimeout, } } @@ -974,13 +1049,28 @@ func labelerFrontendRule(labelerDomain string) request.LoadBalancerFrontendRule } // ensureLBCertificates reconciles TLS certificate bundles on the load balancer. -// It skips domains that already have a TLS config attached and creates missing ones. +// It skips domains that already have a TLS config attached and creates missing +// ones, but only for domains whose DNS resolves to this LB. A dynamic +// (Let's Encrypt) bundle is validated over HTTP at the domain, so a domain +// fronted by a CDN cannot be issued here anyway; requesting it only leaves a +// bundle stuck in validation. The .cr registry domains moved behind Bunny on +// 2026-09-12 and their bundles were removed by hand; on 2026-09-13 this +// function put them straight back. Hence the check. func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error { lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID}) if err != nil { return fmt.Errorf("get load balancer: %w", err) } + lbAddrs := map[string]bool{} + for _, n := range lb.Networks { + if n.Type == upcloud.LoadBalancerNetworkTypePublic && n.DNSName != "" { + for _, a := range lookupHost(n.DNSName) { + lbAddrs[a] = true + } + } + } + // Build set of existing TLS config names on the "https" frontend existing := make(map[string]bool) for _, fe := range lb.Frontends { @@ -997,6 +1087,10 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri fmt.Printf(" TLS certificate: %s (exists)\n", domain) continue } + if !pointsAt(domain, lbAddrs) { + fmt.Printf(" TLS certificate: %s skipped (DNS does not point at this LB; fronted elsewhere or not yet delegated)\n", domain) + continue + } bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{ Type: upcloud.LoadBalancerCertificateBundleTypeDynamic, @@ -1025,11 +1119,32 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri return nil } +// lookupHost resolves name to its IPv4/IPv6 addresses; failures resolve to none. +func lookupHost(name string) []string { + addrs, err := net.LookupHost(name) + if err != nil { + return nil + } + return addrs +} + +// pointsAt reports whether any address domain resolves to is one of lbAddrs. +// An empty lbAddrs (the LB's DNS name did not resolve) matches nothing, so +// no bundle is requested when the precondition cannot be checked. +func pointsAt(domain string, lbAddrs map[string]bool) bool { + for _, a := range lookupHost(domain) { + if lbAddrs[a] { + return true + } + } + return false +} + // ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule. // This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers, // overwriting any pre-existing values (prevents spoofing). -// ensureLBHTTP2 reconciles HTTP/2 and the client timeout onto an LB that -// already exists. +// ensureLBHTTP2 reconciles the HTTP/2 flag (off, see httpFrontendProperties) +// and the client timeout onto an LB that already exists. // // createLoadBalancer only runs when there is no LB yet, so without this the // properties above would reach a fresh deployment and never an existing one. @@ -1045,10 +1160,11 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err } want := httpFrontendProperties() + wantHTTP2 := want.HTTP2Enabled != nil && *want.HTTP2Enabled haveHTTP2 := fe.Properties != nil && fe.Properties.HTTP2Enabled != nil && *fe.Properties.HTTP2Enabled haveTimeout := fe.Properties != nil && fe.Properties.TimeoutClient == want.TimeoutClient - if haveHTTP2 && haveTimeout { - fmt.Println(" Frontend HTTP/2 + timeout: already set") + if haveHTTP2 == wantHTTP2 && haveTimeout { + fmt.Printf(" Frontend HTTP/2 %s + timeout_client=%ds: already set\n", onOff(wantHTTP2), want.TimeoutClient) } else { if _, err := svc.ModifyLoadBalancerFrontend(ctx, &request.ModifyLoadBalancerFrontendRequest{ ServiceUUID: lbUUID, @@ -1059,12 +1175,19 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err }); err != nil { return fmt.Errorf("modify https frontend: %w", err) } - fmt.Printf(" Frontend HTTP/2: enabled (timeout_client=%ds)\n", want.TimeoutClient) + fmt.Printf(" Frontend HTTP/2: %s (timeout_client=%ds)\n", onOff(wantHTTP2), want.TimeoutClient) } return nil } +func onOff(b bool) string { + if b { + return "on" + } + return "off" +} + func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error { rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{ ServiceUUID: lbUUID, @@ -1449,9 +1572,16 @@ func syncCloudInit(name, ip, localScript string) error { } fmt.Printf(" Re-run cloud-init on %s? [Y/n] ", name) - scanner := bufio.NewScanner(os.Stdin) - scanner.Scan() - answer := strings.TrimSpace(strings.ToLower(scanner.Text())) + answer, ok := readLine() + if !ok { + // No answer at all (stdin closed, or a non-interactive run) must not + // re-run setup on a production server. Only a typed yes does. + fmt.Printf(" no answer, skipped\n") + if err := writeRemoteCloudInit(ip, localScript); err != nil { + fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err) + } + return nil + } if answer != "" && answer != "y" && answer != "yes" { fmt.Printf(" Skipped\n") // Still update the remote reference so next provision sees an accurate diff @@ -1479,6 +1609,33 @@ func syncCloudInit(name, ip, localScript string) error { return nil } +// scannerHoldURL is the hold's WebSocket address from the scanner server: the +// hold's private IP over the SDN, never the load balancer (whose 10s client +// timeout would cut the long-lived scan socket). +func scannerHoldURL(holdPrivateIP string) string { + if holdPrivateIP == "" { + return "" + } + return "ws://" + holdPrivateIP + ":8080" +} + +// stdin is shared by every prompt in the tool. Each prompt used to wrap +// os.Stdin in its own bufio.Scanner, and the first one swallowed everything +// that was already buffered (all of a piped "n\nn\n"), so the second prompt +// read EOF, took the default, and re-ran cloud-init on the production hold +// (2026-09-13). One reader keeps the unread lines for the next prompt. +var stdin = bufio.NewReader(os.Stdin) + +// readLine returns the next line typed on stdin, lower-cased and trimmed, and +// false when stdin has nothing more to give. +func readLine() (string, bool) { + line, err := stdin.ReadString('\n') + if err != nil && line == "" { + return "", false + } + return strings.TrimSpace(strings.ToLower(line)), true +} + // generateScannerSecret generates a random 32-byte hex-encoded shared secret // for authenticating scanner-to-hold WebSocket connections. func generateScannerSecret() (string, error) { @@ -1515,11 +1672,25 @@ func waitForSetup(ip, name string) error { time.Sleep(10 * time.Second) } + // Poll rather than `cloud-init status --wait`: runSSH caps one command at + // five minutes, and a fresh server's first boot (apt upgrade plus git, + // gcc, nodejs, npm) takes longer than that. The 2026-09-13 scanner + // provision timed out here with the server healthy and half installed. fmt.Printf(" %s: waiting for cloud-init...\n", name) - _, err := runSSH(ip, "cloud-init status --wait 2>/dev/null || true", false) - if err != nil { - return fmt.Errorf("cloud-init wait on %s: %w", name, err) + deadline := time.Now().Add(20 * time.Minute) + for { + out, err := runSSH(ip, "cloud-init status 2>/dev/null | head -1", false) + status := strings.TrimSpace(out) + switch { + case err == nil && strings.HasPrefix(status, "status: done"): + fmt.Printf(" %s: ready\n", name) + return nil + case err == nil && strings.HasPrefix(status, "status: error"): + return fmt.Errorf("cloud-init on %s finished with errors; see /var/log/cloud-init-output.log on the server", name) + } + if time.Now().After(deadline) { + return fmt.Errorf("cloud-init on %s not done after 20 minutes (last status %q)", name, status) + } + time.Sleep(15 * time.Second) } - fmt.Printf(" %s: ready\n", name) - return nil } diff --git a/deploy/upcloud/scanner_server_test.go b/deploy/upcloud/scanner_server_test.go new file mode 100644 index 0000000..8e7d9c0 --- /dev/null +++ b/deploy/upcloud/scanner_server_test.go @@ -0,0 +1,134 @@ +package main + +import ( + "strings" + "testing" + + "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud" +) + +// The scanner's server is chosen by shape, not by name, because UpCloud +// renames tiers. Developer plans win a tie: same shape, cheapest tier. +func TestMatchPlanPrefersDeveloperTier(t *testing.T) { + plans := []upcloud.Plan{ + {Name: "2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 80}, + {Name: "GPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30, GPUAmount: 1}, + {Name: "HICPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30}, + {Name: "STARTER-2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30}, + {Name: "STARTER-1xCPU-1GB", CoreNumber: 1, MemoryAmount: 1024, StorageSize: 10}, + } + got := matchPlan(plans, scannerPlanSpec) + if strings.Join(got, ",") != "STARTER-2xCPU-4GB,HICPU-2xCPU-4GB-30GB" { + t.Fatalf("matchPlan = %v", got) + } + // Both the current and the pre-2026 tier names take standard storage; + // asking for maxiops on them fails with TIER_INVALID (seen 2026-09-13). + for _, n := range []string{"STARTER-2xCPU-4GB", "DEV-2xCPU-4GB-30GB", "dev-1xcpu-1gb-10gb"} { + if !isStarterPlan(n) { + t.Errorf("%s should be a starter-tier plan", n) + } + } + if isStarterPlan("2xCPU-4GB") || isStarterPlan("HICPU-2xCPU-4GB-30GB") { + t.Error("general purpose and high-CPU plans are not starter tier") + } + if len(matchPlan(plans, planSpec{Cores: 8, MemoryMB: 65536, DiskGB: 1000})) != 0 { + t.Fatal("a shape nobody sells must match nothing, so the picker takes over") + } + if !hasPlan(plans, defaultScannerPlan) || hasPlan(plans, "STARTER-2xCPU-4GB-80GB") { + t.Fatal("hasPlan must match the exact plan name") + } + // The pinned default must be the shape the spec describes, or the two + // paths in resolveScannerPlan would buy different servers. + if got := matchPlan(plans, scannerPlanSpec); got[0] != defaultScannerPlan { + t.Fatalf("defaultScannerPlan %s is not the DEV plan of shape %+v (matcher chose %s)", defaultScannerPlan, scannerPlanSpec, got[0]) + } +} + +// Limits step down from the plan's RAM: MemoryMax leaves the OS ~600 MiB, +// MemoryHigh sits below that, and the Go soft limit below both so the +// collector runs before the kernel reclaims or kills. +func TestScannerMemoryLimits(t *testing.T) { + goLimit, high, maxMem := scannerMemoryLimits(4096) + if goLimit != "2560MiB" || high != "3072M" || maxMem != "3500M" { + t.Fatalf("4 GB plan: got %s / %s / %s", goLimit, high, maxMem) + } + goLimit, _, _ = scannerMemoryLimits(1024) + if goLimit != "512MiB" { + t.Fatalf("the Go limit must never drop below the scanner's own 512 MiB default, got %s", goLimit) + } +} + +func TestScannerCloudInitTargetsHoldPrivateIP(t *testing.T) { + cfg := &InfraConfig{ClientName: "seamark"} + vals := &ConfigValues{ + BasePath: "/var/lib/seamark", + ScannerSecret: "abc", + ScannerHoldURL: scannerHoldURL("10.0.1.3"), + } + script, err := generateScannerCloudInit(cfg, vals) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `url: "ws://10.0.1.3:8080"`, + "mkdir -p /var/lib/seamark/scanner/vulndb /var/lib/seamark/scanner/tmp", + "Environment=GOMEMLIMIT=2560MiB", + "MemorySwapMax=0", + "systemctl enable seamark-scanner", + } { + if !strings.Contains(script, want) { + t.Errorf("scanner cloud-init missing %q", want) + } + } + if strings.Contains(script, "seamark-hold.service") { + t.Error("a standalone scanner must not order itself after a hold unit that is not on its host") + } + + if _, err := generateScannerCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark"}); err == nil { + t.Error("rendering a scanner config with no hold address must fail, not ship ws://:8080") + } + if scannerHoldURL("") != "" { + t.Error("no hold IP must render as no URL so the guard above fires") + } +} + +// The hold's cloud-init no longer carries the scanner: its unit, config and +// directories belong to the scanner server. +func TestHoldCloudInitHasNoScanner(t *testing.T) { + cfg := &InfraConfig{ClientName: "seamark"} + script, err := generateHoldCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark", HoldDomain: "x.cove.seamark.dev"}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(script, "seamark-scanner") || strings.Contains(script, "vulndb") { + t.Error("hold cloud-init still installs the scanner") + } +} + +// Re-enabling frontend HTTP/2 on 2026-09-12 stranded the appview<->hold +// connections through the LB for 25 minutes. Every provision run reconciles +// this property, so the default is the outage switch. +func TestLBFrontendHTTP2StaysOff(t *testing.T) { + p := httpFrontendProperties() + if p.HTTP2Enabled == nil || *p.HTTP2Enabled { + t.Fatal("LB frontend HTTP/2 must be off (see httpFrontendProperties)") + } + if p.TimeoutClient != lbClientTimeout { + t.Fatalf("timeout_client = %d, want %d", p.TimeoutClient, lbClientTimeout) + } +} + +// A domain fronted by a CDN never resolves to the LB, so the tool must not +// request a certificate for it (it re-added the .cr bundles on 2026-09-13). +func TestPointsAtRequiresOverlap(t *testing.T) { + lb := map[string]bool{"203.0.113.10": true} + if pointsAt("localhost", lb) { + t.Fatal("localhost does not resolve to the LB") + } + if pointsAt("localhost", map[string]bool{}) { + t.Fatal("an unresolved LB must match nothing") + } + if !pointsAt("localhost", map[string]bool{"127.0.0.1": true, "::1": true}) { + t.Fatal("a domain resolving to an LB address must match") + } +} diff --git a/deploy/upcloud/state.go b/deploy/upcloud/state.go index f18e024..82f1464 100644 --- a/deploy/upcloud/state.go +++ b/deploy/upcloud/state.go @@ -16,6 +16,7 @@ type InfraState struct { Network StateRef `json:"network"` Appview ServerState `json:"appview"` Hold ServerState `json:"hold"` + Scanner ServerState `json:"scanner,omitempty"` LB StateRef `json:"loadbalancer"` ObjectStorage ObjectStorageState `json:"object_storage"` ScannerEnabled bool `json:"scanner_enabled,omitempty"` diff --git a/deploy/upcloud/status.go b/deploy/upcloud/status.go index 7d5abc8..a463546 100644 --- a/deploy/upcloud/status.go +++ b/deploy/upcloud/status.go @@ -43,7 +43,7 @@ func cmdStatus(token string) error { fmt.Printf("Zone: %s\n\n", state.Zone) // Server status - for _, s := range []struct { + servers := []struct { name string ss ServerState serviceName string @@ -51,7 +51,16 @@ func cmdStatus(token string) error { }{ {"Appview", state.Appview, naming.Appview(), "http://localhost:5000/health"}, {"Hold", state.Hold, naming.Hold(), "http://localhost:8080/xrpc/_health"}, - } { + } + if state.ScannerEnabled || state.Scanner.UUID != "" { + servers = append(servers, struct { + name string + ss ServerState + serviceName string + healthURL string + }{"Scanner", state.Scanner, naming.Scanner(), "http://localhost:9090/healthz"}) + } + for _, s := range servers { fmt.Printf("%-8s UUID: %s\n", s.name, s.ss.UUID) fmt.Printf(" Public: %s\n", s.ss.PublicIP) fmt.Printf(" Private: %s\n", s.ss.PrivateIP) @@ -90,31 +99,6 @@ func cmdStatus(token string) error { fmt.Println() } - // Scanner status (runs on hold server) - if state.ScannerEnabled { - fmt.Printf("Scanner (on hold server)\n") - if state.Hold.PublicIP != "" { - output, err := runSSH(state.Hold.PublicIP, fmt.Sprintf( - "systemctl is-active %s 2>/dev/null || echo 'inactive'; curl -sf http://localhost:9090/healthz > /dev/null 2>&1 && echo 'health:ok' || echo 'health:fail'", - naming.Scanner(), - ), false) - if err != nil { - fmt.Printf(" Service: unreachable\n") - } else { - lines := strings.SplitSeq(strings.TrimSpace(output), "\n") - for line := range lines { - line = strings.TrimSpace(line) - if line == "active" || line == "inactive" { - fmt.Printf(" Service: %s\n", line) - } else if after, ok := strings.CutPrefix(line, "health:"); ok { - fmt.Printf(" Health: %s\n", after) - } - } - } - } - fmt.Println() - } - // LB status if state.LB.UUID != "" { fmt.Printf("Load Balancer: %s\n", state.LB.UUID) diff --git a/deploy/upcloud/systemd/scanner.service.tmpl b/deploy/upcloud/systemd/scanner.service.tmpl index ef1c742..f533e76 100644 --- a/deploy/upcloud/systemd/scanner.service.tmpl +++ b/deploy/upcloud/systemd/scanner.service.tmpl @@ -1,6 +1,6 @@ [Unit] Description={{.DisplayName}} Scanner (Vulnerability Scanning) -After=network-online.target {{.HoldServiceName}}.service +After=network-online.target Wants=network-online.target [Service] @@ -11,38 +11,26 @@ ExecStart={{.BinaryPath}} serve --config {{.ConfigPath}} Restart=on-failure RestartSec=10 -# Memory containment. The scanner sets a 512 MiB Go soft limit itself -# (GOMEMLIMIT, see cmd/scanner/main.go), but that is soft: the runtime -# collects harder to respect it and never fails an allocation to honour it, so -# a large enough image walks straight through it. Without a cgroup cap the -# kernel OOM killer chooses its own victim, and on a shared host the other -# large process is the hold, meaning the scanner's overshoot kills the service -# it reports to. These make the scanner the one that dies. +# The scanner runs on its own server, so it is the only large process on the +# host and the limits below are sized to the box rather than to a neighbour. # -# MemoryHigh throttles and reclaims; MemoryMax kills. Sized from measurement: -# one scan of a node:22-class image peaks at 561 MiB RSS with the Go limit in -# force, so MemoryHigh sits above that and MemoryMax leaves headroom for the -# hold and the OS on a 1 GiB host. Raise both together with scanner.workers. -MemoryHigh=640M -MemoryMax=768M +# GOMEMLIMIT is the Go soft limit: the runtime collects harder as the heap +# approaches it and never fails an allocation to honour it, so a large enough +# image walks straight through. It sits below MemoryHigh so the GC gets a +# chance before the kernel starts reclaiming. Without the variable the scanner +# defaults itself to 512 MiB (cmd/scanner/main.go), which on a 4 GB host only +# wastes CPU on collection. +Environment=GOMEMLIMIT={{.GoMemLimit}} -# Scheduling priority. The scanner is the lowest-value process on a shared -# host: a scan finishing a minute later costs nothing, a hold that cannot -# answer a pull costs a user. Syft's extraction saturates CPU and writes -# several times the compressed image size to disk, so left at the default -# weight it competes evenly with the services that matter. -# -# Weights apply only under contention: the scanner still uses the whole box -# when nothing else wants it, which is what makes this better than the fixed -# inter-job sleep in worker.go (that yields on a timer whether or not anyone -# needs the CPU, and yields nothing while a scan is actually running). -# Everything else runs at the default weight of 100. -# -# IOWeight needs the io controller with a scheduler that honours it (BFQ, or -# io.cost configured); where it is unsupported systemd ignores it silently and -# CPUWeight still applies. -CPUWeight=20 -IOWeight=20 +# MemoryHigh throttles and reclaims; MemoryMax kills. MemorySwapMax=0 is the +# one that matters: with swap available a scanner over its cap is not killed, +# it is paged out, and the host thrashes instead of failing. That is what took +# the shared hold host down for eleven hours on 2026-09-12 (644 MiB resident, +# 1.9 GB swapped). Refusing swap turns that into an OOM kill of one job and a +# clean restart ten seconds later. +MemoryHigh={{.MemoryHigh}} +MemoryMax={{.MemoryMax}} +MemorySwapMax=0 ReadWritePaths={{.DataDir}} ProtectSystem=strict diff --git a/deploy/upcloud/teardown.go b/deploy/upcloud/teardown.go index 7339602..f06a490 100644 --- a/deploy/upcloud/teardown.go +++ b/deploy/upcloud/teardown.go @@ -1,11 +1,8 @@ package main import ( - "bufio" "context" "fmt" - "os" - "strings" "time" "github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request" @@ -39,14 +36,15 @@ func cmdTeardown(token string) error { fmt.Printf(" Zone: %s\n", state.Zone) fmt.Printf(" Appview: %s (%s)\n", state.Appview.UUID, state.Appview.PublicIP) fmt.Printf(" Hold: %s (%s)\n", state.Hold.UUID, state.Hold.PublicIP) + if state.Scanner.UUID != "" { + fmt.Printf(" Scanner: %s (%s)\n", state.Scanner.UUID, state.Scanner.PublicIP) + } fmt.Printf(" Network: %s\n", state.Network.UUID) fmt.Printf(" LB: %s\n", state.LB.UUID) fmt.Println() fmt.Print("Type 'yes' to confirm: ") - scanner := bufio.NewScanner(os.Stdin) - scanner.Scan() - if strings.TrimSpace(scanner.Text()) != "yes" { + if answer, _ := readLine(); answer != "yes" { fmt.Println("Aborted.") return nil } @@ -76,6 +74,7 @@ func cmdTeardown(token string) error { }{ {"appview", state.Appview.UUID}, {"hold", state.Hold.UUID}, + {"scanner", state.Scanner.UUID}, } { if s.uuid == "" { continue diff --git a/deploy/upcloud/update.go b/deploy/upcloud/update.go index 5444b6b..e4a9ef9 100644 --- a/deploy/upcloud/update.go +++ b/deploy/upcloud/update.go @@ -18,7 +18,7 @@ var updateCmd = &cobra.Command{ Use: "update [target]", Short: "Deploy updates to servers", Args: cobra.MaximumNArgs(1), - ValidArgs: []string{"all", "appview", "hold"}, + ValidArgs: []string{"all", "appview", "hold", "scanner"}, RunE: func(cmd *cobra.Command, args []string) error { target := "all" if len(args) > 0 { @@ -37,7 +37,7 @@ var sshCmd = &cobra.Command{ Use: "ssh ", Short: "SSH into a server", Args: cobra.ExactArgs(1), - ValidArgs: []string{"appview", "hold"}, + ValidArgs: []string{"appview", "hold", "scanner"}, RunE: func(cmd *cobra.Command, args []string) error { return cmdSSH(args[0]) }, @@ -45,7 +45,7 @@ var sshCmd = &cobra.Command{ func init() { updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.") - updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold") + updateCmd.Flags().Bool("with-scanner", false, "Enable the vulnerability scanner (its server is created by `provision --with-scanner`)") updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview") updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into /predeploy-/ before replacing them") updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup") @@ -68,7 +68,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u naming := state.Naming() rootDir := projectRoot() - // Enable scanner retroactively via --with-scanner on update + // Enable scanner retroactively via --with-scanner on update. The secret + // lands in the hold's config on this run; the scanner server itself is + // created by provision, which is the only command that creates servers. if withScanner && !state.ScannerEnabled { state.ScannerEnabled = true if state.ScannerSecret == "" { @@ -80,6 +82,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u fmt.Printf("Generated scanner shared secret\n") } _ = saveState(state) + if state.Scanner.UUID == "" { + fmt.Println("Scanner enabled. Run `provision --with-scanner` to create its server, then `update scanner` deploys to it.") + } } // Enable labeler retroactively via --with-labeler on update @@ -90,38 +95,64 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u vals := configValsFromState(state) + // renderUnit is per target because the scanner unit carries memory limits + // the generic serviceUnitParams does not know about. + standardUnit := func(tmpl, binaryName, configPath, serviceName string) func() (string, error) { + return func() (string, error) { + return renderServiceUnit(tmpl, serviceUnitParams{ + DisplayName: naming.DisplayName(), + User: naming.SystemUser(), + BinaryPath: naming.InstallDir() + "/bin/" + binaryName, + ConfigPath: configPath, + DataDir: naming.BasePath(), + ServiceName: serviceName, + }) + } + } + targets := map[string]struct { ip string binaryName string - buildCmd string localBinary string serviceName string healthURL string configTmpl string configPath string - unitTmpl string + renderUnit func() (string, error) + setup string // extra remote shell run before restart; empty for none }{ "appview": { ip: state.Appview.PublicIP, binaryName: naming.Appview(), - buildCmd: "appview", localBinary: "atcr-appview", serviceName: naming.Appview(), healthURL: "http://localhost:5000/health", configTmpl: appviewConfigTmpl, configPath: naming.AppviewConfigPath(), - unitTmpl: appviewServiceTmpl, + renderUnit: standardUnit(appviewServiceTmpl, naming.Appview(), naming.AppviewConfigPath(), naming.Appview()), }, "hold": { ip: state.Hold.PublicIP, binaryName: naming.Hold(), - buildCmd: "hold", localBinary: "atcr-hold", serviceName: naming.Hold(), healthURL: "http://localhost:8080/xrpc/_health", configTmpl: holdConfigTmpl, configPath: naming.HoldConfigPath(), - unitTmpl: holdServiceTmpl, + renderUnit: standardUnit(holdServiceTmpl, naming.Hold(), naming.HoldConfigPath(), naming.Hold()), + }, + "scanner": { + ip: state.Scanner.PublicIP, + binaryName: naming.Scanner(), + localBinary: "atcr-scanner", + serviceName: naming.Scanner(), + healthURL: "http://localhost:9090/healthz", + configTmpl: scannerConfigTmpl, + configPath: naming.ScannerConfigPath(), + renderUnit: func() (string, error) { + return renderScannerServiceUnit(scannerUnitParams(naming)) + }, + setup: scannerDirsScript(naming), }, } @@ -129,10 +160,18 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u switch target { case "all": toUpdate = []string{"appview", "hold"} + if state.Scanner.UUID != "" { + toUpdate = append(toUpdate, "scanner") + } case "appview", "hold": toUpdate = []string{target} + case "scanner": + if state.Scanner.UUID == "" { + return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one") + } + toUpdate = []string{target} default: - return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target) + return fmt.Errorf("unknown target: %s (use: all, appview, hold, scanner)", target) } // Build all binaries via `make build-trixie` so output links against @@ -182,14 +221,7 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u } // Sync systemd service unit - renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{ - DisplayName: naming.DisplayName(), - User: naming.SystemUser(), - BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName, - ConfigPath: t.configPath, - DataDir: naming.BasePath(), - ServiceName: t.serviceName, - }) + renderedUnit, err := t.renderUnit() if err != nil { return fmt.Errorf("render %s service unit: %w", name, err) } @@ -210,58 +242,10 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u daemonReload = "systemctl daemon-reload" } - // Scanner additions for hold server - scannerRestart := "" - scannerHealthCheck := "" - if name == "hold" && state.ScannerEnabled { - // Sync scanner config keys - scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) - if err != nil { - return fmt.Errorf("render scanner config: %w", err) + if t.setup != "" { + if _, err := runSSH(t.ip, t.setup, false); err != nil { + return fmt.Errorf("%s setup: %w", name, err) } - if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML, false); err != nil { - return fmt.Errorf("scanner config sync: %w", err) - } - - // Sync scanner service unit - scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{ - DisplayName: naming.DisplayName(), - User: naming.SystemUser(), - BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(), - ConfigPath: naming.ScannerConfigPath(), - DataDir: naming.BasePath(), - ServiceName: naming.Scanner(), - HoldServiceName: naming.Hold(), - }) - if err != nil { - return fmt.Errorf("render scanner service unit: %w", err) - } - scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit) - if err != nil { - return fmt.Errorf("scanner service unit sync: %w", err) - } - if scannerUnitChanged { - daemonReload = "systemctl daemon-reload" - } - - // Upload scanner binary - scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner") - scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner() - if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil { - return fmt.Errorf("upload scanner: %w", err) - } - - // Ensure scanner data dirs exist on server - scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp -chown -R %s:%s %s`, - naming.ScannerDataDir(), naming.ScannerDataDir(), - naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir()) - if _, err := runSSH(t.ip, scannerSetup, false); err != nil { - return fmt.Errorf("scanner dir setup: %w", err) - } - - scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner()) - scannerHealthCheck = renderHealthProbe("scanner", "http://localhost:9090/healthz", healthProbeTries) } // Labeler additions for appview server @@ -324,9 +308,9 @@ chown -R %s:%s %s`, // status and only shows up as a health URL that never answers. restartScript := fmt.Sprintf(`set -euo pipefail %s -systemctl restart %s%s%s -%s%s%s`, daemonReload, t.serviceName, scannerRestart, labelerRestart, - renderHealthProbe(name, t.healthURL, healthProbeTries), scannerHealthCheck, labelerHealthCheck) +systemctl restart %s%s +%s%s`, daemonReload, t.serviceName, labelerRestart, + renderHealthProbe(name, t.healthURL, healthProbeTries), labelerHealthCheck) output, err := runSSH(t.ip, restartScript, true) if err != nil { @@ -340,9 +324,6 @@ systemctl restart %s%s%s results := parseProbes(output) reportProbe(results, name, t.ip, t.serviceName) - if name == "hold" && state.ScannerEnabled { - reportProbe(results, "scanner", t.ip, naming.Scanner()) - } if name == "appview" && state.LabelerEnabled { reportProbe(results, "labeler", t.ip, naming.Labeler()) } @@ -361,17 +342,18 @@ func configValsFromState(state *InfraState) *ConfigValues { labelerDomain := "labeler." + baseDomain return &ConfigValues{ - S3Endpoint: state.ObjectStorage.Endpoint, - S3Region: state.ObjectStorage.Region, - S3Bucket: state.ObjectStorage.Bucket, - S3AccessKey: state.ObjectStorage.AccessKeyID, - S3SecretKey: "", // not persisted in state; existing value on server is preserved - Zone: state.Zone, - HoldDomain: holdDomain, - HoldDid: "did:web:" + holdDomain, - LabelerDomain: labelerDomain, - BasePath: naming.BasePath(), - ScannerSecret: state.ScannerSecret, + S3Endpoint: state.ObjectStorage.Endpoint, + S3Region: state.ObjectStorage.Region, + S3Bucket: state.ObjectStorage.Bucket, + S3AccessKey: state.ObjectStorage.AccessKeyID, + S3SecretKey: "", // not persisted in state; existing value on server is preserved + Zone: state.Zone, + HoldDomain: holdDomain, + HoldDid: "did:web:" + holdDomain, + LabelerDomain: labelerDomain, + BasePath: naming.BasePath(), + ScannerSecret: state.ScannerSecret, + ScannerHoldURL: scannerHoldURL(state.Hold.PrivateIP), } } @@ -420,8 +402,13 @@ func cmdSSH(target string) error { ip = state.Appview.PublicIP case "hold": ip = state.Hold.PublicIP + case "scanner": + ip = state.Scanner.PublicIP + if ip == "" { + return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one") + } default: - return fmt.Errorf("unknown target: %s (use: appview, hold)", target) + return fmt.Errorf("unknown target: %s (use: appview, hold, scanner)", target) } fmt.Printf("Connecting to %s (%s)...\n", target, ip)