mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-20 17:24:16 +00:00
deploy: give the scanner its own server, and stop the tool from undoing production fixes on provision
The scanner shared the hold's 1 GB host and thrashed it twice: 11 hours on 2026-09-12 and again on the 13th (644 MiB resident plus 1.9 GB of swap, 504 on every repo page). It is memory-bound, not CPU-bound, so it now gets a dedicated STARTER-2xCPU-4GB server: own state entry, own plan flag (pinned name, shape match if UpCloud renames the tier again, picker last), own cloud-init, firewall, `update scanner`, `ssh scanner`, status, backup and teardown. Its config reaches the hold over the private network and its unit sets MemorySwapMax=0 so an overshoot is an OOM kill and a restart, not a wedged host. The hold's cloud-init and update paths no longer carry it. Three defects the first provision run exposed, all fixed here: - Frontend HTTP/2 defaulted to on and was reconciled onto the LB every run. Re-enabling it on the 12th stranded the appview<->hold connections for 25 minutes. Default is now off and reconciled off, with a guard test. - The TLS step requested Let's Encrypt bundles for every registry domain, re-adding the .cr ones that were removed when those moved behind Bunny. It now skips any domain whose DNS does not resolve to the LB. - Each prompt built its own bufio.Scanner on stdin, so the first swallowed every piped answer and the second read EOF and took the default, which re-ran cloud-init on the production hold. One shared reader, and no answer now means skip. Also: STARTER- plans take standard storage (maxiops fails with TIER_INVALID), and the cloud-init wait polls for up to 20 minutes instead of one SSH call capped at five, which a first boot with npm exceeds. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Hho5da4daoCoPBJ9tCrL7s
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
ba1f5eab40
commit
83092d9aee
@@ -91,8 +91,9 @@ func preDeployBackup(ip string, s backupSpec) (string, error) {
|
||||
|
||||
// backupSpecFor lists what a deploy of target replaces on its server: the
|
||||
// service binary, its config and unit, plus the companion service that ships
|
||||
// with it (scanner with hold, labeler with appview). The appview's SQLite
|
||||
// database is snapshotted too because migrations run on the next boot.
|
||||
// with it (labeler with appview). The scanner is its own target on its own
|
||||
// server. The appview's SQLite database is snapshotted too because migrations
|
||||
// run on the next boot.
|
||||
func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec {
|
||||
s := backupSpec{
|
||||
BaseDir: naming.BasePath(),
|
||||
@@ -115,11 +116,10 @@ func backupSpecFor(target string, naming Naming, state *InfraState, now time.Tim
|
||||
s.Binaries = []string{bin + naming.Hold()}
|
||||
s.Configs = []string{naming.HoldConfigPath()}
|
||||
s.Units = []string{naming.Hold()}
|
||||
if state.ScannerEnabled {
|
||||
s.Binaries = append(s.Binaries, bin+naming.Scanner())
|
||||
s.Configs = append(s.Configs, naming.ScannerConfigPath())
|
||||
s.Units = append(s.Units, naming.Scanner())
|
||||
}
|
||||
case "scanner":
|
||||
s.Binaries = []string{bin + naming.Scanner()}
|
||||
s.Configs = []string{naming.ScannerConfigPath()}
|
||||
s.Units = []string{naming.Scanner()}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -30,8 +30,16 @@ func TestBackupSpecForAppviewIncludesDatabaseAndLabeler(t *testing.T) {
|
||||
if h.SQLiteDB != "" {
|
||||
t.Fatalf("hold backup must not snapshot a database, got %q", h.SQLiteDB)
|
||||
}
|
||||
if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold,/opt/seamark/bin/seamark-scanner" {
|
||||
t.Fatalf("hold Binaries = %v", h.Binaries)
|
||||
if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold" {
|
||||
t.Fatalf("hold Binaries = %v (the scanner has its own server and its own backup)", h.Binaries)
|
||||
}
|
||||
|
||||
sc := backupSpecFor("scanner", naming, state, now, 5)
|
||||
if strings.Join(sc.Binaries, ",") != "/opt/seamark/bin/seamark-scanner" {
|
||||
t.Fatalf("scanner Binaries = %v", sc.Binaries)
|
||||
}
|
||||
if strings.Join(sc.Configs, ",") != "/etc/seamark/scanner.yaml" || strings.Join(sc.Units, ",") != "seamark-scanner" {
|
||||
t.Fatalf("scanner Configs = %v Units = %v", sc.Configs, sc.Units)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+85
-57
@@ -57,6 +57,9 @@ type ConfigValues struct {
|
||||
|
||||
// Scanner (auto-generated shared secret)
|
||||
ScannerSecret string // hex-encoded 32-byte secret; empty disables scanning
|
||||
// ScannerHoldURL is the hold's WebSocket address as seen from the scanner
|
||||
// server, e.g. "ws://10.0.1.3:8080" over the private network.
|
||||
ScannerHoldURL string
|
||||
}
|
||||
|
||||
// renderConfig executes a Go template with the given values.
|
||||
@@ -95,15 +98,49 @@ func renderServiceUnit(tmplStr string, p serviceUnitParams) (string, error) {
|
||||
}
|
||||
|
||||
// scannerServiceUnitParams holds values for rendering the scanner systemd unit.
|
||||
// Extends the standard fields with HoldServiceName for the After= dependency.
|
||||
// Extends the standard fields with the memory limits, which are sized to the
|
||||
// scanner's own server (see scannerPlanSpec) rather than shared with a hold.
|
||||
type scannerServiceUnitParams struct {
|
||||
DisplayName string // e.g. "Seamark"
|
||||
User string // e.g. "seamark"
|
||||
BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner"
|
||||
ConfigPath string // e.g. "/etc/seamark/scanner.yaml"
|
||||
DataDir string // e.g. "/var/lib/seamark"
|
||||
ServiceName string // e.g. "seamark-scanner"
|
||||
HoldServiceName string // e.g. "seamark-hold" (After= dependency)
|
||||
DisplayName string // e.g. "Seamark"
|
||||
User string // e.g. "seamark"
|
||||
BinaryPath string // e.g. "/opt/seamark/bin/seamark-scanner"
|
||||
ConfigPath string // e.g. "/etc/seamark/scanner.yaml"
|
||||
DataDir string // e.g. "/var/lib/seamark"
|
||||
ServiceName string // e.g. "seamark-scanner"
|
||||
GoMemLimit string // GOMEMLIMIT, e.g. "2560MiB"
|
||||
MemoryHigh string // systemd MemoryHigh, e.g. "3G"
|
||||
MemoryMax string // systemd MemoryMax, e.g. "3500M"
|
||||
}
|
||||
|
||||
// scannerMemoryLimits sizes the unit's memory knobs from the plan's RAM. The
|
||||
// OS and page cache keep roughly the last 500 MiB; MemoryMax sits just under
|
||||
// that line, MemoryHigh 500 MiB below it so reclaim starts before the kill,
|
||||
// and the Go soft limit another 500 MiB below so the collector gets the first
|
||||
// try. On the 4 GB scanner plan that is 2560MiB / 3G / 3500M.
|
||||
func scannerMemoryLimits(memoryMB int) (goMemLimit, memoryHigh, memoryMax string) {
|
||||
maxMB := memoryMB - 596
|
||||
highMB := maxMB - 428
|
||||
goMB := highMB - 512
|
||||
if goMB < 512 {
|
||||
goMB = 512
|
||||
}
|
||||
return fmt.Sprintf("%dMiB", goMB), fmt.Sprintf("%dM", highMB), fmt.Sprintf("%dM", maxMB)
|
||||
}
|
||||
|
||||
// scannerUnitParams builds the scanner unit parameters for a deployment.
|
||||
func scannerUnitParams(naming Naming) scannerServiceUnitParams {
|
||||
goLimit, high, maxMem := scannerMemoryLimits(scannerPlanSpec.MemoryMB)
|
||||
return scannerServiceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
|
||||
ConfigPath: naming.ScannerConfigPath(),
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: naming.Scanner(),
|
||||
GoMemLimit: goLimit,
|
||||
MemoryHigh: high,
|
||||
MemoryMax: maxMem,
|
||||
}
|
||||
}
|
||||
|
||||
func renderScannerServiceUnit(p scannerServiceUnitParams) (string, error) {
|
||||
@@ -242,10 +279,10 @@ echo "=== Labeler setup complete ==="
|
||||
return script + labelerPhase, nil
|
||||
}
|
||||
|
||||
// generateHoldCloudInit generates the cloud-init user-data script for the hold server.
|
||||
// When withScanner is true, a second phase is appended that creates scanner data
|
||||
// directories and installs a scanner systemd service. Binaries are deployed separately via SCP.
|
||||
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner bool) (string, error) {
|
||||
// generateHoldCloudInit generates the cloud-init user-data script for the hold
|
||||
// server. Binaries are deployed separately via SCP. The scanner has its own
|
||||
// server (generateScannerCloudInit) and no longer rides along here.
|
||||
func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) {
|
||||
naming := cfg.Naming()
|
||||
|
||||
configYAML, err := renderConfig(holdConfigTmpl, vals)
|
||||
@@ -265,7 +302,7 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo
|
||||
return "", fmt.Errorf("hold service unit: %w", err)
|
||||
}
|
||||
|
||||
script, err := generateCloudInit(cloudInitParams{
|
||||
return generateCloudInit(cloudInitParams{
|
||||
BinaryName: naming.Hold(),
|
||||
ServiceUnit: serviceUnit,
|
||||
ConfigYAML: configYAML,
|
||||
@@ -278,69 +315,60 @@ func generateHoldCloudInit(cfg *InfraConfig, vals *ConfigValues, withScanner boo
|
||||
LogFile: naming.LogFile(),
|
||||
DisplayName: naming.DisplayName(),
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
if !withScanner {
|
||||
return script, nil
|
||||
// generateScannerCloudInit generates the cloud-init user-data script for the
|
||||
// scanner's own server. It is the standard base script (packages, swap,
|
||||
// service user, config, unit) plus the vulnerability database and extraction
|
||||
// directories the scanner needs. The binary is deployed separately via SCP.
|
||||
// vals.ScannerHoldURL must already point at the hold's private address.
|
||||
func generateScannerCloudInit(cfg *InfraConfig, vals *ConfigValues) (string, error) {
|
||||
if vals.ScannerHoldURL == "" {
|
||||
return "", fmt.Errorf("scanner cloud-init: hold URL is empty (hold server must exist first)")
|
||||
}
|
||||
naming := cfg.Naming()
|
||||
|
||||
// Render scanner config YAML
|
||||
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
|
||||
configYAML, err := renderConfig(scannerConfigTmpl, vals)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("scanner config: %w", err)
|
||||
}
|
||||
|
||||
// Append scanner setup phase (no build — binary deployed via SCP)
|
||||
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
|
||||
ConfigPath: naming.ScannerConfigPath(),
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: naming.Scanner(),
|
||||
HoldServiceName: naming.Hold(),
|
||||
})
|
||||
serviceUnit, err := renderScannerServiceUnit(scannerUnitParams(naming))
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("scanner service unit: %w", err)
|
||||
}
|
||||
|
||||
// Escape single quotes for heredoc embedding
|
||||
scannerUnit = strings.ReplaceAll(scannerUnit, "'", "'\\''")
|
||||
scannerConfigYAML = strings.ReplaceAll(scannerConfigYAML, "'", "'\\''")
|
||||
script, err := generateCloudInit(cloudInitParams{
|
||||
BinaryName: naming.Scanner(),
|
||||
ServiceUnit: serviceUnit,
|
||||
ConfigYAML: configYAML,
|
||||
ConfigPath: naming.ScannerConfigPath(),
|
||||
ServiceName: naming.Scanner(),
|
||||
DataDir: naming.BasePath(),
|
||||
InstallDir: naming.InstallDir(),
|
||||
SystemUser: naming.SystemUser(),
|
||||
ConfigDir: naming.ConfigDir(),
|
||||
LogFile: naming.LogFile(),
|
||||
DisplayName: naming.DisplayName(),
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
scannerPhase := fmt.Sprintf(`
|
||||
# === Scanner Setup ===
|
||||
return script + scannerDirsScript(naming), nil
|
||||
}
|
||||
|
||||
# Scanner data dirs
|
||||
// scannerDirsScript creates the scanner's data directories. Shared between
|
||||
// cloud-init (first boot) and update (a server whose directories were removed).
|
||||
func scannerDirsScript(naming Naming) string {
|
||||
return fmt.Sprintf(`
|
||||
# Scanner data dirs (vulnerability database, layer extraction scratch)
|
||||
mkdir -p %s/vulndb %s/tmp
|
||||
chown -R %s:%s %s
|
||||
|
||||
# Scanner config
|
||||
cat > %s << 'CFGEOF'
|
||||
%s
|
||||
CFGEOF
|
||||
|
||||
# Scanner systemd service
|
||||
cat > /etc/systemd/system/%s.service << 'SVCEOF'
|
||||
%s
|
||||
SVCEOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable %s
|
||||
|
||||
echo "=== Scanner setup complete ==="
|
||||
`,
|
||||
naming.ScannerDataDir(), naming.ScannerDataDir(),
|
||||
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir(),
|
||||
naming.ScannerConfigPath(),
|
||||
scannerConfigYAML,
|
||||
naming.Scanner(),
|
||||
scannerUnit,
|
||||
naming.Scanner(),
|
||||
)
|
||||
|
||||
return script + scannerPhase, nil
|
||||
}
|
||||
|
||||
type cloudInitParams struct {
|
||||
|
||||
@@ -4,9 +4,11 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
|
||||
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/client"
|
||||
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/service"
|
||||
"go.yaml.in/yaml/v3"
|
||||
@@ -22,6 +24,7 @@ const (
|
||||
type InfraConfig struct {
|
||||
Zone string
|
||||
Plan string
|
||||
ScannerPlan string // plan for the scanner server; resolved by resolveScannerPlan when empty
|
||||
SSHPublicKey string
|
||||
S3SecretKey string
|
||||
|
||||
@@ -39,7 +42,7 @@ func (c *InfraConfig) Naming() Naming {
|
||||
return Naming{ClientName: c.ClientName}
|
||||
}
|
||||
|
||||
func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
|
||||
func loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
|
||||
sshKey, err := readSSHPublicKey(sshKeyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -53,6 +56,7 @@ func loadConfig(zone, plan, sshKeyPath, s3Secret string) (*InfraConfig, error) {
|
||||
return &InfraConfig{
|
||||
Zone: zone,
|
||||
Plan: plan,
|
||||
ScannerPlan: scannerPlan,
|
||||
SSHPublicKey: sshKey,
|
||||
S3SecretKey: s3Secret,
|
||||
ClientName: clientName,
|
||||
@@ -128,6 +132,100 @@ func resolveInteractive(ctx context.Context, svc *service.Service, cfg *InfraCon
|
||||
return nil
|
||||
}
|
||||
|
||||
// scannerPlanSpec is the shape of the scanner's own server. The scanner is
|
||||
// memory-bound, not CPU-bound: Grype keeps a large working set of its 2 GB
|
||||
// vulnerability database resident (400-600 MiB idle) and Syft catalogs an
|
||||
// extracted image in memory on top of that. On the 1 GB hold host the two
|
||||
// together peaked at 644 MiB resident plus 1.9 GB of swap and thrashed the
|
||||
// box for eleven hours (2026-09-12). 4 GB fits the worst scan seen so far with
|
||||
// headroom; 30 GB of disk holds the database plus the ~3.8x extraction
|
||||
// amplification of a 1 GiB image.
|
||||
type planSpec struct {
|
||||
Cores int
|
||||
MemoryMB int
|
||||
DiskGB int
|
||||
}
|
||||
|
||||
var scannerPlanSpec = planSpec{Cores: 2, MemoryMB: 4096, DiskGB: 30}
|
||||
|
||||
// defaultScannerPlan is the UpCloud plan of that shape at the time of
|
||||
// writing (the Starter tier, which UpCloud called Developer / DEV- until
|
||||
// 2026). resolveScannerPlan checks it still exists and falls back to a shape
|
||||
// match if UpCloud renames the tier again.
|
||||
const defaultScannerPlan = "STARTER-2xCPU-4GB"
|
||||
|
||||
// isStarterPlan reports whether a plan is in UpCloud's entry tier, which only
|
||||
// accepts "standard" storage: creating one with maxiops fails with
|
||||
// TIER_INVALID. The tier was named DEV- before it became STARTER-.
|
||||
func isStarterPlan(name string) bool {
|
||||
upper := strings.ToUpper(name)
|
||||
return strings.HasPrefix(upper, "STARTER-") || strings.HasPrefix(upper, "DEV-")
|
||||
}
|
||||
|
||||
// hasPlan reports whether name is one of the plans UpCloud offers.
|
||||
func hasPlan(plans []upcloud.Plan, name string) bool {
|
||||
for _, p := range plans {
|
||||
if p.Name == name {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchPlan returns the plan names whose shape equals spec. Starter-tier
|
||||
// plans sort first because they are the cheapest way to buy that shape.
|
||||
func matchPlan(plans []upcloud.Plan, spec planSpec) []string {
|
||||
var names []string
|
||||
for _, p := range plans {
|
||||
if p.GPUAmount > 0 {
|
||||
continue
|
||||
}
|
||||
if p.CoreNumber == spec.Cores && p.MemoryAmount == spec.MemoryMB && p.StorageSize == spec.DiskGB {
|
||||
names = append(names, p.Name)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(names, func(i, j int) bool {
|
||||
di, dj := isStarterPlan(names[i]), isStarterPlan(names[j])
|
||||
if di != dj {
|
||||
return di
|
||||
}
|
||||
return names[i] < names[j]
|
||||
})
|
||||
return names
|
||||
}
|
||||
|
||||
// resolveScannerPlan fills cfg.ScannerPlan: the --scanner-plan flag wins,
|
||||
// then defaultScannerPlan if UpCloud still offers it, then the first plan
|
||||
// matching scannerPlanSpec, then the interactive picker.
|
||||
func resolveScannerPlan(ctx context.Context, svc *service.Service, cfg *InfraConfig) error {
|
||||
if cfg.ScannerPlan != "" {
|
||||
return nil
|
||||
}
|
||||
resp, err := svc.GetPlans(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch plans: %w", err)
|
||||
}
|
||||
if hasPlan(resp.Plans, defaultScannerPlan) {
|
||||
cfg.ScannerPlan = defaultScannerPlan
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("Plan %s is no longer offered; matching by shape.\n", defaultScannerPlan)
|
||||
if names := matchPlan(resp.Plans, scannerPlanSpec); len(names) > 0 {
|
||||
cfg.ScannerPlan = names[0]
|
||||
fmt.Printf("Scanner plan: %s (%d CPU, %d GB RAM, %d GB disk)\n", names[0],
|
||||
scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("No plan offers %d CPU / %d GB RAM / %d GB disk; pick one for the scanner.\n",
|
||||
scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB)
|
||||
p, err := pickPlan(ctx, svc)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scanner plan picker: %w", err)
|
||||
}
|
||||
cfg.ScannerPlan = p
|
||||
return nil
|
||||
}
|
||||
|
||||
// newService creates an UpCloud API client. If token is non-empty it's used
|
||||
// directly; otherwise credentials are read from UPCLOUD_TOKEN env var.
|
||||
func newService(token string) (*service.Service, error) {
|
||||
|
||||
@@ -10,19 +10,18 @@ log_shipper:
|
||||
server:
|
||||
addr: :9090
|
||||
hold:
|
||||
url: "ws://localhost:8080"
|
||||
# The hold's private-network address. The scanner runs on its own server
|
||||
# and reaches the hold over the UpCloud SDN, never through the load
|
||||
# balancer (the LB frontend has a 10s client timeout that would cut the
|
||||
# scan WebSocket).
|
||||
url: "{{.ScannerHoldURL}}"
|
||||
secret: "{{.ScannerSecret}}"
|
||||
scanner:
|
||||
# One worker, deliberately. Two reasons, and the first one may go away:
|
||||
# - The hold's proactive dispatch loop gates on waitForCapacity() and
|
||||
# hands out one job at a time hold-wide, so a second worker only ever
|
||||
# receives work when two pushes coincide.
|
||||
# - Peak RSS is per concurrent scan. Two concurrent scans of a
|
||||
# node:22-class image measured 687 MiB with the 512 MiB GOMEMLIMIT in
|
||||
# force and 1357 MiB without it. On a host this size that is the
|
||||
# difference between working and OOM-killing the hold.
|
||||
# Raise this only together with MemoryMax in the unit file and the memory
|
||||
# available on the host.
|
||||
# One worker, deliberately: the hold's proactive dispatch loop gates on
|
||||
# waitForCapacity() and hands out one job at a time hold-wide, so a second
|
||||
# worker only ever receives work when two pushes coincide. Peak RSS is per
|
||||
# concurrent scan (two node:22-class scans measured 1357 MiB with no Go
|
||||
# limit), so raise this only together with MemoryMax in the unit file.
|
||||
workers: 1
|
||||
queue_size: 100
|
||||
# Must stay below the hold's 10m scanning timeout, which it measures from
|
||||
@@ -42,14 +41,16 @@ vuln:
|
||||
# roughly 3.8x over their compressed size (measured: node:22, 389 MiB ->
|
||||
# 1493 MiB), so a tmpfs would spend host memory to hold them.
|
||||
tmp_dir: "{{.BasePath}}/scanner/tmp"
|
||||
# 512 MiB compressed. The shipped default is 2 GiB, which no small host
|
||||
# can survive: peak RSS tracks image size, and 389 MiB compressed already
|
||||
# reached 561 MiB RSS with the memory limit applied. Images above this are
|
||||
# rejected before any blob is downloaded.
|
||||
max_image_size: 536870912
|
||||
# 1 GiB compressed. Peak RSS tracks image size (389 MiB compressed reached
|
||||
# 561 MiB RSS with a 512 MiB Go limit) and extraction writes ~3.8x the
|
||||
# compressed size to tmp_dir, so this bounds both memory and disk on the
|
||||
# 4 GB / 30 GB scanner server. Images above it are rejected before any
|
||||
# blob is downloaded. The shipped default is 2 GiB.
|
||||
max_image_size: 1073741824
|
||||
# Reclaim scan directories left behind by a scanner that was killed
|
||||
# mid-scan (a restart or a deploy). Those never run their own cleanup, and
|
||||
# on this host they reached 8.8 GB on a 20 GB disk before scans started
|
||||
# failing with "no space left on device". Above the 8m job timeout, so a
|
||||
# scan in flight elsewhere in the same directory is never touched.
|
||||
# on the old shared host they reached 8.8 GB on a 20 GB disk before scans
|
||||
# started failing with "no space left on device". Above the 8m job
|
||||
# timeout, so a scan in flight elsewhere in the same directory is never
|
||||
# touched.
|
||||
sweep_max_age: 1h
|
||||
|
||||
+225
-54
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -26,28 +27,30 @@ var provisionCmd = &cobra.Command{
|
||||
token, _ := cmd.Root().PersistentFlags().GetString("token")
|
||||
zone, _ := cmd.Flags().GetString("zone")
|
||||
plan, _ := cmd.Flags().GetString("plan")
|
||||
scannerPlan, _ := cmd.Flags().GetString("scanner-plan")
|
||||
sshKey, _ := cmd.Flags().GetString("ssh-key")
|
||||
s3Secret, _ := cmd.Flags().GetString("s3-secret")
|
||||
withScanner, _ := cmd.Flags().GetBool("with-scanner")
|
||||
withLabeler, _ := cmd.Flags().GetBool("with-labeler")
|
||||
withBilling, _ := cmd.Flags().GetBool("with-billing")
|
||||
return cmdProvision(token, zone, plan, sshKey, s3Secret, withScanner, withLabeler, withBilling)
|
||||
return cmdProvision(token, zone, plan, scannerPlan, sshKey, s3Secret, withScanner, withLabeler, withBilling)
|
||||
},
|
||||
}
|
||||
|
||||
func init() {
|
||||
provisionCmd.Flags().String("zone", "", "UpCloud zone (interactive picker if omitted)")
|
||||
provisionCmd.Flags().String("plan", "", "Server plan (interactive picker if omitted)")
|
||||
provisionCmd.Flags().String("plan", "", "Server plan for appview and hold (interactive picker if omitted)")
|
||||
provisionCmd.Flags().String("scanner-plan", "", fmt.Sprintf("Server plan for the scanner's own server (default %s, or whichever plan offers %d CPU / %d GB RAM / %d GB disk if that name is gone)", defaultScannerPlan, scannerPlanSpec.Cores, scannerPlanSpec.MemoryMB/1024, scannerPlanSpec.DiskGB))
|
||||
provisionCmd.Flags().String("ssh-key", "", "Path to SSH public key file (required when creating new servers)")
|
||||
provisionCmd.Flags().String("s3-secret", "", "S3 secret access key (for existing object storage)")
|
||||
provisionCmd.Flags().Bool("with-scanner", false, "Deploy vulnerability scanner alongside hold")
|
||||
provisionCmd.Flags().Bool("with-scanner", false, "Deploy the vulnerability scanner on its own server")
|
||||
provisionCmd.Flags().Bool("with-labeler", false, "Deploy content moderation labeler alongside appview")
|
||||
provisionCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default.")
|
||||
rootCmd.AddCommand(provisionCmd)
|
||||
}
|
||||
|
||||
func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error {
|
||||
cfg, err := loadConfig(zone, plan, sshKeyPath, s3Secret)
|
||||
func cmdProvision(token, zone, plan, scannerPlan, sshKeyPath, s3Secret string, withScanner, withLabeler, withBilling bool) error {
|
||||
cfg, err := loadConfig(zone, plan, scannerPlan, sshKeyPath, s3Secret)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -81,6 +84,16 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
}
|
||||
}
|
||||
|
||||
// The scanner gets its own server, on its own plan. Resolve it only when
|
||||
// that server still has to be created; an existing one keeps its plan.
|
||||
scannerEnabled := withScanner || state.ScannerEnabled
|
||||
needsScannerServer := scannerEnabled && state.Scanner.UUID == ""
|
||||
if needsScannerServer {
|
||||
if err := resolveScannerPlan(ctx, svc, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if state.Zone == "" {
|
||||
state.Zone = cfg.Zone
|
||||
}
|
||||
@@ -111,6 +124,9 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
if needsServers {
|
||||
fmt.Printf("Server plan: %s\n", cfg.Plan)
|
||||
}
|
||||
if needsScannerServer {
|
||||
fmt.Printf("Scanner server plan: %s\n", cfg.ScannerPlan)
|
||||
}
|
||||
fmt.Println()
|
||||
|
||||
// S3 secret key — from flag for existing storage, from API for new
|
||||
@@ -239,7 +255,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
appview, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
|
||||
appview, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Appview(), appviewUserData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create appview: %w", err)
|
||||
}
|
||||
@@ -253,7 +269,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
holdCreated := false
|
||||
if state.Hold.UUID != "" {
|
||||
fmt.Printf("Hold: %s (exists)\n", state.Hold.UUID)
|
||||
holdScript, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
|
||||
holdScript, err := generateHoldCloudInit(cfg, vals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -267,22 +283,13 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
if err := syncConfigKeys("hold", state.Hold.PublicIP, naming.HoldConfigPath(), holdConfigYAML, true); err != nil {
|
||||
return fmt.Errorf("hold config sync: %w", err)
|
||||
}
|
||||
if state.ScannerEnabled {
|
||||
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("render scanner config: %w", err)
|
||||
}
|
||||
if err := syncConfigKeys("scanner", state.Hold.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil {
|
||||
return fmt.Errorf("scanner config sync: %w", err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Creating hold server...")
|
||||
holdUserData, err := generateHoldCloudInit(cfg, vals, state.ScannerEnabled)
|
||||
holdUserData, err := generateHoldCloudInit(cfg, vals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
hold, err := createServer(ctx, svc, cfg, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
|
||||
hold, err := createServer(ctx, svc, cfg, cfg.Plan, templateUUID, state.Network.UUID, naming.Hold(), holdUserData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create hold: %w", err)
|
||||
}
|
||||
@@ -292,15 +299,63 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
fmt.Printf(" Hold: %s (public: %s, private: %s)\n", hold.UUID, hold.PublicIP, hold.PrivateIP)
|
||||
}
|
||||
|
||||
// 4b. Scanner server. Its own box: the scanner is memory-bound (Grype's
|
||||
// database working set plus Syft's in-memory catalog) and sharing a 1 GB
|
||||
// host with the hold swapped that host unresponsive for eleven hours on
|
||||
// 2026-09-12. It reaches the hold over the private network, so the hold's
|
||||
// private IP has to be known before its config can be rendered.
|
||||
scannerCreated := false
|
||||
if scannerEnabled {
|
||||
vals.ScannerHoldURL = scannerHoldURL(state.Hold.PrivateIP)
|
||||
if state.Scanner.UUID != "" {
|
||||
fmt.Printf("Scanner: %s (exists)\n", state.Scanner.UUID)
|
||||
scannerScript, err := generateScannerCloudInit(cfg, vals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := syncCloudInit("scanner", state.Scanner.PublicIP, scannerScript); err != nil {
|
||||
return err
|
||||
}
|
||||
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("render scanner config: %w", err)
|
||||
}
|
||||
if err := syncConfigKeys("scanner", state.Scanner.PublicIP, naming.ScannerConfigPath(), scannerConfigYAML, true); err != nil {
|
||||
return fmt.Errorf("scanner config sync: %w", err)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("Creating scanner server...")
|
||||
scannerUserData, err := generateScannerCloudInit(cfg, vals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
scanner, err := createServer(ctx, svc, cfg, cfg.ScannerPlan, templateUUID, state.Network.UUID, naming.Scanner(), scannerUserData)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create scanner: %w", err)
|
||||
}
|
||||
state.Scanner = *scanner
|
||||
_ = saveState(state)
|
||||
scannerCreated = true
|
||||
fmt.Printf(" Scanner: %s (public: %s, private: %s)\n", scanner.UUID, scanner.PublicIP, scanner.PrivateIP)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Firewall rules (idempotent — replaces all rules)
|
||||
fmt.Println("Configuring firewall rules...")
|
||||
for _, s := range []struct {
|
||||
firewalled := []struct {
|
||||
name string
|
||||
uuid string
|
||||
}{
|
||||
{"appview", state.Appview.UUID},
|
||||
{"hold", state.Hold.UUID},
|
||||
} {
|
||||
}
|
||||
if state.Scanner.UUID != "" {
|
||||
firewalled = append(firewalled, struct {
|
||||
name string
|
||||
uuid string
|
||||
}{"scanner", state.Scanner.UUID})
|
||||
}
|
||||
for _, s := range firewalled {
|
||||
if err := createFirewallRules(ctx, svc, s.uuid, privateNetworkCIDR); err != nil {
|
||||
return fmt.Errorf("firewall %s: %w", s.name, err)
|
||||
}
|
||||
@@ -372,7 +427,7 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
}
|
||||
|
||||
// 7. Build locally and deploy binaries to new servers
|
||||
if appviewCreated || holdCreated {
|
||||
if appviewCreated || holdCreated || scannerCreated {
|
||||
rootDir := projectRoot()
|
||||
|
||||
if err := runMakeBuildTrixie(rootDir, withBilling); err != nil {
|
||||
@@ -393,6 +448,11 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
return err
|
||||
}
|
||||
}
|
||||
if scannerCreated {
|
||||
if err := waitForSetup(state.Scanner.PublicIP, "scanner"); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println("\nDeploying binaries...")
|
||||
if appviewCreated {
|
||||
@@ -415,12 +475,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
if err := scpFile(localPath, state.Hold.PublicIP, remotePath); err != nil {
|
||||
return fmt.Errorf("upload hold: %w", err)
|
||||
}
|
||||
if state.ScannerEnabled {
|
||||
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
|
||||
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
|
||||
if err := scpFile(scannerLocal, state.Hold.PublicIP, scannerRemote); err != nil {
|
||||
return fmt.Errorf("upload scanner: %w", err)
|
||||
}
|
||||
}
|
||||
if scannerCreated {
|
||||
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
|
||||
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
|
||||
if err := scpFile(scannerLocal, state.Scanner.PublicIP, scannerRemote); err != nil {
|
||||
return fmt.Errorf("upload scanner: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -459,9 +519,12 @@ func cmdProvision(token, zone, plan, sshKeyPath, s3Secret string, withScanner, w
|
||||
fmt.Println("SSH access:")
|
||||
fmt.Printf(" ssh root@%s # appview\n", state.Appview.PublicIP)
|
||||
fmt.Printf(" ssh root@%s # hold\n", state.Hold.PublicIP)
|
||||
if state.Scanner.UUID != "" {
|
||||
fmt.Printf(" ssh root@%s # scanner\n", state.Scanner.PublicIP)
|
||||
}
|
||||
fmt.Println()
|
||||
fmt.Println("Next steps:")
|
||||
if appviewCreated || holdCreated {
|
||||
if appviewCreated || holdCreated || scannerCreated {
|
||||
fmt.Println(" 1. Edit configs if needed, then start services:")
|
||||
} else {
|
||||
fmt.Println(" 1. Start services:")
|
||||
@@ -588,12 +651,15 @@ func objectStorageRegion(zone string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
|
||||
func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, plan, templateUUID, networkUUID, title, userData string) (*ServerState, error) {
|
||||
if cfg.SSHPublicKey == "" {
|
||||
return nil, fmt.Errorf("creating server %s requires --ssh-key (path to SSH public key file)", title)
|
||||
}
|
||||
if plan == "" {
|
||||
return nil, fmt.Errorf("creating server %s: no plan selected", title)
|
||||
}
|
||||
storageTier := "maxiops"
|
||||
if strings.HasPrefix(strings.ToUpper(cfg.Plan), "DEV-") {
|
||||
if isStarterPlan(plan) {
|
||||
storageTier = "standard"
|
||||
}
|
||||
|
||||
@@ -602,7 +668,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t
|
||||
plans, err := svc.GetPlans(ctx)
|
||||
if err == nil {
|
||||
for _, p := range plans.Plans {
|
||||
if p.Name == cfg.Plan {
|
||||
if p.Name == plan {
|
||||
diskSize = p.StorageSize
|
||||
break
|
||||
}
|
||||
@@ -613,7 +679,7 @@ func createServer(ctx context.Context, svc *service.Service, cfg *InfraConfig, t
|
||||
Zone: cfg.Zone,
|
||||
Title: title,
|
||||
Hostname: title,
|
||||
Plan: cfg.Plan,
|
||||
Plan: plan,
|
||||
Metadata: upcloud.True,
|
||||
UserData: userData,
|
||||
Firewall: "on",
|
||||
@@ -737,17 +803,26 @@ const lbClientTimeout = 10
|
||||
|
||||
// httpFrontendProperties are the https frontend's properties.
|
||||
//
|
||||
// HTTP/2 is the one that matters for the browser: over HTTP/1.1 a browser
|
||||
// opens at most ~6 connections per origin, so a page that fans out many small
|
||||
// requests (the admin crew tab issues one per member) queues them six at a
|
||||
// time and blocks every other request to the same host behind them. h2
|
||||
// multiplexes them over a single connection and the queue disappears.
|
||||
// HTTP/2 stays OFF on this frontend, and ensureLBHTTP2 turns it off on an LB
|
||||
// that has it on. It was on until 2026-09-12, for the browser's sake (h2
|
||||
// multiplexes the admin crew tab's fan-out over one connection instead of
|
||||
// six). Two things changed the answer:
|
||||
//
|
||||
// Clients that need an HTTP/1.1 upgrade are unaffected: h2 is negotiated per
|
||||
// connection via ALPN, so a WebSocket client simply selects http/1.1.
|
||||
// - Re-enabling it that day caused a 25-minute /auth/token outage. The
|
||||
// appview and the hold call each other through this LB on long-lived Go
|
||||
// HTTP clients; the frontend reconfigure left those connections open at
|
||||
// TCP level but dead, Go's HTTP/2 transport multiplexed every call onto
|
||||
// them, and no transport here sets ReadIdleTimeout to notice. Only
|
||||
// restarting both services recovered. Over HTTP/1.1 a dead connection
|
||||
// fails one request and the pool opens another.
|
||||
// - The 64 KB HTTP/2 stream window capped pushes at ~2.5 MB/s through the
|
||||
// LB; median push fell from 16.5 s to 3.1 s when it went off.
|
||||
//
|
||||
// The registry and web hostnames are fronted by Bunny now, so the browser
|
||||
// case this bought is served by the CDN's own h2 edge rather than by the LB.
|
||||
func httpFrontendProperties() *upcloud.LoadBalancerFrontendProperties {
|
||||
return &upcloud.LoadBalancerFrontendProperties{
|
||||
HTTP2Enabled: new(true),
|
||||
HTTP2Enabled: new(false),
|
||||
TimeoutClient: lbClientTimeout,
|
||||
}
|
||||
}
|
||||
@@ -974,13 +1049,28 @@ func labelerFrontendRule(labelerDomain string) request.LoadBalancerFrontendRule
|
||||
}
|
||||
|
||||
// ensureLBCertificates reconciles TLS certificate bundles on the load balancer.
|
||||
// It skips domains that already have a TLS config attached and creates missing ones.
|
||||
// It skips domains that already have a TLS config attached and creates missing
|
||||
// ones, but only for domains whose DNS resolves to this LB. A dynamic
|
||||
// (Let's Encrypt) bundle is validated over HTTP at the domain, so a domain
|
||||
// fronted by a CDN cannot be issued here anyway; requesting it only leaves a
|
||||
// bundle stuck in validation. The .cr registry domains moved behind Bunny on
|
||||
// 2026-09-12 and their bundles were removed by hand; on 2026-09-13 this
|
||||
// function put them straight back. Hence the check.
|
||||
func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID string, tlsDomains []string) error {
|
||||
lb, err := svc.GetLoadBalancer(ctx, &request.GetLoadBalancerRequest{UUID: lbUUID})
|
||||
if err != nil {
|
||||
return fmt.Errorf("get load balancer: %w", err)
|
||||
}
|
||||
|
||||
lbAddrs := map[string]bool{}
|
||||
for _, n := range lb.Networks {
|
||||
if n.Type == upcloud.LoadBalancerNetworkTypePublic && n.DNSName != "" {
|
||||
for _, a := range lookupHost(n.DNSName) {
|
||||
lbAddrs[a] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build set of existing TLS config names on the "https" frontend
|
||||
existing := make(map[string]bool)
|
||||
for _, fe := range lb.Frontends {
|
||||
@@ -997,6 +1087,10 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri
|
||||
fmt.Printf(" TLS certificate: %s (exists)\n", domain)
|
||||
continue
|
||||
}
|
||||
if !pointsAt(domain, lbAddrs) {
|
||||
fmt.Printf(" TLS certificate: %s skipped (DNS does not point at this LB; fronted elsewhere or not yet delegated)\n", domain)
|
||||
continue
|
||||
}
|
||||
|
||||
bundle, err := svc.CreateLoadBalancerCertificateBundle(ctx, &request.CreateLoadBalancerCertificateBundleRequest{
|
||||
Type: upcloud.LoadBalancerCertificateBundleTypeDynamic,
|
||||
@@ -1025,11 +1119,32 @@ func ensureLBCertificates(ctx context.Context, svc *service.Service, lbUUID stri
|
||||
return nil
|
||||
}
|
||||
|
||||
// lookupHost resolves name to its IPv4/IPv6 addresses; failures resolve to none.
|
||||
func lookupHost(name string) []string {
|
||||
addrs, err := net.LookupHost(name)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return addrs
|
||||
}
|
||||
|
||||
// pointsAt reports whether any address domain resolves to is one of lbAddrs.
|
||||
// An empty lbAddrs (the LB's DNS name did not resolve) matches nothing, so
|
||||
// no bundle is requested when the precondition cannot be checked.
|
||||
func pointsAt(domain string, lbAddrs map[string]bool) bool {
|
||||
for _, a := range lookupHost(domain) {
|
||||
if lbAddrs[a] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ensureLBForwardedHeaders ensures the "https" frontend has a set_forwarded_headers rule.
|
||||
// This makes the LB set X-Forwarded-For, X-Forwarded-Proto, and X-Forwarded-Port headers,
|
||||
// overwriting any pre-existing values (prevents spoofing).
|
||||
// ensureLBHTTP2 reconciles HTTP/2 and the client timeout onto an LB that
|
||||
// already exists.
|
||||
// ensureLBHTTP2 reconciles the HTTP/2 flag (off, see httpFrontendProperties)
|
||||
// and the client timeout onto an LB that already exists.
|
||||
//
|
||||
// createLoadBalancer only runs when there is no LB yet, so without this the
|
||||
// properties above would reach a fresh deployment and never an existing one.
|
||||
@@ -1045,10 +1160,11 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err
|
||||
}
|
||||
|
||||
want := httpFrontendProperties()
|
||||
wantHTTP2 := want.HTTP2Enabled != nil && *want.HTTP2Enabled
|
||||
haveHTTP2 := fe.Properties != nil && fe.Properties.HTTP2Enabled != nil && *fe.Properties.HTTP2Enabled
|
||||
haveTimeout := fe.Properties != nil && fe.Properties.TimeoutClient == want.TimeoutClient
|
||||
if haveHTTP2 && haveTimeout {
|
||||
fmt.Println(" Frontend HTTP/2 + timeout: already set")
|
||||
if haveHTTP2 == wantHTTP2 && haveTimeout {
|
||||
fmt.Printf(" Frontend HTTP/2 %s + timeout_client=%ds: already set\n", onOff(wantHTTP2), want.TimeoutClient)
|
||||
} else {
|
||||
if _, err := svc.ModifyLoadBalancerFrontend(ctx, &request.ModifyLoadBalancerFrontendRequest{
|
||||
ServiceUUID: lbUUID,
|
||||
@@ -1059,12 +1175,19 @@ func ensureLBHTTP2(ctx context.Context, svc *service.Service, lbUUID string) err
|
||||
}); err != nil {
|
||||
return fmt.Errorf("modify https frontend: %w", err)
|
||||
}
|
||||
fmt.Printf(" Frontend HTTP/2: enabled (timeout_client=%ds)\n", want.TimeoutClient)
|
||||
fmt.Printf(" Frontend HTTP/2: %s (timeout_client=%ds)\n", onOff(wantHTTP2), want.TimeoutClient)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func onOff(b bool) string {
|
||||
if b {
|
||||
return "on"
|
||||
}
|
||||
return "off"
|
||||
}
|
||||
|
||||
func ensureLBForwardedHeaders(ctx context.Context, svc *service.Service, lbUUID string) error {
|
||||
rules, err := svc.GetLoadBalancerFrontendRules(ctx, &request.GetLoadBalancerFrontendRulesRequest{
|
||||
ServiceUUID: lbUUID,
|
||||
@@ -1449,9 +1572,16 @@ func syncCloudInit(name, ip, localScript string) error {
|
||||
}
|
||||
|
||||
fmt.Printf(" Re-run cloud-init on %s? [Y/n] ", name)
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
scanner.Scan()
|
||||
answer := strings.TrimSpace(strings.ToLower(scanner.Text()))
|
||||
answer, ok := readLine()
|
||||
if !ok {
|
||||
// No answer at all (stdin closed, or a non-interactive run) must not
|
||||
// re-run setup on a production server. Only a typed yes does.
|
||||
fmt.Printf(" no answer, skipped\n")
|
||||
if err := writeRemoteCloudInit(ip, localScript); err != nil {
|
||||
fmt.Printf(" WARNING: could not update remote cloud-init reference: %v\n", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if answer != "" && answer != "y" && answer != "yes" {
|
||||
fmt.Printf(" Skipped\n")
|
||||
// Still update the remote reference so next provision sees an accurate diff
|
||||
@@ -1479,6 +1609,33 @@ func syncCloudInit(name, ip, localScript string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// scannerHoldURL is the hold's WebSocket address from the scanner server: the
|
||||
// hold's private IP over the SDN, never the load balancer (whose 10s client
|
||||
// timeout would cut the long-lived scan socket).
|
||||
func scannerHoldURL(holdPrivateIP string) string {
|
||||
if holdPrivateIP == "" {
|
||||
return ""
|
||||
}
|
||||
return "ws://" + holdPrivateIP + ":8080"
|
||||
}
|
||||
|
||||
// stdin is shared by every prompt in the tool. Each prompt used to wrap
|
||||
// os.Stdin in its own bufio.Scanner, and the first one swallowed everything
|
||||
// that was already buffered (all of a piped "n\nn\n"), so the second prompt
|
||||
// read EOF, took the default, and re-ran cloud-init on the production hold
|
||||
// (2026-09-13). One reader keeps the unread lines for the next prompt.
|
||||
var stdin = bufio.NewReader(os.Stdin)
|
||||
|
||||
// readLine returns the next line typed on stdin, lower-cased and trimmed, and
|
||||
// false when stdin has nothing more to give.
|
||||
func readLine() (string, bool) {
|
||||
line, err := stdin.ReadString('\n')
|
||||
if err != nil && line == "" {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimSpace(strings.ToLower(line)), true
|
||||
}
|
||||
|
||||
// generateScannerSecret generates a random 32-byte hex-encoded shared secret
|
||||
// for authenticating scanner-to-hold WebSocket connections.
|
||||
func generateScannerSecret() (string, error) {
|
||||
@@ -1515,11 +1672,25 @@ func waitForSetup(ip, name string) error {
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
// Poll rather than `cloud-init status --wait`: runSSH caps one command at
|
||||
// five minutes, and a fresh server's first boot (apt upgrade plus git,
|
||||
// gcc, nodejs, npm) takes longer than that. The 2026-09-13 scanner
|
||||
// provision timed out here with the server healthy and half installed.
|
||||
fmt.Printf(" %s: waiting for cloud-init...\n", name)
|
||||
_, err := runSSH(ip, "cloud-init status --wait 2>/dev/null || true", false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cloud-init wait on %s: %w", name, err)
|
||||
deadline := time.Now().Add(20 * time.Minute)
|
||||
for {
|
||||
out, err := runSSH(ip, "cloud-init status 2>/dev/null | head -1", false)
|
||||
status := strings.TrimSpace(out)
|
||||
switch {
|
||||
case err == nil && strings.HasPrefix(status, "status: done"):
|
||||
fmt.Printf(" %s: ready\n", name)
|
||||
return nil
|
||||
case err == nil && strings.HasPrefix(status, "status: error"):
|
||||
return fmt.Errorf("cloud-init on %s finished with errors; see /var/log/cloud-init-output.log on the server", name)
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("cloud-init on %s not done after 20 minutes (last status %q)", name, status)
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
fmt.Printf(" %s: ready\n", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud"
|
||||
)
|
||||
|
||||
// The scanner's server is chosen by shape, not by name, because UpCloud
|
||||
// renames tiers. Developer plans win a tie: same shape, cheapest tier.
|
||||
func TestMatchPlanPrefersDeveloperTier(t *testing.T) {
|
||||
plans := []upcloud.Plan{
|
||||
{Name: "2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 80},
|
||||
{Name: "GPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30, GPUAmount: 1},
|
||||
{Name: "HICPU-2xCPU-4GB-30GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30},
|
||||
{Name: "STARTER-2xCPU-4GB", CoreNumber: 2, MemoryAmount: 4096, StorageSize: 30},
|
||||
{Name: "STARTER-1xCPU-1GB", CoreNumber: 1, MemoryAmount: 1024, StorageSize: 10},
|
||||
}
|
||||
got := matchPlan(plans, scannerPlanSpec)
|
||||
if strings.Join(got, ",") != "STARTER-2xCPU-4GB,HICPU-2xCPU-4GB-30GB" {
|
||||
t.Fatalf("matchPlan = %v", got)
|
||||
}
|
||||
// Both the current and the pre-2026 tier names take standard storage;
|
||||
// asking for maxiops on them fails with TIER_INVALID (seen 2026-09-13).
|
||||
for _, n := range []string{"STARTER-2xCPU-4GB", "DEV-2xCPU-4GB-30GB", "dev-1xcpu-1gb-10gb"} {
|
||||
if !isStarterPlan(n) {
|
||||
t.Errorf("%s should be a starter-tier plan", n)
|
||||
}
|
||||
}
|
||||
if isStarterPlan("2xCPU-4GB") || isStarterPlan("HICPU-2xCPU-4GB-30GB") {
|
||||
t.Error("general purpose and high-CPU plans are not starter tier")
|
||||
}
|
||||
if len(matchPlan(plans, planSpec{Cores: 8, MemoryMB: 65536, DiskGB: 1000})) != 0 {
|
||||
t.Fatal("a shape nobody sells must match nothing, so the picker takes over")
|
||||
}
|
||||
if !hasPlan(plans, defaultScannerPlan) || hasPlan(plans, "STARTER-2xCPU-4GB-80GB") {
|
||||
t.Fatal("hasPlan must match the exact plan name")
|
||||
}
|
||||
// The pinned default must be the shape the spec describes, or the two
|
||||
// paths in resolveScannerPlan would buy different servers.
|
||||
if got := matchPlan(plans, scannerPlanSpec); got[0] != defaultScannerPlan {
|
||||
t.Fatalf("defaultScannerPlan %s is not the DEV plan of shape %+v (matcher chose %s)", defaultScannerPlan, scannerPlanSpec, got[0])
|
||||
}
|
||||
}
|
||||
|
||||
// Limits step down from the plan's RAM: MemoryMax leaves the OS ~600 MiB,
|
||||
// MemoryHigh sits below that, and the Go soft limit below both so the
|
||||
// collector runs before the kernel reclaims or kills.
|
||||
func TestScannerMemoryLimits(t *testing.T) {
|
||||
goLimit, high, maxMem := scannerMemoryLimits(4096)
|
||||
if goLimit != "2560MiB" || high != "3072M" || maxMem != "3500M" {
|
||||
t.Fatalf("4 GB plan: got %s / %s / %s", goLimit, high, maxMem)
|
||||
}
|
||||
goLimit, _, _ = scannerMemoryLimits(1024)
|
||||
if goLimit != "512MiB" {
|
||||
t.Fatalf("the Go limit must never drop below the scanner's own 512 MiB default, got %s", goLimit)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScannerCloudInitTargetsHoldPrivateIP(t *testing.T) {
|
||||
cfg := &InfraConfig{ClientName: "seamark"}
|
||||
vals := &ConfigValues{
|
||||
BasePath: "/var/lib/seamark",
|
||||
ScannerSecret: "abc",
|
||||
ScannerHoldURL: scannerHoldURL("10.0.1.3"),
|
||||
}
|
||||
script, err := generateScannerCloudInit(cfg, vals)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`url: "ws://10.0.1.3:8080"`,
|
||||
"mkdir -p /var/lib/seamark/scanner/vulndb /var/lib/seamark/scanner/tmp",
|
||||
"Environment=GOMEMLIMIT=2560MiB",
|
||||
"MemorySwapMax=0",
|
||||
"systemctl enable seamark-scanner",
|
||||
} {
|
||||
if !strings.Contains(script, want) {
|
||||
t.Errorf("scanner cloud-init missing %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(script, "seamark-hold.service") {
|
||||
t.Error("a standalone scanner must not order itself after a hold unit that is not on its host")
|
||||
}
|
||||
|
||||
if _, err := generateScannerCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark"}); err == nil {
|
||||
t.Error("rendering a scanner config with no hold address must fail, not ship ws://:8080")
|
||||
}
|
||||
if scannerHoldURL("") != "" {
|
||||
t.Error("no hold IP must render as no URL so the guard above fires")
|
||||
}
|
||||
}
|
||||
|
||||
// The hold's cloud-init no longer carries the scanner: its unit, config and
|
||||
// directories belong to the scanner server.
|
||||
func TestHoldCloudInitHasNoScanner(t *testing.T) {
|
||||
cfg := &InfraConfig{ClientName: "seamark"}
|
||||
script, err := generateHoldCloudInit(cfg, &ConfigValues{BasePath: "/var/lib/seamark", HoldDomain: "x.cove.seamark.dev"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(script, "seamark-scanner") || strings.Contains(script, "vulndb") {
|
||||
t.Error("hold cloud-init still installs the scanner")
|
||||
}
|
||||
}
|
||||
|
||||
// Re-enabling frontend HTTP/2 on 2026-09-12 stranded the appview<->hold
|
||||
// connections through the LB for 25 minutes. Every provision run reconciles
|
||||
// this property, so the default is the outage switch.
|
||||
func TestLBFrontendHTTP2StaysOff(t *testing.T) {
|
||||
p := httpFrontendProperties()
|
||||
if p.HTTP2Enabled == nil || *p.HTTP2Enabled {
|
||||
t.Fatal("LB frontend HTTP/2 must be off (see httpFrontendProperties)")
|
||||
}
|
||||
if p.TimeoutClient != lbClientTimeout {
|
||||
t.Fatalf("timeout_client = %d, want %d", p.TimeoutClient, lbClientTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
// A domain fronted by a CDN never resolves to the LB, so the tool must not
|
||||
// request a certificate for it (it re-added the .cr bundles on 2026-09-13).
|
||||
func TestPointsAtRequiresOverlap(t *testing.T) {
|
||||
lb := map[string]bool{"203.0.113.10": true}
|
||||
if pointsAt("localhost", lb) {
|
||||
t.Fatal("localhost does not resolve to the LB")
|
||||
}
|
||||
if pointsAt("localhost", map[string]bool{}) {
|
||||
t.Fatal("an unresolved LB must match nothing")
|
||||
}
|
||||
if !pointsAt("localhost", map[string]bool{"127.0.0.1": true, "::1": true}) {
|
||||
t.Fatal("a domain resolving to an LB address must match")
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ type InfraState struct {
|
||||
Network StateRef `json:"network"`
|
||||
Appview ServerState `json:"appview"`
|
||||
Hold ServerState `json:"hold"`
|
||||
Scanner ServerState `json:"scanner,omitempty"`
|
||||
LB StateRef `json:"loadbalancer"`
|
||||
ObjectStorage ObjectStorageState `json:"object_storage"`
|
||||
ScannerEnabled bool `json:"scanner_enabled,omitempty"`
|
||||
|
||||
+11
-27
@@ -43,7 +43,7 @@ func cmdStatus(token string) error {
|
||||
fmt.Printf("Zone: %s\n\n", state.Zone)
|
||||
|
||||
// Server status
|
||||
for _, s := range []struct {
|
||||
servers := []struct {
|
||||
name string
|
||||
ss ServerState
|
||||
serviceName string
|
||||
@@ -51,7 +51,16 @@ func cmdStatus(token string) error {
|
||||
}{
|
||||
{"Appview", state.Appview, naming.Appview(), "http://localhost:5000/health"},
|
||||
{"Hold", state.Hold, naming.Hold(), "http://localhost:8080/xrpc/_health"},
|
||||
} {
|
||||
}
|
||||
if state.ScannerEnabled || state.Scanner.UUID != "" {
|
||||
servers = append(servers, struct {
|
||||
name string
|
||||
ss ServerState
|
||||
serviceName string
|
||||
healthURL string
|
||||
}{"Scanner", state.Scanner, naming.Scanner(), "http://localhost:9090/healthz"})
|
||||
}
|
||||
for _, s := range servers {
|
||||
fmt.Printf("%-8s UUID: %s\n", s.name, s.ss.UUID)
|
||||
fmt.Printf(" Public: %s\n", s.ss.PublicIP)
|
||||
fmt.Printf(" Private: %s\n", s.ss.PrivateIP)
|
||||
@@ -90,31 +99,6 @@ func cmdStatus(token string) error {
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// Scanner status (runs on hold server)
|
||||
if state.ScannerEnabled {
|
||||
fmt.Printf("Scanner (on hold server)\n")
|
||||
if state.Hold.PublicIP != "" {
|
||||
output, err := runSSH(state.Hold.PublicIP, fmt.Sprintf(
|
||||
"systemctl is-active %s 2>/dev/null || echo 'inactive'; curl -sf http://localhost:9090/healthz > /dev/null 2>&1 && echo 'health:ok' || echo 'health:fail'",
|
||||
naming.Scanner(),
|
||||
), false)
|
||||
if err != nil {
|
||||
fmt.Printf(" Service: unreachable\n")
|
||||
} else {
|
||||
lines := strings.SplitSeq(strings.TrimSpace(output), "\n")
|
||||
for line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "active" || line == "inactive" {
|
||||
fmt.Printf(" Service: %s\n", line)
|
||||
} else if after, ok := strings.CutPrefix(line, "health:"); ok {
|
||||
fmt.Printf(" Health: %s\n", after)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// LB status
|
||||
if state.LB.UUID != "" {
|
||||
fmt.Printf("Load Balancer: %s\n", state.LB.UUID)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description={{.DisplayName}} Scanner (Vulnerability Scanning)
|
||||
After=network-online.target {{.HoldServiceName}}.service
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
@@ -11,38 +11,26 @@ ExecStart={{.BinaryPath}} serve --config {{.ConfigPath}}
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
|
||||
# Memory containment. The scanner sets a 512 MiB Go soft limit itself
|
||||
# (GOMEMLIMIT, see cmd/scanner/main.go), but that is soft: the runtime
|
||||
# collects harder to respect it and never fails an allocation to honour it, so
|
||||
# a large enough image walks straight through it. Without a cgroup cap the
|
||||
# kernel OOM killer chooses its own victim, and on a shared host the other
|
||||
# large process is the hold, meaning the scanner's overshoot kills the service
|
||||
# it reports to. These make the scanner the one that dies.
|
||||
# The scanner runs on its own server, so it is the only large process on the
|
||||
# host and the limits below are sized to the box rather than to a neighbour.
|
||||
#
|
||||
# MemoryHigh throttles and reclaims; MemoryMax kills. Sized from measurement:
|
||||
# one scan of a node:22-class image peaks at 561 MiB RSS with the Go limit in
|
||||
# force, so MemoryHigh sits above that and MemoryMax leaves headroom for the
|
||||
# hold and the OS on a 1 GiB host. Raise both together with scanner.workers.
|
||||
MemoryHigh=640M
|
||||
MemoryMax=768M
|
||||
# GOMEMLIMIT is the Go soft limit: the runtime collects harder as the heap
|
||||
# approaches it and never fails an allocation to honour it, so a large enough
|
||||
# image walks straight through. It sits below MemoryHigh so the GC gets a
|
||||
# chance before the kernel starts reclaiming. Without the variable the scanner
|
||||
# defaults itself to 512 MiB (cmd/scanner/main.go), which on a 4 GB host only
|
||||
# wastes CPU on collection.
|
||||
Environment=GOMEMLIMIT={{.GoMemLimit}}
|
||||
|
||||
# Scheduling priority. The scanner is the lowest-value process on a shared
|
||||
# host: a scan finishing a minute later costs nothing, a hold that cannot
|
||||
# answer a pull costs a user. Syft's extraction saturates CPU and writes
|
||||
# several times the compressed image size to disk, so left at the default
|
||||
# weight it competes evenly with the services that matter.
|
||||
#
|
||||
# Weights apply only under contention: the scanner still uses the whole box
|
||||
# when nothing else wants it, which is what makes this better than the fixed
|
||||
# inter-job sleep in worker.go (that yields on a timer whether or not anyone
|
||||
# needs the CPU, and yields nothing while a scan is actually running).
|
||||
# Everything else runs at the default weight of 100.
|
||||
#
|
||||
# IOWeight needs the io controller with a scheduler that honours it (BFQ, or
|
||||
# io.cost configured); where it is unsupported systemd ignores it silently and
|
||||
# CPUWeight still applies.
|
||||
CPUWeight=20
|
||||
IOWeight=20
|
||||
# MemoryHigh throttles and reclaims; MemoryMax kills. MemorySwapMax=0 is the
|
||||
# one that matters: with swap available a scanner over its cap is not killed,
|
||||
# it is paged out, and the host thrashes instead of failing. That is what took
|
||||
# the shared hold host down for eleven hours on 2026-09-12 (644 MiB resident,
|
||||
# 1.9 GB swapped). Refusing swap turns that into an OOM kill of one job and a
|
||||
# clean restart ten seconds later.
|
||||
MemoryHigh={{.MemoryHigh}}
|
||||
MemoryMax={{.MemoryMax}}
|
||||
MemorySwapMax=0
|
||||
|
||||
ReadWritePaths={{.DataDir}}
|
||||
ProtectSystem=strict
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/UpCloudLtd/upcloud-go-api/v8/upcloud/request"
|
||||
@@ -39,14 +36,15 @@ func cmdTeardown(token string) error {
|
||||
fmt.Printf(" Zone: %s\n", state.Zone)
|
||||
fmt.Printf(" Appview: %s (%s)\n", state.Appview.UUID, state.Appview.PublicIP)
|
||||
fmt.Printf(" Hold: %s (%s)\n", state.Hold.UUID, state.Hold.PublicIP)
|
||||
if state.Scanner.UUID != "" {
|
||||
fmt.Printf(" Scanner: %s (%s)\n", state.Scanner.UUID, state.Scanner.PublicIP)
|
||||
}
|
||||
fmt.Printf(" Network: %s\n", state.Network.UUID)
|
||||
fmt.Printf(" LB: %s\n", state.LB.UUID)
|
||||
fmt.Println()
|
||||
fmt.Print("Type 'yes' to confirm: ")
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
scanner.Scan()
|
||||
if strings.TrimSpace(scanner.Text()) != "yes" {
|
||||
if answer, _ := readLine(); answer != "yes" {
|
||||
fmt.Println("Aborted.")
|
||||
return nil
|
||||
}
|
||||
@@ -76,6 +74,7 @@ func cmdTeardown(token string) error {
|
||||
}{
|
||||
{"appview", state.Appview.UUID},
|
||||
{"hold", state.Hold.UUID},
|
||||
{"scanner", state.Scanner.UUID},
|
||||
} {
|
||||
if s.uuid == "" {
|
||||
continue
|
||||
|
||||
+75
-88
@@ -18,7 +18,7 @@ var updateCmd = &cobra.Command{
|
||||
Use: "update [target]",
|
||||
Short: "Deploy updates to servers",
|
||||
Args: cobra.MaximumNArgs(1),
|
||||
ValidArgs: []string{"all", "appview", "hold"},
|
||||
ValidArgs: []string{"all", "appview", "hold", "scanner"},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
target := "all"
|
||||
if len(args) > 0 {
|
||||
@@ -37,7 +37,7 @@ var sshCmd = &cobra.Command{
|
||||
Use: "ssh <target>",
|
||||
Short: "SSH into a server",
|
||||
Args: cobra.ExactArgs(1),
|
||||
ValidArgs: []string{"appview", "hold"},
|
||||
ValidArgs: []string{"appview", "hold", "scanner"},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return cmdSSH(args[0])
|
||||
},
|
||||
@@ -45,7 +45,7 @@ var sshCmd = &cobra.Command{
|
||||
|
||||
func init() {
|
||||
updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.")
|
||||
updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
|
||||
updateCmd.Flags().Bool("with-scanner", false, "Enable the vulnerability scanner (its server is created by `provision --with-scanner`)")
|
||||
updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview")
|
||||
updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into <data dir>/predeploy-<stamp>/ before replacing them")
|
||||
updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup")
|
||||
@@ -68,7 +68,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
naming := state.Naming()
|
||||
rootDir := projectRoot()
|
||||
|
||||
// Enable scanner retroactively via --with-scanner on update
|
||||
// Enable scanner retroactively via --with-scanner on update. The secret
|
||||
// lands in the hold's config on this run; the scanner server itself is
|
||||
// created by provision, which is the only command that creates servers.
|
||||
if withScanner && !state.ScannerEnabled {
|
||||
state.ScannerEnabled = true
|
||||
if state.ScannerSecret == "" {
|
||||
@@ -80,6 +82,9 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
fmt.Printf("Generated scanner shared secret\n")
|
||||
}
|
||||
_ = saveState(state)
|
||||
if state.Scanner.UUID == "" {
|
||||
fmt.Println("Scanner enabled. Run `provision --with-scanner` to create its server, then `update scanner` deploys to it.")
|
||||
}
|
||||
}
|
||||
|
||||
// Enable labeler retroactively via --with-labeler on update
|
||||
@@ -90,38 +95,64 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
|
||||
vals := configValsFromState(state)
|
||||
|
||||
// renderUnit is per target because the scanner unit carries memory limits
|
||||
// the generic serviceUnitParams does not know about.
|
||||
standardUnit := func(tmpl, binaryName, configPath, serviceName string) func() (string, error) {
|
||||
return func() (string, error) {
|
||||
return renderServiceUnit(tmpl, serviceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + binaryName,
|
||||
ConfigPath: configPath,
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: serviceName,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
targets := map[string]struct {
|
||||
ip string
|
||||
binaryName string
|
||||
buildCmd string
|
||||
localBinary string
|
||||
serviceName string
|
||||
healthURL string
|
||||
configTmpl string
|
||||
configPath string
|
||||
unitTmpl string
|
||||
renderUnit func() (string, error)
|
||||
setup string // extra remote shell run before restart; empty for none
|
||||
}{
|
||||
"appview": {
|
||||
ip: state.Appview.PublicIP,
|
||||
binaryName: naming.Appview(),
|
||||
buildCmd: "appview",
|
||||
localBinary: "atcr-appview",
|
||||
serviceName: naming.Appview(),
|
||||
healthURL: "http://localhost:5000/health",
|
||||
configTmpl: appviewConfigTmpl,
|
||||
configPath: naming.AppviewConfigPath(),
|
||||
unitTmpl: appviewServiceTmpl,
|
||||
renderUnit: standardUnit(appviewServiceTmpl, naming.Appview(), naming.AppviewConfigPath(), naming.Appview()),
|
||||
},
|
||||
"hold": {
|
||||
ip: state.Hold.PublicIP,
|
||||
binaryName: naming.Hold(),
|
||||
buildCmd: "hold",
|
||||
localBinary: "atcr-hold",
|
||||
serviceName: naming.Hold(),
|
||||
healthURL: "http://localhost:8080/xrpc/_health",
|
||||
configTmpl: holdConfigTmpl,
|
||||
configPath: naming.HoldConfigPath(),
|
||||
unitTmpl: holdServiceTmpl,
|
||||
renderUnit: standardUnit(holdServiceTmpl, naming.Hold(), naming.HoldConfigPath(), naming.Hold()),
|
||||
},
|
||||
"scanner": {
|
||||
ip: state.Scanner.PublicIP,
|
||||
binaryName: naming.Scanner(),
|
||||
localBinary: "atcr-scanner",
|
||||
serviceName: naming.Scanner(),
|
||||
healthURL: "http://localhost:9090/healthz",
|
||||
configTmpl: scannerConfigTmpl,
|
||||
configPath: naming.ScannerConfigPath(),
|
||||
renderUnit: func() (string, error) {
|
||||
return renderScannerServiceUnit(scannerUnitParams(naming))
|
||||
},
|
||||
setup: scannerDirsScript(naming),
|
||||
},
|
||||
}
|
||||
|
||||
@@ -129,10 +160,18 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
switch target {
|
||||
case "all":
|
||||
toUpdate = []string{"appview", "hold"}
|
||||
if state.Scanner.UUID != "" {
|
||||
toUpdate = append(toUpdate, "scanner")
|
||||
}
|
||||
case "appview", "hold":
|
||||
toUpdate = []string{target}
|
||||
case "scanner":
|
||||
if state.Scanner.UUID == "" {
|
||||
return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one")
|
||||
}
|
||||
toUpdate = []string{target}
|
||||
default:
|
||||
return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target)
|
||||
return fmt.Errorf("unknown target: %s (use: all, appview, hold, scanner)", target)
|
||||
}
|
||||
|
||||
// Build all binaries via `make build-trixie` so output links against
|
||||
@@ -182,14 +221,7 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
}
|
||||
|
||||
// Sync systemd service unit
|
||||
renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName,
|
||||
ConfigPath: t.configPath,
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: t.serviceName,
|
||||
})
|
||||
renderedUnit, err := t.renderUnit()
|
||||
if err != nil {
|
||||
return fmt.Errorf("render %s service unit: %w", name, err)
|
||||
}
|
||||
@@ -210,58 +242,10 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts u
|
||||
daemonReload = "systemctl daemon-reload"
|
||||
}
|
||||
|
||||
// Scanner additions for hold server
|
||||
scannerRestart := ""
|
||||
scannerHealthCheck := ""
|
||||
if name == "hold" && state.ScannerEnabled {
|
||||
// Sync scanner config keys
|
||||
scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
|
||||
if err != nil {
|
||||
return fmt.Errorf("render scanner config: %w", err)
|
||||
if t.setup != "" {
|
||||
if _, err := runSSH(t.ip, t.setup, false); err != nil {
|
||||
return fmt.Errorf("%s setup: %w", name, err)
|
||||
}
|
||||
if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML, false); err != nil {
|
||||
return fmt.Errorf("scanner config sync: %w", err)
|
||||
}
|
||||
|
||||
// Sync scanner service unit
|
||||
scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
|
||||
DisplayName: naming.DisplayName(),
|
||||
User: naming.SystemUser(),
|
||||
BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(),
|
||||
ConfigPath: naming.ScannerConfigPath(),
|
||||
DataDir: naming.BasePath(),
|
||||
ServiceName: naming.Scanner(),
|
||||
HoldServiceName: naming.Hold(),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("render scanner service unit: %w", err)
|
||||
}
|
||||
scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scanner service unit sync: %w", err)
|
||||
}
|
||||
if scannerUnitChanged {
|
||||
daemonReload = "systemctl daemon-reload"
|
||||
}
|
||||
|
||||
// Upload scanner binary
|
||||
scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
|
||||
scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
|
||||
if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil {
|
||||
return fmt.Errorf("upload scanner: %w", err)
|
||||
}
|
||||
|
||||
// Ensure scanner data dirs exist on server
|
||||
scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp
|
||||
chown -R %s:%s %s`,
|
||||
naming.ScannerDataDir(), naming.ScannerDataDir(),
|
||||
naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())
|
||||
if _, err := runSSH(t.ip, scannerSetup, false); err != nil {
|
||||
return fmt.Errorf("scanner dir setup: %w", err)
|
||||
}
|
||||
|
||||
scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
|
||||
scannerHealthCheck = renderHealthProbe("scanner", "http://localhost:9090/healthz", healthProbeTries)
|
||||
}
|
||||
|
||||
// Labeler additions for appview server
|
||||
@@ -324,9 +308,9 @@ chown -R %s:%s %s`,
|
||||
// status and only shows up as a health URL that never answers.
|
||||
restartScript := fmt.Sprintf(`set -euo pipefail
|
||||
%s
|
||||
systemctl restart %s%s%s
|
||||
%s%s%s`, daemonReload, t.serviceName, scannerRestart, labelerRestart,
|
||||
renderHealthProbe(name, t.healthURL, healthProbeTries), scannerHealthCheck, labelerHealthCheck)
|
||||
systemctl restart %s%s
|
||||
%s%s`, daemonReload, t.serviceName, labelerRestart,
|
||||
renderHealthProbe(name, t.healthURL, healthProbeTries), labelerHealthCheck)
|
||||
|
||||
output, err := runSSH(t.ip, restartScript, true)
|
||||
if err != nil {
|
||||
@@ -340,9 +324,6 @@ systemctl restart %s%s%s
|
||||
|
||||
results := parseProbes(output)
|
||||
reportProbe(results, name, t.ip, t.serviceName)
|
||||
if name == "hold" && state.ScannerEnabled {
|
||||
reportProbe(results, "scanner", t.ip, naming.Scanner())
|
||||
}
|
||||
if name == "appview" && state.LabelerEnabled {
|
||||
reportProbe(results, "labeler", t.ip, naming.Labeler())
|
||||
}
|
||||
@@ -361,17 +342,18 @@ func configValsFromState(state *InfraState) *ConfigValues {
|
||||
labelerDomain := "labeler." + baseDomain
|
||||
|
||||
return &ConfigValues{
|
||||
S3Endpoint: state.ObjectStorage.Endpoint,
|
||||
S3Region: state.ObjectStorage.Region,
|
||||
S3Bucket: state.ObjectStorage.Bucket,
|
||||
S3AccessKey: state.ObjectStorage.AccessKeyID,
|
||||
S3SecretKey: "", // not persisted in state; existing value on server is preserved
|
||||
Zone: state.Zone,
|
||||
HoldDomain: holdDomain,
|
||||
HoldDid: "did:web:" + holdDomain,
|
||||
LabelerDomain: labelerDomain,
|
||||
BasePath: naming.BasePath(),
|
||||
ScannerSecret: state.ScannerSecret,
|
||||
S3Endpoint: state.ObjectStorage.Endpoint,
|
||||
S3Region: state.ObjectStorage.Region,
|
||||
S3Bucket: state.ObjectStorage.Bucket,
|
||||
S3AccessKey: state.ObjectStorage.AccessKeyID,
|
||||
S3SecretKey: "", // not persisted in state; existing value on server is preserved
|
||||
Zone: state.Zone,
|
||||
HoldDomain: holdDomain,
|
||||
HoldDid: "did:web:" + holdDomain,
|
||||
LabelerDomain: labelerDomain,
|
||||
BasePath: naming.BasePath(),
|
||||
ScannerSecret: state.ScannerSecret,
|
||||
ScannerHoldURL: scannerHoldURL(state.Hold.PrivateIP),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -420,8 +402,13 @@ func cmdSSH(target string) error {
|
||||
ip = state.Appview.PublicIP
|
||||
case "hold":
|
||||
ip = state.Hold.PublicIP
|
||||
case "scanner":
|
||||
ip = state.Scanner.PublicIP
|
||||
if ip == "" {
|
||||
return fmt.Errorf("no scanner server in state.json; run `provision --with-scanner` to create one")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unknown target: %s (use: appview, hold)", target)
|
||||
return fmt.Errorf("unknown target: %s (use: appview, hold, scanner)", target)
|
||||
}
|
||||
|
||||
fmt.Printf("Connecting to %s (%s)...\n", target, ip)
|
||||
|
||||
Reference in New Issue
Block a user