package main import ( "bytes" "fmt" "io" "os" "os/exec" "path/filepath" "slices" "strings" "time" "github.com/spf13/cobra" ) var updateCmd = &cobra.Command{ Use: "update [target]", Short: "Deploy updates to servers", Args: cobra.MaximumNArgs(1), ValidArgs: []string{"all", "appview", "hold"}, RunE: func(cmd *cobra.Command, args []string) error { target := "all" if len(args) > 0 { target = args[0] } withScanner, _ := cmd.Flags().GetBool("with-scanner") withLabeler, _ := cmd.Flags().GetBool("with-labeler") withBilling, _ := cmd.Flags().GetBool("with-billing") skipBackup, _ := cmd.Flags().GetBool("skip-backup") keep, _ := cmd.Flags().GetInt("backup-keep") return cmdUpdate(target, withScanner, withLabeler, withBilling, updateOptions{SkipBackup: skipBackup, BackupKeep: keep}) }, } var sshCmd = &cobra.Command{ Use: "ssh ", Short: "SSH into a server", Args: cobra.ExactArgs(1), ValidArgs: []string{"appview", "hold"}, RunE: func(cmd *cobra.Command, args []string) error { return cmdSSH(args[0]) }, } func init() { updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.") updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold") updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview") updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into /predeploy-/ before replacing them") updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup") rootCmd.AddCommand(updateCmd) rootCmd.AddCommand(sshCmd) } // updateOptions are the deploy knobs that are not about what gets built. type updateOptions struct { SkipBackup bool BackupKeep int } func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts updateOptions) error { state, err := loadState() if err != nil { return err } naming := state.Naming() rootDir := projectRoot() // Enable scanner retroactively via --with-scanner on update if withScanner && !state.ScannerEnabled { state.ScannerEnabled = true if state.ScannerSecret == "" { secret, err := generateScannerSecret() if err != nil { return fmt.Errorf("generate scanner secret: %w", err) } state.ScannerSecret = secret fmt.Printf("Generated scanner shared secret\n") } _ = saveState(state) } // Enable labeler retroactively via --with-labeler on update if withLabeler && !state.LabelerEnabled { state.LabelerEnabled = true _ = saveState(state) } vals := configValsFromState(state) targets := map[string]struct { ip string binaryName string buildCmd string localBinary string serviceName string healthURL string configTmpl string configPath string unitTmpl string }{ "appview": { ip: state.Appview.PublicIP, binaryName: naming.Appview(), buildCmd: "appview", localBinary: "atcr-appview", serviceName: naming.Appview(), healthURL: "http://localhost:5000/health", configTmpl: appviewConfigTmpl, configPath: naming.AppviewConfigPath(), unitTmpl: appviewServiceTmpl, }, "hold": { ip: state.Hold.PublicIP, binaryName: naming.Hold(), buildCmd: "hold", localBinary: "atcr-hold", serviceName: naming.Hold(), healthURL: "http://localhost:8080/xrpc/_health", configTmpl: holdConfigTmpl, configPath: naming.HoldConfigPath(), unitTmpl: holdServiceTmpl, }, } var toUpdate []string switch target { case "all": toUpdate = []string{"appview", "hold"} case "appview", "hold": toUpdate = []string{target} default: return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target) } // Build all binaries via `make build-trixie` so output links against // glibc 2.41 (the deploy target's glibc) regardless of the host's glibc. // build-trixie depends on the `generate` make target, which runs // `go generate ./...` on the host before invoking the trixie container. if err := runMakeBuildTrixie(rootDir, withBilling); err != nil { return fmt.Errorf("build: %w", err) } // Check what was actually produced, not what was asked for. The build tag // is invisible in the binary's name and in its version stamp, so a wrong // one is only discovered in production — as happened here, where a deploy // silently compiled billing into a deployment running without it. if slices.Contains(toUpdate, "appview") { if err := verifyAppviewBilling(filepath.Join(rootDir, "bin", "atcr-appview"), withBilling); err != nil { return err } } // Deploy each target for _, name := range toUpdate { t := targets[name] fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip) // Snapshot what is about to be replaced. The upload below deletes the // remote binary before copying the new one, so without this there is // nothing to roll back to if the deploy fails half way. if opts.SkipBackup { fmt.Printf(" backup: skipped (--skip-backup)\n") } else { spec := backupSpecFor(name, naming, state, time.Now(), opts.BackupKeep) dir, err := preDeployBackup(t.ip, spec) if err != nil { return fmt.Errorf("%s pre-deploy backup: %w", name, err) } fmt.Printf(" backup: %s (restore by copying bin/, etc/, systemd/ back and restarting)\n", dir) } // Sync config keys (adds missing keys from template, never overwrites) configYAML, err := renderConfig(t.configTmpl, vals) if err != nil { return fmt.Errorf("render %s config: %w", name, err) } if err := syncConfigKeys(name, t.ip, t.configPath, configYAML, false); err != nil { return fmt.Errorf("%s config sync: %w", name, err) } // Sync systemd service unit renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName, ConfigPath: t.configPath, DataDir: naming.BasePath(), ServiceName: t.serviceName, }) if err != nil { return fmt.Errorf("render %s service unit: %w", name, err) } unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit) if err != nil { return fmt.Errorf("%s service unit sync: %w", name, err) } // Upload binary localPath := filepath.Join(rootDir, "bin", t.localBinary) remotePath := naming.InstallDir() + "/bin/" + t.binaryName if err := scpFile(localPath, t.ip, remotePath); err != nil { return fmt.Errorf("upload %s: %w", name, err) } daemonReload := "" if unitChanged { daemonReload = "systemctl daemon-reload" } // Scanner additions for hold server scannerRestart := "" scannerHealthCheck := "" if name == "hold" && state.ScannerEnabled { // Sync scanner config keys scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) if err != nil { return fmt.Errorf("render scanner config: %w", err) } if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML, false); err != nil { return fmt.Errorf("scanner config sync: %w", err) } // Sync scanner service unit scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(), ConfigPath: naming.ScannerConfigPath(), DataDir: naming.BasePath(), ServiceName: naming.Scanner(), HoldServiceName: naming.Hold(), }) if err != nil { return fmt.Errorf("render scanner service unit: %w", err) } scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit) if err != nil { return fmt.Errorf("scanner service unit sync: %w", err) } if scannerUnitChanged { daemonReload = "systemctl daemon-reload" } // Upload scanner binary scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner") scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner() if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil { return fmt.Errorf("upload scanner: %w", err) } // Ensure scanner data dirs exist on server scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp chown -R %s:%s %s`, naming.ScannerDataDir(), naming.ScannerDataDir(), naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir()) if _, err := runSSH(t.ip, scannerSetup, false); err != nil { return fmt.Errorf("scanner dir setup: %w", err) } scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner()) scannerHealthCheck = renderHealthProbe("scanner", "http://localhost:9090/healthz", healthProbeTries) } // Labeler additions for appview server labelerRestart := "" labelerHealthCheck := "" if name == "appview" && state.LabelerEnabled { // Sync labeler config keys labelerConfigYAML, err := renderConfig(labelerConfigTmpl, vals) if err != nil { return fmt.Errorf("render labeler config: %w", err) } if err := syncConfigKeys("labeler", t.ip, naming.LabelerConfigPath(), labelerConfigYAML, false); err != nil { return fmt.Errorf("labeler config sync: %w", err) } // Sync labeler service unit labelerUnit, err := renderLabelerServiceUnit(labelerServiceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + naming.Labeler(), ConfigPath: naming.LabelerConfigPath(), DataDir: naming.BasePath(), ServiceName: naming.Labeler(), AppviewServiceName: naming.Appview(), }) if err != nil { return fmt.Errorf("render labeler service unit: %w", err) } labelerUnitChanged, err := syncServiceUnit("labeler", t.ip, naming.Labeler(), labelerUnit) if err != nil { return fmt.Errorf("labeler service unit sync: %w", err) } if labelerUnitChanged { daemonReload = "systemctl daemon-reload" } // Upload labeler binary labelerLocal := filepath.Join(rootDir, "bin", "atcr-labeler") labelerRemote := naming.InstallDir() + "/bin/" + naming.Labeler() if err := scpFile(labelerLocal, t.ip, labelerRemote); err != nil { return fmt.Errorf("upload labeler: %w", err) } // Ensure labeler data dirs exist labelerSetup := fmt.Sprintf(`mkdir -p %s chown -R %s:%s %s`, naming.LabelerDataDir(), naming.SystemUser(), naming.SystemUser(), naming.LabelerDataDir()) if _, err := runSSH(t.ip, labelerSetup, false); err != nil { return fmt.Errorf("labeler dir setup: %w", err) } labelerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Labeler()) labelerHealthCheck = renderHealthProbe("labeler", "http://localhost:5002/.well-known/did.json", healthProbeTries) } // Restart services, then poll each one's health URL for up to 30s. // Every restarted service is probed, the labeler included: a service // that crash-loops on its config is invisible to systemctl's exit // status and only shows up as a health URL that never answers. restartScript := fmt.Sprintf(`set -euo pipefail %s systemctl restart %s%s%s %s%s%s`, daemonReload, t.serviceName, scannerRestart, labelerRestart, renderHealthProbe(name, t.healthURL, healthProbeTries), scannerHealthCheck, labelerHealthCheck) output, err := runSSH(t.ip, restartScript, true) if err != nil { fmt.Printf(" ERROR: %v\n", err) fmt.Printf(" Output: %s\n", output) return fmt.Errorf("restart %s failed", name) } if !probeMarkersPresent(output) { fmt.Printf(" %s: restarted, but the health probes never ran\n", name) } results := parseProbes(output) reportProbe(results, name, t.ip, t.serviceName) if name == "hold" && state.ScannerEnabled { reportProbe(results, "scanner", t.ip, naming.Scanner()) } if name == "appview" && state.LabelerEnabled { reportProbe(results, "labeler", t.ip, naming.Labeler()) } } return nil } // configValsFromState builds ConfigValues from persisted state. // S3SecretKey is intentionally left empty — syncConfigKeys only adds missing // keys and never overwrites, so the server's existing secret is preserved. func configValsFromState(state *InfraState) *ConfigValues { naming := state.Naming() _, baseDomain, _, _ := extractFromAppviewTemplate() holdDomain := state.Zone + ".cove." + baseDomain labelerDomain := "labeler." + baseDomain return &ConfigValues{ S3Endpoint: state.ObjectStorage.Endpoint, S3Region: state.ObjectStorage.Region, S3Bucket: state.ObjectStorage.Bucket, S3AccessKey: state.ObjectStorage.AccessKeyID, S3SecretKey: "", // not persisted in state; existing value on server is preserved Zone: state.Zone, HoldDomain: holdDomain, HoldDid: "did:web:" + holdDomain, LabelerDomain: labelerDomain, BasePath: naming.BasePath(), ScannerSecret: state.ScannerSecret, } } // runMakeBuildTrixie shells out to `make build-trixie`, which builds all // production binaries (appview, hold, credential-helper, labeler, scanner) // inside a Debian 13 container so they link against glibc 2.41. Centralizing // the build recipe in the Makefile keeps deploy and local builds in sync. func runMakeBuildTrixie(rootDir string, withBilling bool) error { fmt.Printf("Running `make build-trixie` (linux/amd64, glibc 2.41, billing: %s)...\n", billingLabel(withBilling)) args := []string{"build-trixie"} if withBilling { args = append(args, "BILLING=1") } cmd := exec.Command("make", args...) cmd.Dir = rootDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } // scpFile uploads a local file to a remote server via SCP. // Removes the remote file first to avoid ETXTBSY when overwriting a running binary. func scpFile(localPath, ip, remotePath string) error { fmt.Printf(" uploading %s → %s:%s\n", filepath.Base(localPath), ip, remotePath) _, _ = runSSH(ip, fmt.Sprintf("rm -f %s", remotePath), false) cmd := exec.Command("scp", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", localPath, "root@"+ip+":"+remotePath, ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func cmdSSH(target string) error { state, err := loadState() if err != nil { return err } var ip string switch target { case "appview": ip = state.Appview.PublicIP case "hold": ip = state.Hold.PublicIP default: return fmt.Errorf("unknown target: %s (use: appview, hold)", target) } fmt.Printf("Connecting to %s (%s)...\n", target, ip) cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=accept-new", "root@"+ip, ) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func runSSH(ip, script string, stream bool) (string, error) { cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", "root@"+ip, "bash -s", ) cmd.Stdin = strings.NewReader(script) var buf bytes.Buffer if stream { cmd.Stdout = io.MultiWriter(os.Stdout, &buf) cmd.Stderr = io.MultiWriter(os.Stderr, &buf) } else { cmd.Stdout = &buf cmd.Stderr = &buf } // Give deploys up to 5 minutes (SCP + restart, much faster than remote builds) done := make(chan error, 1) go func() { done <- cmd.Run() }() select { case err := <-done: return buf.String(), err case <-time.After(5 * time.Minute): _ = cmd.Process.Kill() return buf.String(), fmt.Errorf("SSH command timed out after 5 minutes") } } func billingLabel(on bool) string { if on { return "ENABLED" } return "disabled" } // verifyAppviewBilling refuses to ship an appview whose billing state is not // the one asked for. // // It reads the built binary rather than trusting the flag, because the two can // disagree for reasons the flag cannot see: a stale bin/atcr-appview left by an // earlier build, a Makefile that hardcodes the tag, a builder image that // ignored it. The Stripe SDK is only linked when the `billing` tag is set, so // its symbols are a direct observation of which build this is. // // Getting this wrong is not cosmetic. Compiling billing into a deployment that // runs without it exposes a live Stripe webhook endpoint and a paid tier ladder // on a service whose operator chose neither. func verifyAppviewBilling(binPath string, want bool) error { data, err := os.ReadFile(binPath) if err != nil { return fmt.Errorf("verify appview billing: %w", err) } got := bytes.Contains(data, []byte("stripe-go")) if got != want { return fmt.Errorf( "appview billing mismatch: asked for %s, built binary has billing %s.\n"+ " %s carries %s Stripe symbols. Rebuild before deploying:\n"+ " make build-trixie%s", billingLabel(want), billingLabel(got), binPath, map[bool]string{true: "", false: "no"}[got], map[bool]string{true: " BILLING=1", false: ""}[want], ) } fmt.Printf(" appview billing verified: %s\n", billingLabel(got)) return nil }