package main import ( "bytes" "fmt" "io" "os" "os/exec" "path/filepath" "strings" "time" "github.com/spf13/cobra" ) var updateCmd = &cobra.Command{ Use: "update [target]", Short: "Deploy updates to servers", Args: cobra.MaximumNArgs(1), ValidArgs: []string{"all", "appview", "hold"}, RunE: func(cmd *cobra.Command, args []string) error { target := "all" if len(args) > 0 { target = args[0] } withScanner, _ := cmd.Flags().GetBool("with-scanner") withLabeler, _ := cmd.Flags().GetBool("with-labeler") return cmdUpdate(target, withScanner, withLabeler) }, } var sshCmd = &cobra.Command{ Use: "ssh ", Short: "SSH into a server", Args: cobra.ExactArgs(1), ValidArgs: []string{"appview", "hold"}, RunE: func(cmd *cobra.Command, args []string) error { return cmdSSH(args[0]) }, } func init() { updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold") updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview") rootCmd.AddCommand(updateCmd) rootCmd.AddCommand(sshCmd) } func cmdUpdate(target string, withScanner, withLabeler bool) error { state, err := loadState() if err != nil { return err } naming := state.Naming() rootDir := projectRoot() // Enable scanner retroactively via --with-scanner on update if withScanner && !state.ScannerEnabled { state.ScannerEnabled = true if state.ScannerSecret == "" { secret, err := generateScannerSecret() if err != nil { return fmt.Errorf("generate scanner secret: %w", err) } state.ScannerSecret = secret fmt.Printf("Generated scanner shared secret\n") } _ = saveState(state) } // Enable labeler retroactively via --with-labeler on update if withLabeler && !state.LabelerEnabled { state.LabelerEnabled = true _ = saveState(state) } vals := configValsFromState(state) targets := map[string]struct { ip string binaryName string buildCmd string localBinary string serviceName string healthURL string configTmpl string configPath string unitTmpl string }{ "appview": { ip: state.Appview.PublicIP, binaryName: naming.Appview(), buildCmd: "appview", localBinary: "atcr-appview", serviceName: naming.Appview(), healthURL: "http://localhost:5000/health", configTmpl: appviewConfigTmpl, configPath: naming.AppviewConfigPath(), unitTmpl: appviewServiceTmpl, }, "hold": { ip: state.Hold.PublicIP, binaryName: naming.Hold(), buildCmd: "hold", localBinary: "atcr-hold", serviceName: naming.Hold(), healthURL: "http://localhost:8080/xrpc/_health", configTmpl: holdConfigTmpl, configPath: naming.HoldConfigPath(), unitTmpl: holdServiceTmpl, }, } var toUpdate []string switch target { case "all": toUpdate = []string{"appview", "hold"} case "appview", "hold": toUpdate = []string{target} default: return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target) } // Build all binaries via `make build-trixie` so output links against // glibc 2.41 (the deploy target's glibc) regardless of the host's glibc. // build-trixie depends on the `generate` make target, which runs // `go generate ./...` on the host before invoking the trixie container. if err := runMakeBuildTrixie(rootDir); err != nil { return fmt.Errorf("build: %w", err) } // Deploy each target for _, name := range toUpdate { t := targets[name] fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip) // Sync config keys (adds missing keys from template, never overwrites) configYAML, err := renderConfig(t.configTmpl, vals) if err != nil { return fmt.Errorf("render %s config: %w", name, err) } if err := syncConfigKeys(name, t.ip, t.configPath, configYAML); err != nil { return fmt.Errorf("%s config sync: %w", name, err) } // Sync systemd service unit renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + t.binaryName, ConfigPath: t.configPath, DataDir: naming.BasePath(), ServiceName: t.serviceName, }) if err != nil { return fmt.Errorf("render %s service unit: %w", name, err) } unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit) if err != nil { return fmt.Errorf("%s service unit sync: %w", name, err) } // Upload binary localPath := filepath.Join(rootDir, "bin", t.localBinary) remotePath := naming.InstallDir() + "/bin/" + t.binaryName if err := scpFile(localPath, t.ip, remotePath); err != nil { return fmt.Errorf("upload %s: %w", name, err) } daemonReload := "" if unitChanged { daemonReload = "systemctl daemon-reload" } // Scanner additions for hold server scannerRestart := "" scannerHealthCheck := "" if name == "hold" && state.ScannerEnabled { // Sync scanner config keys scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals) if err != nil { return fmt.Errorf("render scanner config: %w", err) } if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML); err != nil { return fmt.Errorf("scanner config sync: %w", err) } // Sync scanner service unit scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + naming.Scanner(), ConfigPath: naming.ScannerConfigPath(), DataDir: naming.BasePath(), ServiceName: naming.Scanner(), HoldServiceName: naming.Hold(), }) if err != nil { return fmt.Errorf("render scanner service unit: %w", err) } scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit) if err != nil { return fmt.Errorf("scanner service unit sync: %w", err) } if scannerUnitChanged { daemonReload = "systemctl daemon-reload" } // Upload scanner binary scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner") scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner() if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil { return fmt.Errorf("upload scanner: %w", err) } // Ensure scanner data dirs exist on server scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp chown -R %s:%s %s`, naming.ScannerDataDir(), naming.ScannerDataDir(), naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir()) if _, err := runSSH(t.ip, scannerSetup, false); err != nil { return fmt.Errorf("scanner dir setup: %w", err) } scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner()) scannerHealthCheck = ` sleep 2 curl -sf http://localhost:9090/healthz > /dev/null && echo "SCANNER_HEALTH_OK" || echo "SCANNER_HEALTH_FAIL" ` } // Labeler additions for appview server labelerRestart := "" if name == "appview" && state.LabelerEnabled { // Sync labeler config keys labelerConfigYAML, err := renderConfig(labelerConfigTmpl, vals) if err != nil { return fmt.Errorf("render labeler config: %w", err) } if err := syncConfigKeys("labeler", t.ip, naming.LabelerConfigPath(), labelerConfigYAML); err != nil { return fmt.Errorf("labeler config sync: %w", err) } // Sync labeler service unit labelerUnit, err := renderLabelerServiceUnit(labelerServiceUnitParams{ DisplayName: naming.DisplayName(), User: naming.SystemUser(), BinaryPath: naming.InstallDir() + "/bin/" + naming.Labeler(), ConfigPath: naming.LabelerConfigPath(), DataDir: naming.BasePath(), ServiceName: naming.Labeler(), AppviewServiceName: naming.Appview(), }) if err != nil { return fmt.Errorf("render labeler service unit: %w", err) } labelerUnitChanged, err := syncServiceUnit("labeler", t.ip, naming.Labeler(), labelerUnit) if err != nil { return fmt.Errorf("labeler service unit sync: %w", err) } if labelerUnitChanged { daemonReload = "systemctl daemon-reload" } // Upload labeler binary labelerLocal := filepath.Join(rootDir, "bin", "atcr-labeler") labelerRemote := naming.InstallDir() + "/bin/" + naming.Labeler() if err := scpFile(labelerLocal, t.ip, labelerRemote); err != nil { return fmt.Errorf("upload labeler: %w", err) } // Ensure labeler data dirs exist labelerSetup := fmt.Sprintf(`mkdir -p %s chown -R %s:%s %s`, naming.LabelerDataDir(), naming.SystemUser(), naming.SystemUser(), naming.LabelerDataDir()) if _, err := runSSH(t.ip, labelerSetup, false); err != nil { return fmt.Errorf("labeler dir setup: %w", err) } labelerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Labeler()) } // Restart services and health check restartScript := fmt.Sprintf(`set -euo pipefail %s systemctl restart %s%s%s sleep 2 curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL" %s`, daemonReload, t.serviceName, scannerRestart, labelerRestart, t.healthURL, scannerHealthCheck) output, err := runSSH(t.ip, restartScript, true) if err != nil { fmt.Printf(" ERROR: %v\n", err) fmt.Printf(" Output: %s\n", output) return fmt.Errorf("restart %s failed", name) } if strings.Contains(output, "HEALTH_OK") { fmt.Printf(" %s: updated and healthy\n", name) } else if strings.Contains(output, "HEALTH_FAIL") { fmt.Printf(" %s: updated but health check failed!\n", name) fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, t.serviceName) } else { fmt.Printf(" %s: updated (health check inconclusive)\n", name) } // Scanner health reporting if name == "hold" && state.ScannerEnabled { if strings.Contains(output, "SCANNER_HEALTH_OK") { fmt.Printf(" scanner: updated and healthy\n") } else if strings.Contains(output, "SCANNER_HEALTH_FAIL") { fmt.Printf(" scanner: updated but health check failed!\n") fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, naming.Scanner()) } } } return nil } // configValsFromState builds ConfigValues from persisted state. // S3SecretKey is intentionally left empty — syncConfigKeys only adds missing // keys and never overwrites, so the server's existing secret is preserved. func configValsFromState(state *InfraState) *ConfigValues { naming := state.Naming() _, baseDomain, _, _ := extractFromAppviewTemplate() holdDomain := state.Zone + ".cove." + baseDomain labelerDomain := "labeler." + baseDomain return &ConfigValues{ S3Endpoint: state.ObjectStorage.Endpoint, S3Region: state.ObjectStorage.Region, S3Bucket: state.ObjectStorage.Bucket, S3AccessKey: state.ObjectStorage.AccessKeyID, S3SecretKey: "", // not persisted in state; existing value on server is preserved Zone: state.Zone, HoldDomain: holdDomain, HoldDid: "did:web:" + holdDomain, LabelerDomain: labelerDomain, BasePath: naming.BasePath(), ScannerSecret: state.ScannerSecret, } } // runMakeBuildTrixie shells out to `make build-trixie`, which builds all // production binaries (appview, hold, credential-helper, labeler, scanner) // inside a Debian 13 container so they link against glibc 2.41. Centralizing // the build recipe in the Makefile keeps deploy and local builds in sync. func runMakeBuildTrixie(rootDir string) error { fmt.Println("Running `make build-trixie` (linux/amd64, glibc 2.41)...") cmd := exec.Command("make", "build-trixie") cmd.Dir = rootDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } // scpFile uploads a local file to a remote server via SCP. // Removes the remote file first to avoid ETXTBSY when overwriting a running binary. func scpFile(localPath, ip, remotePath string) error { fmt.Printf(" uploading %s → %s:%s\n", filepath.Base(localPath), ip, remotePath) _, _ = runSSH(ip, fmt.Sprintf("rm -f %s", remotePath), false) cmd := exec.Command("scp", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", localPath, "root@"+ip+":"+remotePath, ) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func cmdSSH(target string) error { state, err := loadState() if err != nil { return err } var ip string switch target { case "appview": ip = state.Appview.PublicIP case "hold": ip = state.Hold.PublicIP default: return fmt.Errorf("unknown target: %s (use: appview, hold)", target) } fmt.Printf("Connecting to %s (%s)...\n", target, ip) cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=accept-new", "root@"+ip, ) cmd.Stdin = os.Stdin cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr return cmd.Run() } func runSSH(ip, script string, stream bool) (string, error) { cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10", "root@"+ip, "bash -s", ) cmd.Stdin = strings.NewReader(script) var buf bytes.Buffer if stream { cmd.Stdout = io.MultiWriter(os.Stdout, &buf) cmd.Stderr = io.MultiWriter(os.Stderr, &buf) } else { cmd.Stdout = &buf cmd.Stderr = &buf } // Give deploys up to 5 minutes (SCP + restart, much faster than remote builds) done := make(chan error, 1) go func() { done <- cmd.Run() }() select { case err := <-done: return buf.String(), err case <-time.After(5 * time.Minute): _ = cmd.Process.Kill() return buf.String(), fmt.Errorf("SSH command timed out after 5 minutes") } }