at-container-registry/deploy/upcloud/update.go

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"github.com/spf13/cobra"
)

var updateCmd = &cobra.Command{
	Use:       "update [target]",
	Short:     "Deploy updates to servers",
	Args:      cobra.MaximumNArgs(1),
	ValidArgs: []string{"all", "appview", "hold"},
	RunE: func(cmd *cobra.Command, args []string) error {
		target := "all"
		if len(args) > 0 {
			target = args[0]
		}
		withScanner, _ := cmd.Flags().GetBool("with-scanner")
		withLabeler, _ := cmd.Flags().GetBool("with-labeler")
		withBilling, _ := cmd.Flags().GetBool("with-billing")
		skipBackup, _ := cmd.Flags().GetBool("skip-backup")
		keep, _ := cmd.Flags().GetInt("backup-keep")
		return cmdUpdate(target, withScanner, withLabeler, withBilling, updateOptions{SkipBackup: skipBackup, BackupKeep: keep})
	},
}

var sshCmd = &cobra.Command{
	Use:       "ssh <target>",
	Short:     "SSH into a server",
	Args:      cobra.ExactArgs(1),
	ValidArgs: []string{"appview", "hold"},
	RunE: func(cmd *cobra.Command, args []string) error {
		return cmdSSH(args[0])
	},
}

func init() {
	updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.")
	updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
	updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview")
	updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into <data dir>/predeploy-<stamp>/ before replacing them")
	updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup")
	rootCmd.AddCommand(updateCmd)
	rootCmd.AddCommand(sshCmd)
}

// updateOptions are the deploy knobs that are not about what gets built.
type updateOptions struct {
	SkipBackup bool
	BackupKeep int
}

func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts updateOptions) error {
	state, err := loadState()
	if err != nil {
		return err
	}

	naming := state.Naming()
	rootDir := projectRoot()

	// Enable scanner retroactively via --with-scanner on update
	if withScanner && !state.ScannerEnabled {
		state.ScannerEnabled = true
		if state.ScannerSecret == "" {
			secret, err := generateScannerSecret()
			if err != nil {
				return fmt.Errorf("generate scanner secret: %w", err)
			}
			state.ScannerSecret = secret
			fmt.Printf("Generated scanner shared secret\n")
		}
		_ = saveState(state)
	}

	// Enable labeler retroactively via --with-labeler on update
	if withLabeler && !state.LabelerEnabled {
		state.LabelerEnabled = true
		_ = saveState(state)
	}

	vals := configValsFromState(state)

	targets := map[string]struct {
		ip          string
		binaryName  string
		buildCmd    string
		localBinary string
		serviceName string
		healthURL   string
		configTmpl  string
		configPath  string
		unitTmpl    string
	}{
		"appview": {
			ip:          state.Appview.PublicIP,
			binaryName:  naming.Appview(),
			buildCmd:    "appview",
			localBinary: "atcr-appview",
			serviceName: naming.Appview(),
			healthURL:   "http://localhost:5000/health",
			configTmpl:  appviewConfigTmpl,
			configPath:  naming.AppviewConfigPath(),
			unitTmpl:    appviewServiceTmpl,
		},
		"hold": {
			ip:          state.Hold.PublicIP,
			binaryName:  naming.Hold(),
			buildCmd:    "hold",
			localBinary: "atcr-hold",
			serviceName: naming.Hold(),
			healthURL:   "http://localhost:8080/xrpc/_health",
			configTmpl:  holdConfigTmpl,
			configPath:  naming.HoldConfigPath(),
			unitTmpl:    holdServiceTmpl,
		},
	}

	var toUpdate []string
	switch target {
	case "all":
		toUpdate = []string{"appview", "hold"}
	case "appview", "hold":
		toUpdate = []string{target}
	default:
		return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target)
	}

	// Build all binaries via `make build-trixie` so output links against
	// glibc 2.41 (the deploy target's glibc) regardless of the host's glibc.
	// build-trixie depends on the `generate` make target, which runs
	// `go generate ./...` on the host before invoking the trixie container.
	if err := runMakeBuildTrixie(rootDir, withBilling); err != nil {
		return fmt.Errorf("build: %w", err)
	}

	// Check what was actually produced, not what was asked for. The build tag
	// is invisible in the binary's name and in its version stamp, so a wrong
	// one is only discovered in production — as happened here, where a deploy
	// silently compiled billing into a deployment running without it.
	if slices.Contains(toUpdate, "appview") {
		if err := verifyAppviewBilling(filepath.Join(rootDir, "bin", "atcr-appview"), withBilling); err != nil {
			return err
		}
	}

	// Deploy each target
	for _, name := range toUpdate {
		t := targets[name]
		fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip)

		// Snapshot what is about to be replaced. The upload below deletes the
		// remote binary before copying the new one, so without this there is
		// nothing to roll back to if the deploy fails half way.
		if opts.SkipBackup {
			fmt.Printf("  backup: skipped (--skip-backup)\n")
		} else {
			spec := backupSpecFor(name, naming, state, time.Now(), opts.BackupKeep)
			dir, err := preDeployBackup(t.ip, spec)
			if err != nil {
				return fmt.Errorf("%s pre-deploy backup: %w", name, err)
			}
			fmt.Printf("  backup: %s (restore by copying bin/, etc/, systemd/ back and restarting)\n", dir)
		}

		// Sync config keys (adds missing keys from template, never overwrites)
		configYAML, err := renderConfig(t.configTmpl, vals)
		if err != nil {
			return fmt.Errorf("render %s config: %w", name, err)
		}
		if err := syncConfigKeys(name, t.ip, t.configPath, configYAML, false); err != nil {
			return fmt.Errorf("%s config sync: %w", name, err)
		}

		// Sync systemd service unit
		renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
			DisplayName: naming.DisplayName(),
			User:        naming.SystemUser(),
			BinaryPath:  naming.InstallDir() + "/bin/" + t.binaryName,
			ConfigPath:  t.configPath,
			DataDir:     naming.BasePath(),
			ServiceName: t.serviceName,
		})
		if err != nil {
			return fmt.Errorf("render %s service unit: %w", name, err)
		}
		unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit)
		if err != nil {
			return fmt.Errorf("%s service unit sync: %w", name, err)
		}

		// Upload binary
		localPath := filepath.Join(rootDir, "bin", t.localBinary)
		remotePath := naming.InstallDir() + "/bin/" + t.binaryName
		if err := scpFile(localPath, t.ip, remotePath); err != nil {
			return fmt.Errorf("upload %s: %w", name, err)
		}

		daemonReload := ""
		if unitChanged {
			daemonReload = "systemctl daemon-reload"
		}

		// Scanner additions for hold server
		scannerRestart := ""
		scannerHealthCheck := ""
		if name == "hold" && state.ScannerEnabled {
			// Sync scanner config keys
			scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
			if err != nil {
				return fmt.Errorf("render scanner config: %w", err)
			}
			if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML, false); err != nil {
				return fmt.Errorf("scanner config sync: %w", err)
			}

			// Sync scanner service unit
			scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
				DisplayName:     naming.DisplayName(),
				User:            naming.SystemUser(),
				BinaryPath:      naming.InstallDir() + "/bin/" + naming.Scanner(),
				ConfigPath:      naming.ScannerConfigPath(),
				DataDir:         naming.BasePath(),
				ServiceName:     naming.Scanner(),
				HoldServiceName: naming.Hold(),
			})
			if err != nil {
				return fmt.Errorf("render scanner service unit: %w", err)
			}
			scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
			if err != nil {
				return fmt.Errorf("scanner service unit sync: %w", err)
			}
			if scannerUnitChanged {
				daemonReload = "systemctl daemon-reload"
			}

			// Upload scanner binary
			scannerLocal := filepath.Join(rootDir, "bin", "atcr-scanner")
			scannerRemote := naming.InstallDir() + "/bin/" + naming.Scanner()
			if err := scpFile(scannerLocal, t.ip, scannerRemote); err != nil {
				return fmt.Errorf("upload scanner: %w", err)
			}

			// Ensure scanner data dirs exist on server
			scannerSetup := fmt.Sprintf(`mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s`,
				naming.ScannerDataDir(), naming.ScannerDataDir(),
				naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())
			if _, err := runSSH(t.ip, scannerSetup, false); err != nil {
				return fmt.Errorf("scanner dir setup: %w", err)
			}

			scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
			scannerHealthCheck = renderHealthProbe("scanner", "http://localhost:9090/healthz", healthProbeTries)
		}

		// Labeler additions for appview server
		labelerRestart := ""
		labelerHealthCheck := ""
		if name == "appview" && state.LabelerEnabled {
			// Sync labeler config keys
			labelerConfigYAML, err := renderConfig(labelerConfigTmpl, vals)
			if err != nil {
				return fmt.Errorf("render labeler config: %w", err)
			}
			if err := syncConfigKeys("labeler", t.ip, naming.LabelerConfigPath(), labelerConfigYAML, false); err != nil {
				return fmt.Errorf("labeler config sync: %w", err)
			}

			// Sync labeler service unit
			labelerUnit, err := renderLabelerServiceUnit(labelerServiceUnitParams{
				DisplayName:        naming.DisplayName(),
				User:               naming.SystemUser(),
				BinaryPath:         naming.InstallDir() + "/bin/" + naming.Labeler(),
				ConfigPath:         naming.LabelerConfigPath(),
				DataDir:            naming.BasePath(),
				ServiceName:        naming.Labeler(),
				AppviewServiceName: naming.Appview(),
			})
			if err != nil {
				return fmt.Errorf("render labeler service unit: %w", err)
			}
			labelerUnitChanged, err := syncServiceUnit("labeler", t.ip, naming.Labeler(), labelerUnit)
			if err != nil {
				return fmt.Errorf("labeler service unit sync: %w", err)
			}
			if labelerUnitChanged {
				daemonReload = "systemctl daemon-reload"
			}

			// Upload labeler binary
			labelerLocal := filepath.Join(rootDir, "bin", "atcr-labeler")
			labelerRemote := naming.InstallDir() + "/bin/" + naming.Labeler()
			if err := scpFile(labelerLocal, t.ip, labelerRemote); err != nil {
				return fmt.Errorf("upload labeler: %w", err)
			}

			// Ensure labeler data dirs exist
			labelerSetup := fmt.Sprintf(`mkdir -p %s
chown -R %s:%s %s`,
				naming.LabelerDataDir(),
				naming.SystemUser(), naming.SystemUser(), naming.LabelerDataDir())
			if _, err := runSSH(t.ip, labelerSetup, false); err != nil {
				return fmt.Errorf("labeler dir setup: %w", err)
			}

			labelerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Labeler())
			labelerHealthCheck = renderHealthProbe("labeler", "http://localhost:5002/.well-known/did.json", healthProbeTries)
		}

		// Restart services, then poll each one's health URL for up to 30s.
		// Every restarted service is probed, the labeler included: a service
		// that crash-loops on its config is invisible to systemctl's exit
		// status and only shows up as a health URL that never answers.
		restartScript := fmt.Sprintf(`set -euo pipefail
%s
systemctl restart %s%s%s
%s%s%s`, daemonReload, t.serviceName, scannerRestart, labelerRestart,
			renderHealthProbe(name, t.healthURL, healthProbeTries), scannerHealthCheck, labelerHealthCheck)

		output, err := runSSH(t.ip, restartScript, true)
		if err != nil {
			fmt.Printf("  ERROR: %v\n", err)
			fmt.Printf("  Output: %s\n", output)
			return fmt.Errorf("restart %s failed", name)
		}
		if !probeMarkersPresent(output) {
			fmt.Printf("  %s: restarted, but the health probes never ran\n", name)
		}

		results := parseProbes(output)
		reportProbe(results, name, t.ip, t.serviceName)
		if name == "hold" && state.ScannerEnabled {
			reportProbe(results, "scanner", t.ip, naming.Scanner())
		}
		if name == "appview" && state.LabelerEnabled {
			reportProbe(results, "labeler", t.ip, naming.Labeler())
		}
	}

	return nil
}

// configValsFromState builds ConfigValues from persisted state.
// S3SecretKey is intentionally left empty — syncConfigKeys only adds missing
// keys and never overwrites, so the server's existing secret is preserved.
func configValsFromState(state *InfraState) *ConfigValues {
	naming := state.Naming()
	_, baseDomain, _, _ := extractFromAppviewTemplate()
	holdDomain := state.Zone + ".cove." + baseDomain
	labelerDomain := "labeler." + baseDomain

	return &ConfigValues{
		S3Endpoint:    state.ObjectStorage.Endpoint,
		S3Region:      state.ObjectStorage.Region,
		S3Bucket:      state.ObjectStorage.Bucket,
		S3AccessKey:   state.ObjectStorage.AccessKeyID,
		S3SecretKey:   "", // not persisted in state; existing value on server is preserved
		Zone:          state.Zone,
		HoldDomain:    holdDomain,
		HoldDid:       "did:web:" + holdDomain,
		LabelerDomain: labelerDomain,
		BasePath:      naming.BasePath(),
		ScannerSecret: state.ScannerSecret,
	}
}

// runMakeBuildTrixie shells out to `make build-trixie`, which builds all
// production binaries (appview, hold, credential-helper, labeler, scanner)
// inside a Debian 13 container so they link against glibc 2.41. Centralizing
// the build recipe in the Makefile keeps deploy and local builds in sync.
func runMakeBuildTrixie(rootDir string, withBilling bool) error {
	fmt.Printf("Running `make build-trixie` (linux/amd64, glibc 2.41, billing: %s)...\n", billingLabel(withBilling))
	args := []string{"build-trixie"}
	if withBilling {
		args = append(args, "BILLING=1")
	}
	cmd := exec.Command("make", args...)
	cmd.Dir = rootDir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// scpFile uploads a local file to a remote server via SCP.
// Removes the remote file first to avoid ETXTBSY when overwriting a running binary.
func scpFile(localPath, ip, remotePath string) error {
	fmt.Printf("  uploading %s → %s:%s\n", filepath.Base(localPath), ip, remotePath)
	_, _ = runSSH(ip, fmt.Sprintf("rm -f %s", remotePath), false)
	cmd := exec.Command("scp",
		"-o", "StrictHostKeyChecking=accept-new",
		"-o", "ConnectTimeout=10",
		localPath,
		"root@"+ip+":"+remotePath,
	)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func cmdSSH(target string) error {
	state, err := loadState()
	if err != nil {
		return err
	}

	var ip string
	switch target {
	case "appview":
		ip = state.Appview.PublicIP
	case "hold":
		ip = state.Hold.PublicIP
	default:
		return fmt.Errorf("unknown target: %s (use: appview, hold)", target)
	}

	fmt.Printf("Connecting to %s (%s)...\n", target, ip)
	cmd := exec.Command("ssh",
		"-o", "StrictHostKeyChecking=accept-new",
		"root@"+ip,
	)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func runSSH(ip, script string, stream bool) (string, error) {
	cmd := exec.Command("ssh",
		"-o", "StrictHostKeyChecking=accept-new",
		"-o", "ConnectTimeout=10",
		"root@"+ip,
		"bash -s",
	)
	cmd.Stdin = strings.NewReader(script)

	var buf bytes.Buffer
	if stream {
		cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
		cmd.Stderr = io.MultiWriter(os.Stderr, &buf)
	} else {
		cmd.Stdout = &buf
		cmd.Stderr = &buf
	}

	// Give deploys up to 5 minutes (SCP + restart, much faster than remote builds)
	done := make(chan error, 1)
	go func() { done <- cmd.Run() }()

	select {
	case err := <-done:
		return buf.String(), err
	case <-time.After(5 * time.Minute):
		_ = cmd.Process.Kill()
		return buf.String(), fmt.Errorf("SSH command timed out after 5 minutes")
	}
}

func billingLabel(on bool) string {
	if on {
		return "ENABLED"
	}
	return "disabled"
}

// verifyAppviewBilling refuses to ship an appview whose billing state is not
// the one asked for.
//
// It reads the built binary rather than trusting the flag, because the two can
// disagree for reasons the flag cannot see: a stale bin/atcr-appview left by an
// earlier build, a Makefile that hardcodes the tag, a builder image that
// ignored it. The Stripe SDK is only linked when the `billing` tag is set, so
// its symbols are a direct observation of which build this is.
//
// Getting this wrong is not cosmetic. Compiling billing into a deployment that
// runs without it exposes a live Stripe webhook endpoint and a paid tier ladder
// on a service whose operator chose neither.
func verifyAppviewBilling(binPath string, want bool) error {
	data, err := os.ReadFile(binPath)
	if err != nil {
		return fmt.Errorf("verify appview billing: %w", err)
	}
	got := bytes.Contains(data, []byte("stripe-go"))
	if got != want {
		return fmt.Errorf(
			"appview billing mismatch: asked for %s, built binary has billing %s.\n"+
				"  %s carries %s Stripe symbols. Rebuild before deploying:\n"+
				"    make build-trixie%s",
			billingLabel(want), billingLabel(got),
			binPath, map[bool]string{true: "", false: "no"}[got],
			map[bool]string{true: " BILLING=1", false: ""}[want],
		)
	}
	fmt.Printf("  appview billing verified: %s\n", billingLabel(got))
	return nil
}