at-container-registry/deploy/upcloud/update.go

package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/spf13/cobra"
)

var updateCmd = &cobra.Command{
	Use:       "update [target]",
	Short:     "Deploy updates to servers",
	Args:      cobra.MaximumNArgs(1),
	ValidArgs: []string{"all", "appview", "hold"},
	RunE: func(cmd *cobra.Command, args []string) error {
		target := "all"
		if len(args) > 0 {
			target = args[0]
		}
		withScanner, _ := cmd.Flags().GetBool("with-scanner")
		return cmdUpdate(target, withScanner)
	},
}

var sshCmd = &cobra.Command{
	Use:       "ssh <target>",
	Short:     "SSH into a server",
	Args:      cobra.ExactArgs(1),
	ValidArgs: []string{"appview", "hold"},
	RunE: func(cmd *cobra.Command, args []string) error {
		return cmdSSH(args[0])
	},
}

func init() {
	updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
	rootCmd.AddCommand(updateCmd)
	rootCmd.AddCommand(sshCmd)
}

func cmdUpdate(target string, withScanner bool) error {
	state, err := loadState()
	if err != nil {
		return err
	}

	naming := state.Naming()
	branch := state.Branch()

	goVersion, err := requiredGoVersion()
	if err != nil {
		return err
	}

	// Enable scanner retroactively via --with-scanner on update
	if withScanner && !state.ScannerEnabled {
		state.ScannerEnabled = true
		if state.ScannerSecret == "" {
			secret, err := generateScannerSecret()
			if err != nil {
				return fmt.Errorf("generate scanner secret: %w", err)
			}
			state.ScannerSecret = secret
			fmt.Printf("Generated scanner shared secret\n")
		}
		saveState(state)
	}

	vals := configValsFromState(state)

	targets := map[string]struct {
		ip          string
		binaryName  string
		buildCmd    string
		serviceName string
		healthURL   string
		configTmpl  string
		configPath  string
		unitTmpl    string
	}{
		"appview": {
			ip:          state.Appview.PublicIP,
			binaryName:  naming.Appview(),
			buildCmd:    "appview",
			serviceName: naming.Appview(),
			healthURL:   "http://localhost:5000/health",
			configTmpl:  appviewConfigTmpl,
			configPath:  naming.AppviewConfigPath(),
			unitTmpl:    appviewServiceTmpl,
		},
		"hold": {
			ip:          state.Hold.PublicIP,
			binaryName:  naming.Hold(),
			buildCmd:    "hold",
			serviceName: naming.Hold(),
			healthURL:   "http://localhost:8080/xrpc/_health",
			configTmpl:  holdConfigTmpl,
			configPath:  naming.HoldConfigPath(),
			unitTmpl:    holdServiceTmpl,
		},
	}

	var toUpdate []string
	switch target {
	case "all":
		toUpdate = []string{"appview", "hold"}
	case "appview", "hold":
		toUpdate = []string{target}
	default:
		return fmt.Errorf("unknown target: %s (use: all, appview, hold)", target)
	}

	for _, name := range toUpdate {
		t := targets[name]
		fmt.Printf("Updating %s (%s)...\n", name, t.ip)

		// Sync config keys (adds missing keys from template, never overwrites)
		configYAML, err := renderConfig(t.configTmpl, vals)
		if err != nil {
			return fmt.Errorf("render %s config: %w", name, err)
		}
		if err := syncConfigKeys(name, t.ip, t.configPath, configYAML); err != nil {
			return fmt.Errorf("%s config sync: %w", name, err)
		}

		// Sync systemd service unit
		renderedUnit, err := renderServiceUnit(t.unitTmpl, serviceUnitParams{
			DisplayName: naming.DisplayName(),
			User:        naming.SystemUser(),
			BinaryPath:  naming.InstallDir() + "/bin/" + t.binaryName,
			ConfigPath:  t.configPath,
			DataDir:     naming.BasePath(),
			ServiceName: t.serviceName,
		})
		if err != nil {
			return fmt.Errorf("render %s service unit: %w", name, err)
		}
		unitChanged, err := syncServiceUnit(name, t.ip, t.serviceName, renderedUnit)
		if err != nil {
			return fmt.Errorf("%s service unit sync: %w", name, err)
		}

		daemonReload := ""
		if unitChanged {
			daemonReload = "systemctl daemon-reload"
		}

		// Scanner additions for hold server
		scannerBuild := ""
		scannerRestart := ""
		scannerHealthCheck := ""
		if name == "hold" && state.ScannerEnabled {
			// Sync scanner config keys
			scannerConfigYAML, err := renderConfig(scannerConfigTmpl, vals)
			if err != nil {
				return fmt.Errorf("render scanner config: %w", err)
			}
			if err := syncConfigKeys("scanner", t.ip, naming.ScannerConfigPath(), scannerConfigYAML); err != nil {
				return fmt.Errorf("scanner config sync: %w", err)
			}

			// Sync scanner service unit
			scannerUnit, err := renderScannerServiceUnit(scannerServiceUnitParams{
				DisplayName:     naming.DisplayName(),
				User:            naming.SystemUser(),
				BinaryPath:      naming.InstallDir() + "/bin/" + naming.Scanner(),
				ConfigPath:      naming.ScannerConfigPath(),
				DataDir:         naming.BasePath(),
				ServiceName:     naming.Scanner(),
				HoldServiceName: naming.Hold(),
			})
			if err != nil {
				return fmt.Errorf("render scanner service unit: %w", err)
			}
			scannerUnitChanged, err := syncServiceUnit("scanner", t.ip, naming.Scanner(), scannerUnit)
			if err != nil {
				return fmt.Errorf("scanner service unit sync: %w", err)
			}
			if scannerUnitChanged {
				daemonReload = "systemctl daemon-reload"
			}

			scannerBuild = fmt.Sprintf(`
# Build scanner
cd %s/scanner
CGO_ENABLED=1 go build \
    -ldflags="-s -w" \
    -trimpath \
    -o ../bin/%s ./cmd/scanner
cd %s

# Ensure scanner data dirs exist
mkdir -p %s/vulndb %s/tmp
chown -R %s:%s %s
`, naming.InstallDir(), naming.Scanner(), naming.InstallDir(),
				naming.ScannerDataDir(), naming.ScannerDataDir(),
				naming.SystemUser(), naming.SystemUser(), naming.ScannerDataDir())

			scannerRestart = fmt.Sprintf("\nsystemctl restart %s", naming.Scanner())
			scannerHealthCheck = fmt.Sprintf(`
sleep 2
curl -sf http://localhost:9090/healthz > /dev/null && echo "SCANNER_HEALTH_OK" || echo "SCANNER_HEALTH_FAIL"
`)
		}

		updateScript := fmt.Sprintf(`set -euo pipefail
export PATH=$PATH:/usr/local/go/bin
export GOTMPDIR=/var/tmp

# Update Go if needed
CURRENT_GO=$(go version 2>/dev/null | grep -oP 'go\K[0-9.]+' || echo "none")
REQUIRED_GO="%s"
if [ "$CURRENT_GO" != "$REQUIRED_GO" ]; then
    echo "Updating Go: $CURRENT_GO -> $REQUIRED_GO"
    rm -rf /usr/local/go
    curl -fsSL https://go.dev/dl/go${REQUIRED_GO}.linux-amd64.tar.gz | tar -C /usr/local -xz
fi

cd %s
git pull origin %s
npm ci
go generate ./...
CGO_ENABLED=1 go build \
    -ldflags="-s -w -linkmode external -extldflags '-static'" \
    -tags sqlite_omit_load_extension -trimpath \
    -o bin/%s ./cmd/%s
%s
%s
systemctl restart %s
%s
sleep 2
curl -sf %s > /dev/null && echo "HEALTH_OK" || echo "HEALTH_FAIL"
%s
`, goVersion, naming.InstallDir(), branch, t.binaryName, t.buildCmd,
			scannerBuild, daemonReload, t.serviceName, scannerRestart,
			t.healthURL, scannerHealthCheck)

		output, err := runSSH(t.ip, updateScript, true)
		if err != nil {
			fmt.Printf("  ERROR: %v\n", err)
			fmt.Printf("  Output: %s\n", output)
			return fmt.Errorf("update %s failed", name)
		}

		if strings.Contains(output, "HEALTH_OK") {
			fmt.Printf("  %s: updated and healthy\n", name)
		} else if strings.Contains(output, "HEALTH_FAIL") {
			fmt.Printf("  %s: updated but health check failed!\n", name)
			fmt.Printf("  Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, t.serviceName)
		} else {
			fmt.Printf("  %s: updated (health check inconclusive)\n", name)
		}

		// Scanner health reporting
		if name == "hold" && state.ScannerEnabled {
			if strings.Contains(output, "SCANNER_HEALTH_OK") {
				fmt.Printf("  scanner: updated and healthy\n")
			} else if strings.Contains(output, "SCANNER_HEALTH_FAIL") {
				fmt.Printf("  scanner: updated but health check failed!\n")
				fmt.Printf("  Check: ssh root@%s journalctl -u %s -n 50\n", t.ip, naming.Scanner())
			}
		}
	}

	return nil
}

// configValsFromState builds ConfigValues from persisted state.
// S3SecretKey is intentionally left empty — syncConfigKeys only adds missing
// keys and never overwrites, so the server's existing secret is preserved.
func configValsFromState(state *InfraState) *ConfigValues {
	naming := state.Naming()
	_, baseDomain, _, _ := extractFromAppviewTemplate()
	holdDomain := state.Zone + ".cove." + baseDomain

	return &ConfigValues{
		S3Endpoint:    state.ObjectStorage.Endpoint,
		S3Region:      state.ObjectStorage.Region,
		S3Bucket:      state.ObjectStorage.Bucket,
		S3AccessKey:   state.ObjectStorage.AccessKeyID,
		S3SecretKey:   "", // not persisted in state; existing value on server is preserved
		Zone:          state.Zone,
		HoldDomain:    holdDomain,
		HoldDid:       "did:web:" + holdDomain,
		BasePath:      naming.BasePath(),
		ScannerSecret: state.ScannerSecret,
	}
}

func cmdSSH(target string) error {
	state, err := loadState()
	if err != nil {
		return err
	}

	var ip string
	switch target {
	case "appview":
		ip = state.Appview.PublicIP
	case "hold":
		ip = state.Hold.PublicIP
	default:
		return fmt.Errorf("unknown target: %s (use: appview, hold)", target)
	}

	fmt.Printf("Connecting to %s (%s)...\n", target, ip)
	cmd := exec.Command("ssh",
		"-o", "StrictHostKeyChecking=accept-new",
		"root@"+ip,
	)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func runSSH(ip, script string, stream bool) (string, error) {
	cmd := exec.Command("ssh",
		"-o", "StrictHostKeyChecking=accept-new",
		"-o", "ConnectTimeout=10",
		"root@"+ip,
		"bash -s",
	)
	cmd.Stdin = strings.NewReader(script)

	var buf bytes.Buffer
	if stream {
		cmd.Stdout = io.MultiWriter(os.Stdout, &buf)
		cmd.Stderr = io.MultiWriter(os.Stderr, &buf)
	} else {
		cmd.Stdout = &buf
		cmd.Stderr = &buf
	}

	// Give builds up to 10 minutes
	done := make(chan error, 1)
	go func() { done <- cmd.Run() }()

	select {
	case err := <-done:
		return buf.String(), err
	case <-time.After(10 * time.Minute):
		cmd.Process.Kill()
		return buf.String(), fmt.Errorf("SSH command timed out after 10 minutes")
	}
}