deploy: snapshot binaries, configs, units and the appview database before every update

The update command deleted the remote binary before uploading the new one and took no backup, so a deploy that failed half way left nothing to roll back to. Previous production deploys made a predeploy-<stamp> directory by hand. Each target now gets that snapshot automatically before anything is replaced: the service binary and its companion (scanner with hold, labeler with appview), their configs and systemd units, the vcs.revision of each old binary, and for the appview a sqlite3 .backup of ui.db because migrations run on the next boot. A backup that does not complete aborts the deploy. The newest five per server are kept; --backup-keep changes that and --skip-backup turns it off. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
2026-09-18 16:24:16 +00:00 · 2026-09-11 19:19:40 -05:00
parent 7ecb09465d
commit dcad0f8626
3 changed files with 221 additions and 2 deletions
@@ -0,0 +1,125 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// backupSpec describes what a pre-deploy backup captures on one server.
+// Everything the update step is about to replace is copied first, so a bad
+// deploy can be reverted by copying the files back and restarting.
+type backupSpec struct {
+	BaseDir  string   // parent for predeploy-* directories, e.g. /var/lib/seamark
+	Stamp    string   // UTC timestamp used in the directory name
+	Binaries []string // absolute paths; missing files are skipped
+	Configs  []string // absolute paths; missing files are skipped
+	Units    []string // systemd service names, without .service
+	SQLiteDB string   // optional database to snapshot with sqlite3 .backup
+	Keep     int      // how many predeploy-* directories to keep, including this one
+}
+
+// Dir is the backup directory this spec writes to.
+func (s backupSpec) Dir() string {
+	return s.BaseDir + "/predeploy-" + s.Stamp
+}
+
+// backupStamp names a backup directory the same way earlier manual backups
+// did (predeploy-20260903-005611), so old and new ones sort together.
+func backupStamp(now time.Time) string {
+	return now.UTC().Format("20060102-150405")
+}
+
+// renderBackupScript produces the bash run on the server. It never fails on a
+// file that is not there (a fresh server has no previous binary), but it does
+// fail if a copy or the database snapshot fails, which aborts the deploy
+// before anything is replaced.
+func renderBackupScript(s backupSpec) string {
+	var b strings.Builder
+	dir := s.Dir()
+	fmt.Fprintf(&b, "set -euo pipefail\nDIR=%q\nmkdir -p \"$DIR/bin\" \"$DIR/etc\" \"$DIR/systemd\"\n", dir)
+
+	for _, bin := range s.Binaries {
+		// The vcs.revision stamp Go embeds is the only record of which commit a
+		// binary came from; keep it next to the copy so a rollback is
+		// identifiable without a go toolchain on the server.
+		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/bin/\"; printf '%%s %%s\\n' %q \"$(grep -aoE 'vcs.revision=[0-9a-f]+' %q | head -1 || true)\" >> \"$DIR/REVISIONS\"; fi\n",
+			bin, bin, bin, bin)
+	}
+	for _, cfg := range s.Configs {
+		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/etc/\"; fi\n", cfg, cfg)
+	}
+	for _, unit := range s.Units {
+		path := "/etc/systemd/system/" + unit + ".service"
+		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/systemd/\"; fi\n", path, path)
+	}
+	if s.SQLiteDB != "" {
+		// sqlite3 .backup takes a consistent snapshot of a live WAL database.
+		// Without the CLI, fall back to copying the file set and say so: the
+		// copy is only consistent if the service is quiet, which it is not
+		// guaranteed to be.
+		fmt.Fprintf(&b, `if [ -f %[1]q ]; then
+  if command -v sqlite3 >/dev/null 2>&1; then
+    sqlite3 %[1]q ".backup '$DIR/$(basename %[1]q)'"
+  else
+    echo "WARN: sqlite3 not installed, copying database files raw"
+    cp -p %[1]q %[1]q-wal %[1]q-shm "$DIR/" 2>/dev/null || cp -p %[1]q "$DIR/"
+  fi
+fi
+`, s.SQLiteDB)
+	}
+	if s.Keep > 0 {
+		// Prune oldest first, never the one just written.
+		fmt.Fprintf(&b, "ls -1d %q/predeploy-* 2>/dev/null | sort | head -n -%d | xargs -r rm -rf\n", s.BaseDir, s.Keep)
+	}
+	b.WriteString("du -sh \"$DIR\" | cut -f1\necho \"BACKUP_OK $DIR\"\n")
+	return b.String()
+}
+
+// preDeployBackup runs the backup on the server and returns the directory it
+// wrote. A backup that does not report BACKUP_OK is treated as a failure.
+func preDeployBackup(ip string, s backupSpec) (string, error) {
+	out, err := runSSH(ip, renderBackupScript(s), false)
+	if err != nil {
+		return "", fmt.Errorf("%w\n%s", err, strings.TrimSpace(out))
+	}
+	if !strings.Contains(out, "BACKUP_OK") {
+		return "", fmt.Errorf("backup did not complete:\n%s", strings.TrimSpace(out))
+	}
+	return s.Dir(), nil
+}
+
+// backupSpecFor lists what a deploy of target replaces on its server: the
+// service binary, its config and unit, plus the companion service that ships
+// with it (scanner with hold, labeler with appview). The appview's SQLite
+// database is snapshotted too because migrations run on the next boot.
+func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec {
+	s := backupSpec{
+		BaseDir: naming.BasePath(),
+		Stamp:   backupStamp(now),
+		Keep:    keep,
+	}
+	bin := naming.InstallDir() + "/bin/"
+	switch target {
+	case "appview":
+		s.Binaries = []string{bin + naming.Appview()}
+		s.Configs = []string{naming.AppviewConfigPath()}
+		s.Units = []string{naming.Appview()}
+		s.SQLiteDB = naming.BasePath() + "/ui.db"
+		if state.LabelerEnabled {
+			s.Binaries = append(s.Binaries, bin+naming.Labeler())
+			s.Configs = append(s.Configs, naming.LabelerConfigPath())
+			s.Units = append(s.Units, naming.Labeler())
+		}
+	case "hold":
+		s.Binaries = []string{bin + naming.Hold()}
+		s.Configs = []string{naming.HoldConfigPath()}
+		s.Units = []string{naming.Hold()}
+		if state.ScannerEnabled {
+			s.Binaries = append(s.Binaries, bin+naming.Scanner())
+			s.Configs = append(s.Configs, naming.ScannerConfigPath())
+			s.Units = append(s.Units, naming.Scanner())
+		}
+	}
+	return s
+}
@@ -0,0 +1,70 @@
+package main
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestBackupSpecForAppviewIncludesDatabaseAndLabeler(t *testing.T) {
+	naming := Naming{ClientName: "seamark"}
+	state := &InfraState{LabelerEnabled: true, ScannerEnabled: true}
+	now := time.Date(2026, 9, 11, 23, 0, 5, 0, time.UTC)
+
+	s := backupSpecFor("appview", naming, state, now, 5)
+	if s.Dir() != "/var/lib/seamark/predeploy-20260911-230005" {
+		t.Fatalf("dir = %q", s.Dir())
+	}
+	if s.SQLiteDB != "/var/lib/seamark/ui.db" {
+		t.Fatalf("SQLiteDB = %q", s.SQLiteDB)
+	}
+	want := []string{"/opt/seamark/bin/seamark-appview", "/opt/seamark/bin/seamark-labeler"}
+	if strings.Join(s.Binaries, ",") != strings.Join(want, ",") {
+		t.Fatalf("Binaries = %v", s.Binaries)
+	}
+	if strings.Join(s.Units, ",") != "seamark-appview,seamark-labeler" {
+		t.Fatalf("Units = %v", s.Units)
+	}
+
+	h := backupSpecFor("hold", naming, state, now, 5)
+	if h.SQLiteDB != "" {
+		t.Fatalf("hold backup must not snapshot a database, got %q", h.SQLiteDB)
+	}
+	if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold,/opt/seamark/bin/seamark-scanner" {
+		t.Fatalf("hold Binaries = %v", h.Binaries)
+	}
+}
+
+func TestRenderBackupScript(t *testing.T) {
+	s := backupSpec{
+		BaseDir:  "/var/lib/seamark",
+		Stamp:    "20260911-230005",
+		Binaries: []string{"/opt/seamark/bin/seamark-appview"},
+		Configs:  []string{"/etc/seamark/appview.yaml"},
+		Units:    []string{"seamark-appview"},
+		SQLiteDB: "/var/lib/seamark/ui.db",
+		Keep:     5,
+	}
+	script := renderBackupScript(s)
+	for _, want := range []string{
+		`DIR="/var/lib/seamark/predeploy-20260911-230005"`,
+		`cp -p "/opt/seamark/bin/seamark-appview" "$DIR/bin/"`,
+		`vcs.revision=`,
+		`cp -p "/etc/seamark/appview.yaml" "$DIR/etc/"`,
+		`cp -p "/etc/systemd/system/seamark-appview.service" "$DIR/systemd/"`,
+		`sqlite3 "/var/lib/seamark/ui.db" ".backup`,
+		`head -n -5 | xargs -r rm -rf`,
+		`echo "BACKUP_OK $DIR"`,
+	} {
+		if !strings.Contains(script, want) {
+			t.Errorf("script missing %q\n%s", want, script)
+		}
+	}
+
+	s.SQLiteDB = ""
+	s.Keep = 0
+	script = renderBackupScript(s)
+	if strings.Contains(script, "sqlite3") || strings.Contains(script, "xargs -r rm -rf") {
+		t.Errorf("no database and no pruning expected without SQLiteDB/Keep:\n%s", script)
+	}
+}
@@ -27,7 +27,9 @@ var updateCmd = &cobra.Command{
 		withScanner, _ := cmd.Flags().GetBool("with-scanner")
 		withLabeler, _ := cmd.Flags().GetBool("with-labeler")
 		withBilling, _ := cmd.Flags().GetBool("with-billing")
-		return cmdUpdate(target, withScanner, withLabeler, withBilling)
+		skipBackup, _ := cmd.Flags().GetBool("skip-backup")
+		keep, _ := cmd.Flags().GetInt("backup-keep")
+		return cmdUpdate(target, withScanner, withLabeler, withBilling, updateOptions{SkipBackup: skipBackup, BackupKeep: keep})
 	},
 }

@@ -45,11 +47,19 @@ func init() {
 	updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.")
 	updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold")
 	updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview")
+	updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into <data dir>/predeploy-<stamp>/ before replacing them")
+	updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup")
 	rootCmd.AddCommand(updateCmd)
 	rootCmd.AddCommand(sshCmd)
 }

-func cmdUpdate(target string, withScanner, withLabeler, withBilling bool) error {
+// updateOptions are the deploy knobs that are not about what gets built.
+type updateOptions struct {
+	SkipBackup bool
+	BackupKeep int
+}
+
+func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts updateOptions) error {
 	state, err := loadState()
 	if err != nil {
 		return err
@@ -148,6 +158,20 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool) error
 		t := targets[name]
 		fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip)

+		// Snapshot what is about to be replaced. The upload below deletes the
+		// remote binary before copying the new one, so without this there is
+		// nothing to roll back to if the deploy fails half way.
+		if opts.SkipBackup {
+			fmt.Printf("  backup: skipped (--skip-backup)\n")
+		} else {
+			spec := backupSpecFor(name, naming, state, time.Now(), opts.BackupKeep)
+			dir, err := preDeployBackup(t.ip, spec)
+			if err != nil {
+				return fmt.Errorf("%s pre-deploy backup: %w", name, err)
+			}
+			fmt.Printf("  backup: %s (restore by copying bin/, etc/, systemd/ back and restarting)\n", dir)
+		}
+
 		// Sync config keys (adds missing keys from template, never overwrites)
 		configYAML, err := renderConfig(t.configTmpl, vals)
 		if err != nil {