From dcad0f8626a8194eebb2e526d2106e055725cfb2 Mon Sep 17 00:00:00 2001 From: Evan Jarrett Date: Fri, 11 Sep 2026 19:19:40 -0500 Subject: [PATCH] deploy: snapshot binaries, configs, units and the appview database before every update The update command deleted the remote binary before uploading the new one and took no backup, so a deploy that failed half way left nothing to roll back to. Previous production deploys made a predeploy- directory by hand. Each target now gets that snapshot automatically before anything is replaced: the service binary and its companion (scanner with hold, labeler with appview), their configs and systemd units, the vcs.revision of each old binary, and for the appview a sqlite3 .backup of ui.db because migrations run on the next boot. A backup that does not complete aborts the deploy. The newest five per server are kept; --backup-keep changes that and --skip-backup turns it off. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt --- deploy/upcloud/backup.go | 125 ++++++++++++++++++++++++++++++++++ deploy/upcloud/backup_test.go | 70 +++++++++++++++++++ deploy/upcloud/update.go | 28 +++++++- 3 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 deploy/upcloud/backup.go create mode 100644 deploy/upcloud/backup_test.go diff --git a/deploy/upcloud/backup.go b/deploy/upcloud/backup.go new file mode 100644 index 0000000..4ba1ff5 --- /dev/null +++ b/deploy/upcloud/backup.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + "strings" + "time" +) + +// backupSpec describes what a pre-deploy backup captures on one server. +// Everything the update step is about to replace is copied first, so a bad +// deploy can be reverted by copying the files back and restarting. +type backupSpec struct { + BaseDir string // parent for predeploy-* directories, e.g. /var/lib/seamark + Stamp string // UTC timestamp used in the directory name + Binaries []string // absolute paths; missing files are skipped + Configs []string // absolute paths; missing files are skipped + Units []string // systemd service names, without .service + SQLiteDB string // optional database to snapshot with sqlite3 .backup + Keep int // how many predeploy-* directories to keep, including this one +} + +// Dir is the backup directory this spec writes to. +func (s backupSpec) Dir() string { + return s.BaseDir + "/predeploy-" + s.Stamp +} + +// backupStamp names a backup directory the same way earlier manual backups +// did (predeploy-20260903-005611), so old and new ones sort together. +func backupStamp(now time.Time) string { + return now.UTC().Format("20060102-150405") +} + +// renderBackupScript produces the bash run on the server. It never fails on a +// file that is not there (a fresh server has no previous binary), but it does +// fail if a copy or the database snapshot fails, which aborts the deploy +// before anything is replaced. +func renderBackupScript(s backupSpec) string { + var b strings.Builder + dir := s.Dir() + fmt.Fprintf(&b, "set -euo pipefail\nDIR=%q\nmkdir -p \"$DIR/bin\" \"$DIR/etc\" \"$DIR/systemd\"\n", dir) + + for _, bin := range s.Binaries { + // The vcs.revision stamp Go embeds is the only record of which commit a + // binary came from; keep it next to the copy so a rollback is + // identifiable without a go toolchain on the server. + fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/bin/\"; printf '%%s %%s\\n' %q \"$(grep -aoE 'vcs.revision=[0-9a-f]+' %q | head -1 || true)\" >> \"$DIR/REVISIONS\"; fi\n", + bin, bin, bin, bin) + } + for _, cfg := range s.Configs { + fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/etc/\"; fi\n", cfg, cfg) + } + for _, unit := range s.Units { + path := "/etc/systemd/system/" + unit + ".service" + fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/systemd/\"; fi\n", path, path) + } + if s.SQLiteDB != "" { + // sqlite3 .backup takes a consistent snapshot of a live WAL database. + // Without the CLI, fall back to copying the file set and say so: the + // copy is only consistent if the service is quiet, which it is not + // guaranteed to be. + fmt.Fprintf(&b, `if [ -f %[1]q ]; then + if command -v sqlite3 >/dev/null 2>&1; then + sqlite3 %[1]q ".backup '$DIR/$(basename %[1]q)'" + else + echo "WARN: sqlite3 not installed, copying database files raw" + cp -p %[1]q %[1]q-wal %[1]q-shm "$DIR/" 2>/dev/null || cp -p %[1]q "$DIR/" + fi +fi +`, s.SQLiteDB) + } + if s.Keep > 0 { + // Prune oldest first, never the one just written. + fmt.Fprintf(&b, "ls -1d %q/predeploy-* 2>/dev/null | sort | head -n -%d | xargs -r rm -rf\n", s.BaseDir, s.Keep) + } + b.WriteString("du -sh \"$DIR\" | cut -f1\necho \"BACKUP_OK $DIR\"\n") + return b.String() +} + +// preDeployBackup runs the backup on the server and returns the directory it +// wrote. A backup that does not report BACKUP_OK is treated as a failure. +func preDeployBackup(ip string, s backupSpec) (string, error) { + out, err := runSSH(ip, renderBackupScript(s), false) + if err != nil { + return "", fmt.Errorf("%w\n%s", err, strings.TrimSpace(out)) + } + if !strings.Contains(out, "BACKUP_OK") { + return "", fmt.Errorf("backup did not complete:\n%s", strings.TrimSpace(out)) + } + return s.Dir(), nil +} + +// backupSpecFor lists what a deploy of target replaces on its server: the +// service binary, its config and unit, plus the companion service that ships +// with it (scanner with hold, labeler with appview). The appview's SQLite +// database is snapshotted too because migrations run on the next boot. +func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec { + s := backupSpec{ + BaseDir: naming.BasePath(), + Stamp: backupStamp(now), + Keep: keep, + } + bin := naming.InstallDir() + "/bin/" + switch target { + case "appview": + s.Binaries = []string{bin + naming.Appview()} + s.Configs = []string{naming.AppviewConfigPath()} + s.Units = []string{naming.Appview()} + s.SQLiteDB = naming.BasePath() + "/ui.db" + if state.LabelerEnabled { + s.Binaries = append(s.Binaries, bin+naming.Labeler()) + s.Configs = append(s.Configs, naming.LabelerConfigPath()) + s.Units = append(s.Units, naming.Labeler()) + } + case "hold": + s.Binaries = []string{bin + naming.Hold()} + s.Configs = []string{naming.HoldConfigPath()} + s.Units = []string{naming.Hold()} + if state.ScannerEnabled { + s.Binaries = append(s.Binaries, bin+naming.Scanner()) + s.Configs = append(s.Configs, naming.ScannerConfigPath()) + s.Units = append(s.Units, naming.Scanner()) + } + } + return s +} diff --git a/deploy/upcloud/backup_test.go b/deploy/upcloud/backup_test.go new file mode 100644 index 0000000..3cadf6d --- /dev/null +++ b/deploy/upcloud/backup_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +func TestBackupSpecForAppviewIncludesDatabaseAndLabeler(t *testing.T) { + naming := Naming{ClientName: "seamark"} + state := &InfraState{LabelerEnabled: true, ScannerEnabled: true} + now := time.Date(2026, 9, 11, 23, 0, 5, 0, time.UTC) + + s := backupSpecFor("appview", naming, state, now, 5) + if s.Dir() != "/var/lib/seamark/predeploy-20260911-230005" { + t.Fatalf("dir = %q", s.Dir()) + } + if s.SQLiteDB != "/var/lib/seamark/ui.db" { + t.Fatalf("SQLiteDB = %q", s.SQLiteDB) + } + want := []string{"/opt/seamark/bin/seamark-appview", "/opt/seamark/bin/seamark-labeler"} + if strings.Join(s.Binaries, ",") != strings.Join(want, ",") { + t.Fatalf("Binaries = %v", s.Binaries) + } + if strings.Join(s.Units, ",") != "seamark-appview,seamark-labeler" { + t.Fatalf("Units = %v", s.Units) + } + + h := backupSpecFor("hold", naming, state, now, 5) + if h.SQLiteDB != "" { + t.Fatalf("hold backup must not snapshot a database, got %q", h.SQLiteDB) + } + if strings.Join(h.Binaries, ",") != "/opt/seamark/bin/seamark-hold,/opt/seamark/bin/seamark-scanner" { + t.Fatalf("hold Binaries = %v", h.Binaries) + } +} + +func TestRenderBackupScript(t *testing.T) { + s := backupSpec{ + BaseDir: "/var/lib/seamark", + Stamp: "20260911-230005", + Binaries: []string{"/opt/seamark/bin/seamark-appview"}, + Configs: []string{"/etc/seamark/appview.yaml"}, + Units: []string{"seamark-appview"}, + SQLiteDB: "/var/lib/seamark/ui.db", + Keep: 5, + } + script := renderBackupScript(s) + for _, want := range []string{ + `DIR="/var/lib/seamark/predeploy-20260911-230005"`, + `cp -p "/opt/seamark/bin/seamark-appview" "$DIR/bin/"`, + `vcs.revision=`, + `cp -p "/etc/seamark/appview.yaml" "$DIR/etc/"`, + `cp -p "/etc/systemd/system/seamark-appview.service" "$DIR/systemd/"`, + `sqlite3 "/var/lib/seamark/ui.db" ".backup`, + `head -n -5 | xargs -r rm -rf`, + `echo "BACKUP_OK $DIR"`, + } { + if !strings.Contains(script, want) { + t.Errorf("script missing %q\n%s", want, script) + } + } + + s.SQLiteDB = "" + s.Keep = 0 + script = renderBackupScript(s) + if strings.Contains(script, "sqlite3") || strings.Contains(script, "xargs -r rm -rf") { + t.Errorf("no database and no pruning expected without SQLiteDB/Keep:\n%s", script) + } +} diff --git a/deploy/upcloud/update.go b/deploy/upcloud/update.go index f260219..601f010 100644 --- a/deploy/upcloud/update.go +++ b/deploy/upcloud/update.go @@ -27,7 +27,9 @@ var updateCmd = &cobra.Command{ withScanner, _ := cmd.Flags().GetBool("with-scanner") withLabeler, _ := cmd.Flags().GetBool("with-labeler") withBilling, _ := cmd.Flags().GetBool("with-billing") - return cmdUpdate(target, withScanner, withLabeler, withBilling) + skipBackup, _ := cmd.Flags().GetBool("skip-backup") + keep, _ := cmd.Flags().GetInt("backup-keep") + return cmdUpdate(target, withScanner, withLabeler, withBilling, updateOptions{SkipBackup: skipBackup, BackupKeep: keep}) }, } @@ -45,11 +47,19 @@ func init() { updateCmd.Flags().Bool("with-billing", false, "Compile Stripe billing into the appview (pkg/billing, `billing` build tag). Off by default: a deployment that runs without billing must not gain it because someone redeployed.") updateCmd.Flags().Bool("with-scanner", false, "Enable and deploy vulnerability scanner alongside hold") updateCmd.Flags().Bool("with-labeler", false, "Enable and deploy content moderation labeler alongside appview") + updateCmd.Flags().Bool("skip-backup", false, "Do not snapshot the server's binaries, configs, units and appview database into /predeploy-/ before replacing them") + updateCmd.Flags().Int("backup-keep", 5, "How many predeploy-* backups to keep per server; older ones are pruned after a successful backup") rootCmd.AddCommand(updateCmd) rootCmd.AddCommand(sshCmd) } -func cmdUpdate(target string, withScanner, withLabeler, withBilling bool) error { +// updateOptions are the deploy knobs that are not about what gets built. +type updateOptions struct { + SkipBackup bool + BackupKeep int +} + +func cmdUpdate(target string, withScanner, withLabeler, withBilling bool, opts updateOptions) error { state, err := loadState() if err != nil { return err @@ -148,6 +158,20 @@ func cmdUpdate(target string, withScanner, withLabeler, withBilling bool) error t := targets[name] fmt.Printf("\nDeploying %s (%s)...\n", name, t.ip) + // Snapshot what is about to be replaced. The upload below deletes the + // remote binary before copying the new one, so without this there is + // nothing to roll back to if the deploy fails half way. + if opts.SkipBackup { + fmt.Printf(" backup: skipped (--skip-backup)\n") + } else { + spec := backupSpecFor(name, naming, state, time.Now(), opts.BackupKeep) + dir, err := preDeployBackup(t.ip, spec) + if err != nil { + return fmt.Errorf("%s pre-deploy backup: %w", name, err) + } + fmt.Printf(" backup: %s (restore by copying bin/, etc/, systemd/ back and restarting)\n", dir) + } + // Sync config keys (adds missing keys from template, never overwrites) configYAML, err := renderConfig(t.configTmpl, vals) if err != nil {