at-container-registry/deploy/upcloud/backup.go

package main

import (
	"fmt"
	"strings"
	"time"
)

// backupSpec describes what a pre-deploy backup captures on one server.
// Everything the update step is about to replace is copied first, so a bad
// deploy can be reverted by copying the files back and restarting.
type backupSpec struct {
	BaseDir  string   // parent for predeploy-* directories, e.g. /var/lib/seamark
	Stamp    string   // UTC timestamp used in the directory name
	Binaries []string // absolute paths; missing files are skipped
	Configs  []string // absolute paths; missing files are skipped
	Units    []string // systemd service names, without .service
	SQLiteDB string   // optional database to snapshot with sqlite3 .backup
	Keep     int      // how many predeploy-* directories to keep, including this one
}

// Dir is the backup directory this spec writes to.
func (s backupSpec) Dir() string {
	return s.BaseDir + "/predeploy-" + s.Stamp
}

// backupStamp names a backup directory the same way earlier manual backups
// did (predeploy-20260903-005611), so old and new ones sort together.
func backupStamp(now time.Time) string {
	return now.UTC().Format("20060102-150405")
}

// renderBackupScript produces the bash run on the server. It never fails on a
// file that is not there (a fresh server has no previous binary), but it does
// fail if a copy or the database snapshot fails, which aborts the deploy
// before anything is replaced.
func renderBackupScript(s backupSpec) string {
	var b strings.Builder
	dir := s.Dir()
	fmt.Fprintf(&b, "set -euo pipefail\nDIR=%q\nmkdir -p \"$DIR/bin\" \"$DIR/etc\" \"$DIR/systemd\"\n", dir)

	for _, bin := range s.Binaries {
		// The vcs.revision stamp Go embeds is the only record of which commit a
		// binary came from; keep it next to the copy so a rollback is
		// identifiable without a go toolchain on the server.
		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/bin/\"; printf '%%s %%s\\n' %q \"$(grep -aoE 'vcs.revision=[0-9a-f]+' %q | head -1 || true)\" >> \"$DIR/REVISIONS\"; fi\n",
			bin, bin, bin, bin)
	}
	for _, cfg := range s.Configs {
		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/etc/\"; fi\n", cfg, cfg)
	}
	for _, unit := range s.Units {
		path := "/etc/systemd/system/" + unit + ".service"
		fmt.Fprintf(&b, "if [ -f %q ]; then cp -p %q \"$DIR/systemd/\"; fi\n", path, path)
	}
	if s.SQLiteDB != "" {
		// sqlite3 .backup takes a consistent snapshot of a live WAL database.
		// Without the CLI, fall back to copying the file set and say so: the
		// copy is only consistent if the service is quiet, which it is not
		// guaranteed to be.
		fmt.Fprintf(&b, `if [ -f %[1]q ]; then
  if command -v sqlite3 >/dev/null 2>&1; then
    sqlite3 %[1]q ".backup '$DIR/$(basename %[1]q)'"
  else
    echo "WARN: sqlite3 not installed, copying database files raw"
    cp -p %[1]q %[1]q-wal %[1]q-shm "$DIR/" 2>/dev/null || cp -p %[1]q "$DIR/"
  fi
fi
`, s.SQLiteDB)
	}
	if s.Keep > 0 {
		// Prune oldest first, never the one just written.
		fmt.Fprintf(&b, "ls -1d %q/predeploy-* 2>/dev/null | sort | head -n -%d | xargs -r rm -rf\n", s.BaseDir, s.Keep)
	}
	b.WriteString("du -sh \"$DIR\" | cut -f1\necho \"BACKUP_OK $DIR\"\n")
	return b.String()
}

// preDeployBackup runs the backup on the server and returns the directory it
// wrote. A backup that does not report BACKUP_OK is treated as a failure.
func preDeployBackup(ip string, s backupSpec) (string, error) {
	out, err := runSSH(ip, renderBackupScript(s), false)
	if err != nil {
		return "", fmt.Errorf("%w\n%s", err, strings.TrimSpace(out))
	}
	if !strings.Contains(out, "BACKUP_OK") {
		return "", fmt.Errorf("backup did not complete:\n%s", strings.TrimSpace(out))
	}
	return s.Dir(), nil
}

// backupSpecFor lists what a deploy of target replaces on its server: the
// service binary, its config and unit, plus the companion service that ships
// with it (labeler with appview). The scanner is its own target on its own
// server. The appview's SQLite database is snapshotted too because migrations
// run on the next boot.
func backupSpecFor(target string, naming Naming, state *InfraState, now time.Time, keep int) backupSpec {
	s := backupSpec{
		BaseDir: naming.BasePath(),
		Stamp:   backupStamp(now),
		Keep:    keep,
	}
	bin := naming.InstallDir() + "/bin/"
	switch target {
	case "appview":
		s.Binaries = []string{bin + naming.Appview()}
		s.Configs = []string{naming.AppviewConfigPath()}
		s.Units = []string{naming.Appview()}
		s.SQLiteDB = naming.BasePath() + "/ui.db"
		if state.LabelerEnabled {
			s.Binaries = append(s.Binaries, bin+naming.Labeler())
			s.Configs = append(s.Configs, naming.LabelerConfigPath())
			s.Units = append(s.Units, naming.Labeler())
		}
	case "hold":
		s.Binaries = []string{bin + naming.Hold()}
		s.Configs = []string{naming.HoldConfigPath()}
		s.Units = []string{naming.Hold()}
	case "scanner":
		s.Binaries = []string{bin + naming.Scanner()}
		s.Configs = []string{naming.ScannerConfigPath()}
		s.Units = []string{naming.Scanner()}
	}
	return s
}