diff --git a/deploy/upcloud/cloudinit.go b/deploy/upcloud/cloudinit.go index 8cc0953..b0e2a7b 100644 --- a/deploy/upcloud/cloudinit.go +++ b/deploy/upcloud/cloudinit.go @@ -417,15 +417,28 @@ func syncServiceUnit(name, ip, serviceName, renderedUnit string) (bool, error) { // syncConfigKeys fetches the existing config from a server and merges in any // missing keys from the rendered template. Existing values are never overwritten. -func syncConfigKeys(name, ip, configPath, templateYAML string) error { +// +// installIfMissing is true only during provision. On an update a missing +// config is never a first install: the service has been running with one, so +// the file has been moved or deleted, and writing the template in its place +// would hand the service empty identity fields (owner DID, DID, keys) that it +// refuses to start with. That is exactly what happened to the labeler on +// 2026-09-09, and it only surfaced at the next restart three days later. +func syncConfigKeys(name, ip, configPath, templateYAML string, installIfMissing bool) error { remote, err := runSSH(ip, fmt.Sprintf("cat %s 2>/dev/null || echo '__MISSING__'", configPath), false) if err != nil { + if !installIfMissing { + return fmt.Errorf("read %s config %s: %w", name, configPath, err) + } fmt.Printf(" config sync: could not reach %s (%v)\n", name, err) return nil } remote = strings.TrimSpace(remote) if remote == "__MISSING__" { + if !installIfMissing { + return fmt.Errorf("%s config %s is missing on the server; refusing to install the template during an update because its identity fields are empty. Restore the file from the newest predeploy-* backup under the data directory and rerun", name, configPath) + } // First-time install: write the rendered template as-is. Subsequent // runs use the merge-keys path below to preserve operator edits. dir := configPath[:strings.LastIndex(configPath, "/")] diff --git a/deploy/upcloud/health.go b/deploy/upcloud/health.go new file mode 100644 index 0000000..3282176 --- /dev/null +++ b/deploy/upcloud/health.go @@ -0,0 +1,72 @@ +package main + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// healthProbeTries is how many 2-second polls a service gets to answer its +// health URL after a restart. The hold takes several seconds to open its +// listener (PLC check, carstore open), so a single probe two seconds after +// restart reported a failure on a hold that was healthy moments later. +const healthProbeTries = 15 + +// renderHealthProbe returns a bash snippet that polls url every two seconds +// until it answers success or the tries run out, then prints one line: +// +// PROBE