mirror of
https://tangled.org/evan.jarrett.net/at-container-registry
synced 2026-09-25 03:34:14 +00:00
Evan JarrettandClaude Fable 5.1
db6f37a070
On 2026-09-09 the labeler's config was replaced with the rendered template, which has empty identity fields (owner DID, DID, rotation key). The running process kept its in-memory config, so nothing failed until the next restart on 2026-09-12, when the labeler crash-looped on "labeler.owner_did is required" and nobody was told: the deploy tool never probed the labeler at all. The sync's missing-file branch is the only code that writes a whole template, so an update now refuses when the file is gone and says to restore it from the predeploy backup; provision keeps the first-install behaviour. The health check was a single curl two seconds after restart. The hold takes longer than that to open its listener, so tonight's deploy printed HEALTH_FAIL for a hold that answered seconds later. Worse, the verdict was a substring match on HEALTH_OK, which the scanner's SCANNER_HEALTH_OK line also satisfies, so a failed hold next to a healthy scanner was reported healthy. Each restarted service (hold, scanner, appview, labeler) is now polled every two seconds for up to thirty, and reports on its own whole-label line. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EvFJr4Dwz8p2NDAeXmgmBt
73 lines
2.5 KiB
Go
73 lines
2.5 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// healthProbeTries is how many 2-second polls a service gets to answer its
|
|
// health URL after a restart. The hold takes several seconds to open its
|
|
// listener (PLC check, carstore open), so a single probe two seconds after
|
|
// restart reported a failure on a hold that was healthy moments later.
|
|
const healthProbeTries = 15
|
|
|
|
// renderHealthProbe returns a bash snippet that polls url every two seconds
|
|
// until it answers success or the tries run out, then prints one line:
|
|
//
|
|
// PROBE <label> OK <seconds>
|
|
// PROBE <label> FAIL <seconds>
|
|
//
|
|
// Labels are matched whole by parseProbes, so a healthy scanner can never be
|
|
// mistaken for a healthy hold the way a substring match on HEALTH_OK could.
|
|
func renderHealthProbe(label, url string, tries int) string {
|
|
return fmt.Sprintf(`for __i in $(seq 1 %d); do
|
|
if curl -sf %q > /dev/null 2>&1; then echo "PROBE %s OK $((__i * 2))"; break; fi
|
|
if [ "$__i" -eq %d ]; then echo "PROBE %s FAIL $((__i * 2))"; fi
|
|
sleep 2
|
|
done
|
|
`, tries, url, label, tries, label)
|
|
}
|
|
|
|
var probeLine = regexp.MustCompile(`(?m)^PROBE (\S+) (OK|FAIL) (\d+)$`)
|
|
|
|
// probeResult is one service's health verdict after a restart.
|
|
type probeResult struct {
|
|
Status string // OK or FAIL
|
|
Seconds int
|
|
}
|
|
|
|
// parseProbes reads the PROBE lines out of a restart script's output.
|
|
func parseProbes(output string) map[string]probeResult {
|
|
results := map[string]probeResult{}
|
|
for _, m := range probeLine.FindAllStringSubmatch(output, -1) {
|
|
secs, _ := strconv.Atoi(m[3]) // the regexp guarantees digits
|
|
results[m[1]] = probeResult{Status: m[2], Seconds: secs}
|
|
}
|
|
return results
|
|
}
|
|
|
|
// reportProbe prints the verdict for one service, with the journal hint on
|
|
// failure. A service with no PROBE line at all was never probed, which is
|
|
// reported rather than assumed healthy.
|
|
func reportProbe(results map[string]probeResult, label, ip, serviceName string) {
|
|
r, ok := results[label]
|
|
switch {
|
|
case !ok:
|
|
fmt.Printf(" %s: updated (health check inconclusive)\n", label)
|
|
case r.Status == "OK":
|
|
fmt.Printf(" %s: updated and healthy (answered after %ds)\n", label, r.Seconds)
|
|
default:
|
|
fmt.Printf(" %s: updated but health check failed after %ds!\n", label, r.Seconds)
|
|
fmt.Printf(" Check: ssh root@%s journalctl -u %s -n 50\n", ip, serviceName)
|
|
}
|
|
}
|
|
|
|
// probeMarkersPresent reports whether output carries any PROBE line, which
|
|
// distinguishes a restart script that ran to completion from one that died
|
|
// before probing.
|
|
func probeMarkersPresent(output string) bool {
|
|
return strings.Contains(output, "PROBE ")
|
|
}
|