package maintenance import ( "context" "log/slog" "time" ) // WatchdogInterval is how often the watchdog checks the maintenance // flag's age. One minute strikes a balance: cluster-maintenance // windows are usually measured in hours, so once-per-minute is plenty, // and log collectors aggregating every 5 minutes will see at least // one warn-line per collection window once the rail is tripped. const WatchdogInterval = time.Minute // Watchdog warns (via slog) whenever the cluster-wide maintenance flag // has been set longer than maxDuration. It runs until ctx is cancelled. // // The whole point of the rail is that a "forgotten maintenance on" // silently masks real outages (rebalancer and requeue sweeper stay // no-op'd). A warning every WatchdogInterval ensures the operator's // log collector will surface the situation before enough time passes // for it to matter. // // maxDuration <= 0 disables the watchdog (returns immediately). The // default from config is 1h; operators running very long planned // upgrades should bump it rather than silencing the warning. func Watchdog(ctx context.Context, mgr *Manager, maxDuration time.Duration) { if mgr == nil || maxDuration <= 0 { return } t := time.NewTicker(WatchdogInterval) defer t.Stop() for { select { case <-ctx.Done(): return case <-t.C: check(ctx, mgr, maxDuration) } } } func check(ctx context.Context, mgr *Manager, maxDuration time.Duration) { flag, on, err := mgr.IsOn(ctx) if err != nil { // Non-fatal — the KV bucket may be transiently unavailable. // The next tick will retry. return } if !on || flag == nil { return } age := time.Since(flag.SetAt) if age <= maxDuration { return } slog.Warn("maintenance: cluster-wide flag has been on longer than maxDuration — rebalancer + sweeper are silently paused", "age", age.Round(time.Second), "max_duration", maxDuration, "reason", flag.Reason, "set_by", flag.SetBy, "set_at", flag.SetAt, "expires_at", flag.ExpiresAt) }