From e984ee12081f4e112601c3f718c9d81c112fe718 Mon Sep 17 00:00:00 2001 From: Mike Bogdan Date: Sat, 17 Oct 2026 21:40:19 +0400 Subject: [PATCH] feat(alerts): add TLS certificate expiry alert for network monitors (#2627) Adds a NetworkMonitorCert alert with a threshold in days, evaluated on the hub from the certInfo stored by #2401. Follows the NetworkMonitorLoss alert: per-monitor incidents, alert history, recovery notifications. Refs #391 --- internal/alerts/alerts_api.go | 6 +- internal/alerts/alerts_cache.go | 2 +- internal/alerts/alerts_history.go | 4 +- .../alerts_network_monitor_cert_test.go | 310 ++++++++++++++++++ internal/alerts/alerts_network_monitors.go | 171 ++++++++-- internal/alerts/alerts_system.go | 2 +- internal/alerts/network_monitor_cache.go | 49 ++- internal/hub/collections_test.go | 2 +- internal/hub/systems/system_manager.go | 2 +- .../1791926360_network_monitor_cert_alert.go | 34 ++ .../site/src/components/active-alerts.tsx | 7 + .../src/components/alerts-history-columns.tsx | 9 +- .../src/components/alerts/alerts-sheet.tsx | 11 +- internal/site/src/lib/alerts.ts | 12 +- 14 files changed, 573 insertions(+), 48 deletions(-) create mode 100644 internal/alerts/alerts_network_monitor_cert_test.go create mode 100644 internal/migrations/1791926360_network_monitor_cert_alert.go diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.go index 7a44a22f5..4d21b39be 100644 --- a/internal/alerts/alerts_api.go +++ b/internal/alerts/alerts_api.go @@ -29,9 +29,9 @@ func UpsertUserAlerts(e *core.RequestEvent) error { return e.BadRequestError("Bad data", err) } - if reqData.Name == alertNameNetworkMonitorLoss { - if reqData.Value < 0 || reqData.Value >= 100 { - return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil) + if isNetworkMonitorAlert(reqData.Name) { + if msg := networkMonitorThresholdError(reqData.Name, reqData.Value); msg != "" { + return e.BadRequestError(msg, nil) } reqData.Min = 0 } diff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.go index facc401b3..0dfcd5025 100644 --- a/internal/alerts/alerts_cache.go +++ b/internal/alerts/alerts_cache.go @@ -34,7 +34,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) { a.Triggered = record.GetBool("triggered") a.Min = uint8(record.GetInt("min")) a.PendingSince = record.GetDateTime("pending_since").Time() - if a.Name == alertNameNetworkMonitorLoss { + if isNetworkMonitorAlert(a.Name) { var state networkMonitorAlertState a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil a.MonitorStates = state.Monitors diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.go index f6b1bd4d2..d8283f3c3 100644 --- a/internal/alerts/alerts_history.go +++ b/internal/alerts/alerts_history.go @@ -9,7 +9,7 @@ import ( // On triggered alert record delete, set matching alert history record to resolved func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { - if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + if isNetworkMonitorAlert(e.Record.GetString("name")) { if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil { return err } @@ -25,7 +25,7 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { // On alert record update, update alert history record func updateHistoryOnAlertUpdate(e *core.RecordEvent) error { // Network monitor incidents have separate history entries per monitor. - if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + if isNetworkMonitorAlert(e.Record.GetString("name")) { return e.Next() } original := e.Record.Original() diff --git a/internal/alerts/alerts_network_monitor_cert_test.go b/internal/alerts/alerts_network_monitor_cert_test.go new file mode 100644 index 000000000..154cff894 --- /dev/null +++ b/internal/alerts/alerts_network_monitor_cert_test.go @@ -0,0 +1,310 @@ +//go:build testing + +package alerts_test + +import ( + "strconv" + "testing" + "time" + + "github.com/henrygd/beszel/internal/alerts" + "github.com/henrygd/beszel/internal/entities/monitor" + beszelTests "github.com/henrygd/beszel/internal/tests" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + pbTests "github.com/pocketbase/pocketbase/tests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// certAlertSetup creates a 30 day certificate alert and an HTTPS monitor without certificate info. +func certAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, *core.Record) { + t.Helper() + hub, system, alert := systemdTestSetup(t, false) + t.Cleanup(hub.Cleanup) + alert.Set("name", "NetworkMonitorCert") + alert.Set("value", 30) + require.NoError(t, hub.Save(alert)) + m, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": system.Id, "target": "https://example.com", "protocol": "http", "interval": 60, "enabled": true, + }) + require.NoError(t, err) + // Avoid starting a system update worker in tests. + _, err = hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + system.Set("status", "up") + return hub, system, alert, m +} + +// certExpiringIn returns certificate info for a certificate with the given days +// left, plus an hour so that the day count is stable during the test. +func certExpiringIn(days int) *monitor.CertInfo { + return &monitor.CertInfo{Expires: time.Now().Add(time.Duration(days)*24*time.Hour + time.Hour).UnixMilli()} +} + +func setMonitorCert(t *testing.T, hub *beszelTests.TestHub, m *core.Record, cert *monitor.CertInfo) { + t.Helper() + m.Set("certInfo", cert) + require.NoError(t, hub.Save(m)) +} + +func TestNetworkMonitorCertAlertLifecycle(t *testing.T) { + hub, system, alert, m := certAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + count := hub.TestMailer.TotalSend() + check := func(active bool, open, sent int) { + t.Helper() + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + record, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.Equal(t, active, record.GetBool("triggered")) + total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""}) + require.NoError(t, err) + assert.EqualValues(t, open, total) + assert.Equal(t, count+sent, hub.TestMailer.TotalSend()) + } + lastMessage := func() string { + return hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text + } + // Unknown expiry never triggers. + check(false, 0, 0) + setMonitorCert(t, hub, m, certExpiringIn(40)) + check(false, 0, 0) + // Expiry is checked against the clock, without needing a new result. + cert := certExpiringIn(25) + setMonitorCert(t, hub, m, cert) + check(true, 1, 1) + assert.Contains(t, lastMessage(), "https://example.com") + assert.Contains(t, lastMessage(), "expires in 25 days ("+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly)+")") + assert.Contains(t, lastMessage(), "within the 30-day threshold") + // Repeated checks and a hub restart do not notify again. + check(true, 1, 1) + am = alerts.NewTestAlertManagerWithoutWorker(hub) + check(true, 1, 1) + // A renewed certificate resolves the incident. + setMonitorCert(t, hub, m, certExpiringIn(90)) + check(false, 0, 2) + assert.Contains(t, lastMessage(), "expires in 90 days") + assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Subject, "Certificate expiry recovered") + assert.Contains(t, lastMessage(), "beyond the 30-day threshold") + // A new incident opens when expiry nears again. + setMonitorCert(t, hub, m, certExpiringIn(20)) + check(true, 1, 3) + histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id}) + require.NoError(t, err) + require.Len(t, histories, 2) + for _, history := range histories { + assert.Equal(t, "NetworkMonitorCert", history.GetString("name")) + assert.Equal(t, "https://example.com", history.GetString("monitor_name")) + } +} + +func TestNetworkMonitorCertAlertThresholdBoundary(t *testing.T) { + for _, tc := range []struct { + name string + offset time.Duration + triggered bool + }{ + {name: "a minute beyond", offset: time.Minute, triggered: false}, + {name: "23 hours beyond", offset: 23 * time.Hour, triggered: false}, + {name: "a minute within", offset: -time.Minute, triggered: true}, + } { + t.Run(tc.name, func(t *testing.T) { + hub, system, alert, m := certAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + // Expiry beyond 30 days still displays as 30 days left, but must not trigger a 30 day threshold. + setMonitorCert(t, hub, m, &monitor.CertInfo{Expires: time.Now().Add(30*24*time.Hour + tc.offset).UnixMilli()}) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + alert, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.Equal(t, tc.triggered, alert.GetBool("triggered")) + }) + } +} + +func TestNetworkMonitorCertAlertExpired(t *testing.T) { + hub, system, _, m := certAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + count := hub.TestMailer.TotalSend() + cert := &monitor.CertInfo{Expires: time.Now().Add(-72 * time.Hour).UnixMilli()} + setMonitorCert(t, hub, m, cert) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + require.Equal(t, count+1, hub.TestMailer.TotalSend()) + message := hub.TestMailer.Messages()[count] + assert.Contains(t, message.Subject, "Certificate expired") + assert.Contains(t, message.Text, "certificate expired on "+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly)) + histories, err := hub.FindAllRecords("alerts_history") + require.NoError(t, err) + require.Len(t, histories, 1) + assert.Negative(t, histories[0].GetFloat("value")) +} + +func TestNetworkMonitorCertAlertIgnoresMonitorsWithoutCert(t *testing.T) { + hub, system, alert, _ := certAlertSetup(t) + alert.Set("value", 365) + require.NoError(t, hub.Save(alert)) + _, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": system.Id, "target": "gateway.example.com", "protocol": "icmp", "interval": 60, "enabled": true, + }) + require.NoError(t, err) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + count := hub.TestMailer.TotalSend() + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + alert, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.False(t, alert.GetBool("triggered")) +} + +// The hub stores certificate info with direct SQL, which bypasses record hooks. +func TestNetworkMonitorCertAlertSQLUpdatedCert(t *testing.T) { + hub, system, alert, m := certAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + setMonitorCert(t, hub, m, certExpiringIn(60)) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + count := hub.TestMailer.TotalSend() + cert := certExpiringIn(10) + _, err := hub.DB().Update("network_monitors", dbx.Params{"certInfo": `{"expires":` + strconv.FormatInt(cert.Expires, 10) + `}`}, dbx.HashExp{"id": m.Id}).Execute() + require.NoError(t, err) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: {Cert: cert}})) + assert.Equal(t, count+1, hub.TestMailer.TotalSend()) + alert, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.True(t, alert.GetBool("triggered")) +} + +func TestNetworkMonitorCertAlertSteadyStateNoDatabaseWork(t *testing.T) { + hub, system, _, m := certAlertSetup(t) + counted := &monitorCountingHub{TestHub: hub} + am := alerts.NewTestAlertManagerWithoutWorker(counted) + sql := monitorSQLCounter(t, hub) + cert := certExpiringIn(10) + setMonitorCert(t, hub, m, cert) + // The agent resends unchanged certificate info for certificates about to expire. + result := monitorResult(0) + result.Cert = cert + results := map[string]monitor.Result{m.Id: result} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + sql.Store(0) + counted.transactions.Store(0) + for range 100 { + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + } + assert.Zero(t, sql.Load(), "steady state must not issue SQL") + assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions") +} + +func TestNetworkMonitorCertAlertIndependentOfLossAlert(t *testing.T) { + hub, system, certAlert, m := certAlertSetup(t) + lossAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{ + "name": "NetworkMonitorLoss", "system": system.Id, "user": certAlert.GetString("user"), "value": 5, + }) + require.NoError(t, err) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + setMonitorCert(t, hub, m, certExpiringIn(10)) + results := map[string]monitor.Result{m.Id: monitorResult(0)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + results[m.Id] = monitorResult(50) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + // Recovering loss must leave the certificate incident open. + results[m.Id] = monitorResult(0) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + for id, triggered := range map[string]bool{certAlert.Id: true, lossAlert.Id: false} { + record, err := hub.FindRecordById("alerts", id) + require.NoError(t, err) + assert.Equal(t, triggered, record.GetBool("triggered"), record.GetString("name")) + } + open, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""}) + require.NoError(t, err) + assert.EqualValues(t, 1, open) +} + +func TestNetworkMonitorCertAlertCleanup(t *testing.T) { + for _, scenario := range []string{"disable monitor", "delete monitor", "down"} { + t.Run(scenario, func(t *testing.T) { + hub, system, alert, m := certAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + setMonitorCert(t, hub, m, certExpiringIn(10)) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + count := hub.TestMailer.TotalSend() + switch scenario { + case "disable monitor": + m.Set("enabled", false) + require.NoError(t, hub.Save(m)) + case "delete monitor": + require.NoError(t, hub.Delete(m)) + case "down": + _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + system.Set("status", "down") + } + require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil)) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + record, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""}) + require.NoError(t, err) + // A disconnected system keeps its incident; removed monitors close it silently. + assert.Equal(t, scenario == "down", record.GetBool("triggered")) + assert.Equal(t, scenario == "down", open == 1) + }) + } +} + +func TestNetworkMonitorCertAlertAPI(t *testing.T) { + for _, tc := range []struct { + name string + value float64 + status int + }{ + {name: "valid threshold", value: 14, status: 200}, + {name: "maximum threshold", value: 365, status: 200}, + {name: "zero threshold", value: 0, status: 400}, + {name: "fractional threshold", value: 14.5, status: 400}, + {name: "too large threshold", value: 366, status: 400}, + } { + t.Run(tc.name, func(t *testing.T) { + hub, user := beszelTests.GetHubWithUser(t) + defer hub.Cleanup() + systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused") + require.NoError(t, err) + token, err := user.NewAuthToken() + require.NoError(t, err) + for _, direct := range []bool{false, true} { + body := map[string]any{"name": "NetworkMonitorCert", "value": tc.value, "min": 10, "systems": []string{systems[0].Id}, "overwrite": true} + url := "/api/beszel/user-alerts" + if direct { + url = "/api/collections/alerts/records" + body["system"], body["user"] = systems[0].Id, user.Id + existing, err := hub.FindAllRecords("alerts") + require.NoError(t, err) + for _, record := range existing { + require.NoError(t, hub.Delete(record)) + } + } + content := `"status":400` + if tc.status == 200 { + content = `"NetworkMonitorCert"` + if !direct { + content = `"success":true` + } + } + scenario := beszelTests.ApiScenario{ + Name: tc.name, Method: "POST", URL: url, Body: jsonReader(body), + Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content}, + TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp }, + } + scenario.Test(t) + records, err := hub.FindAllRecords("alerts") + require.NoError(t, err) + if tc.status != 200 { + assert.Empty(t, records) + continue + } + require.Len(t, records, 1) + assert.Equal(t, tc.value, records[0].GetFloat("value")) + assert.Zero(t, records[0].GetInt("min")) + } + }) + } +} diff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.go index fb7723eb0..3e6612c0e 100644 --- a/internal/alerts/alerts_network_monitors.go +++ b/internal/alerts/alerts_network_monitors.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "net" + "slices" "strconv" "time" @@ -14,7 +15,12 @@ import ( "github.com/pocketbase/pocketbase/core" ) -const alertNameNetworkMonitorLoss = "NetworkMonitorLoss" +const ( + alertNameNetworkMonitorLoss = "NetworkMonitorLoss" + alertNameNetworkMonitorCert = "NetworkMonitorCert" + // maxCertAlertDays is the largest certificate expiry threshold in days. + maxCertAlertDays = 365 +) // networkMonitorAlertState is this alert type's persisted runtime state. // Monitor IDs map to their open history entries independently of history retention. @@ -22,20 +28,48 @@ type networkMonitorAlertState struct { Monitors map[string]string `json:"monitors"` } +// monitorObservation is one monitor's current value for an alert threshold. +type monitorObservation struct { + triggered bool + // value is stored in alert history: loss percentage or days left. + value float64 + // certExpires is the certificate expiry in Unix milliseconds (certificate alerts only). + certExpires int64 +} + +// isNetworkMonitorAlert reports whether an alert tracks incidents per network monitor. +func isNetworkMonitorAlert(name string) bool { + return name == alertNameNetworkMonitorLoss || name == alertNameNetworkMonitorCert +} + +// networkMonitorThresholdError describes why a threshold is invalid, or returns "" if it is valid. +func networkMonitorThresholdError(name string, value float64) string { + switch name { + case alertNameNetworkMonitorLoss: + if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 { + return "Monitor loss threshold must be at least 0 and below 100" + } + case alertNameNetworkMonitorCert: + if math.IsNaN(value) || value < 1 || value > maxCertAlertDays || value != math.Trunc(value) { + return fmt.Sprintf("Certificate expiry threshold must be a whole number of days from 1 to %d", maxCertAlertDays) + } + } + return "" +} + func (am *AlertManager) bindNetworkMonitorAlertEvents() { // Hidden fields are still writable through the record API unless protected. protectState := func(e *core.RecordRequestEvent) error { e.Record.Set("state", e.Record.Original().Get("state")) oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name") - if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) && + if oldName != "" && (isNetworkMonitorAlert(oldName) || isNetworkMonitorAlert(newName)) && (oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) { return e.BadRequestError("Delete and recreate the alert to change its type or system", nil) } - if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + if isNetworkMonitorAlert(newName) { e.Record.Set("triggered", e.Record.Original().GetBool("triggered")) - value := e.Record.GetFloat("value") - if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 { - return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil) + if msg := networkMonitorThresholdError(newName, e.Record.GetFloat("value")); msg != "" { + return e.BadRequestError(msg, nil) } e.Record.Set("min", 0) } @@ -62,10 +96,20 @@ func (am *AlertManager) bindNetworkMonitorAlertEvents() { // using its exact payload (dashboard requests can replace the cached payload). // Omitted results and disconnected systems never imply recovery. func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error { + // Certificate info is stored by direct SQL without record hooks, so drop the + // cached expiry times when a result carries a different expiry. The agent + // resends unchanged info hourly for certificates that are about to expire. + for id, result := range results { + if result.Cert != nil && am.networkMonitors.certChanged(systemRecord.Id, id, result.Cert.Expires) { + am.networkMonitors.invalidate(systemRecord.Id) + break + } + } if systemRecord.GetString("status") != "up" { return nil } - alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss) + alerts := am.alertsCache.GetSystemAlerts(systemRecord.Id) + alerts = slices.DeleteFunc(alerts, func(a CachedAlertData) bool { return !isNetworkMonitorAlert(a.Name) }) if len(alerts) == 0 { return nil } @@ -82,7 +126,7 @@ func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, re } // networkMonitorTransitionPending does no IO and never mutates cached maps. -func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool { +func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]cachedMonitor, results map[string]monitor.Result, now time.Time) bool { for _, alert := range alerts { if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) { return true @@ -92,9 +136,23 @@ func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[stri return true } } + if alert.Name == alertNameNetworkMonitorCert { + // Expiry depends on the clock, so check stored certificates rather than results. + cutoff := certExpiryCutoff(alert.Value, now) + for id, m := range monitors { + if m.certExpires <= 0 { + continue + } + _, active := alert.MonitorStates[id] + if (m.certExpires <= cutoff) != active { + return true + } + } + continue + } for id, result := range results { - interval, enabled := monitors[id] - if !enabled || !monitorResultReady(result, interval, now) { + m, enabled := monitors[id] + if !enabled || !monitorResultReady(result, m.interval, now) { continue } _, active := alert.MonitorStates[id] @@ -111,7 +169,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri err := app.RunInTransaction(func(tx core.App) error { // Read configuration inside the transaction so concurrent threshold changes, // disabling, and evaluations cannot overwrite each other's incident state. - alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss}) + alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": []any{alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert}}) if err != nil || len(alerts) == 0 { return err } @@ -156,21 +214,27 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri } } if system.GetString("status") == "up" { + name, threshold := alert.GetString("name"), alert.GetFloat("value") for _, m := range monitors { - result, ok := results[m.Id] - if !ok || !monitorResultReady(result, m.GetInt("interval"), now) { + var obs monitorObservation + var ok bool + if name == alertNameNetworkMonitorCert { + obs, ok = observeMonitorCert(m, threshold, now) + } else { + obs, ok = observeMonitorLoss(results[m.Id], m, threshold, now) + } + if !ok { continue } historyID, active := states[m.Id] - triggered := result.PacketLoss1h > alert.GetFloat("value") - if triggered == active { + if obs.triggered == active { continue } label := m.GetString("target") if m.GetString("protocol") == "tcp" { label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port"))) } - if triggered { + if obs.triggered { collection, err := tx.FindCachedCollectionByNameOrId("alerts_history") if err != nil { return err @@ -178,7 +242,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri history := core.NewRecord(collection) history.Load(map[string]any{ "alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID, - "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h, + "name": name, "monitor_name": label, "value": obs.value, }) if err := tx.Save(history); err != nil { return err @@ -191,15 +255,11 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri delete(states, m.Id) } changed = true - state, comparison := "loss", "exceeds" - if !triggered { - state, comparison = "recovered", "is at or below" - } + title, message := networkMonitorAlertMessage(name, obs, label, system.GetString("name"), threshold) messages = append(messages, AlertMessageData{ UserID: alert.GetString("user"), SystemID: systemID, - Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label), - Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")), - Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"), + Title: title, Message: message, + Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"), }) } } @@ -226,6 +286,69 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri return nil } +// observeMonitorLoss compares a fresh result's one hour loss with the threshold. +func observeMonitorLoss(result monitor.Result, m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) { + if !monitorResultReady(result, m.GetInt("interval"), now) { + return monitorObservation{}, false + } + return monitorObservation{triggered: result.PacketLoss1h > threshold, value: result.PacketLoss1h}, true +} + +// observeMonitorCert compares the stored certificate expiry with a threshold in days. +// Monitors without certificate info (non-HTTPS or never checked) are skipped. +func observeMonitorCert(m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) { + var cert monitor.CertInfo + if err := m.UnmarshalJSONField("certInfo", &cert); err != nil || cert.Expires <= 0 { + return monitorObservation{}, false + } + triggered := cert.Expires <= certExpiryCutoff(threshold, now) + return monitorObservation{triggered: triggered, value: float64(certDaysLeft(cert.Expires, now)), certExpires: cert.Expires}, true +} + +// certExpiryCutoff returns the latest expiry, in Unix milliseconds, that is within the given +// number of days. It uses exact times, so a threshold of 14 does not trigger with 14 days and +// 23 hours left. +func certExpiryCutoff(days float64, now time.Time) int64 { + return now.Add(time.Duration(days * float64(24*time.Hour))).UnixMilli() +} + +// certDaysLeft returns whole days until expiry for display, matching the days shown in the UI. +// It is negative once the certificate has expired. +func certDaysLeft(expires int64, now time.Time) int { + return int(math.Floor(float64(expires-now.UnixMilli()) / float64(time.Hour.Milliseconds()*24))) +} + +func networkMonitorAlertMessage(name string, obs monitorObservation, label, systemName string, threshold float64) (title, message string) { + if name == alertNameNetworkMonitorCert { + days := int(obs.value) + date := time.UnixMilli(obs.certExpires).UTC().Format(time.DateOnly) + switch { + case days < 0: + return fmt.Sprintf("Certificate expired on %s: %s", systemName, label), + fmt.Sprintf("%s on %s: certificate expired on %s.", label, systemName, date) + case obs.triggered: + return fmt.Sprintf("Certificate expiring on %s: %s", systemName, label), + fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is within the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold)) + default: + return fmt.Sprintf("Certificate expiry recovered on %s: %s", systemName, label), + fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is beyond the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold)) + } + } + state, comparison := "loss", "exceeds" + if !obs.triggered { + state, comparison = "recovered", "is at or below" + } + return fmt.Sprintf("Network monitor %s on %s: %s", state, systemName, label), + fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, systemName, obs.value, comparison, threshold) +} + +func pluralDays(days int) string { + if days == 1 { + return "1 day" + } + return fmt.Sprintf("%d days", days) +} + func monitorResultReady(result monitor.Result, interval int, now time.Time) bool { // Three completed attempts provide a short warm-up, including after an agent // restart. diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.go index ad47d0bd5..39cf3ddc8 100644 --- a/internal/alerts/alerts_system.go +++ b/internal/alerts/alerts_system.go @@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst return nil } - alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss) + alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert) if len(alerts) == 0 { return nil } diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.go index 2ca3718e7..e449c58c7 100644 --- a/internal/alerts/network_monitor_cache.go +++ b/internal/alerts/network_monitor_cache.go @@ -1,23 +1,34 @@ package alerts import ( + "encoding/json" "sync" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/types" ) -// networkMonitorCache keeps just the enabled monitor IDs and probe intervals -// needed for the alert fast path. Names and targets are read only on transitions. -// Returned maps are immutable; configuration changes invalidate the whole entry. +// cachedMonitor holds the monitor fields needed for the alert fast path. +type cachedMonitor struct { + interval int + // certExpires is the stored certificate expiry in Unix milliseconds, or zero if unknown. + certExpires int64 +} + +// networkMonitorCache keeps just the enabled monitor IDs, probe intervals, and +// certificate expiry times needed for the alert fast path. Names and targets are +// read only on transitions. Returned maps are immutable; configuration and +// certificate changes invalidate the whole entry. type networkMonitorCache struct { app core.App mu sync.RWMutex - systems map[string]map[string]int + systems map[string]map[string]cachedMonitor } func newNetworkMonitorCache(app core.App) *networkMonitorCache { - c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)} + c := &networkMonitorCache{app: app, systems: make(map[string]map[string]cachedMonitor)} invalidate := func(e *core.RecordEvent) error { c.invalidate(e.Record.GetString("system")) return e.Next() @@ -29,7 +40,8 @@ func newNetworkMonitorCache(app core.App) *networkMonitorCache { // Realtime metric saves also invoke this hook. They must not evict config. if old.GetString("system") != e.Record.GetString("system") || old.GetBool("enabled") != e.Record.GetBool("enabled") || - old.GetInt("interval") != e.Record.GetInt("interval") { + old.GetInt("interval") != e.Record.GetInt("interval") || + old.GetString("certInfo") != e.Record.GetString("certInfo") { c.invalidate(old.GetString("system")) c.invalidate(e.Record.GetString("system")) } @@ -48,7 +60,16 @@ func (c *networkMonitorCache) invalidate(systemID string) { c.mu.Unlock() } -func (c *networkMonitorCache) get(systemID string) (map[string]int, error) { +// certChanged reports whether a cached monitor's certificate expiry differs from +// expires. Systems that are not cached load fresh data when next needed. +func (c *networkMonitorCache) certChanged(systemID, monitorID string, expires int64) bool { + c.mu.RLock() + defer c.mu.RUnlock() + m, ok := c.systems[systemID][monitorID] + return ok && m.certExpires != expires +} + +func (c *networkMonitorCache) get(systemID string) (map[string]cachedMonitor, error) { c.mu.RLock() monitors, ok := c.systems[systemID] c.mu.RUnlock() @@ -63,16 +84,20 @@ func (c *networkMonitorCache) get(systemID string) (map[string]int, error) { // Keep the lock through the load so a concurrent config change cannot be // invalidated first and then overwritten by the older query result. var rows []struct { - ID string `db:"id"` - Interval int `db:"interval"` + ID string `db:"id"` + Interval int `db:"interval"` + CertInfo types.JSONRaw `db:"certInfo"` } - if err := c.app.DB().Select("id", "interval").From("network_monitors"). + if err := c.app.DB().Select("id", "interval", "certInfo").From("network_monitors"). Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil { return nil, err } - monitors = make(map[string]int, len(rows)) + monitors = make(map[string]cachedMonitor, len(rows)) for _, row := range rows { - monitors[row.ID] = row.Interval + var cert monitor.CertInfo + // Missing or malformed info leaves the expiry unknown. + _ = json.Unmarshal(row.CertInfo, &cert) + monitors[row.ID] = cachedMonitor{interval: row.Interval, certExpires: cert.Expires} } c.systems[systemID] = monitors return monitors, nil diff --git a/internal/hub/collections_test.go b/internal/hub/collections_test.go index 2acd245c0..f39947484 100644 --- a/internal/hub/collections_test.go +++ b/internal/hub/collections_test.go @@ -51,7 +51,7 @@ func TestCollectionRulesDefault(t *testing.T) { assert.Equal(t, isUserMatchesUser+` && @request.body.user:changed = false && @request.body.system:changed = false && system.users.id ?= @request.auth.id`, *alertsCollection.UpdateRule) assert.Equal(t, isUserMatchesUser, *alertsCollection.DeleteRule) alertNames := alertsCollection.Fields.GetByName("name").(*core.SelectField).Values - for _, name := range []string{"CPUIOWait", "CPUSteal"} { + for _, name := range []string{"CPUIOWait", "CPUSteal", "NetworkMonitorCert"} { assert.Contains(t, alertNames, name) } for _, name := range []string{"CPUSystem", "CPUUser", "CPUIdle", "CPUOther"} { diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.go index a01897aa5..a2875ec67 100644 --- a/internal/hub/systems/system_manager.go +++ b/internal/hub/systems/system_manager.go @@ -427,7 +427,7 @@ func deactivateAlerts(app core.App, systemID string, preserveStatusAlert bool) e // Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API // _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute() - alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0) + alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss' && name != 'NetworkMonitorCert'", systemID), "", -1, 0) if err != nil { return err } diff --git a/internal/migrations/1791926360_network_monitor_cert_alert.go b/internal/migrations/1791926360_network_monitor_cert_alert.go new file mode 100644 index 000000000..fa3c3e2f1 --- /dev/null +++ b/internal/migrations/1791926360_network_monitor_cert_alert.go @@ -0,0 +1,34 @@ +package migrations + +import ( + "slices" + + "github.com/pocketbase/pocketbase/core" + m "github.com/pocketbase/pocketbase/migrations" +) + +const networkMonitorCertAlertName = "NetworkMonitorCert" + +func init() { + m.Register(func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("alerts") + if err != nil { + return err + } + field := collection.Fields.GetByName("name").(*core.SelectField) + if !slices.Contains(field.Values, networkMonitorCertAlertName) { + field.Values = append(field.Values, networkMonitorCertAlertName) + } + return app.Save(collection) + }, func(app core.App) error { + collection, err := app.FindCollectionByNameOrId("alerts") + if err != nil { + return err + } + field := collection.Fields.GetByName("name").(*core.SelectField) + field.Values = slices.DeleteFunc(field.Values, func(name string) bool { + return name == networkMonitorCertAlertName + }) + return app.Save(collection) + }) +} diff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsx index e00efd269..c1bdd3584 100644 --- a/internal/site/src/components/active-alerts.tsx +++ b/internal/site/src/components/active-alerts.tsx @@ -17,6 +17,13 @@ function AlertTriggeredDesc({ alert }: { alert: AlertRecord }) { if (alert.name === "NetworkMonitorLoss") { return One or more monitors exceed {alert.value}% loss } + if (alert.name === "NetworkMonitorCert") { + return ( + + One or more certificates expire within + + ) + } if (alert.name === "Status") { return Connection is down } diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsx index 2ae9bd97e..b87bd2a56 100644 --- a/internal/site/src/components/alerts-history-columns.tsx +++ b/internal/site/src/components/alerts-history-columns.tsx @@ -1,5 +1,5 @@ import { t } from "@lingui/core/macro" -import { Trans } from "@lingui/react/macro" +import { Plural, Trans } from "@lingui/react/macro" import type { ColumnDef } from "@tanstack/react-table" import { Badge } from "@/components/ui/badge" import { Button } from "@/components/ui/button" @@ -69,6 +69,13 @@ export const alertsHistoryColumns: ColumnDef[] = [ return {t`Down`} } const value = getValue() as number + if (name === "NetworkMonitorCert") { + return ( + + {value < 0 ? Expired : } + + ) + } const unit = info?.unit return ( diff --git a/internal/site/src/components/alerts/alerts-sheet.tsx b/internal/site/src/components/alerts/alerts-sheet.tsx index 845aa0746..bd0ea8975 100644 --- a/internal/site/src/components/alerts/alerts-sheet.tsx +++ b/internal/site/src/components/alerts/alerts-sheet.tsx @@ -350,7 +350,14 @@ export function AlertContent({ {!noThreshold && (

- {alertData.invert ? ( + {alertKey === "NetworkMonitorCert" ? ( + + Expires within{" "} + + + + + ) : alertData.invert ? ( Average drops below{" "} @@ -386,6 +393,8 @@ export function AlertContent({ if (!Number.isNaN(val)) { // keep values on the step grid so the unit can be inferred on reload if (unit) val = roundToStep(val, inputStep) + // certificate thresholds are whole days + else if (alertKey === "NetworkMonitorCert") val = Math.round(val) if (valueMax != null) val = Math.min(val, valueMax) if (inputMin != null) val = Math.max(val, inputMin) setValue(val) diff --git a/internal/site/src/lib/alerts.ts b/internal/site/src/lib/alerts.ts index 39c6a0f32..f19c0e830 100644 --- a/internal/site/src/lib/alerts.ts +++ b/internal/site/src/lib/alerts.ts @@ -1,5 +1,5 @@ import { t } from "@lingui/core/macro" -import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react" +import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon, ShieldCheckIcon } from "lucide-react" import type { RecordSubscription } from "pocketbase" import { EthernetIcon, GpuIcon } from "@/components/ui/icons" import { $alerts } from "@/lib/stores" @@ -71,6 +71,16 @@ export const alertInfo: Record = { step: 0.1, start: 5, }, + NetworkMonitorCert: { + name: () => t`Certificate Expiration`, + unit: " days", + icon: ShieldCheckIcon, + desc: () => t`Triggers when an HTTPS certificate nears expiration`, + noDuration: true, + min: 1, + max: 365, + start: 14, + }, GPU: { name: () => t`GPU Usage`, unit: "%",