feat(alerts): add TLS certificate expiry alert for network monitors (#2627)

Adds a NetworkMonitorCert alert with a threshold in days, evaluated on the hub from the certInfo stored by #2401. Follows the NetworkMonitorLoss alert: per-monitor incidents, alert history, recovery notifications.

Refs #391
This commit is contained in:
Mike Bogdan
2026-10-17 13:40:19 -04:00
committed by GitHub
parent c479c3e79c
commit e984ee1208
14 changed files with 573 additions and 48 deletions
+3 -3
View File
@@ -29,9 +29,9 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err)
}
if reqData.Name == alertNameNetworkMonitorLoss {
if reqData.Value < 0 || reqData.Value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
if isNetworkMonitorAlert(reqData.Name) {
if msg := networkMonitorThresholdError(reqData.Name, reqData.Value); msg != "" {
return e.BadRequestError(msg, nil)
}
reqData.Min = 0
}
+1 -1
View File
@@ -34,7 +34,7 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time()
if a.Name == alertNameNetworkMonitorLoss {
if isNetworkMonitorAlert(a.Name) {
var state networkMonitorAlertState
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
a.MonitorStates = state.Monitors
+2 -2
View File
@@ -9,7 +9,7 @@ import (
// On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if isNetworkMonitorAlert(e.Record.GetString("name")) {
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
return err
}
@@ -25,7 +25,7 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
// Network monitor incidents have separate history entries per monitor.
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if isNetworkMonitorAlert(e.Record.GetString("name")) {
return e.Next()
}
original := e.Record.Original()
@@ -0,0 +1,310 @@
//go:build testing
package alerts_test
import (
"strconv"
"testing"
"time"
"github.com/henrygd/beszel/internal/alerts"
"github.com/henrygd/beszel/internal/entities/monitor"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
pbTests "github.com/pocketbase/pocketbase/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// certAlertSetup creates a 30 day certificate alert and an HTTPS monitor without certificate info.
func certAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, *core.Record) {
t.Helper()
hub, system, alert := systemdTestSetup(t, false)
t.Cleanup(hub.Cleanup)
alert.Set("name", "NetworkMonitorCert")
alert.Set("value", 30)
require.NoError(t, hub.Save(alert))
m, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "target": "https://example.com", "protocol": "http", "interval": 60, "enabled": true,
})
require.NoError(t, err)
// Avoid starting a system update worker in tests.
_, err = hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
system.Set("status", "up")
return hub, system, alert, m
}
// certExpiringIn returns certificate info for a certificate with the given days
// left, plus an hour so that the day count is stable during the test.
func certExpiringIn(days int) *monitor.CertInfo {
return &monitor.CertInfo{Expires: time.Now().Add(time.Duration(days)*24*time.Hour + time.Hour).UnixMilli()}
}
func setMonitorCert(t *testing.T, hub *beszelTests.TestHub, m *core.Record, cert *monitor.CertInfo) {
t.Helper()
m.Set("certInfo", cert)
require.NoError(t, hub.Save(m))
}
func TestNetworkMonitorCertAlertLifecycle(t *testing.T) {
hub, system, alert, m := certAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
check := func(active bool, open, sent int) {
t.Helper()
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.Equal(t, active, record.GetBool("triggered"))
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
assert.EqualValues(t, open, total)
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
}
lastMessage := func() string {
return hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text
}
// Unknown expiry never triggers.
check(false, 0, 0)
setMonitorCert(t, hub, m, certExpiringIn(40))
check(false, 0, 0)
// Expiry is checked against the clock, without needing a new result.
cert := certExpiringIn(25)
setMonitorCert(t, hub, m, cert)
check(true, 1, 1)
assert.Contains(t, lastMessage(), "https://example.com")
assert.Contains(t, lastMessage(), "expires in 25 days ("+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly)+")")
assert.Contains(t, lastMessage(), "within the 30-day threshold")
// Repeated checks and a hub restart do not notify again.
check(true, 1, 1)
am = alerts.NewTestAlertManagerWithoutWorker(hub)
check(true, 1, 1)
// A renewed certificate resolves the incident.
setMonitorCert(t, hub, m, certExpiringIn(90))
check(false, 0, 2)
assert.Contains(t, lastMessage(), "expires in 90 days")
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Subject, "Certificate expiry recovered")
assert.Contains(t, lastMessage(), "beyond the 30-day threshold")
// A new incident opens when expiry nears again.
setMonitorCert(t, hub, m, certExpiringIn(20))
check(true, 1, 3)
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 2)
for _, history := range histories {
assert.Equal(t, "NetworkMonitorCert", history.GetString("name"))
assert.Equal(t, "https://example.com", history.GetString("monitor_name"))
}
}
func TestNetworkMonitorCertAlertThresholdBoundary(t *testing.T) {
for _, tc := range []struct {
name string
offset time.Duration
triggered bool
}{
{name: "a minute beyond", offset: time.Minute, triggered: false},
{name: "23 hours beyond", offset: 23 * time.Hour, triggered: false},
{name: "a minute within", offset: -time.Minute, triggered: true},
} {
t.Run(tc.name, func(t *testing.T) {
hub, system, alert, m := certAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
// Expiry beyond 30 days still displays as 30 days left, but must not trigger a 30 day threshold.
setMonitorCert(t, hub, m, &monitor.CertInfo{Expires: time.Now().Add(30*24*time.Hour + tc.offset).UnixMilli()})
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
alert, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.Equal(t, tc.triggered, alert.GetBool("triggered"))
})
}
}
func TestNetworkMonitorCertAlertExpired(t *testing.T) {
hub, system, _, m := certAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
cert := &monitor.CertInfo{Expires: time.Now().Add(-72 * time.Hour).UnixMilli()}
setMonitorCert(t, hub, m, cert)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
require.Equal(t, count+1, hub.TestMailer.TotalSend())
message := hub.TestMailer.Messages()[count]
assert.Contains(t, message.Subject, "Certificate expired")
assert.Contains(t, message.Text, "certificate expired on "+time.UnixMilli(cert.Expires).UTC().Format(time.DateOnly))
histories, err := hub.FindAllRecords("alerts_history")
require.NoError(t, err)
require.Len(t, histories, 1)
assert.Negative(t, histories[0].GetFloat("value"))
}
func TestNetworkMonitorCertAlertIgnoresMonitorsWithoutCert(t *testing.T) {
hub, system, alert, _ := certAlertSetup(t)
alert.Set("value", 365)
require.NoError(t, hub.Save(alert))
_, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "target": "gateway.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
assert.Equal(t, count, hub.TestMailer.TotalSend())
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, alert.GetBool("triggered"))
}
// The hub stores certificate info with direct SQL, which bypasses record hooks.
func TestNetworkMonitorCertAlertSQLUpdatedCert(t *testing.T) {
hub, system, alert, m := certAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
setMonitorCert(t, hub, m, certExpiringIn(60))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
count := hub.TestMailer.TotalSend()
cert := certExpiringIn(10)
_, err := hub.DB().Update("network_monitors", dbx.Params{"certInfo": `{"expires":` + strconv.FormatInt(cert.Expires, 10) + `}`}, dbx.HashExp{"id": m.Id}).Execute()
require.NoError(t, err)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: {Cert: cert}}))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, alert.GetBool("triggered"))
}
func TestNetworkMonitorCertAlertSteadyStateNoDatabaseWork(t *testing.T) {
hub, system, _, m := certAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
sql := monitorSQLCounter(t, hub)
cert := certExpiringIn(10)
setMonitorCert(t, hub, m, cert)
// The agent resends unchanged certificate info for certificates about to expire.
result := monitorResult(0)
result.Cert = cert
results := map[string]monitor.Result{m.Id: result}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
sql.Store(0)
counted.transactions.Store(0)
for range 100 {
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
}
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
}
func TestNetworkMonitorCertAlertIndependentOfLossAlert(t *testing.T) {
hub, system, certAlert, m := certAlertSetup(t)
lossAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
"name": "NetworkMonitorLoss", "system": system.Id, "user": certAlert.GetString("user"), "value": 5,
})
require.NoError(t, err)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
setMonitorCert(t, hub, m, certExpiringIn(10))
results := map[string]monitor.Result{m.Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
results[m.Id] = monitorResult(50)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
// Recovering loss must leave the certificate incident open.
results[m.Id] = monitorResult(0)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
for id, triggered := range map[string]bool{certAlert.Id: true, lossAlert.Id: false} {
record, err := hub.FindRecordById("alerts", id)
require.NoError(t, err)
assert.Equal(t, triggered, record.GetBool("triggered"), record.GetString("name"))
}
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"resolved": ""})
require.NoError(t, err)
assert.EqualValues(t, 1, open)
}
func TestNetworkMonitorCertAlertCleanup(t *testing.T) {
for _, scenario := range []string{"disable monitor", "delete monitor", "down"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, m := certAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
setMonitorCert(t, hub, m, certExpiringIn(10))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
count := hub.TestMailer.TotalSend()
switch scenario {
case "disable monitor":
m.Set("enabled", false)
require.NoError(t, hub.Save(m))
case "delete monitor":
require.NoError(t, hub.Delete(m))
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
system.Set("status", "down")
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, nil))
assert.Equal(t, count, hub.TestMailer.TotalSend())
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
// A disconnected system keeps its incident; removed monitors close it silently.
assert.Equal(t, scenario == "down", record.GetBool("triggered"))
assert.Equal(t, scenario == "down", open == 1)
})
}
}
func TestNetworkMonitorCertAlertAPI(t *testing.T) {
for _, tc := range []struct {
name string
value float64
status int
}{
{name: "valid threshold", value: 14, status: 200},
{name: "maximum threshold", value: 365, status: 200},
{name: "zero threshold", value: 0, status: 400},
{name: "fractional threshold", value: 14.5, status: 400},
{name: "too large threshold", value: 366, status: 400},
} {
t.Run(tc.name, func(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "paused")
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
for _, direct := range []bool{false, true} {
body := map[string]any{"name": "NetworkMonitorCert", "value": tc.value, "min": 10, "systems": []string{systems[0].Id}, "overwrite": true}
url := "/api/beszel/user-alerts"
if direct {
url = "/api/collections/alerts/records"
body["system"], body["user"] = systems[0].Id, user.Id
existing, err := hub.FindAllRecords("alerts")
require.NoError(t, err)
for _, record := range existing {
require.NoError(t, hub.Delete(record))
}
}
content := `"status":400`
if tc.status == 200 {
content = `"NetworkMonitorCert"`
if !direct {
content = `"success":true`
}
}
scenario := beszelTests.ApiScenario{
Name: tc.name, Method: "POST", URL: url, Body: jsonReader(body),
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
}
scenario.Test(t)
records, err := hub.FindAllRecords("alerts")
require.NoError(t, err)
if tc.status != 200 {
assert.Empty(t, records)
continue
}
require.Len(t, records, 1)
assert.Equal(t, tc.value, records[0].GetFloat("value"))
assert.Zero(t, records[0].GetInt("min"))
}
})
}
}
+147 -24
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"math"
"net"
"slices"
"strconv"
"time"
@@ -14,7 +15,12 @@ import (
"github.com/pocketbase/pocketbase/core"
)
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
const (
alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
alertNameNetworkMonitorCert = "NetworkMonitorCert"
// maxCertAlertDays is the largest certificate expiry threshold in days.
maxCertAlertDays = 365
)
// networkMonitorAlertState is this alert type's persisted runtime state.
// Monitor IDs map to their open history entries independently of history retention.
@@ -22,20 +28,48 @@ type networkMonitorAlertState struct {
Monitors map[string]string `json:"monitors"`
}
// monitorObservation is one monitor's current value for an alert threshold.
type monitorObservation struct {
triggered bool
// value is stored in alert history: loss percentage or days left.
value float64
// certExpires is the certificate expiry in Unix milliseconds (certificate alerts only).
certExpires int64
}
// isNetworkMonitorAlert reports whether an alert tracks incidents per network monitor.
func isNetworkMonitorAlert(name string) bool {
return name == alertNameNetworkMonitorLoss || name == alertNameNetworkMonitorCert
}
// networkMonitorThresholdError describes why a threshold is invalid, or returns "" if it is valid.
func networkMonitorThresholdError(name string, value float64) string {
switch name {
case alertNameNetworkMonitorLoss:
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
return "Monitor loss threshold must be at least 0 and below 100"
}
case alertNameNetworkMonitorCert:
if math.IsNaN(value) || value < 1 || value > maxCertAlertDays || value != math.Trunc(value) {
return fmt.Sprintf("Certificate expiry threshold must be a whole number of days from 1 to %d", maxCertAlertDays)
}
}
return ""
}
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// Hidden fields are still writable through the record API unless protected.
protectState := func(e *core.RecordRequestEvent) error {
e.Record.Set("state", e.Record.Original().Get("state"))
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
if oldName != "" && (isNetworkMonitorAlert(oldName) || isNetworkMonitorAlert(newName)) &&
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
}
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if isNetworkMonitorAlert(newName) {
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
value := e.Record.GetFloat("value")
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
if msg := networkMonitorThresholdError(newName, e.Record.GetFloat("value")); msg != "" {
return e.BadRequestError(msg, nil)
}
e.Record.Set("min", 0)
}
@@ -62,10 +96,20 @@ func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// using its exact payload (dashboard requests can replace the cached payload).
// Omitted results and disconnected systems never imply recovery.
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
// Certificate info is stored by direct SQL without record hooks, so drop the
// cached expiry times when a result carries a different expiry. The agent
// resends unchanged info hourly for certificates that are about to expire.
for id, result := range results {
if result.Cert != nil && am.networkMonitors.certChanged(systemRecord.Id, id, result.Cert.Expires) {
am.networkMonitors.invalidate(systemRecord.Id)
break
}
}
if systemRecord.GetString("status") != "up" {
return nil
}
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
alerts := am.alertsCache.GetSystemAlerts(systemRecord.Id)
alerts = slices.DeleteFunc(alerts, func(a CachedAlertData) bool { return !isNetworkMonitorAlert(a.Name) })
if len(alerts) == 0 {
return nil
}
@@ -82,7 +126,7 @@ func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, re
}
// networkMonitorTransitionPending does no IO and never mutates cached maps.
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]cachedMonitor, results map[string]monitor.Result, now time.Time) bool {
for _, alert := range alerts {
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
return true
@@ -92,9 +136,23 @@ func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[stri
return true
}
}
if alert.Name == alertNameNetworkMonitorCert {
// Expiry depends on the clock, so check stored certificates rather than results.
cutoff := certExpiryCutoff(alert.Value, now)
for id, m := range monitors {
if m.certExpires <= 0 {
continue
}
_, active := alert.MonitorStates[id]
if (m.certExpires <= cutoff) != active {
return true
}
}
continue
}
for id, result := range results {
interval, enabled := monitors[id]
if !enabled || !monitorResultReady(result, interval, now) {
m, enabled := monitors[id]
if !enabled || !monitorResultReady(result, m.interval, now) {
continue
}
_, active := alert.MonitorStates[id]
@@ -111,7 +169,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
err := app.RunInTransaction(func(tx core.App) error {
// Read configuration inside the transaction so concurrent threshold changes,
// disabling, and evaluations cannot overwrite each other's incident state.
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": []any{alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert}})
if err != nil || len(alerts) == 0 {
return err
}
@@ -156,21 +214,27 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
}
}
if system.GetString("status") == "up" {
name, threshold := alert.GetString("name"), alert.GetFloat("value")
for _, m := range monitors {
result, ok := results[m.Id]
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
var obs monitorObservation
var ok bool
if name == alertNameNetworkMonitorCert {
obs, ok = observeMonitorCert(m, threshold, now)
} else {
obs, ok = observeMonitorLoss(results[m.Id], m, threshold, now)
}
if !ok {
continue
}
historyID, active := states[m.Id]
triggered := result.PacketLoss1h > alert.GetFloat("value")
if triggered == active {
if obs.triggered == active {
continue
}
label := m.GetString("target")
if m.GetString("protocol") == "tcp" {
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
}
if triggered {
if obs.triggered {
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
@@ -178,7 +242,7 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
history := core.NewRecord(collection)
history.Load(map[string]any{
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
"name": name, "monitor_name": label, "value": obs.value,
})
if err := tx.Save(history); err != nil {
return err
@@ -191,15 +255,11 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
delete(states, m.Id)
}
changed = true
state, comparison := "loss", "exceeds"
if !triggered {
state, comparison = "recovered", "is at or below"
}
title, message := networkMonitorAlertMessage(name, obs, label, system.GetString("name"), threshold)
messages = append(messages, AlertMessageData{
UserID: alert.GetString("user"), SystemID: systemID,
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
Title: title, Message: message,
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
})
}
}
@@ -226,6 +286,69 @@ func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID stri
return nil
}
// observeMonitorLoss compares a fresh result's one hour loss with the threshold.
func observeMonitorLoss(result monitor.Result, m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {
if !monitorResultReady(result, m.GetInt("interval"), now) {
return monitorObservation{}, false
}
return monitorObservation{triggered: result.PacketLoss1h > threshold, value: result.PacketLoss1h}, true
}
// observeMonitorCert compares the stored certificate expiry with a threshold in days.
// Monitors without certificate info (non-HTTPS or never checked) are skipped.
func observeMonitorCert(m *core.Record, threshold float64, now time.Time) (monitorObservation, bool) {
var cert monitor.CertInfo
if err := m.UnmarshalJSONField("certInfo", &cert); err != nil || cert.Expires <= 0 {
return monitorObservation{}, false
}
triggered := cert.Expires <= certExpiryCutoff(threshold, now)
return monitorObservation{triggered: triggered, value: float64(certDaysLeft(cert.Expires, now)), certExpires: cert.Expires}, true
}
// certExpiryCutoff returns the latest expiry, in Unix milliseconds, that is within the given
// number of days. It uses exact times, so a threshold of 14 does not trigger with 14 days and
// 23 hours left.
func certExpiryCutoff(days float64, now time.Time) int64 {
return now.Add(time.Duration(days * float64(24*time.Hour))).UnixMilli()
}
// certDaysLeft returns whole days until expiry for display, matching the days shown in the UI.
// It is negative once the certificate has expired.
func certDaysLeft(expires int64, now time.Time) int {
return int(math.Floor(float64(expires-now.UnixMilli()) / float64(time.Hour.Milliseconds()*24)))
}
func networkMonitorAlertMessage(name string, obs monitorObservation, label, systemName string, threshold float64) (title, message string) {
if name == alertNameNetworkMonitorCert {
days := int(obs.value)
date := time.UnixMilli(obs.certExpires).UTC().Format(time.DateOnly)
switch {
case days < 0:
return fmt.Sprintf("Certificate expired on %s: %s", systemName, label),
fmt.Sprintf("%s on %s: certificate expired on %s.", label, systemName, date)
case obs.triggered:
return fmt.Sprintf("Certificate expiring on %s: %s", systemName, label),
fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is within the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))
default:
return fmt.Sprintf("Certificate expiry recovered on %s: %s", systemName, label),
fmt.Sprintf("%s on %s: certificate expires in %s (%s), which is beyond the %d-day threshold.", label, systemName, pluralDays(days), date, int(threshold))
}
}
state, comparison := "loss", "exceeds"
if !obs.triggered {
state, comparison = "recovered", "is at or below"
}
return fmt.Sprintf("Network monitor %s on %s: %s", state, systemName, label),
fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, systemName, obs.value, comparison, threshold)
}
func pluralDays(days int) string {
if days == 1 {
return "1 day"
}
return fmt.Sprintf("%d days", days)
}
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
// Three completed attempts provide a short warm-up, including after an agent
// restart.
+1 -1
View File
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss, alertNameNetworkMonitorCert)
if len(alerts) == 0 {
return nil
}
+37 -12
View File
@@ -1,23 +1,34 @@
package alerts
import (
"encoding/json"
"sync"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
)
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
// needed for the alert fast path. Names and targets are read only on transitions.
// Returned maps are immutable; configuration changes invalidate the whole entry.
// cachedMonitor holds the monitor fields needed for the alert fast path.
type cachedMonitor struct {
interval int
// certExpires is the stored certificate expiry in Unix milliseconds, or zero if unknown.
certExpires int64
}
// networkMonitorCache keeps just the enabled monitor IDs, probe intervals, and
// certificate expiry times needed for the alert fast path. Names and targets are
// read only on transitions. Returned maps are immutable; configuration and
// certificate changes invalidate the whole entry.
type networkMonitorCache struct {
app core.App
mu sync.RWMutex
systems map[string]map[string]int
systems map[string]map[string]cachedMonitor
}
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]cachedMonitor)}
invalidate := func(e *core.RecordEvent) error {
c.invalidate(e.Record.GetString("system"))
return e.Next()
@@ -29,7 +40,8 @@ func newNetworkMonitorCache(app core.App) *networkMonitorCache {
// Realtime metric saves also invoke this hook. They must not evict config.
if old.GetString("system") != e.Record.GetString("system") ||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
old.GetInt("interval") != e.Record.GetInt("interval") {
old.GetInt("interval") != e.Record.GetInt("interval") ||
old.GetString("certInfo") != e.Record.GetString("certInfo") {
c.invalidate(old.GetString("system"))
c.invalidate(e.Record.GetString("system"))
}
@@ -48,7 +60,16 @@ func (c *networkMonitorCache) invalidate(systemID string) {
c.mu.Unlock()
}
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
// certChanged reports whether a cached monitor's certificate expiry differs from
// expires. Systems that are not cached load fresh data when next needed.
func (c *networkMonitorCache) certChanged(systemID, monitorID string, expires int64) bool {
c.mu.RLock()
defer c.mu.RUnlock()
m, ok := c.systems[systemID][monitorID]
return ok && m.certExpires != expires
}
func (c *networkMonitorCache) get(systemID string) (map[string]cachedMonitor, error) {
c.mu.RLock()
monitors, ok := c.systems[systemID]
c.mu.RUnlock()
@@ -63,16 +84,20 @@ func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
// Keep the lock through the load so a concurrent config change cannot be
// invalidated first and then overwritten by the older query result.
var rows []struct {
ID string `db:"id"`
Interval int `db:"interval"`
ID string `db:"id"`
Interval int `db:"interval"`
CertInfo types.JSONRaw `db:"certInfo"`
}
if err := c.app.DB().Select("id", "interval").From("network_monitors").
if err := c.app.DB().Select("id", "interval", "certInfo").From("network_monitors").
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
return nil, err
}
monitors = make(map[string]int, len(rows))
monitors = make(map[string]cachedMonitor, len(rows))
for _, row := range rows {
monitors[row.ID] = row.Interval
var cert monitor.CertInfo
// Missing or malformed info leaves the expiry unknown.
_ = json.Unmarshal(row.CertInfo, &cert)
monitors[row.ID] = cachedMonitor{interval: row.Interval, certExpires: cert.Expires}
}
c.systems[systemID] = monitors
return monitors, nil
+1 -1
View File
@@ -51,7 +51,7 @@ func TestCollectionRulesDefault(t *testing.T) {
assert.Equal(t, isUserMatchesUser+` && @request.body.user:changed = false && @request.body.system:changed = false && system.users.id ?= @request.auth.id`, *alertsCollection.UpdateRule)
assert.Equal(t, isUserMatchesUser, *alertsCollection.DeleteRule)
alertNames := alertsCollection.Fields.GetByName("name").(*core.SelectField).Values
for _, name := range []string{"CPUIOWait", "CPUSteal"} {
for _, name := range []string{"CPUIOWait", "CPUSteal", "NetworkMonitorCert"} {
assert.Contains(t, alertNames, name)
}
for _, name := range []string{"CPUSystem", "CPUUser", "CPUIdle", "CPUOther"} {
+1 -1
View File
@@ -427,7 +427,7 @@ func deactivateAlerts(app core.App, systemID string, preserveStatusAlert bool) e
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss' && name != 'NetworkMonitorCert'", systemID), "", -1, 0)
if err != nil {
return err
}
@@ -0,0 +1,34 @@
package migrations
import (
"slices"
"github.com/pocketbase/pocketbase/core"
m "github.com/pocketbase/pocketbase/migrations"
)
const networkMonitorCertAlertName = "NetworkMonitorCert"
func init() {
m.Register(func(app core.App) error {
collection, err := app.FindCollectionByNameOrId("alerts")
if err != nil {
return err
}
field := collection.Fields.GetByName("name").(*core.SelectField)
if !slices.Contains(field.Values, networkMonitorCertAlertName) {
field.Values = append(field.Values, networkMonitorCertAlertName)
}
return app.Save(collection)
}, func(app core.App) error {
collection, err := app.FindCollectionByNameOrId("alerts")
if err != nil {
return err
}
field := collection.Fields.GetByName("name").(*core.SelectField)
field.Values = slices.DeleteFunc(field.Values, func(name string) bool {
return name == networkMonitorCertAlertName
})
return app.Save(collection)
})
}
@@ -17,6 +17,13 @@ function AlertTriggeredDesc({ alert }: { alert: AlertRecord }) {
if (alert.name === "NetworkMonitorLoss") {
return <Trans>One or more monitors exceed {alert.value}% loss</Trans>
}
if (alert.name === "NetworkMonitorCert") {
return (
<Trans>
One or more certificates expire within <Plural value={alert.value} one="# day" other="# days" />
</Trans>
)
}
if (alert.name === "Status") {
return <Trans>Connection is down</Trans>
}
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
import { Trans } from "@lingui/react/macro"
import { Plural, Trans } from "@lingui/react/macro"
import type { ColumnDef } from "@tanstack/react-table"
import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
@@ -69,6 +69,13 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
return <span className="ps-2">{t`Down`}</span>
}
const value = getValue() as number
if (name === "NetworkMonitorCert") {
return (
<span className="tabular-nums ps-2.5">
{value < 0 ? <Trans>Expired</Trans> : <Plural value={value} one="# day" other="# days" />}
</span>
)
}
const unit = info?.unit
return (
<span className="tabular-nums ps-2.5">
@@ -350,7 +350,14 @@ export function AlertContent({
{!noThreshold && (
<div className={cn(noDuration && "col-span-full")}>
<p id={`v${name}`} className="text-sm block h-6">
{alertData.invert ? (
{alertKey === "NetworkMonitorCert" ? (
<Trans>
Expires within{" "}
<strong className="text-foreground">
<Plural value={value} one="# day" other="# days" />
</strong>
</Trans>
) : alertData.invert ? (
<Trans>
Average drops below{" "}
<strong className="text-foreground">
@@ -386,6 +393,8 @@ export function AlertContent({
if (!Number.isNaN(val)) {
// keep values on the step grid so the unit can be inferred on reload
if (unit) val = roundToStep(val, inputStep)
// certificate thresholds are whole days
else if (alertKey === "NetworkMonitorCert") val = Math.round(val)
if (valueMax != null) val = Math.min(val, valueMax)
if (inputMin != null) val = Math.max(val, inputMin)
setValue(val)
+11 -1
View File
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon, ShieldCheckIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase"
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
import { $alerts } from "@/lib/stores"
@@ -71,6 +71,16 @@ export const alertInfo: Record<string, AlertInfo> = {
step: 0.1,
start: 5,
},
NetworkMonitorCert: {
name: () => t`Certificate Expiration`,
unit: " days",
icon: ShieldCheckIcon,
desc: () => t`Triggers when an HTTPS certificate nears expiration`,
noDuration: true,
min: 1,
max: 365,
start: 14,
},
GPU: {
name: () => t`GPU Usage`,
unit: "%",