From bb1b39928e317592159e65dbcac1de12510d49b8 Mon Sep 17 00:00:00 2001 From: hank Date: Fri, 18 Sep 2026 13:22:50 -0400 Subject: [PATCH] feat: network monitoring from agents (#2266, #1911) Co-authored-by: Sven van Ginkel Co-authored-by: xiaomiku01 --- .dockerignore | 4 +- agent/agent.go | 9 + agent/connection_manager.go | 1 + agent/gpu_test.go | 3 +- agent/handlers.go | 20 + agent/network_monitor.go | 176 ++++ agent/network_monitor_history.go | 274 ++++++ agent/network_monitor_history_test.go | 154 ++++ agent/network_monitor_ping.go | 312 +++++++ agent/network_monitor_ping_test.go | 433 ++++++++++ agent/network_monitor_probe.go | 105 +++ agent/network_monitor_resume.go | 88 ++ agent/network_monitor_resume_test.go | 121 +++ agent/network_monitor_schedule.go | 60 ++ agent/network_monitor_schedule_test.go | 167 ++++ agent/network_monitor_task.go | 116 +++ agent/network_monitor_task_test.go | 79 ++ agent/network_monitor_test.go | 524 ++++++++++++ beszel.go | 5 +- internal/alerts/alerts.go | 15 +- internal/alerts/alerts_api.go | 7 + internal/alerts/alerts_cache.go | 50 +- internal/alerts/alerts_history.go | 10 + internal/alerts/alerts_network_monitors.go | 269 ++++++ .../alerts/alerts_network_monitors_test.go | 513 ++++++++++++ internal/alerts/alerts_system.go | 2 +- internal/alerts/alerts_test_helpers.go | 5 +- internal/alerts/network_monitor_cache.go | 79 ++ internal/common/common-ws.go | 2 + internal/entities/monitor/monitor.go | 102 +++ internal/entities/system/system.go | 4 +- internal/hub/collections.go | 12 +- internal/hub/hub.go | 2 + internal/hub/network_monitors.go | 158 ++++ internal/hub/network_monitors_test.go | 225 +++++ .../hub/systems/network_monitor_stats_test.go | 224 +++++ .../hub/systems/network_monitor_sync_test.go | 144 ++++ internal/hub/systems/network_monitors.go | 58 ++ internal/hub/systems/system.go | 145 +++- internal/hub/systems/system_manager.go | 30 +- internal/hub/systems/system_realtime.go | 20 +- internal/hub/systems/system_smart.go | 4 +- internal/hub/systems/system_smart_test.go | 6 +- internal/hub/systems/system_systemd_test.go | 18 +- internal/hub/systems/system_zfs.go | 4 +- internal/hub/systems/system_zfs_test.go | 12 +- internal/hub/utils/utils.go | 29 +- ...=> 0_collections_snapshot_0_20_0_dev_5.go} | 425 +++++++++- internal/records/monitor_averaging_test.go | 226 +++++ internal/records/records.go | 168 +++- internal/records/records_deletion.go | 28 +- internal/records/records_rollup_test.go | 86 ++ internal/site/package.json | 4 +- .../site/src/components/active-alerts.tsx | 2 + .../src/components/alerts-history-columns.tsx | 3 +- .../src/components/alerts/alerts-sheet.tsx | 6 +- .../site/src/components/charts/area-chart.tsx | 8 +- .../components/charts/chart-time-select.tsx | 13 +- .../site/src/components/charts/line-chart.tsx | 47 +- .../site/src/components/command-palette.tsx | 15 + .../containers-table-columns.tsx | 11 +- internal/site/src/components/navbar.tsx | 23 + .../network-monitors-table/monitor-dialog.tsx | 778 ++++++++++++++++++ .../network-monitors-columns.tsx | 411 +++++++++ .../network-monitors-table.tsx | 575 +++++++++++++ internal/site/src/components/router.tsx | 1 + .../site/src/components/routes/monitors.tsx | 33 + .../settings/alerts-history-data-table.tsx | 4 +- .../site/src/components/routes/system.tsx | 36 +- .../components/routes/system/chart-card.tsx | 1 + .../components/routes/system/chart-data.ts | 55 +- .../routes/system/charts/monitors-charts.tsx | 212 +++++ .../components/routes/system/lazy-tables.tsx | 17 + .../components/routes/system/smart-table.tsx | 50 +- .../routes/system/use-system-data.ts | 13 +- .../systems-table/systems-table-columns.tsx | 12 +- internal/site/src/components/ui/chart.tsx | 90 +- .../site/src/components/ui/dropdown-menu.tsx | 8 +- internal/site/src/components/ui/table.tsx | 2 +- internal/site/src/lib/alerts.ts | 395 ++++----- internal/site/src/lib/api.ts | 5 +- .../site/src/lib/network-monitor-utils.ts | 17 + internal/site/src/lib/stores.ts | 9 +- internal/site/src/lib/systemsManager.ts | 33 +- internal/site/src/lib/use-network-monitors.ts | 351 ++++++++ internal/site/src/lib/utils.ts | 86 +- internal/site/src/main.tsx | 3 + internal/site/src/types.d.ts | 58 +- .../site/tests/network-monitor-utils.test.ts | 36 + 89 files changed, 8699 insertions(+), 457 deletions(-) create mode 100644 agent/network_monitor.go create mode 100644 agent/network_monitor_history.go create mode 100644 agent/network_monitor_history_test.go create mode 100644 agent/network_monitor_ping.go create mode 100644 agent/network_monitor_ping_test.go create mode 100644 agent/network_monitor_probe.go create mode 100644 agent/network_monitor_resume.go create mode 100644 agent/network_monitor_resume_test.go create mode 100644 agent/network_monitor_schedule.go create mode 100644 agent/network_monitor_schedule_test.go create mode 100644 agent/network_monitor_task.go create mode 100644 agent/network_monitor_task_test.go create mode 100644 agent/network_monitor_test.go create mode 100644 internal/alerts/alerts_network_monitors.go create mode 100644 internal/alerts/alerts_network_monitors_test.go create mode 100644 internal/alerts/network_monitor_cache.go create mode 100644 internal/entities/monitor/monitor.go create mode 100644 internal/hub/network_monitors.go create mode 100644 internal/hub/network_monitors_test.go create mode 100644 internal/hub/systems/network_monitor_stats_test.go create mode 100644 internal/hub/systems/network_monitor_sync_test.go create mode 100644 internal/hub/systems/network_monitors.go rename internal/migrations/{0_collections_snapshot_0_19_0.go => 0_collections_snapshot_0_20_0_dev_5.go} (81%) create mode 100644 internal/records/monitor_averaging_test.go create mode 100644 internal/records/records_rollup_test.go create mode 100644 internal/site/src/components/network-monitors-table/monitor-dialog.tsx create mode 100644 internal/site/src/components/network-monitors-table/network-monitors-columns.tsx create mode 100644 internal/site/src/components/network-monitors-table/network-monitors-table.tsx create mode 100644 internal/site/src/components/routes/monitors.tsx create mode 100644 internal/site/src/components/routes/system/charts/monitors-charts.tsx create mode 100644 internal/site/src/lib/network-monitor-utils.ts create mode 100644 internal/site/src/lib/use-network-monitors.ts create mode 100644 internal/site/tests/network-monitor-utils.test.ts diff --git a/.dockerignore b/.dockerignore index d0ebc4d7..9ded2435 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ # Node.js dependencies -node_modules -internalsite/node_modules +node_modules/ +**/node_modules/ # Go build artifacts and binaries build diff --git a/agent/agent.go b/agent/agent.go index 86abb8a3..454c565e 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -48,6 +48,7 @@ type Agent struct { keys []gossh.PublicKey // SSH public keys smartManager *SmartManager // Manages SMART data systemdManager *systemdManager // Manages systemd services + monitorManager *MonitorManager // Manages network monitors storagePoolManager *StoragePoolManager // Manages storage pool and dataset data } @@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) { // initialize handler registry agent.handlerRegistry = NewHandlerRegistry() + // initialize monitor manager + agent.monitorManager = newMonitorManager() + agent.storagePoolManager = newStoragePoolManager() // Retain ZFS_INTERVAL for the shared storage pool detail refresh interval. @@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD } } + if a.monitorManager != nil { + data.Monitors = a.monitorManager.GetResults(cacheTimeMs) + slog.Debug("Monitors", "data", data.Monitors) + } + // skip updating systemd services if cache time is not the default 60sec interval if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs { totalCount := uint16(a.systemdManager.getServiceStatsCount()) diff --git a/agent/connection_manager.go b/agent/connection_manager.go index 28ea98ae..0f29bd24 100644 --- a/agent/connection_manager.go +++ b/agent/connection_manager.go @@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error { // } func (c *ConnectionManager) stop() error { _ = c.agent.StopServer() + c.agent.monitorManager.Stop() c.closeWebSocket() return health.CleanUp() } diff --git a/agent/gpu_test.go b/agent/gpu_test.go index 1ac6e018..889f6c26 100644 --- a/agent/gpu_test.go +++ b/agent/gpu_test.go @@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) { } func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { - // Save original PATH hasAmdSysfs := (&GPUManager{}).hasAmdSysfs() tests := []struct { @@ -1213,7 +1212,7 @@ echo "[]"` { name: "no gpu tools available", setupCommands: func(_ string) error { - t.Setenv("PATH", "") + // The subtest already restricts PATH to its empty temporary directory. return nil }, wantErr: true, diff --git a/agent/handlers.go b/agent/handlers.go index 9da87e54..67b69a73 100644 --- a/agent/handlers.go +++ b/agent/handlers.go @@ -7,6 +7,7 @@ import ( "github.com/fxamacker/cbor/v2" "github.com/henrygd/beszel/internal/common" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/smart" "log/slog" @@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry { registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{}) registry.Register(common.GetSmartData, &GetSmartDataHandler{}) registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{}) + registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{}) registry.Register(common.GetZfsData, &GetZfsDataHandler{}) return registry @@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error { return hctx.SendResponse(details, hctx.RequestID) } + +//////////////////////////////////////////////////////////////////////////// +//////////////////////////////////////////////////////////////////////////// + +// SyncNetworkMonitorsHandler handles monitor configuration sync from hub +type SyncNetworkMonitorsHandler struct{} + +func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error { + var req monitor.SyncRequest + if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil { + return err + } + resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req) + if err != nil { + return err + } + return hctx.SendResponse(resp, hctx.RequestID) +} diff --git a/agent/network_monitor.go b/agent/network_monitor.go new file mode 100644 index 00000000..d5257675 --- /dev/null +++ b/agent/network_monitor.go @@ -0,0 +1,176 @@ +package agent + +import ( + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" +) + +// MonitorManager manages network monitor configurations and task lifetimes. +type MonitorManager struct { + mu sync.RWMutex + monitors map[string]*monitorTask // keyed by monitor ID + probe monitorProbe + resumeGuard monitorResumeGuard +} + +func newMonitorManager() *MonitorManager { + return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout})) +} + +func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager { + return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe} +} + +// SyncMonitors replaces all monitor tasks with the given configs. +func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) { + pm.mu.Lock() + defer pm.mu.Unlock() + + // Build set of new keys + newKeys := make(map[string]monitor.Config, len(configs)) + for _, cfg := range configs { + if cfg.ID == "" { + continue + } + newKeys[cfg.ID] = cfg + } + + // Stop removed monitors + for key, task := range pm.monitors { + if _, exists := newKeys[key]; !exists { + task.cancel() + delete(pm.monitors, key) + } + } + + // Start new monitors and restart tasks whose config changed. + for key, cfg := range newKeys { + task, exists := pm.monitors[key] + if exists && task.config == cfg { + continue + } + if exists { + task.cancel() + } + task = newMonitorTaskFromExisting(cfg, task) + task.resumeGuard = &pm.resumeGuard + pm.resumeGuard.start() + pm.monitors[key] = task + pm.startMonitor(task) + } + if len(pm.monitors) == 0 { + pm.resumeGuard.shutdown() + } +} + +// HandleSyncRequest applies a full or incremental monitor sync request. +func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) { + switch req.Action { + case monitor.SyncActionReplace: + pm.SyncMonitors(req.Configs) + return monitor.SyncResponse{}, nil + case monitor.SyncActionUpsert: + result, err := pm.UpsertMonitor(req.Config, req.RunNow) + if err != nil { + return monitor.SyncResponse{}, err + } + if result == nil { + return monitor.SyncResponse{}, nil + } + return monitor.SyncResponse{Result: *result}, nil + case monitor.SyncActionDelete: + if req.Config.ID == "" { + return monitor.SyncResponse{}, errors.New("missing monitor ID for delete") + } + pm.DeleteMonitor(req.Config.ID) + return monitor.SyncResponse{}, nil + default: + return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action) + } +} + +// UpsertMonitor creates or replaces a single monitor task. +func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) { + if config.ID == "" { + return nil, errors.New("missing monitor ID") + } + + pm.mu.Lock() + task, exists := pm.monitors[config.ID] + if exists && task.config == config { + pm.mu.Unlock() + if !runNow { + return nil, nil + } + return task.runProbe(pm.probe), nil + } + if exists { + task.cancel() + } + task = newMonitorTaskFromExisting(config, task) + task.resumeGuard = &pm.resumeGuard + pm.resumeGuard.start() + pm.monitors[config.ID] = task + pm.mu.Unlock() + + if runNow { + result := task.runProbe(pm.probe) + pm.startMonitor(task) + return result, nil + } + pm.startMonitor(task) + return nil, nil +} + +// DeleteMonitor stops and removes a single monitor task. +func (pm *MonitorManager) DeleteMonitor(id string) { + if id == "" { + return + } + pm.mu.Lock() + defer pm.mu.Unlock() + if task, exists := pm.monitors[id]; exists { + task.cancel() + delete(pm.monitors, id) + } + if len(pm.monitors) == 0 { + pm.resumeGuard.shutdown() + } +} + +// GetResults returns aggregated results for all monitors over the last supplied duration in ms. +func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result { + pm.mu.RLock() + defer pm.mu.RUnlock() + + results := make(map[string]monitor.Result, len(pm.monitors)) + now := time.Now() + duration := time.Duration(durationMs) * time.Millisecond + + for _, task := range pm.monitors { + result, ok := task.history.result(duration, now) + + if !ok { + continue + } + results[task.config.ID] = result + } + + return results +} + +// Stop stops all monitor tasks. +func (pm *MonitorManager) Stop() { + pm.mu.Lock() + defer pm.mu.Unlock() + for key, task := range pm.monitors { + task.cancel() + delete(pm.monitors, key) + } + pm.resumeGuard.shutdown() +} diff --git a/agent/network_monitor_history.go b/agent/network_monitor_history.go new file mode 100644 index 00000000..9db1639a --- /dev/null +++ b/agent/network_monitor_history.go @@ -0,0 +1,274 @@ +package agent + +import ( + "math" + "sync" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" +) + +// Monitors run at user-defined intervals (e.g., every 10s). +// To keep memory usage low and constant, data is stored in two layers: +// 1. Raw samples: The most recent individual results (kept for monitorRawRetention). +// 2. Minute buckets: A ring buffer of 61 buckets, each representing one +// wall-clock minute. Samples collected within the same minute are aggregated +// (sum, min, max, count) into a single bucket. +// +// Short-term requests (<= 61s) use raw samples. +// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands +// of individual data points. + +const ( + // monitorRawRetention is the duration to keep individual samples + monitorRawRetention = 61 * time.Second + // monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials) + monitorMinuteBucketLen int32 = 61 +) + +// monitorHistory owns retention and aggregation, independently of probe execution. +type monitorHistory struct { + mu sync.Mutex + sampleCount int64 + samples []monitorSample + buckets [monitorMinuteBucketLen]monitorBucket +} + +func newMonitorHistory() *monitorHistory { + // Start small for typical intervals; append grows the buffer for faster probes. + return &monitorHistory{samples: make([]monitorSample, 0, 4)} +} + +func (h *monitorHistory) clone() *monitorHistory { + h.mu.Lock() + defer h.mu.Unlock() + cloned := newMonitorHistory() + cloned.samples = append(cloned.samples, h.samples...) + cloned.buckets = h.buckets + cloned.sampleCount = h.sampleCount + return cloned +} + +func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) { + h.mu.Lock() + defer h.mu.Unlock() + return h.resultLocked(duration, now) +} + +func (h *monitorHistory) record(sample monitorSample) monitor.Result { + h.mu.Lock() + defer h.mu.Unlock() + h.addSampleLocked(sample) + result, _ := h.resultLocked(time.Minute, sample.timestamp) + return result +} + +// monitorSample stores one monitor attempt and its collection time. +type monitorSample struct { + responseUs int64 // -1 means loss + timestamp time.Time +} + +// monitorBucket stores one minute of aggregated monitor data. +type monitorBucket struct { + minute int32 + filled bool + stats monitorAggregate +} + +// monitorAggregate accumulates successful response stats and total sample counts. +type monitorAggregate struct { + sumUs int64 + minUs int64 + maxUs int64 + totalCount int64 + successCount int64 +} + +// newMonitorAggregate initializes an aggregate with an unset minimum value. +func newMonitorAggregate() monitorAggregate { + return monitorAggregate{minUs: math.MaxInt64} +} + +// addResponse folds a single monitor sample into the aggregate. +func (agg *monitorAggregate) addResponse(responseUs int64) { + agg.totalCount++ + if responseUs < 0 { + return + } + agg.successCount++ + agg.sumUs += responseUs + if responseUs < agg.minUs { + agg.minUs = responseUs + } + if responseUs > agg.maxUs { + agg.maxUs = responseUs + } +} + +// addAggregate merges another aggregate into this one. +func (agg *monitorAggregate) addAggregate(other monitorAggregate) { + if other.totalCount == 0 { + return + } + agg.totalCount += other.totalCount + agg.successCount += other.successCount + agg.sumUs += other.sumUs + if other.successCount == 0 { + return + } + if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs { + agg.minUs = other.minUs + } + if other.maxUs > agg.maxUs { + agg.maxUs = other.maxUs + } +} + +// hasData reports whether the aggregate contains any samples. +func (agg monitorAggregate) hasData() bool { + return agg.totalCount > 0 +} + +// result converts the aggregate into the monitor result format. +func (agg monitorAggregate) result() monitor.Result { + avg := agg.avgResponse() + result := monitor.Result{ + AvgResponse: avg, + MinResponse: agg.minUs, + MaxResponse: agg.maxUs, + PacketLoss: agg.lossPercentage(), + TotalCount: agg.totalCount, + SuccessCount: agg.successCount, + ResponseSum: agg.sumUs, + } + if agg.successCount == 0 { + result.MinResponse, result.MaxResponse = 0, 0 + } + return result +} + +// avgResponse returns the rounded average of successful samples. +func (agg monitorAggregate) avgResponse() int64 { + if agg.successCount == 0 { + return 0 + } + return agg.sumUs / agg.successCount + +} + +// lossPercentage returns the rounded failure rate for the aggregate. +func (agg monitorAggregate) lossPercentage() float64 { + if agg.totalCount == 0 { + return 0 + } + return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100 +} + +// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available. +func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) { + agg := h.aggregateLocked(duration, now) + if !agg.hasData() { + // short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall + // between monitor samples since monitors run at longer, user-defined intervals; fall back to + // the most recent sample so realtime requests still report current status. + agg = h.latestSampleAggregateLocked() + } + hourAgg := h.aggregateLocked(time.Hour, now) + if !agg.hasData() { + return monitor.Result{}, false + } + + result := agg.result() + if len(h.samples) > 0 { + result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli() + } + + result.AvgResponse1h = hourAgg.avgResponse() + result.MinResponse1h = hourAgg.minUs + result.MaxResponse1h = hourAgg.maxUs + result.PacketLoss1h = hourAgg.lossPercentage() + result.SampleCount = h.sampleCount + + if hourAgg.successCount == 0 { + result.MinResponse1h, result.MaxResponse1h = 0, 0 + } + return result, true +} + +// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any. +func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate { + agg := newMonitorAggregate() + if len(h.samples) == 0 { + return agg + } + agg.addResponse(h.samples[len(h.samples)-1].responseUs) + return agg +} + +// aggregateLocked collects monitor data for the requested time window. +func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate { + cutoff := now.Add(-duration) + // Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention. + if duration <= monitorRawRetention { + return aggregateSamplesSince(h.samples, cutoff) + } + return aggregateBucketsSince(h.buckets[:], cutoff, now) +} + +// aggregateSamplesSince aggregates raw samples newer than the cutoff. +func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate { + agg := newMonitorAggregate() + for _, sample := range samples { + if sample.timestamp.Before(cutoff) { + continue + } + agg.addResponse(sample.responseUs) + } + return agg +} + +// aggregateBucketsSince aggregates minute buckets overlapping the requested window. +func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate { + agg := newMonitorAggregate() + startMinute := int32(cutoff.Unix() / 60) + endMinute := int32(now.Unix() / 60) + for _, bucket := range buckets { + if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute { + continue + } + agg.addAggregate(bucket.stats) + } + return agg +} + +// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers. +func (h *monitorHistory) addSampleLocked(sample monitorSample) { + h.sampleCount++ + cutoff := sample.timestamp.Add(-monitorRawRetention) + start := 0 + for i := range h.samples { + if !h.samples[i].timestamp.Before(cutoff) { + start = i + break + } + if i == len(h.samples)-1 { + start = len(h.samples) + } + } + if start > 0 { + size := copy(h.samples, h.samples[start:]) + h.samples = h.samples[:size] + } + h.samples = append(h.samples, sample) + + minute := int32(sample.timestamp.Unix() / 60) + // Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor. + bucket := &h.buckets[minute%monitorMinuteBucketLen] + if !bucket.filled || bucket.minute != minute { + bucket.minute = minute + bucket.filled = true + bucket.stats = newMonitorAggregate() + } + bucket.stats.addResponse(sample.responseUs) +} diff --git a/agent/network_monitor_history_test.go b/agent/network_monitor_history_test.go new file mode 100644 index 00000000..12a944ab --- /dev/null +++ b/agent/network_monitor_history_test.go @@ -0,0 +1,154 @@ +package agent + +import ( + "testing" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMonitorHistoryWindowCounts(t *testing.T) { + history := newMonitorHistory() + now := time.Now() + // This older success counts toward lifetime warm-up, but not this window. + history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)}) + history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)}) + history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)}) + history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)}) + result, ok := history.result(time.Minute, now) + require.True(t, ok) + assert.EqualValues(t, 4, result.SampleCount) + assert.EqualValues(t, 3, result.TotalCount) + assert.EqualValues(t, 2, result.SuccessCount) + assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding") + assert.EqualValues(t, 15, result.AvgResponse) + assert.Equal(t, 33.33, result.PacketLoss) + + encoded, err := cbor.Marshal(result) + require.NoError(t, err) + var decoded monitor.Result + require.NoError(t, cbor.Unmarshal(encoded, &decoded)) + assert.Equal(t, result, decoded) + stats := monitor.Stats{}.FromResult(decoded) + assert.Equal(t, result.TotalCount, stats.TotalCount) + assert.Equal(t, result.SuccessCount, stats.SuccessCount) + assert.Equal(t, result.ResponseSum, stats.ResponseSum) + + // Reads do not consume samples. A short window's latest-sample fallback + // carries the count for that single failure, not the minute or lifetime count. + repeated, _ := history.result(time.Minute, now) + assert.Equal(t, result, repeated) + fallback, ok := history.result(time.Second, now) + require.True(t, ok) + assert.EqualValues(t, 1, fallback.TotalCount) + assert.Zero(t, fallback.SuccessCount) + assert.Zero(t, fallback.ResponseSum) + assert.Equal(t, 100.0, fallback.PacketLoss) + assert.EqualValues(t, 4, fallback.SampleCount) +} + +func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) { + now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC) + history := newMonitorHistory() + + history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)}) + history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)}) + history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)}) + + agg := history.aggregateLocked(time.Minute, now) + require.True(t, agg.hasData()) + assert.Equal(t, int64(2), agg.totalCount) + assert.Equal(t, int64(1), agg.successCount) + result := agg.result() + assert.Equal(t, int64(20), result.AvgResponse) + assert.Equal(t, int64(20), result.MinResponse) + assert.Equal(t, int64(20), result.MaxResponse) + assert.Equal(t, 50.0, result.PacketLoss) +} + +func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) { + now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC) + history := newMonitorHistory() + + history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)}) + history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)}) + history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)}) + history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)}) + history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)}) + + agg := history.aggregateLocked(10*time.Minute, now) + require.True(t, agg.hasData()) + assert.Equal(t, int64(4), agg.totalCount) + assert.Equal(t, int64(3), agg.successCount) + result := agg.result() + assert.Equal(t, int64(30), result.AvgResponse) + assert.Equal(t, int64(20), result.MinResponse) + assert.Equal(t, int64(40), result.MaxResponse) + assert.Equal(t, 25.0, result.PacketLoss) +} + +func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) { + now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC) + history := newMonitorHistory() + + history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)}) + history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now}) + + require.Len(t, history.samples, 1) + assert.Equal(t, int64(20), history.samples[0].responseUs) + + agg := history.aggregateLocked(10*time.Minute, now) + require.True(t, agg.hasData()) + assert.Equal(t, int64(2), agg.totalCount) + assert.Equal(t, int64(2), agg.successCount) + result := agg.result() + assert.Equal(t, int64(15), result.AvgResponse) + assert.Equal(t, int64(10), result.MinResponse) + assert.Equal(t, int64(20), result.MaxResponse) + assert.Equal(t, 0.0, result.PacketLoss) +} + +func TestMonitorHistoryProbeTimestamp(t *testing.T) { + history := newMonitorHistory() + start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC) + _, ok := history.result(time.Minute, start) + require.False(t, ok) + first := history.record(monitorSample{responseUs: 20, timestamp: start}) + assert.Equal(t, start.UnixMilli(), first.LastProbeAt) + for minute := 0; minute < 5; minute++ { + now := start.Add(time.Duration(minute)*time.Minute + time.Second) + // Realtime reads must not consume freshness for the persistence request. + for _, window := range []time.Duration{time.Second, time.Minute} { + result, ok := history.result(window, now) + require.True(t, ok) + assert.Equal(t, first.LastProbeAt, result.LastProbeAt) + assert.Equal(t, int64(20), result.AvgResponse) + } + } + next := start.Add(5 * time.Minute) + failed := history.record(monitorSample{responseUs: -1, timestamp: next}) + assert.Equal(t, next.UnixMilli(), failed.LastProbeAt) + assert.Equal(t, float64(100), failed.PacketLoss) + repeated, ok := history.result(time.Minute, next.Add(2*time.Minute)) + require.True(t, ok) + assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt) + assert.Equal(t, float64(100), repeated.PacketLoss) +} + +func TestMonitorHistorySampleCount(t *testing.T) { + history := newMonitorHistory() + now := time.Now() + // Both failed and successful probes count, including older samples so + // monitors with hourly intervals can finish warming up. + history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)}) + for i, response := range []int64{10, -1, 20} { + result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)}) + assert.EqualValues(t, i+2, result.SampleCount) + } + result, ok := history.clone().result(time.Minute, now.Add(3*time.Second)) + require.True(t, ok) + assert.EqualValues(t, 4, result.SampleCount) +} diff --git a/agent/network_monitor_ping.go b/agent/network_monitor_ping.go new file mode 100644 index 00000000..632afd66 --- /dev/null +++ b/agent/network_monitor_ping.go @@ -0,0 +1,312 @@ +package agent + +import ( + "bytes" + "context" + "crypto/rand" + "errors" + "fmt" + "math" + "net" + "os" + "os/exec" + "regexp" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" + + "log/slog" +) + +// Match the numeric RTT independently of the localized label used by Windows. +var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`) + +var icmpSequence atomic.Uint32 + +type icmpPacketConn interface { + Close() error +} + +// icmpMethod tracks which ICMP approach to use. Once a method succeeds or +// all native methods fail, the choice is cached so subsequent monitors skip +// the trial-and-error overhead. +type icmpMethod uint8 + +const ( + icmpUntried icmpMethod = iota // haven't tried yet + icmpRaw // privileged raw socket + icmpDatagram // unprivileged datagram socket + icmpExecFallback // shell out to system ping command +) + +// icmpFamily holds the network parameters and cached detection result for one address family. +type icmpFamily struct { + rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp" + dgramNetwork string // e.g. "udp4" or "udp6" + listenAddr string // "0.0.0.0" or "::" + echoType icmp.Type // outgoing echo request type + replyType icmp.Type // expected echo reply type + proto int // IANA protocol number for parsing replies + isIPv6 bool + mode icmpMethod // cached detection result (guarded by icmpModeMu) +} + +var ( + icmpV4 = icmpFamily{ + rawNetwork: "ip4:icmp", + dgramNetwork: "udp4", + listenAddr: "0.0.0.0", + echoType: ipv4.ICMPTypeEcho, + replyType: ipv4.ICMPTypeEchoReply, + proto: 1, + } + icmpV6 = icmpFamily{ + rawNetwork: "ip6:ipv6-icmp", + dgramNetwork: "udp6", + listenAddr: "::", + echoType: ipv6.ICMPTypeEchoRequest, + replyType: ipv6.ICMPTypeEchoReply, + proto: 58, + isIPv6: true, + } + icmpModeMu sync.Mutex + icmpListen = func(network, listenAddr string) (icmpPacketConn, error) { + return icmp.ListenPacket(network, listenAddr) + } +) + +// monitorICMP sends an ICMP echo request and measures round-trip response. +// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket, +// unprivileged datagram, or exec fallback) is detected once per address +// family and cached for subsequent monitors. +// Returns response in microseconds, or -1 and an error on failure. +func monitorICMP(ctx context.Context, target string) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + family, ip, err := resolveICMPTarget(ctx, target) + if err != nil { + return -1, err + } + + icmpModeMu.Lock() + if family.mode == icmpUntried { + family.mode = detectICMPMode(family, icmpListen) + } + mode := family.mode + icmpModeMu.Unlock() + + switch mode { + case icmpRaw: + return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip}) + case icmpDatagram: + return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip}) + case icmpExecFallback: + return monitorICMPExec(ctx, ip.String(), family.isIPv6) + default: + return -1, errors.New("unsupported ICMP mode") + } +} + +// resolveICMPTarget resolves a target hostname or IP to determine the address +// family and concrete IP address. Prefers IPv4 for dual-stack hostnames. +func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) { + if ip := net.ParseIP(target); ip != nil { + if ip.To4() != nil { + return &icmpV4, ip.To4(), nil + } + return &icmpV6, ip, nil + } + + ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target) + if err != nil || len(ips) == 0 { + return nil, nil, err + } + for _, ip := range ips { + if v4 := ip.To4(); v4 != nil { + return &icmpV4, v4, nil + } + } + return &icmpV6, ips[0], nil +} + +func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod { + label := "IPv4" + if family.isIPv6 { + label = "IPv6" + } + + conn, err := listen(family.rawNetwork, family.listenAddr) + slog.Debug("ICMP raw socket test", "family", label, "err", err) + if err == nil { + conn.Close() + return icmpRaw + } + + conn, err = listen(family.dgramNetwork, family.listenAddr) + slog.Debug("ICMP datagram socket test", "family", label, "err", err) + if err == nil { + conn.Close() + return icmpDatagram + } + + return icmpExecFallback +} + +// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package. +func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) { + conn, err := icmp.ListenPacket(network, family.listenAddr) + if err != nil { + return -1, err + } + defer conn.Close() + + return monitorICMPPacket(ctx, conn, family, dst) +} + +func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) { + if err := ctx.Err(); err != nil { + return -1, err + } + // Closing the socket interrupts both reads and writes on cancellation. + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() + + // Prepare correlation data before starting the round-trip timer. The token + // also distinguishes delayed replies after the 16-bit sequence wraps. + token := make([]byte, 16) + if _, err := rand.Read(token); err != nil { + return -1, err + } + echo := &icmp.Echo{ + ID: os.Getpid() & 0xffff, + Seq: int(icmpSequence.Add(1) & 0xffff), + Data: token, + } + // Linux ping sockets replace the Echo ID with their bound port. Darwin + // datagram sockets and raw sockets preserve the supplied ID. + if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" { + echo.ID = local.Port + } + targetIP := icmpAddrIP(dst) + msg := &icmp.Message{ + Type: family.echoType, + Code: 0, + Body: echo, + } + msgBytes, err := msg.Marshal(nil) + if err != nil { + return -1, err + } + + // Set deadline before sending + if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil { + return -1, err + } + + buf := make([]byte, 1500) + start := time.Now() + if _, err := conn.WriteTo(msgBytes, dst); err != nil { + return -1, err + } + + // Read reply + for { + n, peer, err := conn.ReadFrom(buf) + received := time.Now() + if err != nil { + return -1, err + } + if !targetIP.Equal(icmpAddrIP(peer)) { + continue + } + + reply, err := icmp.ParseMessage(family.proto, buf[:n]) + if err != nil || reply.Type != family.replyType || reply.Code != 0 { + continue + } + + body, ok := reply.Body.(*icmp.Echo) + if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) { + return received.Sub(start).Microseconds(), nil + } + // Keep waiting for our reply without extending the original deadline. + } +} + +func icmpAddrIP(addr net.Addr) net.IP { + switch addr := addr.(type) { + case *net.IPAddr: + return addr.IP + case *net.UDPAddr: + return addr.IP + default: + return nil + } +} + +// pingCommand selects the executable and arguments for the supported agent platforms. +// The context deadline enforces the timeout: -W has incompatible meanings across +// Linux, BSD IPv4 ping, and macOS ping6. +func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) { + family := "-4" + if isIPv6 { + family = "-6" + } + switch goos { + case "windows": + return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil + case "linux": + return "ping", []string{family, "-n", "-c", "1", target}, nil + case "darwin", "freebsd", "openbsd": + command := "ping" + if isIPv6 { + command = "ping6" + } + return command, []string{"-n", "-c", "1", target}, nil + default: + return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos) + } +} + +// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure. +func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + name, args, err := pingCommand(runtime.GOOS, target, isIPv6) + if err != nil { + return -1, err + } + cmd := exec.CommandContext(ctx, name, args...) + // Keep Unix output and decimal formatting stable. Windows ignores LC_ALL. + cmd.Env = append(os.Environ(), "LC_ALL=C") + output, err := cmd.Output() + if ctx.Err() != nil { + return -1, ctx.Err() + } + if err != nil { + return -1, fmt.Errorf("%s failed: %w", name, err) + } + return parsePingResponse(output) +} + +// parsePingResponse returns the reported RTT, never subprocess execution time. +// For a bounded value such as Windows' time<1ms, retain the reported upper bound. +func parsePingResponse(output []byte) (int64, error) { + matches := pingTimeRegex.FindSubmatch(output) + if len(matches) < 2 { + return -1, errors.New("ping output contains no round-trip time") + } + ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64) + if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 { + return -1, errors.New("invalid round-trip time in ping output") + } + return int64(math.Round(ms * 1000)), nil +} diff --git a/agent/network_monitor_ping_test.go b/agent/network_monitor_ping_test.go new file mode 100644 index 00000000..89ff92e6 --- /dev/null +++ b/agent/network_monitor_ping_test.go @@ -0,0 +1,433 @@ +//go:build testing + +package agent + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/icmp" +) + +type testICMPPacketConn struct{} + +func (testICMPPacketConn) Close() error { return nil } + +type blockingICMPConn struct { + net.PacketConn + reading chan struct{} +} + +func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) { + return len(p), nil +} + +func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) { + close(c.reading) + return c.PacketConn.ReadFrom(p) +} + +func TestMonitorICMPPacketCancellation(t *testing.T) { + conn, err := net.ListenPacket("udp4", "127.0.0.1:0") + require.NoError(t, err) + defer conn.Close() + blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr()) + done <- err + }() + select { + case <-blocking.reading: + case <-time.After(time.Second): + t.Fatal("probe did not begin reading") + } + cancel() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("cancellation did not interrupt the socket read") + } +} + +func TestMonitorICMPExecCancellation(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test uses a POSIX shell stub for ping") + } + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := monitorICMPExec(ctx, "127.0.0.1", false) + done <- err + }() + select { + case err := <-done: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(time.Second): + t.Fatal("cancellation did not terminate ping") + } +} + +func TestPingCommand(t *testing.T) { + for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} { + for _, ipv6 := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) { + target, family := "192.0.2.1", "-4" + if ipv6 { + target, family = "2001:db8::1", "-6" + } + name, args, err := pingCommand(goos, target, ipv6) + require.NoError(t, err) + wantName := "ping" + wantArgs := []string{"-n", "-c", "1", target} + switch goos { + case "windows": + wantArgs = []string{family, "-n", "1", "-w", "3000", target} + case "linux": + wantArgs = append([]string{family}, wantArgs...) + default: + if ipv6 { + wantName = "ping6" + } + } + assert.Equal(t, wantName, name) + assert.Equal(t, wantArgs, args) + }) + } + } + _, _, err := pingCommand("unsupported", "192.0.2.1", false) + require.Error(t, err) +} + +func TestParsePingResponse(t *testing.T) { + for _, tc := range []struct { + name string + output string + wantUs int64 + }{ + {"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345}, + {"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23}, + {"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234}, + {"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000}, + {"windows submillisecond", "Reply from ::1: time<1ms", 1000}, + {"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000}, + {"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234}, + {"rounding", "time=0.1236 ms", 124}, + {"empty", "", -1}, + {"timeout", "Request timed out.", -1}, + {"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1}, + {"malformed", "time=oops ms", -1}, + {"negative", "time=-1 ms", -1}, + {"overflow", "time=999999999999999999999 ms", -1}, + } { + t.Run(tc.name, func(t *testing.T) { + responseUs, err := parsePingResponse([]byte(tc.output)) + if tc.wantUs < 0 { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantUs, responseUs) + }) + } +} + +func TestMonitorICMPExecOutput(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test uses a POSIX shell stub for ping") + } + for _, tc := range []struct { + name string + output string + exit int + wantUs int64 + }{ + {"success", "time=1.234 ms", 0, 1234}, + {"missing RTT", "unrecognized output", 0, -1}, + {"failed command with RTT", "time=1.234 ms", 1, -1}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + // Also verify an inherited locale cannot override the C locale. + script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit) + require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755)) + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("LC_ALL", "de_DE.UTF-8") + responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false) + if tc.wantUs < 0 { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.wantUs, responseUs) + }) + } +} + +type icmpTestReply struct { + data []byte + peer net.Addr +} + +type scriptedICMPConn struct { + net.PacketConn + local net.Addr + onWrite func([]byte, net.Addr) + replies []icmpTestReply + reads int + deadlineSets int +} + +func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local } + +func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error { + c.deadlineSets++ + return nil +} + +func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) { + c.onWrite(data, dst) + return len(data), nil +} + +func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) { + c.reads++ + if len(c.replies) == 0 { + return 0, nil, os.ErrDeadlineExceeded + } + reply := c.replies[0] + c.replies = c.replies[1:] + return copy(buf, reply.data), reply.peer, nil +} + +func TestMonitorICMPReplyCorrelation(t *testing.T) { + for _, family := range []*icmpFamily{&icmpV4, &icmpV6} { + for _, datagram := range []bool{false, true} { + network := family.rawNetwork + ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2") + if family.isIPv6 { + ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2") + } + var dst net.Addr = &net.IPAddr{IP: ip} + var wrongPeer net.Addr = &net.IPAddr{IP: other} + if datagram { + network = family.dgramNetwork + dst = &net.UDPAddr{IP: ip} + wrongPeer = &net.UDPAddr{IP: other} + } + for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} { + for _, eventuallyMatches := range []bool{false, true} { + ending := "timeout" + if eventuallyMatches { + ending = "success" + } + t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) { + conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}} + if datagram { + conn.local = &net.UDPAddr{Port: 12345} + if runtime.GOOS == "linux" { + // Deliberately differ from the process ID. + conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1} + } + } + conn.onWrite = func(data []byte, target net.Addr) { + require.Equal(t, dst, target) + request, err := icmp.ParseMessage(family.proto, data) + require.NoError(t, err) + echo := request.Body.(*icmp.Echo) + expectedID := os.Getpid() & 0xffff + if datagram && runtime.GOOS == "linux" { + expectedID = conn.local.(*net.UDPAddr).Port + } + require.Equal(t, expectedID, echo.ID) + reply := &icmp.Message{Type: family.replyType, Body: echo} + valid, err := reply.Marshal(nil) + require.NoError(t, err) + peer := dst + switch mismatch { + case "source": + peer = wrongPeer + case "id": + echo.ID ^= 1 + case "sequence": + echo.Seq ^= 1 + case "payload": + echo.Data[0] ^= 1 + case "type": + reply.Type = family.echoType + case "code": + reply.Code = 1 + } + invalid, err := reply.Marshal(nil) + require.NoError(t, err) + if mismatch == "malformed" { + invalid = invalid[:2] + } + conn.replies = []icmpTestReply{{invalid, peer}} + if eventuallyMatches { + conn.replies = append(conn.replies, icmpTestReply{valid, dst}) + } + } + elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst) + if eventuallyMatches { + require.NoError(t, err) + assert.GreaterOrEqual(t, elapsed, int64(0)) + } else { + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + assert.Equal(t, int64(-1), elapsed) + } + assert.Equal(t, 2, conn.reads) + assert.Equal(t, 1, conn.deadlineSets) + }) + } + } + } + } +} + +func TestMonitorICMPLoopback(t *testing.T) { + for _, family := range []*icmpFamily{&icmpV4, &icmpV6} { + for _, network := range []string{family.rawNetwork, family.dgramNetwork} { + t.Run(network, func(t *testing.T) { + conn, err := icmp.ListenPacket(network, family.listenAddr) + if err != nil { + t.Skipf("ICMP socket unavailable: %v", err) + } + defer conn.Close() + ip := net.ParseIP("127.0.0.1") + if family.isIPv6 { + ip = net.ParseIP("::1") + } + var dst net.Addr = &net.IPAddr{IP: ip} + if network == family.dgramNetwork { + dst = &net.UDPAddr{IP: ip} + } + elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst) + require.NoError(t, err) + assert.GreaterOrEqual(t, elapsed, int64(0)) + }) + } + } +} + +func TestDetectICMPMode(t *testing.T) { + tests := []struct { + name string + family *icmpFamily + rawErr error + udpErr error + want icmpMethod + wantNetworks []string + }{ + { + name: "IPv4 prefers raw socket when available", + family: &icmpV4, + want: icmpRaw, + wantNetworks: []string{"ip4:icmp"}, + }, + { + name: "IPv4 uses datagram when raw unavailable", + family: &icmpV4, + rawErr: errors.New("operation not permitted"), + want: icmpDatagram, + wantNetworks: []string{"ip4:icmp", "udp4"}, + }, + { + name: "IPv4 falls back to exec when both unavailable", + family: &icmpV4, + rawErr: errors.New("operation not permitted"), + udpErr: errors.New("protocol not supported"), + want: icmpExecFallback, + wantNetworks: []string{"ip4:icmp", "udp4"}, + }, + { + name: "IPv6 prefers raw socket when available", + family: &icmpV6, + want: icmpRaw, + wantNetworks: []string{"ip6:ipv6-icmp"}, + }, + { + name: "IPv6 uses datagram when raw unavailable", + family: &icmpV6, + rawErr: errors.New("operation not permitted"), + want: icmpDatagram, + wantNetworks: []string{"ip6:ipv6-icmp", "udp6"}, + }, + { + name: "IPv6 falls back to exec when both unavailable", + family: &icmpV6, + rawErr: errors.New("operation not permitted"), + udpErr: errors.New("protocol not supported"), + want: icmpExecFallback, + wantNetworks: []string{"ip6:ipv6-icmp", "udp6"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := make([]string, 0, 2) + listen := func(network, listenAddr string) (icmpPacketConn, error) { + require.Equal(t, tt.family.listenAddr, listenAddr) + calls = append(calls, network) + switch network { + case tt.family.rawNetwork: + if tt.rawErr != nil { + return nil, tt.rawErr + } + case tt.family.dgramNetwork: + if tt.udpErr != nil { + return nil, tt.udpErr + } + default: + t.Fatalf("unexpected network %q", network) + } + return testICMPPacketConn{}, nil + } + + assert.Equal(t, tt.want, detectICMPMode(tt.family, listen)) + assert.Equal(t, tt.wantNetworks, calls) + }) + } +} + +func TestResolveICMPTarget(t *testing.T) { + t.Run("IPv4 literal", func(t *testing.T) { + family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1") + require.NoError(t, err) + require.NotNil(t, family) + assert.False(t, family.isIPv6) + assert.Equal(t, "127.0.0.1", ip.String()) + }) + + t.Run("IPv6 literal", func(t *testing.T) { + family, ip, err := resolveICMPTarget(context.Background(), "::1") + require.NoError(t, err) + require.NotNil(t, family) + assert.True(t, family.isIPv6) + assert.Equal(t, "::1", ip.String()) + }) + + t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) { + family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1") + require.NoError(t, err) + require.NotNil(t, family) + assert.False(t, family.isIPv6) + assert.Equal(t, "127.0.0.1", ip.String()) + }) +} diff --git a/agent/network_monitor_probe.go b/agent/network_monitor_probe.go new file mode 100644 index 00000000..187a6bf3 --- /dev/null +++ b/agent/network_monitor_probe.go @@ -0,0 +1,105 @@ +package agent + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" +) + +// monitorProbe performs one check. Errors are recorded as loss by the task runner. +// Implementations must honor cancellation and bound their execution time. +type monitorProbe func(context.Context, monitor.Config) (int64, error) + +func networkMonitorProbe(client *http.Client) monitorProbe { + return func(ctx context.Context, config monitor.Config) (int64, error) { + switch config.Protocol { + case "icmp": + return monitorICMP(ctx, config.Target) + case "tcp": + return monitorTCP(ctx, config.Target, config.Port) + case "http": + return monitorHTTP(ctx, client, config.Target) + case "dns": + return monitorDNS(ctx, config.Target) + default: + return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol) + } + } +} + +// monitorTCP measures connection establishment time, including address fallback +// but excluding DNS resolution. +// Returns -1 and an error on failure. +func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + // Resolve DNS first, outside the timing window but within the probe deadline. + ips, err := net.DefaultResolver.LookupHost(ctx, target) + if err != nil { + return -1, err + } + if len(ips) == 0 { + return -1, errors.New("no addresses resolved for TCP monitor") + } + portString := fmt.Sprintf("%d", port) + deadline, _ := ctx.Deadline() + + // Share the remaining probe budget across addresses so an unresponsive + // first address cannot consume all the time available for alternatives. + start := time.Now() + for i, ip := range ips { + if err := ctx.Err(); err != nil { + return -1, err + } + dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)} + var conn net.Conn + conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString)) + if err != nil { + continue + } + responseUs := time.Since(start).Microseconds() + conn.Close() + return responseUs, nil + } + return -1, err +} + +// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure. +func monitorDNS(ctx context.Context, target string) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + start := time.Now() + ips, err := net.DefaultResolver.LookupHost(ctx, target) + if err != nil || len(ips) == 0 { + return -1, err + } + return time.Since(start).Microseconds(), nil +} + +// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure. +func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) { + if client == nil { + client = http.DefaultClient + } + start := time.Now() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return -1, err + } + resp, err := client.Do(req) + if err != nil { + return -1, err + } + resp.Body.Close() + if resp.StatusCode >= 400 { + return -1, fmt.Errorf("HTTP error: %s", resp.Status) + } + return time.Since(start).Microseconds(), nil +} diff --git a/agent/network_monitor_resume.go b/agent/network_monitor_resume.go new file mode 100644 index 00000000..49526623 --- /dev/null +++ b/agent/network_monitor_resume.go @@ -0,0 +1,88 @@ +package agent + +import ( + "sync" + "time" +) + +const ( + monitorResumeHeartbeat = 10 * time.Second + // Allow scheduling jitter without mistaking an ordinary tick for resume. + monitorResumeGap = 2 * monitorResumeHeartbeat + monitorResumePause = 10 * time.Second +) + +// monitorResumeGuard detects likely suspend/resume using wall time. A long +// process stall or forward clock adjustment can also trigger the bounded pause. +// One heartbeat is shared by all configured monitors. +type monitorResumeGuard struct { + mu sync.Mutex + stop chan struct{} + lastTick time.Time + pauseUntil time.Time + generation uint32 +} + +func (g *monitorResumeGuard) start() { + g.mu.Lock() + defer g.mu.Unlock() + if g.stop != nil { + return + } + stop := make(chan struct{}) + g.stop = stop + g.lastTick = time.Now().Round(0) + g.pauseUntil = time.Time{} + go func() { + ticker := time.NewTicker(monitorResumeHeartbeat) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + g.mu.Lock() + if g.stop == stop { + g.observe(time.Now()) + } + g.mu.Unlock() + } + } + }() +} + +func (g *monitorResumeGuard) shutdown() { + g.mu.Lock() + defer g.mu.Unlock() + if g.stop != nil { + close(g.stop) + g.stop = nil + g.generation++ + } +} + +// observe requires mu. Strip the monotonic component because it can stop during +// suspend. Read the current time rather than the ticker's queued timestamp. +func (g *monitorResumeGuard) observe(now time.Time) { + now = now.Round(0) + if now.Sub(g.lastTick) > monitorResumeGap { + g.pauseUntil = now.Add(monitorResumePause) + g.generation++ + } + g.lastTick = now +} + +// snapshot also observes time so a probe waking before the heartbeat detects +// resume itself. A changed generation invalidates probes spanning suspend. +func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) { + if g == nil { + return 0, true + } + g.mu.Lock() + defer g.mu.Unlock() + if g.stop == nil { + return g.generation, true + } + g.observe(time.Now()) + return g.generation, !g.lastTick.Before(g.pauseUntil) +} diff --git a/agent/network_monitor_resume_test.go b/agent/network_monitor_resume_test.go new file mode 100644 index 00000000..f1a637c4 --- /dev/null +++ b/agent/network_monitor_resume_test.go @@ -0,0 +1,121 @@ +//go:build testing + +package agent + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func simulateMonitorSleep(g *monitorResumeGuard) { + g.mu.Lock() + g.lastTick = time.Now().Add(-time.Hour).Round(0) + g.mu.Unlock() +} + +func TestMonitorResumePause(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var g monitorResumeGuard + g.start() + defer g.shutdown() + generation, allowed := g.snapshot() + require.True(t, allowed) + // Heartbeats alone must keep the guard current between infrequent probes. + time.Sleep(time.Minute) + synctest.Wait() + steadyGeneration, allowed := g.snapshot() + require.True(t, allowed) + require.Equal(t, generation, steadyGeneration) + // The probe, rather than the heartbeat, must detect this gap. + simulateMonitorSleep(&g) + next, allowed := g.snapshot() + assert.False(t, allowed) + assert.NotEqual(t, generation, next) + time.Sleep(9 * time.Second) + _, allowed = g.snapshot() + assert.False(t, allowed) + time.Sleep(time.Second) + end, allowed := g.snapshot() + assert.True(t, allowed) + assert.Equal(t, next, end) + }) +} + +func TestMonitorResumeGuardLifecycle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil }) + defer pm.Stop() + assert.Nil(t, pm.resumeGuard.stop) + pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}}) + stop := pm.resumeGuard.stop + require.NotNil(t, stop) + pm.DeleteMonitor("a") + assert.Equal(t, stop, pm.resumeGuard.stop) + pm.DeleteMonitor("b") + assert.Nil(t, pm.resumeGuard.stop) + select { + case <-stop: + default: + t.Fatal("heartbeat was not stopped") + } + time.Sleep(time.Hour) + _, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false) + require.NoError(t, err) + _, allowed := pm.resumeGuard.snapshot() + assert.True(t, allowed, "idle time must not trigger a resume pause") + pm.SyncMonitors(nil) + assert.Nil(t, pm.resumeGuard.stop) + }) +} + +func TestMonitorResumeDiscardsInflightProbe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var g monitorResumeGuard + g.start() + defer g.shutdown() + task := newMonitorTask(monitor.Config{ID: "test"}) + defer task.cancel() + task.resumeGuard = &g + result := task.runProbe(func(context.Context, monitor.Config) (int64, error) { + simulateMonitorSleep(&g) + return 0, errors.New("network not ready") + }) + assert.Nil(t, result) + assert.Empty(t, task.history.samples) + // Explicit requests may still run during the pause and record real failures. + result = task.runProbe(func(context.Context, monitor.Config) (int64, error) { + return 0, errors.New("unreachable") + }) + require.NotNil(t, result) + assert.Equal(t, 100.0, result.PacketLoss) + }) +} + +func TestMonitorResumeSkipsScheduledProbes(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { + calls.Add(1) + return 1, nil + }) + defer pm.Stop() + pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}}) + simulateMonitorSleep(&pm.resumeGuard) + pm.resumeGuard.snapshot() + time.Sleep(9 * time.Second) + synctest.Wait() + assert.Zero(t, calls.Load()) + assert.Empty(t, pm.GetResults(1000)) + time.Sleep(2 * time.Second) + synctest.Wait() + assert.Positive(t, calls.Load()) + }) +} diff --git a/agent/network_monitor_schedule.go b/agent/network_monitor_schedule.go new file mode 100644 index 00000000..fa53c36c --- /dev/null +++ b/agent/network_monitor_schedule.go @@ -0,0 +1,60 @@ +package agent + +import ( + "context" + "log/slog" + "math/rand" + "time" +) + +func (pm *MonitorManager) startMonitor(task *monitorTask) { + interval := time.Duration(task.config.Interval) * time.Second + if interval < time.Second { + interval = 30 * time.Second + } + delay := getStagger(interval.Milliseconds()) + slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval) + go runMonitorSchedule(task.ctx, interval, delay, func() { + if _, allowed := task.resumeGuard.snapshot(); allowed { + task.runProbe(pm.probe) + } + }) +} + +// runMonitorSchedule owns only timing. Checks run serially, and slow checks +// naturally drop missed ticks rather than building an execution backlog. +func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return + case <-timer.C: + } + if ctx.Err() != nil { + return + } + run() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if ctx.Err() != nil { + return + } + run() + } + } +} + +// getStagger returns an initial delay between half an interval and one interval. +func getStagger(intervalMilli int64) time.Duration { + delay := rand.Intn(int(intervalMilli)) + if delay < int(intervalMilli)/2 { + delay += int(intervalMilli) / 2 + } + return time.Duration(delay) * time.Millisecond +} diff --git a/agent/network_monitor_schedule_test.go b/agent/network_monitor_schedule_test.go new file mode 100644 index 00000000..a66a59ab --- /dev/null +++ b/agent/network_monitor_schedule_test.go @@ -0,0 +1,167 @@ +//go:build testing + +package agent + +import ( + "context" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMonitorScheduleTiming(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var calls atomic.Int32 + go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) }) + synctest.Wait() + time.Sleep(4 * time.Second) + synctest.Wait() + assert.Equal(t, 0, int(calls.Load())) + time.Sleep(time.Second) + synctest.Wait() + assert.Equal(t, 1, int(calls.Load())) + time.Sleep(10 * time.Second) + synctest.Wait() + assert.Equal(t, 2, int(calls.Load())) + cancel() + synctest.Wait() + time.Sleep(time.Minute) + synctest.Wait() + assert.Equal(t, 2, int(calls.Load())) + }) +} + +func TestMonitorScheduleSlowProbe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var calls atomic.Int32 + release := make(chan struct{}) + go runMonitorSchedule(ctx, time.Second, 0, func() { + calls.Add(1) + select { + case <-release: + case <-ctx.Done(): + } + }) + synctest.Wait() + assert.Equal(t, 1, int(calls.Load())) + time.Sleep(time.Minute) + synctest.Wait() + assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks") + close(release) + synctest.Wait() + assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog") + time.Sleep(time.Second) + synctest.Wait() + assert.Equal(t, 2, int(calls.Load())) + cancel() + synctest.Wait() + }) +} + +func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var calls atomic.Int32 + release := make(chan struct{}) + cfg := monitor.Config{ID: "test", Interval: 10} + pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) { + assert.Equal(t, cfg, config) + calls.Add(1) + <-release + return 42, nil + }) + defer pm.Stop() + task := newMonitorTask(cfg) + pm.monitors[cfg.ID] = task + go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) }) + synctest.Wait() + results := make(chan *monitor.Result, 2) + for range 2 { + go func() { + result, _ := pm.UpsertMonitor(cfg, true) + results <- result + }() + } + synctest.Wait() + assert.Equal(t, 1, int(calls.Load())) + assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O") + close(release) + synctest.Wait() + first, second := <-results, <-results + require.NotNil(t, first) + require.NotNil(t, second) + assert.Equal(t, int64(42), first.AvgResponse) + assert.Equal(t, first, second) + assert.NotSame(t, first, second, "callers must not share mutable result pointers") + assert.Len(t, task.history.samples, 1) + // A later explicit request must still perform a fresh probe. + _, err := pm.UpsertMonitor(cfg, true) + require.NoError(t, err) + assert.Equal(t, 2, int(calls.Load())) + assert.Len(t, task.history.samples, 2) + }) +} + +func TestMonitorReplacementCancelsSharedProbe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + cfg := monitor.Config{ID: "test", Interval: 10} + pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) { + if config.Interval == 10 { + <-ctx.Done() + return 0, ctx.Err() + } + return 30, nil + }) + defer pm.Stop() + task := newMonitorTask(cfg) + task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()}) + pm.monitors[cfg.ID] = task + results := make(chan *monitor.Result, 2) + for range 2 { + go func() { + result, _ := pm.UpsertMonitor(cfg, true) + results <- result + }() + } + synctest.Wait() + updated := cfg + updated.Interval = 20 + result, err := pm.UpsertMonitor(updated, true) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, int64(20), result.AvgResponse) + assert.Zero(t, result.PacketLoss) + synctest.Wait() + assert.Nil(t, <-results) + assert.Nil(t, <-results) + assert.Len(t, task.history.samples, 1) + assert.Len(t, pm.monitors[cfg.ID].history.samples, 2) + }) +} + +func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + <-ctx.Done() + return 0, ctx.Err() + }) + defer pm.Stop() + start := time.Now() + result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, 3*time.Second, time.Since(start)) + assert.Equal(t, 100.0, result.PacketLoss) + assert.NoError(t, pm.monitors["test"].ctx.Err()) + }) +} diff --git a/agent/network_monitor_task.go b/agent/network_monitor_task.go new file mode 100644 index 00000000..2c3469ab --- /dev/null +++ b/agent/network_monitor_task.go @@ -0,0 +1,116 @@ +package agent + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" +) + +const monitorFailureLogInterval = 5 * time.Minute + +// monitorTask coordinates a probe and its history for one immutable configuration. +type monitorTask struct { + config monitor.Config + ctx context.Context + cancel context.CancelFunc + history *monitorHistory + resumeGuard *monitorResumeGuard + runMu sync.Mutex + inflight *monitorRun + lastFailureLog int64 // Unix nanoseconds +} + +type monitorRun struct { + done chan struct{} + result *monitor.Result // published by closing done; never mutated afterwards +} + +func newMonitorTask(config monitor.Config) *monitorTask { + ctx, cancel := context.WithCancel(context.Background()) + task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()} + // Serialize cancellation with publication, so canceled probes cannot enter + // history copied into a replacement task. + task.cancel = func() { + task.runMu.Lock() + cancel() + task.runMu.Unlock() + } + return task +} + +func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask { + task := newMonitorTask(config) + if existing != nil { + task.history = existing.history.clone() + } + return task +} + +// runProbe shares an in-flight check between scheduled and immediate requests. +// Every completed check contributes exactly one sample, regardless of how many +// callers were waiting for it. No task or history lock is held during network I/O. +func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result { + task.runMu.Lock() + if task.ctx.Err() != nil { + task.runMu.Unlock() + return nil + } + if run := task.inflight; run != nil { + task.runMu.Unlock() + select { + case <-task.ctx.Done(): + return nil + case <-run.done: + if task.ctx.Err() != nil { + return nil + } + return copyMonitorResult(run.result) + } + } + run := &monitorRun{done: make(chan struct{})} + task.inflight = run + task.runMu.Unlock() + + generation, _ := task.resumeGuard.snapshot() + responseUs, err := probe(task.ctx, task.config) + var logFailure bool + task.runMu.Lock() + currentGeneration, _ := task.resumeGuard.snapshot() + if task.ctx.Err() == nil && generation == currentGeneration { + now := time.Now() + if err != nil { + responseUs = -1 + logAt := now.UnixNano() + if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) { + logFailure = true + task.lastFailureLog = logAt + } + } else { + task.lastFailureLog = 0 + } + result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now}) + run.result = &result + } + + task.inflight = nil + close(run.done) + task.runMu.Unlock() + if logFailure { + slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol) + } + if task.ctx.Err() != nil { + return nil + } + return copyMonitorResult(run.result) +} + +func copyMonitorResult(result *monitor.Result) *monitor.Result { + if result == nil { + return nil + } + copy := *result + return © +} diff --git a/agent/network_monitor_task_test.go b/agent/network_monitor_task_test.go new file mode 100644 index 00000000..427f71d5 --- /dev/null +++ b/agent/network_monitor_task_test.go @@ -0,0 +1,79 @@ +//go:build testing + +package agent + +import ( + "bytes" + "context" + "errors" + "log/slog" + "testing" + "testing/synctest" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMonitorFailureLogCooldown(t *testing.T) { + var logs bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil))) + t.Cleanup(func() { slog.SetDefault(previous) }) + + synctest.Test(t, func(t *testing.T) { + task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"}) + defer task.cancel() + failure := errors.New("connection refused") + probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure } + var samples int64 + check := func(wantLog bool) { + t.Helper() + logs.Reset() + result := task.runProbe(probe) + require.NotNil(t, result) + samples++ + assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples") + if !wantLog { + assert.Empty(t, logs.String()) + } else { + assert.Contains(t, logs.String(), `msg="monitor failed"`) + assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n"))) + } + } + + check(true) + check(false) + time.Sleep(5*time.Minute - time.Nanosecond) + check(false) + time.Sleep(time.Nanosecond) + check(true) + check(false) + time.Sleep(5 * time.Minute) + check(true) + check(false) + + // Recovery clears the cooldown. + failure = nil + check(false) + failure = errors.New("connection refused again") + check(true) + + // Another monitor has its own cooldown. + other := newMonitorTask(task.config) + defer other.cancel() + logs.Reset() + require.NotNil(t, other.runProbe(probe)) + assert.Contains(t, logs.String(), `msg="monitor failed"`) + + // A canceled probe must not publish a failure or emit a warning. + logs.Reset() + result := other.runProbe(func(context.Context, monitor.Config) (int64, error) { + other.cancel() + return -1, context.Canceled + }) + assert.Nil(t, result) + assert.Empty(t, logs.String()) + }) +} diff --git a/agent/network_monitor_test.go b/agent/network_monitor_test.go new file mode 100644 index 00000000..b24ab043 --- /dev/null +++ b/agent/network_monitor_test.go @@ -0,0 +1,524 @@ +package agent + +import ( + "context" + "encoding/binary" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/net/dns/dnsmessage" +) + +func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) { + now := time.Now().UTC() + task := newMonitorTask(monitor.Config{ID: "monitor-1"}) + task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)}) + task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)}) + task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)}) + task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)}) + task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)}) + + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{"icmp:example.com": task} + + results := pm.GetResults(uint16(time.Minute / time.Millisecond)) + result, ok := results["monitor-1"] + require.True(t, ok) + assert.Equal(t, int64(30), result.AvgResponse) + assert.Equal(t, int64(25), result.AvgResponse1h) + assert.Equal(t, int64(30), result.MinResponse) + assert.Equal(t, int64(10), result.MinResponse1h) + assert.Equal(t, int64(30), result.MaxResponse) + assert.Equal(t, int64(40), result.MaxResponse1h) + assert.Equal(t, 50.0, result.PacketLoss) + assert.Equal(t, 20.0, result.PacketLoss1h) +} + +func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) { + now := time.Now().UTC() + task := newMonitorTask(monitor.Config{ID: "monitor-1"}) + task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)}) + task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)}) + + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{"icmp:example.com": task} + + results := pm.GetResults(uint16(time.Minute / time.Millisecond)) + result, ok := results["monitor-1"] + require.True(t, ok) + assert.Equal(t, int64(0), result.AvgResponse) + assert.Equal(t, int64(0), result.AvgResponse1h) + assert.Equal(t, int64(0), result.MinResponse) + assert.Equal(t, int64(0), result.MinResponse1h) + assert.Equal(t, int64(0), result.MaxResponse) + assert.Equal(t, int64(0), result.MaxResponse1h) + assert.Equal(t, 100.0, result.PacketLoss) + assert.Equal(t, 100.0, result.PacketLoss1h) +} + +func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) { + cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10} + assert.Equal(t, "monitor-1", cfg.ID) +} + +func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) { + validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10} + invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10} + + pm := newMonitorManager() + pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg}) + defer pm.Stop() + + _, validExists := pm.monitors[validCfg.ID] + _, invalidExists := pm.monitors[invalidCfg.ID] + assert.True(t, validExists) + assert.False(t, invalidExists) +} + +func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) { + keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10} + removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10} + + keptTask := newMonitorTask(keepCfg) + removedTask := newMonitorTask(removeCfg) + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{ + keepCfg.ID: keptTask, + removeCfg.ID: removedTask, + } + + pm.SyncMonitors([]monitor.Config{keepCfg}) + + assert.Same(t, keptTask, pm.monitors[keepCfg.ID]) + _, exists := pm.monitors[removeCfg.ID] + assert.False(t, exists) + + select { + case <-removedTask.ctx.Done(): + default: + t.Fatal("expected removed monitor task to be cancelled") + } + + select { + case <-keptTask.ctx.Done(): + t.Fatal("expected existing monitor task to remain active") + default: + } +} + +func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) { + originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10} + updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10} + originalTask := newMonitorTask(originalCfg) + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{ + originalCfg.ID: originalTask, + } + + pm.SyncMonitors([]monitor.Config{updatedCfg}) + defer pm.Stop() + + restartedTask := pm.monitors[updatedCfg.ID] + assert.NotSame(t, originalTask, restartedTask) + assert.Equal(t, updatedCfg, restartedTask.config) + + select { + case <-originalTask.ctx.Done(): + default: + t.Fatal("expected changed monitor task to be cancelled") + } +} + +func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + pm := &MonitorManager{ + monitors: make(map[string]*monitorTask), + probe: networkMonitorProbe(server.Client()), + } + + resp, err := pm.HandleSyncRequest(monitor.SyncRequest{ + Action: monitor.SyncActionUpsert, + Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10}, + RunNow: true, + }) + defer pm.Stop() + + require.NoError(t, err) + assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0)) + assert.Equal(t, 0.0, resp.Result.PacketLoss) + assert.Equal(t, 0.0, resp.Result.PacketLoss1h) + + task := pm.monitors["monitor-1"] + require.NotNil(t, task) + task.history.mu.Lock() + defer task.history.mu.Unlock() + require.Len(t, task.history.samples, 1) +} + +func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) { + originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10} + updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30} + now := time.Now().UTC() + + existingTask := newMonitorTask(originalCfg) + existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)}) + existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)}) + + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask} + + result, err := pm.UpsertMonitor(updatedCfg, false) + defer pm.Stop() + + require.NoError(t, err) + assert.Nil(t, result) + + updatedTask := pm.monitors[updatedCfg.ID] + require.NotNil(t, updatedTask) + assert.NotSame(t, existingTask, updatedTask) + assert.Equal(t, updatedCfg, updatedTask.config) + + updatedTask.history.mu.Lock() + defer updatedTask.history.mu.Unlock() + require.Len(t, updatedTask.history.samples, 1) + assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs) + + agg := updatedTask.history.aggregateLocked(time.Hour, now) + require.True(t, agg.hasData()) + assert.Equal(t, int64(2), agg.totalCount) + assert.Equal(t, int64(2), agg.successCount) + assert.Equal(t, int64(18), agg.avgResponse()) + + select { + case <-existingTask.ctx.Done(): + default: + t.Fatal("expected original monitor task to be cancelled") + } +} + +func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) { + config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10} + task := newMonitorTask(config) + pm := newMonitorManager() + pm.monitors = map[string]*monitorTask{config.ID: task} + + _, err := pm.HandleSyncRequest(monitor.SyncRequest{ + Action: monitor.SyncActionDelete, + Config: monitor.Config{ID: config.ID}, + }) + + require.NoError(t, err) + _, exists := pm.monitors[config.ID] + assert.False(t, exists) + + select { + case <-task.ctx.Done(): + default: + t.Fatal("expected deleted monitor task to be cancelled") + } +} + +func TestMonitorManagerGetRandomDelay(t *testing.T) { + for i := 1000; i < 360_000; i += 1000 { + delay := getStagger(int64(i)) + assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond) + assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond) + } +} + +func TestMonitorHTTP(t *testing.T) { + t.Run("success", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL) + require.NoError(t, err) + assert.GreaterOrEqual(t, responseUs, int64(0)) + }) + + t.Run("server error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + })) + defer server.Close() + + responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL) + assert.Equal(t, int64(-1), responseUs) + require.Error(t, err) + }) +} + +func TestMonitorTCP(t *testing.T) { + t.Run("success", func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + + accepted := make(chan struct{}) + go func() { + defer close(accepted) + conn, err := listener.Accept() + if err == nil { + _ = conn.Close() + } + }() + + port := uint16(listener.Addr().(*net.TCPAddr).Port) + responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port) + require.NoError(t, err) + assert.GreaterOrEqual(t, responseUs, int64(0)) + <-accepted + }) + + t.Run("connection failure", func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + port := uint16(listener.Addr().(*net.TCPAddr).Port) + require.NoError(t, listener.Close()) + + responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port) + assert.Equal(t, int64(-1), responseUs) + require.Error(t, err) + }) +} + +func TestMonitorTCPAddressFallback(t *testing.T) { + for _, tc := range []struct { + name string + ips []string + loss bool + }{ + {"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false}, + {"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false}, + {"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + listener, err := net.Listen("tcp4", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + original := net.DefaultResolver + net.DefaultResolver = tcpMonitorTestResolver(tc.ips) + defer func() { net.DefaultResolver = original }() + + // Verify the resolver preserves the intended order, so success cannot + // accidentally bypass the failed first address in the regression case. + ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.") + require.NoError(t, err) + require.Equal(t, tc.ips, ips) + responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port)) + if tc.loss { + require.Error(t, err) + assert.Equal(t, int64(-1), responseUs) + } else { + require.NoError(t, err) + assert.GreaterOrEqual(t, responseUs, int64(0)) + } + }) + } +} + +// tcpMonitorTestResolver supplies multiple A records without external DNS. +func tcpMonitorTestResolver(ips []string) *net.Resolver { + return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + client, server := net.Pipe() + go func() { + defer server.Close() + // net.Resolver uses TCP framing when its connection is not a PacketConn. + var size uint16 + if err := binary.Read(server, binary.BigEndian, &size); err != nil { + return + } + packet := make([]byte, size) + if _, err := io.ReadFull(server, packet); err != nil { + return + } + var msg dnsmessage.Message + if err := msg.Unpack(packet); err != nil { + return + } + msg.Header.Response = true + msg.Header.RecursionAvailable = true + for _, question := range msg.Questions { + if question.Type != dnsmessage.TypeA { + continue + } + for _, ip := range ips { + msg.Answers = append(msg.Answers, dnsmessage.Resource{ + Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET}, + Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())}, + }) + } + } + packet, err := msg.Pack() + if err != nil { + return + } + response := binary.BigEndian.AppendUint16(nil, uint16(len(packet))) + _, _ = server.Write(append(response, packet...)) + }() + return client, nil + }} +} + +func TestMonitorDNS(t *testing.T) { + t.Run("success", func(t *testing.T) { + responseUs, err := monitorDNS(context.Background(), "localhost") + require.NoError(t, err) + assert.GreaterOrEqual(t, responseUs, int64(0)) + }) + + t.Run("lookup failure", func(t *testing.T) { + responseUs, err := monitorDNS(context.Background(), "") + assert.Equal(t, int64(-1), responseUs) + require.Error(t, err) + }) +} + +func TestMonitorManagerCancelsActiveProbe(t *testing.T) { + for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} { + t.Run(action, func(t *testing.T) { + started := make(chan struct{}) + canceled := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(started) + select { + case <-r.Context().Done(): + close(canceled) + case <-release: + } + })) + defer server.Close() + defer close(release) + pm := newMonitorManager() + defer pm.Stop() + cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600} + task := newMonitorTask(cfg) + // Seed history to ensure a canceled RunNow does not return an old result. + task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()}) + pm.monitors[cfg.ID] = task + done := make(chan *monitor.Result, 1) + go func() { + result, _ := pm.UpsertMonitor(cfg, true) + done <- result + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("probe did not start") + } + updated := cfg + updated.Interval-- + switch action { + case "stop": + pm.Stop() + case "delete": + pm.DeleteMonitor(cfg.ID) + case "upsert": + _, err := pm.UpsertMonitor(updated, false) + require.NoError(t, err) + case "sync replace": + pm.SyncMonitors([]monitor.Config{updated}) + case "sync remove": + pm.SyncMonitors(nil) + } + select { + case <-canceled: + case <-time.After(time.Second): + t.Fatal("active HTTP request was not canceled") + } + select { + case result := <-done: + assert.Nil(t, result) + case <-time.After(time.Second): + t.Fatal("RunNow did not return after cancellation") + } + task.history.mu.Lock() + assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss") + task.history.mu.Unlock() + }) + } +} + +func TestMonitorResolutionCancellation(t *testing.T) { + for _, protocol := range []string{"tcp", "dns", "icmp"} { + t.Run(protocol, func(t *testing.T) { + started := make(chan struct{}, 1) + original := net.DefaultResolver + net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + select { + case started <- struct{}{}: + default: + } + <-ctx.Done() + return nil, ctx.Err() + }} + defer func() { net.DefaultResolver = original }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { + var err error + switch protocol { + case "tcp": + _, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80) + case "dns": + _, err = monitorDNS(ctx, "monitor-cancellation.invalid.") + case "icmp": + _, err = monitorICMP(ctx, "monitor-cancellation.invalid.") + } + done <- err + }() + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("lookup did not start") + } + cancel() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("lookup did not cancel") + } + }) + } +} + +func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + })) + defer server.Close() + defer close(release) + pm := newMonitorManager() + pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond}) + task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL}) + defer task.cancel() + + result := task.runProbe(pm.probe) + require.NotNil(t, result) + assert.Equal(t, 100.0, result.PacketLoss) + assert.Equal(t, 100.0, result.PacketLoss1h) + require.Len(t, task.history.samples, 1) + assert.Equal(t, int64(-1), task.history.samples[0].responseUs) + assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task") +} diff --git a/beszel.go b/beszel.go index 57d8f4e1..93f19b47 100644 --- a/beszel.go +++ b/beszel.go @@ -6,7 +6,7 @@ import "github.com/blang/semver" const ( // Version is the current version of the application. - Version = "0.19.0" + Version = "0.20.0" // AppName is the name of the application. AppName = "beszel" ) @@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0") // MinVersionZfsData is the minimum agent version that supports ZFS detail requests. var MinVersionZfsData = semver.MustParse("0.18.9") + +// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync. +var MinVersionNetworkMonitors = semver.MustParse("0.20.0") diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 8857addf..589ec836 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -20,10 +20,11 @@ type hubLike interface { } type AlertManager struct { - hub hubLike - stopOnce sync.Once - pendingAlerts sync.Map - alertsCache *AlertsCache + hub hubLike + stopOnce sync.Once + pendingAlerts sync.Map + alertsCache *AlertsCache + networkMonitors *networkMonitorCache } type AlertMessageData struct { @@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{ // NewAlertManager creates a new AlertManager instance. func NewAlertManager(app hubLike) *AlertManager { am := &AlertManager{ - hub: app, - alertsCache: NewAlertsCache(app), + hub: app, + alertsCache: NewAlertsCache(app), + networkMonitors: newNetworkMonitorCache(app), } am.bindEvents() return am @@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager { // Bind events to the alerts collection lifecycle func (am *AlertManager) bindEvents() { + am.bindNetworkMonitorAlertEvents() am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate) am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete) am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert) diff --git a/internal/alerts/alerts_api.go b/internal/alerts/alerts_api.go index 6e98c3bc..7a44a22f 100644 --- a/internal/alerts/alerts_api.go +++ b/internal/alerts/alerts_api.go @@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error { return e.BadRequestError("Bad data", err) } + if reqData.Name == alertNameNetworkMonitorLoss { + if reqData.Value < 0 || reqData.Value >= 100 { + return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil) + } + reqData.Min = 0 + } + alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts") if err != nil { return err diff --git a/internal/alerts/alerts_cache.go b/internal/alerts/alerts_cache.go index 0fce5c64..58550ff0 100644 --- a/internal/alerts/alerts_cache.go +++ b/internal/alerts/alerts_cache.go @@ -1,6 +1,7 @@ package alerts import ( + "sync" "time" "github.com/pocketbase/dbx" @@ -18,6 +19,9 @@ type CachedAlertData struct { Triggered bool Min uint8 PendingSince time.Time + // Immutable after publication; decoded only when the alert record changes. + MonitorStates map[string]string + MonitorStatesValid bool // Created types.DateTime } @@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) { a.Triggered = record.GetBool("triggered") a.Min = uint8(record.GetInt("min")) a.PendingSince = record.GetDateTime("pending_since").Time() + if a.Name == alertNameNetworkMonitorLoss { + var state networkMonitorAlertState + a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil + a.MonitorStates = state.Monitors + } // a.Created = record.GetDateTime("created") } // AlertsCache provides an in-memory cache for system alerts. type AlertsCache struct { + // Serialize lazy loads with updates so a late load cannot replace newer state. + loadMu sync.Mutex app core.App store *store.Store[string, *store.Store[string, CachedAlertData]] populated bool @@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache { // PopulateFromDB clears current entries and loads all alerts from the database into the cache. func (c *AlertsCache) PopulateFromDB(force bool) error { + c.loadMu.Lock() + defer c.loadMu.Unlock() if !force && c.populated { return nil } @@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error { } c.store.RemoveAll() for _, record := range records { - c.Update(record) + c.update(record) } c.populated = true return nil @@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error { // Update adds or updates an alert record in the cache. func (c *AlertsCache) Update(record *core.Record) { + c.loadMu.Lock() + defer c.loadMu.Unlock() + c.update(record) +} + +func (c *AlertsCache) update(record *core.Record) { systemID := record.GetString("system") if systemID == "" { return @@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) { // Delete removes an alert record from the cache. func (c *AlertsCache) Delete(record *core.Record) { + c.loadMu.Lock() + defer c.loadMu.Unlock() systemID := record.GetString("system") if systemID == "" { return @@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) { func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData { systemStore, ok := c.store.GetOk(systemID) if !ok { - // Populate cache for this system - records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID})) - if err != nil { - return nil + c.loadMu.Lock() + defer c.loadMu.Unlock() + systemStore, ok = c.store.GetOk(systemID) + if !ok { + // Populate cache for this system + records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID})) + if err != nil { + return nil + } + systemStore = store.New(map[string]CachedAlertData{}) + for _, record := range records { + var ca CachedAlertData + ca.PopulateFromRecord(record) + systemStore.Set(record.Id, ca) + } + c.store.Set(systemID, systemStore) } - systemStore = store.New(map[string]CachedAlertData{}) - for _, record := range records { - var ca CachedAlertData - ca.PopulateFromRecord(record) - systemStore.Set(record.Id, ca) - } - c.store.Set(systemID, systemStore) } all := systemStore.GetAll() alerts := make([]CachedAlertData, 0, len(all)) diff --git a/internal/alerts/alerts_history.go b/internal/alerts/alerts_history.go index 9654456f..c29475f6 100644 --- a/internal/alerts/alerts_history.go +++ b/internal/alerts/alerts_history.go @@ -9,6 +9,12 @@ import ( // On triggered alert record delete, set matching alert history record to resolved func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { + if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil { + return err + } + return e.Next() + } if !e.Record.GetBool("triggered") { return e.Next() } @@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { // On alert record update, update alert history record func updateHistoryOnAlertUpdate(e *core.RecordEvent) error { + // Network monitor incidents have separate history entries per monitor. + if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + return e.Next() + } original := e.Record.Original() new := e.Record diff --git a/internal/alerts/alerts_network_monitors.go b/internal/alerts/alerts_network_monitors.go new file mode 100644 index 00000000..6b69c3cc --- /dev/null +++ b/internal/alerts/alerts_network_monitors.go @@ -0,0 +1,269 @@ +package alerts + +import ( + "database/sql" + "errors" + "fmt" + "math" + "net" + "strconv" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" +) + +const alertNameNetworkMonitorLoss = "NetworkMonitorLoss" + +// networkMonitorAlertState is this alert type's persisted runtime state. +// Monitor IDs map to their open history entries independently of history retention. +type networkMonitorAlertState struct { + Monitors map[string]string `json:"monitors"` +} + +func (am *AlertManager) bindNetworkMonitorAlertEvents() { + // Hidden fields are still writable through the record API unless protected. + protectState := func(e *core.RecordRequestEvent) error { + e.Record.Set("state", e.Record.Original().Get("state")) + oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name") + if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) && + (oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) { + return e.BadRequestError("Delete and recreate the alert to change its type or system", nil) + } + if e.Record.GetString("name") == alertNameNetworkMonitorLoss { + if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) { + return e.ForbiddenError("You do not have access to this system", nil) + } + e.Record.Set("triggered", e.Record.Original().GetBool("triggered")) + value := e.Record.GetFloat("value") + if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 { + return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil) + } + e.Record.Set("min", 0) + } + return e.Next() + } + am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState) + am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState) + cleanup := func(e *core.RecordEvent) error { + if err := e.Next(); err != nil { + return err + } + return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil) + } + am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup) + am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error { + if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") { + return e.Next() + } + return cleanup(e) + }) +} + +// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits, +// using its exact payload (dashboard requests can replace the cached payload). +// Omitted results and disconnected systems never imply recovery. +func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error { + if systemRecord.GetString("status") != "up" { + return nil + } + alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss) + if len(alerts) == 0 { + return nil + } + monitors, err := am.networkMonitors.get(systemRecord.Id) + if err != nil { + return err + } + if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) { + return nil + } + // The cache only predicts a transition. Reload and recheck under the DB + // transaction before persisting, including current system/monitor status. + return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results) +} + +// networkMonitorTransitionPending does no IO and never mutates cached maps. +func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool { + for _, alert := range alerts { + if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) { + return true + } + for id := range alert.MonitorStates { + if _, enabled := monitors[id]; !enabled { + return true + } + } + for id, result := range results { + interval, enabled := monitors[id] + if !enabled || !monitorResultReady(result, interval, now) { + continue + } + _, active := alert.MonitorStates[id] + if (result.PacketLoss1h > alert.Value) != active { + return true + } + } + } + return false +} + +func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error { + var messages []AlertMessageData + err := app.RunInTransaction(func(tx core.App) error { + // Read configuration inside the transaction so concurrent threshold changes, + // disabling, and evaluations cannot overwrite each other's incident state. + alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss}) + if err != nil || len(alerts) == 0 { + return err + } + system, err := tx.FindRecordById("systems", systemID) + if errors.Is(err, sql.ErrNoRows) { + // System deletion cascades to its alerts. + return nil + } + if err != nil { + return err + } + monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true}) + if err != nil { + return err + } + enabled := make(map[string]*core.Record, len(monitors)) + for _, m := range monitors { + enabled[m.Id] = m + } + now := time.Now() + for _, alert := range alerts { + var state networkMonitorAlertState + if err := alert.UnmarshalJSONField("state", &state); err != nil { + return err + } + states := state.Monitors + if states == nil { + states = map[string]string{} + } + changed := false + // Removing or disabling a monitor closes its incident silently. + for id, historyID := range states { + if _, ok := enabled[id]; !ok { + if err := resolveMonitorIncident(tx, historyID, now); err != nil { + return err + } + delete(states, id) + changed = true + } + } + if system.GetString("status") == "up" { + for _, m := range monitors { + result, ok := results[m.Id] + if !ok || !monitorResultReady(result, m.GetInt("interval"), now) { + continue + } + historyID, active := states[m.Id] + triggered := result.PacketLoss1h > alert.GetFloat("value") + if triggered == active { + continue + } + label := m.GetString("target") + if m.GetString("protocol") == "tcp" { + label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port"))) + } + if triggered { + collection, err := tx.FindCachedCollectionByNameOrId("alerts_history") + if err != nil { + return err + } + history := core.NewRecord(collection) + history.Load(map[string]any{ + "alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID, + "name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h, + }) + if err := tx.Save(history); err != nil { + return err + } + states[m.Id] = history.Id + } else { + if err := resolveMonitorIncident(tx, historyID, now); err != nil { + return err + } + delete(states, m.Id) + } + changed = true + state, comparison := "loss", "exceeds" + if !triggered { + state, comparison = "recovered", "is at or below" + } + messages = append(messages, AlertMessageData{ + UserID: alert.GetString("user"), SystemID: systemID, + Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label), + Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")), + Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"), + }) + } + } + if changed || alert.GetBool("triggered") != (len(states) > 0) { + alert.Set("state", networkMonitorAlertState{Monitors: states}) + alert.Set("triggered", len(states) > 0) + if err := tx.Save(alert); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + return err + } + // Match other alert types: persist transitions before delivery, and respect + // the user's existing notification destinations and quiet hours. + for _, message := range messages { + if err := am.SendAlert(message); err != nil { + app.Logger().Error("Failed to send network monitor alert", "err", err) + } + } + return nil +} + +func monitorResultReady(result monitor.Result, interval int, now time.Time) bool { + // Three completed attempts provide a short warm-up, including after an agent + // restart. + if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 { + return false + } + // Never interpret an empty one-hour window as zero loss. + maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute)) + age := now.Sub(time.UnixMilli(result.LastProbeAt)) + return age >= -time.Minute && age <= maxAge +} + +func resolveMonitorIncident(app core.App, id string, now time.Time) error { + record, err := app.FindRecordById("alerts_history", id) + if errors.Is(err, sql.ErrNoRows) { + // History can be purged independently. + return nil + } + if err != nil { + return err + } + if !record.GetDateTime("resolved").IsZero() { + return nil + } + record.Set("resolved", now.UTC()) + return app.Save(record) +} + +func resolveNetworkMonitorHistory(app core.App, alertID string) error { + records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""}) + if err != nil { + return err + } + for _, record := range records { + record.Set("resolved", time.Now().UTC()) + if err := app.Save(record); err != nil { + return err + } + } + return nil +} diff --git a/internal/alerts/alerts_network_monitors_test.go b/internal/alerts/alerts_network_monitors_test.go new file mode 100644 index 00000000..aa02cf43 --- /dev/null +++ b/internal/alerts/alerts_network_monitors_test.go @@ -0,0 +1,513 @@ +//go:build testing + +package alerts_test + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/henrygd/beszel/internal/alerts" + "github.com/henrygd/beszel/internal/entities/monitor" + beszelTests "github.com/henrygd/beszel/internal/tests" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" + pbTests "github.com/pocketbase/pocketbase/tests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) { + t.Helper() + hub, system, alert := systemdTestSetup(t, false) + t.Cleanup(hub.Cleanup) + alert.Set("name", "NetworkMonitorLoss") + alert.Set("value", 5) + require.NoError(t, hub.Save(alert)) + var monitors []*core.Record + for _, name := range []string{"gateway", "website"} { + record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true, + }) + require.NoError(t, err) + monitors = append(monitors, record) + } + // Avoid starting a system update worker in tests. + _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + system.Set("status", "up") + return hub, system, alert, monitors +} + +func monitorResult(loss float64) monitor.Result { + return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss} +} + +func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + count := hub.TestMailer.TotalSend() + results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)} + check := func(active bool, open, sent int) { + t.Helper() + record, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.Equal(t, active, record.GetBool("triggered")) + total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""}) + require.NoError(t, err) + assert.EqualValues(t, open, total) + assert.Equal(t, count+sent, hub.TestMailer.TotalSend()) + } + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + check(true, 1, 1) + message := hub.TestMailer.Messages()[count] + assert.Contains(t, message.Text, "gateway.example.com") + assert.Contains(t, message.Text, "10.00%") + assert.Contains(t, message.Text, "5.00%") + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + check(true, 1, 1) + // Persisted monitor state prevents duplicate notifications after a hub restart. + am = alerts.NewTestAlertManagerWithoutWorker(hub) + results[monitors[1].Id] = monitorResult(20) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + check(true, 2, 2) + results[monitors[0].Id] = monitorResult(5) // Equality is a recovery. + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + check(true, 1, 3) + results[monitors[1].Id] = monitorResult(0) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + check(false, 0, 4) + histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id}) + require.NoError(t, err) + require.Len(t, histories, 2) + for _, history := range histories { + assert.NotEmpty(t, history.GetString("monitor_name")) + } +} + +func TestNetworkMonitorAlertTargetLabel(t *testing.T) { + for _, tc := range []struct { + protocol, target, label string + port int + }{ + {"icmp", "gateway.example.com", "gateway.example.com", 0}, + {"http", "https://example.com/health", "https://example.com/health", 0}, + {"tcp", "example.com", "example.com:8443", 8443}, + {"tcp", "2001:db8::1", "[2001:db8::1]:443", 443}, + } { + t.Run(tc.label, func(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + m := monitors[0] + m.Set("protocol", tc.protocol) + m.Set("target", tc.target) + m.Set("port", tc.port) + require.NoError(t, hub.Save(m)) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)})) + histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id}) + require.NoError(t, err) + require.Len(t, histories, 1) + assert.Equal(t, tc.label, histories[0].GetString("monitor_name")) + assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label) + }) + } +} + +func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) { + for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} { + t.Run(scenario, func(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + apply := func(loss float64) { + result := monitorResult(loss) + results := map[string]monitor.Result{monitors[0].Id: result} + switch scenario { + case "missing": + results = nil + case "stale": + result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli() + results[monitors[0].Id] = result + case "expired hourly window": + monitors[0].Set("interval", 3600) + require.NoError(t, hub.Save(monitors[0])) + result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli() + results[monitors[0].Id] = result + case "future": + result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli() + results[monitors[0].Id] = result + case "warmup": + result.SampleCount = 2 + results[monitors[0].Id] = result + case "no probes": + result.SampleCount = 0 + results[monitors[0].Id] = result + case "down": + _, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + case "paused": + record, err := hub.FindRecordById("systems", system.Id) + require.NoError(t, err) + record.Set("status", "paused") + require.NoError(t, hub.Save(record)) + } + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + } + count := hub.TestMailer.TotalSend() + apply(100) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + _, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)})) + apply(0) + assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident") + record, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.True(t, record.GetBool("triggered")) + }) + } +} + +func TestNetworkMonitorAlertCleanup(t *testing.T) { + for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} { + t.Run(scenario, func(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + count := hub.TestMailer.TotalSend() + switch scenario { + case "disable monitor": + monitors[0].Set("enabled", false) + require.NoError(t, hub.Save(monitors[0])) + case "delete monitor": + require.NoError(t, hub.Delete(monitors[0])) + case "disable alert": + require.NoError(t, hub.Delete(alert)) + case "delete system": + require.NoError(t, hub.Delete(system)) + case "purge history": + history, err := hub.FindAllRecords("alerts_history") + require.NoError(t, err) + for _, record := range history { + require.NoError(t, hub.Delete(record)) + } + } + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""}) + require.NoError(t, err) + if scenario == "disable monitor" || scenario == "delete monitor" { + assert.EqualValues(t, 1, open) + record, err := hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.True(t, record.GetBool("triggered")) + require.NoError(t, hub.Delete(monitors[1])) + record, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.False(t, record.GetBool("triggered")) + } else { + assert.Zero(t, open) + } + }) + } +} + +func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password") + require.NoError(t, err) + other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20}) + require.NoError(t, err) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + other, err = hub.FindRecordById("alerts", other.Id) + require.NoError(t, err) + assert.False(t, other.GetBool("triggered")) + // Editing the threshold re-evaluates on the next batch, without losing state. + alert, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + alert.Set("value", 15) + require.NoError(t, hub.Save(alert)) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + alert, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + assert.False(t, alert.GetBool("triggered")) +} + +func TestNetworkMonitorAlertAPI(t *testing.T) { + for _, tc := range []struct { + name string + value float64 + direct, denied, patch bool + status int + }{ + {name: "zero threshold", value: 0, status: 200}, + {name: "fractional threshold", value: 5.5, status: 200}, + {name: "negative threshold", value: -1, status: 400}, + {name: "unreachable threshold", value: 100, status: 400}, + {name: "bulk inaccessible system", value: 5, denied: true, status: 200}, + {name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403}, + {name: "direct invalid threshold", value: -1, direct: true, status: 400}, + {name: "direct private state", value: 5, direct: true, status: 200}, + {name: "patch preserves state", value: 10, direct: true, patch: true, status: 200}, + } { + t.Run(tc.name, func(t *testing.T) { + hub, user := beszelTests.GetHubWithUser(t) + defer hub.Cleanup() + owner := user.Id + if tc.denied { + other, err := beszelTests.CreateUser(hub, "other@example.com", "password") + require.NoError(t, err) + owner = other.Id + } + systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused") + require.NoError(t, err) + token, err := user.NewAuthToken() + require.NoError(t, err) + body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true} + url, method := "/api/beszel/user-alerts", "POST" + if tc.direct { + url = "/api/collections/alerts/records" + body["system"], body["user"] = systems[0].Id, user.Id + body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true + } + if tc.patch { + alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}}) + require.NoError(t, err) + url += "/" + alert.Id + method = "PATCH" + body["triggered"] = false + } + content := `"success":true` + if tc.direct { + content = `"name":"NetworkMonitorLoss"` + } + if tc.status == 400 { + content = `"status":400` + } + if tc.status == 403 { + content = `"status":403` + } + scenario := beszelTests.ApiScenario{ + Name: tc.name, Method: method, URL: url, Body: jsonReader(body), + Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content}, + TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp }, + } + scenario.Test(t) + records, err := hub.FindAllRecords("alerts") + require.NoError(t, err) + if tc.status != 200 || tc.denied { + assert.Empty(t, records) + return + } + require.Len(t, records, 1) + assert.Equal(t, tc.value, records[0].GetFloat("value")) + assert.Zero(t, records[0].GetInt("min")) + state := struct { + Monitors map[string]string `json:"monitors"` + }{} + require.NoError(t, records[0].UnmarshalJSONField("state", &state)) + states := state.Monitors + if tc.patch { + assert.Equal(t, map[string]string{"real": "history"}, states) + assert.True(t, records[0].GetBool("triggered")) + } else { + assert.Empty(t, states) + assert.False(t, records[0].GetBool("triggered")) + } + }) + } +} + +type monitorCountingHub struct { + *beszelTests.TestHub + transactions atomic.Int64 + beforeTransaction func() +} + +func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error { + h.transactions.Add(1) + if h.beforeTransaction != nil { + h.beforeTransaction() + } + return h.App.RunInTransaction(fn) +} + +// Count actual SQL on both DB connections, including queries through record APIs. +func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 { + t.Helper() + count := &atomic.Int64{} + for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} { + db := builder.(*dbx.DB) + old := db.LogFunc + db.LogFunc = func(string, ...any) { count.Add(1) } + t.Cleanup(func() { db.LogFunc = old }) + } + return count +} + +func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + counted := &monitorCountingHub{TestHub: hub} + am := alerts.NewTestAlertManagerWithoutWorker(counted) + sql := monitorSQLCounter(t, hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)} + evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) } + noWork := func() { + t.Helper() + sql.Store(0) + counted.transactions.Store(0) + for range 100 { + evaluate() + } + assert.Zero(t, sql.Load(), "steady state must not issue SQL") + assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions") + } + // One-time lazy loads are permitted, including on hub restart. + evaluate() + assert.Positive(t, sql.Load()) + noWork() + // Realtime metric saves invoke record hooks but must not invalidate config. + fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id) + require.NoError(t, err) + monitors[0] = fresh + monitors[0].Set("loss1h", 0) + monitors[0].Set("res", 100) + require.NoError(t, hub.Save(monitors[0])) + noWork() + results[monitors[0].Id] = monitorResult(10) + evaluate() + assert.Positive(t, sql.Load(), "transitions must still be persisted") + assert.EqualValues(t, 1, counted.transactions.Load()) + noWork() + // Missing and stale observations must not enter the transaction either. + results = nil + noWork() + results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}} + noWork() + results[monitors[0].Id] = monitorResult(0) + evaluate() + noWork() + require.NoError(t, hub.Delete(alert)) + noWork() +} + +func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + count := hub.TestMailer.TotalSend() + // Widening the interval makes this observation fresh. A stale interval cache + // would miss the failure indefinitely, even though results keep arriving. + result := monitorResult(10) + result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli() + results[monitors[0].Id] = result + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + monitors[0].Set("interval", 120) + require.NoError(t, hub.Save(monitors[0])) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count+1, hub.TestMailer.TotalSend()) + // Disable, then re-enable the same ID: its new failure must be detected. + monitors[0].Set("enabled", false) + require.NoError(t, hub.Save(monitors[0])) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + monitors[0].Set("enabled", true) + require.NoError(t, hub.Save(monitors[0])) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count+2, hub.TestMailer.TotalSend()) + // A new monitor must also become eligible without restarting the hub. + created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true, + }) + require.NoError(t, err) + results[created.Id] = monitorResult(10) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count+3, hub.TestMailer.TotalSend()) + // Threshold changes refresh cached config and preserve the active incidents. + alert, err = hub.FindRecordById("alerts", alert.Id) + require.NoError(t, err) + alert.Set("value", 15) + require.NoError(t, hub.Save(alert)) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count+5, hub.TestMailer.TotalSend()) +} + +func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) { + for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} { + t.Run(change, func(t *testing.T) { + hub, system, alert, monitors := networkAlertSetup(t) + counted := &monitorCountingHub{TestHub: hub} + am := alerts.NewTestAlertManagerWithoutWorker(counted) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + count := hub.TestMailer.TotalSend() + // Change the DB after the cache predicts a transition, before its transaction. + counted.beforeTransaction = func() { + counted.beforeTransaction = nil + switch change { + case "threshold": + alert.Set("value", 20) + require.NoError(t, hub.Save(alert)) + case "disable alert": + require.NoError(t, hub.Delete(alert)) + case "disable monitor": + monitors[0].Set("enabled", false) + require.NoError(t, hub.Save(monitors[0])) + case "down": + _, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute() + require.NoError(t, err) + } + } + results[monitors[0].Id] = monitorResult(10) + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.EqualValues(t, 1, counted.transactions.Load()) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + histories, err := hub.CountRecords("alerts_history") + require.NoError(t, err) + assert.Zero(t, histories) + }) + } +} + +func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) { + hub, system, _, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)} + count := hub.TestMailer.TotalSend() + var wg sync.WaitGroup + errs := make(chan error, 8) + for range 8 { + wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) }) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + assert.Equal(t, count+1, hub.TestMailer.TotalSend()) +} + +func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) { + hub, system, _, monitors := networkAlertSetup(t) + am := alerts.NewTestAlertManagerWithoutWorker(hub) + results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)} + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + _, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute() + require.NoError(t, err) + count := hub.TestMailer.TotalSend() + results[monitors[0].Id] = monitorResult(10) + require.Error(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count, hub.TestMailer.TotalSend()) + histories, err := hub.CountRecords("alerts_history") + require.NoError(t, err) + assert.Zero(t, histories) + _, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute() + require.NoError(t, err) + // A failed transition must not be published to the cache and mask the retry. + require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) + assert.Equal(t, count+1, hub.TestMailer.TotalSend()) +} diff --git a/internal/alerts/alerts_system.go b/internal/alerts/alerts_system.go index ef0a61d9..ad47d0bd 100644 --- a/internal/alerts/alerts_system.go +++ b/internal/alerts/alerts_system.go @@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst return nil } - alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName) + alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss) if len(alerts) == 0 { return nil } diff --git a/internal/alerts/alerts_test_helpers.go b/internal/alerts/alerts_test_helpers.go index df796b36..89f067c2 100644 --- a/internal/alerts/alerts_test_helpers.go +++ b/internal/alerts/alerts_test_helpers.go @@ -11,8 +11,9 @@ import ( func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager { return &AlertManager{ - hub: app, - alertsCache: NewAlertsCache(app), + hub: app, + alertsCache: NewAlertsCache(app), + networkMonitors: newNetworkMonitorCache(app), } } diff --git a/internal/alerts/network_monitor_cache.go b/internal/alerts/network_monitor_cache.go new file mode 100644 index 00000000..2ca3718e --- /dev/null +++ b/internal/alerts/network_monitor_cache.go @@ -0,0 +1,79 @@ +package alerts + +import ( + "sync" + + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/core" +) + +// networkMonitorCache keeps just the enabled monitor IDs and probe intervals +// needed for the alert fast path. Names and targets are read only on transitions. +// Returned maps are immutable; configuration changes invalidate the whole entry. +type networkMonitorCache struct { + app core.App + mu sync.RWMutex + systems map[string]map[string]int +} + +func newNetworkMonitorCache(app core.App) *networkMonitorCache { + c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)} + invalidate := func(e *core.RecordEvent) error { + c.invalidate(e.Record.GetString("system")) + return e.Next() + } + app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate) + app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate) + app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error { + old := e.Record.Original() + // Realtime metric saves also invoke this hook. They must not evict config. + if old.GetString("system") != e.Record.GetString("system") || + old.GetBool("enabled") != e.Record.GetBool("enabled") || + old.GetInt("interval") != e.Record.GetInt("interval") { + c.invalidate(old.GetString("system")) + c.invalidate(e.Record.GetString("system")) + } + return e.Next() + }) + app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error { + c.invalidate(e.Record.Id) + return e.Next() + }) + return c +} + +func (c *networkMonitorCache) invalidate(systemID string) { + c.mu.Lock() + delete(c.systems, systemID) + c.mu.Unlock() +} + +func (c *networkMonitorCache) get(systemID string) (map[string]int, error) { + c.mu.RLock() + monitors, ok := c.systems[systemID] + c.mu.RUnlock() + if ok { + return monitors, nil + } + c.mu.Lock() + defer c.mu.Unlock() + if monitors, ok := c.systems[systemID]; ok { + return monitors, nil + } + // Keep the lock through the load so a concurrent config change cannot be + // invalidated first and then overwritten by the older query result. + var rows []struct { + ID string `db:"id"` + Interval int `db:"interval"` + } + if err := c.app.DB().Select("id", "interval").From("network_monitors"). + Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil { + return nil, err + } + monitors = make(map[string]int, len(rows)) + for _, row := range rows { + monitors[row.ID] = row.Interval + } + c.systems[systemID] = monitors + return monitors, nil +} diff --git a/internal/common/common-ws.go b/internal/common/common-ws.go index 3f69ba88..b9e8aea0 100644 --- a/internal/common/common-ws.go +++ b/internal/common/common-ws.go @@ -24,6 +24,8 @@ const ( GetSystemdInfo // Request ZFS detail data from agent GetZfsData + // Sync network monitor configuration to agent + SyncNetworkMonitors // Add new actions here... ) diff --git a/internal/entities/monitor/monitor.go b/internal/entities/monitor/monitor.go new file mode 100644 index 00000000..46995ce8 --- /dev/null +++ b/internal/entities/monitor/monitor.go @@ -0,0 +1,102 @@ +package monitor + +import "time" + +// MaxProbeTimeout is the longest agent probe timeout (currently HTTP). +// Hub requests that run a probe must allow this time in addition to transport overhead. +const MaxProbeTimeout = 10 * time.Second + +type SyncAction uint8 + +const ( + // SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system. + SyncActionReplace SyncAction = iota + // SyncActionUpsert indicates an incremental sync where the provided config should be added or updated. + SyncActionUpsert + // SyncActionDelete indicates an incremental sync where the provided config should be removed. + SyncActionDelete +) + +// Config defines a network monitor task sent from hub to agent. +type Config struct { + // ID is the stable network_monitors record ID generated by the hub. + ID string `cbor:"0,keyasint"` + Target string `cbor:"1,keyasint"` + Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns" + Port uint16 `cbor:"3,keyasint,omitempty"` + Interval uint16 `cbor:"4,keyasint"` // seconds +} + +// SyncRequest defines an incremental or full monitor sync request sent to the agent. +type SyncRequest struct { + Action SyncAction `cbor:"0,keyasint"` + Config Config `cbor:"1,keyasint,omitempty"` + Configs []Config `cbor:"2,keyasint,omitempty"` + RunNow bool `cbor:"3,keyasint,omitempty"` +} + +// SyncResponse returns the immediate result for an upsert when requested. +type SyncResponse struct { + Result Result `cbor:"0,keyasint,omitempty"` +} + +// Result holds aggregated monitor results for a single target. +// +// 0: avg response in microseconds +// +// 1: 1h average response in microseconds +// +// 2: min response in microseconds +// +// 3: 1h min response in microseconds +// +// 4: max response in microseconds +// +// 5: 1h max response in microseconds +// +// 6: packet loss percentage (0-100) +// +// 7: 1h packet loss percentage (0-100) +type Result struct { + AvgResponse int64 `cbor:"0,keyasint,omitempty"` + AvgResponse1h int64 `cbor:"1,keyasint,omitempty"` + MinResponse int64 `cbor:"2,keyasint,omitempty"` + MinResponse1h int64 `cbor:"3,keyasint,omitempty"` + MaxResponse int64 `cbor:"4,keyasint,omitempty"` + MaxResponse1h int64 `cbor:"5,keyasint,omitempty"` + PacketLoss float64 `cbor:"6,keyasint,omitempty"` + PacketLoss1h float64 `cbor:"7,keyasint,omitempty"` + // LastProbeAt is the latest completed probe's Unix timestamp in milliseconds. + LastProbeAt int64 `cbor:"8,keyasint"` + // SampleCount includes all completed probes since this monitor started. + // Used for alert warm-up even when the interval is longer than 20 minutes. + SampleCount int64 `cbor:"9,keyasint,omitempty"` + // Counts and sum cover the current response window (or latest-sample + // fallback), not the hourly window or lifetime SampleCount. + TotalCount int64 `cbor:"10,keyasint"` + SuccessCount int64 `cbor:"11,keyasint"` + ResponseSum int64 `cbor:"12,keyasint"` +} + +// Stats holds response times in microseconds and packet loss percentage (0-100). +type Stats struct { + ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored. + ResMin float64 `json:"res_min" db:"res_min"` + ResMax float64 `json:"res_max" db:"res_max"` + Loss float64 `json:"loss" db:"-"` // Derived for display; not stored. + TotalCount int64 `json:"-" db:"total_count"` + SuccessCount int64 `json:"-" db:"success_count"` + ResponseSum int64 `json:"-" db:"res_sum"` +} + +func (s Stats) FromResult(result Result) Stats { + return Stats{ + ResAvg: float64(result.AvgResponse), + ResMin: float64(result.MinResponse), + ResMax: float64(result.MaxResponse), + Loss: result.PacketLoss, + TotalCount: result.TotalCount, + SuccessCount: result.SuccessCount, + ResponseSum: result.ResponseSum, + } +} diff --git a/internal/entities/system/system.go b/internal/entities/system/system.go index fd3c42e4..c6e8d0ea 100644 --- a/internal/entities/system/system.go +++ b/internal/entities/system/system.go @@ -7,6 +7,7 @@ import ( "time" "github.com/henrygd/beszel/internal/entities/container" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/systemd" ) @@ -210,5 +211,6 @@ type CombinedData struct { Details *Details `cbor:"4,keyasint,omitempty"` // SystemdServicesUpdated distinguishes a fresh empty snapshot from a response // that omitted systemd data (for example, a short-cache dashboard request). - SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"` + SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"` + Monitors map[string]monitor.Result `cbor:"6,keyasint"` } diff --git a/internal/hub/collections.go b/internal/hub/collections.go index 667f4bf8..3a0c3e85 100644 --- a/internal/hub/collections.go +++ b/internal/hub/collections.go @@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error { return err } - if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{ + if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{ list: &systemScopedReadRule, }); err != nil { return err @@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error { return err } + if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{ + list: &systemScopedReadRule, + view: &systemScopedReadRule, + create: &systemScopedWriteRule, + update: &systemScopedWriteRule, + delete: &systemScopedWriteRule, + }); err != nil { + return err + } + if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{ list: &systemScopedReadRule, view: &systemScopedReadRule, diff --git a/internal/hub/hub.go b/internal/hub/hub.go index 30548cf2..3f5f241b 100644 --- a/internal/hub/hub.go +++ b/internal/hub/hub.go @@ -109,6 +109,8 @@ func (h *Hub) StartHub() error { h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole) h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings) + bindNetworkMonitorsEvents(h) + pb, ok := h.App.(*pocketbase.PocketBase) if !ok { return errors.New("not a pocketbase app") diff --git a/internal/hub/network_monitors.go b/internal/hub/network_monitors.go new file mode 100644 index 00000000..25ad1b65 --- /dev/null +++ b/internal/hub/network_monitors.go @@ -0,0 +1,158 @@ +package hub + +import ( + "strconv" + "time" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/henrygd/beszel/internal/hub/systems" + "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/types" +) + +// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to. +func generateMonitorID(systemId string, config monitor.Config) string { + args := []string{systemId, config.Target, config.Protocol} + // only use port for TCP monitors, since for other protocols it's not relevant as standalone value + if config.Protocol == "tcp" { + args = append(args, strconv.FormatUint(uint64(config.Port), 10)) + } + return systems.MakeStableHashId(args...) +} + +// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync. +func bindNetworkMonitorsEvents(hub *Hub) { + // on create, make sure the id is set to a stable hash + hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error { + systemID := e.Record.GetString("system") + config := monitorConfigFromRecord(e.Record) + id := generateMonitorID(systemID, *config) + e.Record.Set("id", id) + return e.Next() + }) + + // sync monitor to agent on creation and persist the first result immediately when available + hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error { + err := e.Next() + if err != nil { + return err + } + if !e.Record.GetBool("enabled") { + return nil + } + // If connected, run the monitor immediately. Paused systems may be absent + // from the manager; their monitors will sync when they reconnect. + system, err := hub.sm.GetSystem(e.Record.GetString("system")) + if err == nil && system.Status == "up" { + go hub.upsertNetworkMonitor(e.Record, true) + } + return nil + }) + + // On API update requests, if the monitor config changed in a way that requires a new ID, create a new + // record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent. + hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error { + systemID := e.Record.GetString("system") + // only tcp uses port - set other protocols port to zero + if e.Record.GetString("protocol") != "tcp" { + e.Record.Set("port", 0) + } + ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record)) + if ID != e.Record.Id { + newRecord := copyMonitorToNewRecord(e.Record, ID) + if err := e.App.Save(newRecord); err != nil { + return err + } + if err := e.App.Delete(e.Record); err != nil { + return err + } + return nil + } + err := e.Next() + if err != nil { + return err + } + if e.Record.GetBool("enabled") { + // if the monitor is enabled, sync the updated config to the agent now + runNow := !e.Record.Original().GetBool("enabled") + err = hub.upsertNetworkMonitor(e.Record, runNow) + } else { + // if the monitor is paused, remove it from the agent + err = hub.deleteNetworkMonitor(e.Record) + } + if err != nil { + hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err) + } + return nil + }) + + // sync monitor to agent on delete + hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error { + if err := hub.deleteNetworkMonitor(e.Record); err != nil { + hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err) + } + return e.Next() + }) +} + +// monitorConfigFromRecord builds a monitor config from a network_monitors record. +func monitorConfigFromRecord(record *core.Record) *monitor.Config { + return &monitor.Config{ + ID: record.Id, + Target: record.GetString("target"), + Protocol: record.GetString("protocol"), + Port: uint16(record.GetInt("port")), + Interval: uint16(record.GetInt("interval")), + } +} + +// setMonitorResultFields stores the latest monitor result values on the record. +func setMonitorResultFields(record *core.Record, result monitor.Result) { + nowString := time.Now().UTC().Format(types.DefaultDateLayout) + record.Set("res", result.AvgResponse) + record.Set("resAvg1h", result.AvgResponse1h) + record.Set("resMin1h", result.MinResponse1h) + record.Set("resMax1h", result.MaxResponse1h) + record.Set("loss1h", result.PacketLoss1h) + record.Set("updated", nowString) +} + +// copyMonitorToNewRecord creates a new record with the same field values as the old one. +// This is used when the monitor config changes in a way that requires a new ID, so we need +// to create a new record with the new ID and delete the old one. +func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record { + collection := oldRecord.Collection() + newRecord := core.NewRecord(collection) + newRecord.Id = newID + fields := []string{"system", "target", "protocol", "port", "interval", "enabled"} + for _, field := range fields { + newRecord.Set(field, oldRecord.Get(field)) + } + return newRecord +} + +// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow +// is true, it will also trigger an immediate monitor run and update the record with the result. +func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error { + systemID := record.GetString("system") + system, err := h.sm.GetSystem(systemID) + if err != nil { + return err + } + result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow) + if err != nil || result == nil { + return err + } + setMonitorResultFields(record, *result) + return h.App.SaveNoValidate(record) +} + +// deleteNetworkMonitor removes the record's monitor from the target system. +func (h *Hub) deleteNetworkMonitor(record *core.Record) error { + systemID := record.GetString("system") + system, err := h.sm.GetSystem(systemID) + if err != nil { + return err + } + return system.DeleteNetworkMonitor(record.Id) +} diff --git a/internal/hub/network_monitors_test.go b/internal/hub/network_monitors_test.go new file mode 100644 index 00000000..f206ff84 --- /dev/null +++ b/internal/hub/network_monitors_test.go @@ -0,0 +1,225 @@ +package hub + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/pocketbase/pocketbase/apis" + "github.com/pocketbase/pocketbase/core" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) { + for _, batch := range []bool{false, true} { + name := "single" + if batch { + name = "batch" + } + t.Run(name, func(t *testing.T) { + hub, testApp, err := createTestHub(t) + require.NoError(t, err) + defer cleanupTestHub(hub, testApp) + bindNetworkMonitorsEvents(hub) + + user, err := createTestUser(hub) + require.NoError(t, err) + system, err := createTestRecord(hub, "systems", map[string]any{ + "name": "Paused", "host": "localhost", "port": "45876", + "status": "paused", "users": []string{user.Id}, + }) + require.NoError(t, err) + // Paused systems are not loaded into the manager at startup. + _, err = hub.sm.GetSystem(system.Id) + require.Error(t, err) + + payload := func(target string) map[string]any { + return map[string]any{ + "system": system.Id, "target": target, "protocol": "icmp", + "interval": 60, "enabled": true, + } + } + url := "/api/collections/network_monitors/records" + var body any = payload("1.1.1.1") + count := 1 + if batch { + body = map[string]any{"requests": []map[string]any{ + {"method": "POST", "url": url, "body": payload("1.1.1.1")}, + {"method": "POST", "url": url, "body": payload("8.8.8.8")}, + }} + url = "/api/batch" + count = 2 + } + data, err := json.Marshal(body) + require.NoError(t, err) + token, err := user.NewAuthToken() + require.NoError(t, err) + router, err := apis.NewRouter(hub) + require.NoError(t, err) + handler, err := router.BuildMux() + require.NoError(t, err) + request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data)) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Authorization", token) + response := httptest.NewRecorder() + handler.ServeHTTP(response, request) + assert.Equal(t, http.StatusOK, response.Code, response.Body.String()) + + records, err := hub.FindAllRecords("network_monitors") + require.NoError(t, err) + require.Len(t, records, count) + for _, record := range records { + assert.Equal(t, system.Id, record.GetString("system")) + assert.True(t, record.GetBool("enabled")) + } + }) + } +} + +func TestGenerateMonitorID(t *testing.T) { + tests := []struct { + name string + systemID string + config monitor.Config + expected string + }{ + { + name: "HTTP monitor on example.com", + systemID: "sys123", + config: monitor.Config{ + Protocol: "http", + Target: "example.com", + Port: 0, + Interval: 60, + }, + expected: "a20a5827", + }, + { + name: "HTTP monitor on example.com with different port", + systemID: "sys123", + config: monitor.Config{ + Protocol: "http", + Target: "example.com", + Port: 8080, + Interval: 60, + }, + expected: "a20a5827", + }, + { + name: "HTTP monitor on example.com with different system ID", + systemID: "sys1234", + config: monitor.Config{ + Protocol: "http", + Target: "example.com", + Port: 80, + Interval: 60, + }, + expected: "ab602ae7", + }, + { + name: "Same monitor, different interval", + systemID: "sys1234", + config: monitor.Config{ + Protocol: "http", + Target: "example.com", + Port: 80, + Interval: 120, + }, + expected: "ab602ae7", + }, + { + name: "ICMP monitor on 1.1.1.1", + systemID: "sys456", + config: monitor.Config{ + Protocol: "icmp", + Target: "1.1.1.1", + Port: 0, + Interval: 10, + }, + expected: "6d13a4a4", + }, { + name: "ICMP monitor on 1.1.1.1 with different system ID", + systemID: "sys4567", + config: monitor.Config{ + Protocol: "icmp", + Target: "1.1.1.1", + Port: 0, + Interval: 10, + }, + expected: "ddd6c81", + }, + { + name: "TCP monitor on example.com with port 443", + systemID: "sys789", + config: monitor.Config{ + Protocol: "tcp", + Target: "example.com", + Port: 443, + Interval: 30, + }, + expected: "677b991", + }, + { + name: "TCP monitor on example.com with port 8443", + systemID: "sys789", + config: monitor.Config{ + Protocol: "tcp", + Target: "example.com", + Port: 8443, + Interval: 30, + }, + expected: "84167969", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := generateMonitorID(tt.systemID, tt.config) + assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected) + }) + } +} + +func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) { + hub, testApp, err := createTestHub(t) + require.NoError(t, err) + defer cleanupTestHub(hub, testApp) + + collection, err := hub.FindCachedCollectionByNameOrId("network_monitors") + require.NoError(t, err) + assert.Nil(t, collection.Fields.GetByName("name")) + + oldRecord := core.NewRecord(collection) + oldRecord.Load(map[string]any{ + "system": "sys123", + "target": "https://example.com", + "protocol": "http", + "port": 443, + "interval": 60, + "enabled": true, + "res": 1200, + "resAvg1h": 1300, + "resMin1h": 900, + "resMax1h": 1600, + "loss1h": 5, + "updated": "2026-04-29 12:00:00.000Z", + }) + + newRecord := copyMonitorToNewRecord(oldRecord, "next12345") + + assert.Equal(t, "next12345", newRecord.Id) + assert.Equal(t, "https://example.com", newRecord.GetString("target")) + assert.Equal(t, "http", newRecord.GetString("protocol")) + assert.Equal(t, 443, newRecord.GetInt("port")) + assert.True(t, newRecord.GetBool("enabled")) + assert.Zero(t, newRecord.GetFloat("res")) + assert.Zero(t, newRecord.GetFloat("resAvg1h")) + assert.Zero(t, newRecord.GetFloat("resMin1h")) + assert.Zero(t, newRecord.GetFloat("resMax1h")) + assert.Zero(t, newRecord.GetFloat("loss1h")) + assert.Equal(t, "", newRecord.GetString("updated")) +} diff --git a/internal/hub/systems/network_monitor_stats_test.go b/internal/hub/systems/network_monitor_stats_test.go new file mode 100644 index 00000000..38bc8cb0 --- /dev/null +++ b/internal/hub/systems/network_monitor_stats_test.go @@ -0,0 +1,224 @@ +//go:build testing + +package systems + +import ( + "fmt" + "testing" + "time" + + "github.com/fxamacker/cbor/v2" + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/henrygd/beszel/internal/entities/system" + "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/subscriptions" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNetworkMonitorProbePruning(t *testing.T) { + for _, tc := range []struct { + name string + monitors map[string]monitor.Result + fail bool + want map[string]int64 + }{ + {"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}}, + {"empty report", map[string]monitor.Result{}, false, map[string]int64{}}, + {"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}}, + {"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}}, + } { + t.Run(tc.name, func(t *testing.T) { + sys, app := newTestSystemWithHub(t) + sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000} + // Preserve the distinction between nil and empty across the agent transport. + encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors}) + require.NoError(t, err) + var data system.CombinedData + require.NoError(t, cbor.Unmarshal(encoded, &data)) + if tc.fail { + _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute() + require.NoError(t, err) + } + _, err = sys.createRecords(&data) + if tc.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tc.want, sys.lastSavedMonitorProbe) + }) + } +} + +func TestNetworkMonitorStatsFreshness(t *testing.T) { + for _, realtime := range []bool{false, true} { + name := "sql" + if realtime { + name = "realtime" + } + t.Run(name, func(t *testing.T) { + sys, app := newTestSystemWithHub(t) + if realtime { + client := subscriptions.NewDefaultClient() + client.Subscribe("network_monitors/*") + app.SubscriptionsBroker().Register(client) + t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) }) + } + col, err := app.FindCachedCollectionByNameOrId("network_monitors") + require.NoError(t, err) + for _, id := range []string{"monitor1", "monitor2"} { + record := core.NewRecord(col) + record.Id = id + record.Set("system", sys.Id) + require.NoError(t, app.SaveNoValidate(record)) + } + data := &system.CombinedData{Monitors: map[string]monitor.Result{ + "monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123}, + "monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1}, + }} + count := func(want int64) { + t.Helper() + got, err := app.CountRecords("network_monitor_stats") + require.NoError(t, err) + assert.Equal(t, want, got) + } + save := func() { + t.Helper() + _, err := sys.createRecords(data) + require.NoError(t, err) + } + save() + count(2) + stored, err := app.FindAllRecords("network_monitor_stats") + require.NoError(t, err) + for _, record := range stored { + result := data.Monitors[record.GetString("monitor")] + assert.EqualValues(t, result.TotalCount, record.GetInt("total_count")) + assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count")) + assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum")) + } + // A resume can overlap the scheduled update with the same probe. + errs := make(chan error, 4) + for range 4 { + go func() { + _, err := sys.createRecords(data) + errs <- err + }() + } + for range 4 { + require.NoError(t, <-errs) + } + count(2) + + // A rolling hourly value can change without a new probe. + result := data.Monitors["monitor1"] + result.AvgResponse1h = 42 + data.Monitors["monitor1"] = result + save() + count(2) + record, err := app.FindRecordById("network_monitors", "monitor1") + require.NoError(t, err) + assert.Equal(t, 42, record.GetInt("resAvg1h")) + + // Identical response values and failed probes still count as new measurements. + for id, result := range data.Monitors { + result.LastProbeAt = 301000 + data.Monitors[id] = result + } + save() + count(4) + + // A failed individual insert must remain retryable, even if others commit. + _, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute() + require.NoError(t, err) + for id, result := range data.Monitors { + result.LastProbeAt = 601000 + data.Monitors[id] = result + } + save() + count(5) + assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"]) + assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"]) + _, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute() + require.NoError(t, err) + save() + count(6) + + // Failure after inserting stats rolls back the whole transaction and its markers. + _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute() + require.NoError(t, err) + result = data.Monitors["monitor1"] + result.LastProbeAt = 901000 + data.Monitors["monitor1"] = result + _, err = sys.createRecords(data) + require.Error(t, err) + count(6) + assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"]) + _, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute() + require.NoError(t, err) + save() + count(7) + + // Clock rollback is a new probe identity, not a reason to stall writes. + result.LastProbeAt = 500 + data.Monitors["monitor1"] = result + save() + count(8) + + // Recreated systems intentionally accept the first result without restoring state. + sys = &System{Id: sys.Id, manager: sys.manager} + save() + count(10) + }) + } +} + +// Observes the committed DB through the hub, not the transaction's app. +type monitorAlertHub struct { + stubHub + handle func(*core.Record, map[string]monitor.Result) error +} + +func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error { + return h.handle(record, results) +} + +func TestNetworkMonitorAlertsAfterCommit(t *testing.T) { + for _, realtime := range []bool{false, true} { + t.Run(fmt.Sprint(realtime), func(t *testing.T) { + sys, app := newTestSystemWithHub(t) + if realtime { + client := subscriptions.NewDefaultClient() + client.Subscribe("network_monitors/*") + app.SubscriptionsBroker().Register(client) + } + collection, err := app.FindCachedCollectionByNameOrId("network_monitors") + require.NoError(t, err) + record := core.NewRecord(collection) + record.Set("system", sys.Id) + require.NoError(t, app.SaveNoValidate(record)) + called := 0 + result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10} + sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error { + called++ + assert.Equal(t, sys.Id, systemRecord.Id) + assert.Equal(t, result, results[record.Id]) + saved, err := app.FindRecordById("network_monitors", record.Id) + require.NoError(t, err) + assert.Equal(t, 10.0, saved.GetFloat("loss1h")) + return nil + }} + data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}} + _, err = sys.createRecords(data) + require.NoError(t, err) + assert.Equal(t, 1, called) + // A transaction that fails after writing monitor stats must not notify. + _, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute() + require.NoError(t, err) + _, err = sys.createRecords(data) + require.Error(t, err) + assert.Equal(t, 1, called) + }) + } +} diff --git a/internal/hub/systems/network_monitor_sync_test.go b/internal/hub/systems/network_monitor_sync_test.go new file mode 100644 index 00000000..f9c7666e --- /dev/null +++ b/internal/hub/systems/network_monitor_sync_test.go @@ -0,0 +1,144 @@ +//go:build testing + +package systems + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/blang/semver" + "github.com/fxamacker/cbor/v2" + "github.com/henrygd/beszel/internal/common" + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/henrygd/beszel/internal/hub/ws" + "github.com/lxzan/gws" + "github.com/pocketbase/pocketbase/core" + "github.com/stretchr/testify/require" +) + +type monitorSyncClient struct { + gws.BuiltinEventHandler + requests chan common.HubRequest[monitor.SyncRequest] +} + +func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) { + defer message.Close() + var req common.HubRequest[monitor.SyncRequest] + if err := cbor.Unmarshal(message.Bytes(), &req); err != nil { + return + } + c.requests <- req + data, _ := cbor.Marshal(monitor.SyncResponse{}) + response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data}) + _ = conn.WriteMessage(gws.OpcodeBinary, response) +} + +// Avoid the production delayed disconnect notification; these tests explicitly +// remove each connection from the manager before reconnecting. +type monitorSyncServer struct{ ws.Handler } + +func (*monitorSyncServer) OnClose(*gws.Conn, error) {} + +func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) { + for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} { + t.Run(version, func(t *testing.T) { + // No transport: attempting to send any request would fail. + sys := &System{agentVersion: semver.MustParse(version)} + require.NoError(t, sys.SyncNetworkMonitors(nil)) + result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true) + require.NoError(t, err) + require.Nil(t, result) + require.NoError(t, sys.DeleteNetworkMonitor("test")) + }) + } +} + +func TestNetworkMonitorReconnectSync(t *testing.T) { + for _, change := range []string{"delete", "disable"} { + t.Run(change, func(t *testing.T) { + sys, app := newTestSystemWithHub(t) + record, err := app.FindRecordById("systems", sys.Id) + require.NoError(t, err) + // Suppress unrelated system-stat requests while exercising reconnects. + record.Set("status", paused) + require.NoError(t, app.SaveNoValidate(record)) + collection, err := app.FindCachedCollectionByNameOrId("network_monitors") + require.NoError(t, err) + probe := core.NewRecord(collection) + probe.Load(map[string]any{ + "system": sys.Id, "target": "localhost", "protocol": "tcp", + "port": 80, "interval": 60, "enabled": true, + }) + require.NoError(t, app.SaveNoValidate(probe)) + + sm := NewSystemManager(stubHub{app}) + t.Cleanup(func() { + sm.cancel() + _ = sm.RemoveSystem(sys.Id) + sm.smartFetchMap.StopCleaner() + sm.zfsFetchMap.StopCleaner() + }) + version := semver.MustParse("0.20.0") + connections := make(chan *ws.WsConn, 1) + upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r) + if err != nil { + t.Error(err) + return + } + wsConn := ws.NewWsConnection(conn, version) + conn.Session().Store("wsConn", wsConn) + connections <- wsConn + conn.ReadLoop() + })) + t.Cleanup(server.Close) + client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)} + connect := func() monitor.SyncRequest { + t.Helper() + conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.NetConn().Close() }) + go conn.ReadLoop() + select { + case wsConn := <-connections: + require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn)) + case <-time.After(3 * time.Second): + t.Fatal("websocket connection was not established") + } + select { + case req := <-client.requests: + require.Equal(t, common.SyncNetworkMonitors, req.Action) + require.Equal(t, monitor.SyncActionReplace, req.Data.Action) + return req.Data + case <-time.After(3 * time.Second): + t.Fatal("reconnected agent did not receive a monitor replacement") + return monitor.SyncRequest{} + } + } + + initial := connect() + require.Len(t, initial.Configs, 1) + require.Equal(t, probe.Id, initial.Configs[0].ID) + require.NoError(t, sm.RemoveSystem(sys.Id)) + if change == "delete" { + require.NoError(t, app.Delete(probe)) + } else { + probe.Set("enabled", false) + require.NoError(t, app.SaveNoValidate(probe)) + } + require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe") + }) + } +} + +func TestGetMonitorConfigsForSystemQueryError(t *testing.T) { + sys, app := newTestSystemWithHub(t) + _, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute() + require.NoError(t, err) + _, err = sys.manager.GetMonitorConfigsForSystem(sys.Id) + require.Error(t, err, "a failed query must not be treated as an empty monitor set") +} diff --git a/internal/hub/systems/network_monitors.go b/internal/hub/systems/network_monitors.go new file mode 100644 index 00000000..0fb44ed4 --- /dev/null +++ b/internal/hub/systems/network_monitors.go @@ -0,0 +1,58 @@ +package systems + +import ( + "context" + "time" + + "github.com/henrygd/beszel" + "github.com/henrygd/beszel/internal/common" + "github.com/henrygd/beszel/internal/entities/monitor" +) + +// SyncNetworkMonitors sends monitor configurations to the agent. +func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error { + _, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs}) + return err +} + +// UpsertNetworkMonitor sends a single monitor configuration change to the agent. +func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) { + resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{ + Action: monitor.SyncActionUpsert, + Config: config, + RunNow: runNow, + }) + if err != nil { + return nil, err + } + if resp.Result == (monitor.Result{}) { + return nil, nil + } + result := resp.Result + return &result, nil +} + +// DeleteNetworkMonitor removes a single monitor task from the agent. +func (sys *System) DeleteNetworkMonitor(id string) error { + _, err := sys.syncNetworkMonitors(monitor.SyncRequest{ + Action: monitor.SyncActionDelete, + Config: monitor.Config{ID: id}, + }) + return err +} + +func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) { + if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) { + return monitor.SyncResponse{}, nil + } + timeout := 5 * time.Second + if req.Action == monitor.SyncActionUpsert && req.RunNow { + // Allow the probe to finish, including a timeout result, while preserving + // the normal request budget for transport and response handling. + timeout += monitor.MaxProbeTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var result monitor.SyncResponse + return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result) +} diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.go index 2ba22169..6b0ca713 100644 --- a/internal/hub/systems/system.go +++ b/internal/hub/systems/system.go @@ -9,6 +9,7 @@ import ( "math/rand" "net" "strings" + "sync" "sync/atomic" "time" @@ -18,6 +19,7 @@ import ( "github.com/henrygd/beszel/internal/hub/ws" "github.com/henrygd/beszel/internal/entities/container" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/systemd" @@ -30,6 +32,8 @@ import ( "github.com/lxzan/gws" "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/security" + "github.com/pocketbase/pocketbase/tools/types" "golang.org/x/crypto/ssh" ) @@ -52,6 +56,10 @@ type System struct { smartInterval time.Duration // Interval for periodic SMART data updates zfsFetching atomic.Bool // True if ZFS pools are currently being fetched zfsInterval time.Duration // Interval for periodic ZFS detail data updates + // Serialize persistence from scheduled updates and resumes through commit. + recordsMu sync.Mutex + // Protected by recordsMu; realtime reads don't consume probes. + lastSavedMonitorProbe map[string]int64 } func (sm *SystemManager) NewSystem(systemId string) *System { @@ -211,11 +219,15 @@ func (sys *System) handlePaused() { // createRecords updates the system record and adds system_stats and container_stats records func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) { + sys.recordsMu.Lock() + defer sys.recordsMu.Unlock() + systemRecord, err := sys.getRecord(sys.manager.hub) if err != nil { return nil, err } hub := sys.manager.hub + savedMonitorProbes := make(map[string]int64) err = hub.RunInTransaction(func(txApp core.App) error { // add system_stats record systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats") @@ -266,6 +278,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error } } + if data.Monitors != nil { + if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil { + return err + } + } + if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil { return err } @@ -287,6 +305,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error return nil }) + // Publish only successful inserts after the entire transaction commits. + if err == nil && len(savedMonitorProbes) > 0 { + if sys.lastSavedMonitorProbe == nil { + sys.lastSavedMonitorProbe = savedMonitorProbes + } else { + for id, timestamp := range savedMonitorProbes { + sys.lastSavedMonitorProbe[id] = timestamp + } + } + } + // A non-nil report includes cached results for all remaining monitors. + if err == nil && data.Monitors != nil { + for id := range sys.lastSavedMonitorProbe { + if _, exists := data.Monitors[id]; !exists { + delete(sys.lastSavedMonitorProbe, id) + } + } + } + if err == nil { + if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil { + hub.Logger().Error("Error handling network monitor alerts", "err", alertErr) + } + } return systemRecord, err } @@ -337,7 +378,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s } suffix := fmt.Sprintf("%d", i) valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix)) - params["id"+suffix] = makeStableHashId(systemId, service.Name) + params["id"+suffix] = MakeStableHashId(systemId, service.Name) params["name"+suffix] = service.Name params["state"+suffix] = service.State params["sub"+suffix] = service.Sub @@ -363,6 +404,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s return err } +func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error { + if len(monitorResults) == 0 { + return nil + } + var err error + systemId := sys.Id + const monitorCollectionName = "network_monitors" + + // If realtime updates are active, we save via PocketBase records to trigger realtime events. + // Otherwise we can do a more efficient direct update via SQL + realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool { + return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId) + }) + + now := time.Now().UTC() + nowMilli := now.UnixMilli() + nowString := now.Format(types.DefaultDateLayout) + var db dbx.Builder + var updateQuery *dbx.Query + if !realtimeActive { + db = app.DB() + monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"} + setClauses := make([]string, len(monitorFields)) + for i, f := range monitorFields { + setClauses[i] = fmt.Sprintf("%s={:%s}", f, f) + } + queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", ")) + updateQuery = db.NewQuery(queryString) + } + + // update network_monitors records + for id, result := range monitorResults { + monitorData := map[string]any{ + "id": id, + "res": result.AvgResponse, + "resAvg1h": result.AvgResponse1h, + "resMin1h": result.MinResponse1h, + "resMax1h": result.MaxResponse1h, + "loss1h": result.PacketLoss1h, + "updated": nowString, + } + switch realtimeActive { + case true: + var record *core.Record + record, err = app.FindRecordById(monitorCollectionName, id) + if err == nil { + record.Load(monitorData) + err = app.SaveNoValidate(record) + } + default: + _, err = updateQuery.Bind(dbx.Params(monitorData)).Execute() + } + if err != nil { + app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err) + } + } + + // handle stats collection — one record per monitor + const statsCollectionName = "network_monitor_stats" + + var statsCollection *core.Collection + if realtimeActive { + statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName) + } + + for monitorId, result := range monitorResults { + // Compare identity, not ordering, so agent clock changes don't stall writes. + if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] { + continue + } + statsRecordData := map[string]any{ + "system": systemId, + "monitor": monitorId, + "type": "1m", + "created": nowMilli, + "res_min": result.MinResponse, + "res_max": result.MaxResponse, + "total_count": result.TotalCount, + "success_count": result.SuccessCount, + "res_sum": result.ResponseSum, + } + switch realtimeActive { + case true: + record := core.NewRecord(statsCollection) + record.Load(statsRecordData) + err = app.SaveNoValidate(record) + default: + statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet) + _, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute() + } + if err != nil { + app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err) + } else { + savedProbes[monitorId] = result.LastProbeAt + } + } + + return nil +} + // createContainerRecords creates container records func createContainerRecords(app core.App, data []*container.Stats, systemId string) error { if len(data) == 0 { @@ -622,7 +763,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) { return &result, err } -func makeStableHashId(strings ...string) string { +func MakeStableHashId(strings ...string) string { hash := fnv.New32a() for _, str := range strings { hash.Write([]byte(str)) diff --git a/internal/hub/systems/system_manager.go b/internal/hub/systems/system_manager.go index c1a4cde4..db279609 100644 --- a/internal/hub/systems/system_manager.go +++ b/internal/hub/systems/system_manager.go @@ -9,6 +9,7 @@ import ( "github.com/henrygd/beszel/internal/hub/ws" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/hub/expirymap" @@ -17,6 +18,7 @@ import ( "github.com/henrygd/beszel" "github.com/blang/semver" + "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/tools/store" "golang.org/x/crypto/ssh" @@ -62,6 +64,7 @@ type hubLike interface { core.App GetSSHKey(dataDir string) (ssh.Signer, error) HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error + HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error HandleStatusAlerts(status string, systemRecord *core.Record) error HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error CancelPendingStatusAlerts(systemID string) @@ -350,6 +353,20 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver if err := sm.AddRecord(systemRecord, system); err != nil { return err } + + // Sync network monitors to the newly connected agent + go func() { + configs, err := sm.GetMonitorConfigsForSystem(systemId) + if err != nil { + sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err) + return + } + // An empty set must also replace any probes retained across a disconnect. + if err := system.SyncNetworkMonitors(configs); err != nil { + sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err) + } + }() + return nil } @@ -362,6 +379,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) { } } +// GetMonitorConfigsForSystem returns all enabled monitor configs for a system. +func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) { + var configs []monitor.Config + err := sm.hub.DB(). + NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true"). + Bind(dbx.Params{"system": systemID}). + All(&configs) + return configs, err +} + // resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh // agent reconnect retries ZFS discovery immediately after configuration changes. func (sm *SystemManager) resetFailedZfsFetchState(systemID string) { @@ -397,11 +424,12 @@ func (sm *SystemManager) createSSHClientConfig() error { // deactivateAlerts finds all triggered alerts for a system and sets them to inactive. // This is called when a system is paused or goes offline to prevent continued alerts. +// Monitor incidents remain open: a missing observation does not establish recovery. func deactivateAlerts(app core.App, systemID string) error { // Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API // _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute() - alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0) + alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0) if err != nil { return err } diff --git a/internal/hub/systems/system_realtime.go b/internal/hub/systems/system_realtime.go index 0fb6b6f9..f290eda6 100644 --- a/internal/hub/systems/system_realtime.go +++ b/internal/hub/systems/system_realtime.go @@ -6,6 +6,8 @@ import ( "time" "github.com/henrygd/beszel/internal/common" + "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/hub/utils" "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/apis" @@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() { if err != nil { return } - bytes, err := json.Marshal(data) + bytes, err := marshalRealtimeData(data) if err == nil { notify(sm.hub, system, fetch.subscription, bytes) } @@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) { } } +// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting +// the per-monitor results into the derived metric fields the frontend charts expect. +func marshalRealtimeData(data *system.CombinedData) ([]byte, error) { + if len(data.Monitors) == 0 { + return json.Marshal(data) + } + monitorStats := make(map[string]monitor.Stats, len(data.Monitors)) + for id, result := range data.Monitors { + monitorStats[id] = monitor.Stats{}.FromResult(result) + } + return json.Marshal(struct { + *system.CombinedData + Monitors map[string]monitor.Stats `json:"Monitors"` + }{data, monitorStats}) +} + // notify broadcasts realtime data to all clients subscribed to a specific subscription. // Custom topics bypass collection rules, so check current access for every // recipient, including clients whose authentication or membership was revoked. diff --git a/internal/hub/systems/system_smart.go b/internal/hub/systems/system_smart.go index 68b750fe..9a6db8de 100644 --- a/internal/hub/systems/system_smart.go +++ b/internal/hub/systems/system_smart.go @@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple currentIDs := make(map[string]struct{}, len(smartData)) for deviceKey := range smartData { - currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{} + currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{} } err = hub.RunInTransaction(func(txApp core.App) error { @@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple } func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error { - recordID := makeStableHashId(sys.Id, deviceKey) + recordID := MakeStableHashId(sys.Id, deviceKey) record, err := app.FindRecordById(collection, recordID) if err != nil { diff --git a/internal/hub/systems/system_smart_test.go b/internal/hub/systems/system_smart_test.go index aa98633d..460a38f3 100644 --- a/internal/hub/systems/system_smart_test.go +++ b/internal/hub/systems/system_smart_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/smart" esystem "github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/hub/expirymap" @@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil } func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error { return nil } -func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil } +func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil } +func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil } func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error { return nil } @@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) { }, false)) assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2) - recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA")) + recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA")) require.NoError(t, err) assert.EqualValues(t, 42, recordA.GetInt("temp")) } diff --git a/internal/hub/systems/system_systemd_test.go b/internal/hub/systems/system_systemd_test.go index c2d890d5..d5b103e0 100644 --- a/internal/hub/systems/system_systemd_test.go +++ b/internal/hub/systems/system_systemd_test.go @@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) { serviceName := "nginx.service" // Call multiple times and ensure same result - id1 := makeStableHashId(systemId, serviceName) - id2 := makeStableHashId(systemId, serviceName) - id3 := makeStableHashId(systemId, serviceName) + id1 := MakeStableHashId(systemId, serviceName) + id2 := MakeStableHashId(systemId, serviceName) + id3 := MakeStableHashId(systemId, serviceName) assert.Equal(t, id1, id2) assert.Equal(t, id2, id3) @@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) { serviceName1 := "nginx.service" serviceName2 := "apache.service" - id1 := makeStableHashId(systemId1, serviceName1) - id2 := makeStableHashId(systemId2, serviceName1) - id3 := makeStableHashId(systemId1, serviceName2) - id4 := makeStableHashId(systemId2, serviceName2) + id1 := MakeStableHashId(systemId1, serviceName1) + id2 := MakeStableHashId(systemId2, serviceName1) + id3 := MakeStableHashId(systemId1, serviceName2) + id4 := MakeStableHashId(systemId2, serviceName2) // All IDs should be different assert.NotEqual(t, id1, id2) @@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) { } for _, tc := range testCases { - id := makeStableHashId(tc.systemId, tc.serviceName) + id := MakeStableHashId(tc.systemId, tc.serviceName) // FNV-32 produces 8 hex characters assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName) } }) t.Run("hexadecimal output", func(t *testing.T) { - id := makeStableHashId("test-system", "test-service") + id := MakeStableHashId("test-system", "test-service") assert.NotEmpty(t, id) // Should only contain hexadecimal characters diff --git a/internal/hub/systems/system_zfs.go b/internal/hub/systems/system_zfs.go index 30f57502..93465711 100644 --- a/internal/hub/systems/system_zfs.go +++ b/internal/hub/systems/system_zfs.go @@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error { } func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error { - recordID := makeStableHashId(sys.Id, pool.Name) + recordID := MakeStableHashId(sys.Id, pool.Name) record, err := app.FindRecordById(collection, recordID) if err != nil { @@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP if pool == nil { continue } - recordID := makeStableHashId(sys.Id, name) + recordID := MakeStableHashId(sys.Id, name) record, err := app.FindRecordById(collection, recordID) if err != nil { if !errors.Is(err, sql.ErrNoRows) { diff --git a/internal/hub/systems/system_zfs_test.go b/internal/hub/systems/system_zfs_test.go index b61d687a..b035dd80 100644 --- a/internal/hub/systems/system_zfs_test.go +++ b/internal/hub/systems/system_zfs_test.go @@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) { {Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10}, }} require.NoError(t, sys.saveZfsPools(initial)) - failedID := makeStableHashId(sys.Id, failedKey) + failedID := MakeStableHashId(sys.Id, failedKey) before, err := app.FindRecordById("zfs_pools", failedID) require.NoError(t, err) partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{ {Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99}, }} assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData) - fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey)) + fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey)) require.NoError(t, err) assert.EqualValues(t, 20, fresh.GetInt("alloc")) cached, err := app.FindRecordById("zfs_pools", failedID) @@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) { require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{ "tank": {Total: 100, Used: 25, Health: "ONLINE"}, })) - record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank")) + record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank")) require.NoError(t, err) firstUpdated := record.GetDateTime("updated") assert.Equal(t, "ONLINE", record.GetString("health")) @@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) { func TestZfsRawCapacityPersistence(t *testing.T) { sys, app := newTestSystemWithHub(t) require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}})) - record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs")) + record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs")) require.NoError(t, err) require.True(t, record.GetBool("raw")) require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}})) @@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) { key: {DisplayName: "tank", Health: "ONLINE"}, "tank": {Health: "ONLINE"}, })) - id := makeStableHashId(sys.Id, key) + id := MakeStableHashId(sys.Id, key) record, err := app.FindRecordById("zfs_pools", id) require.NoError(t, err) assert.Equal(t, "tank", record.GetString("display_name")) @@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) { record, err = app.FindRecordById("zfs_pools", id) require.NoError(t, err) assert.Equal(t, "detail name", record.GetString("display_name")) - _, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank")) + _, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank")) require.NoError(t, err) } diff --git a/internal/hub/utils/utils.go b/internal/hub/utils/utils.go index 43838868..0cfc1fa9 100644 --- a/internal/hub/utils/utils.go +++ b/internal/hub/utils/utils.go @@ -1,7 +1,11 @@ // Package utils provides utility functions for the hub. package utils -import "os" +import ( + "os" + + "github.com/pocketbase/pocketbase/core" +) // GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key. func GetEnv(key string) (value string, exists bool) { @@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) { } return os.LookupEnv(key) } + +// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection. +func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool { + broker := app.SubscriptionsBroker() + if broker.TotalClients() == 0 { + return false + } + for _, client := range broker.Clients() { + subs := client.Subscriptions(collectionName) + if len(subs) > 0 { + if validateFn == nil { + return true + } + for k := range subs { + filter := subs[k].Query["filter"] + if validateFn(filter) { + return true + } + } + } + } + return false +} diff --git a/internal/migrations/0_collections_snapshot_0_19_0.go b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.go similarity index 81% rename from internal/migrations/0_collections_snapshot_0_19_0.go rename to internal/migrations/0_collections_snapshot_0_20_0_dev_5.go index 73b89d2b..c11985a5 100644 --- a/internal/migrations/0_collections_snapshot_0_19_0.go +++ b/internal/migrations/0_collections_snapshot_0_20_0_dev_5.go @@ -83,7 +83,8 @@ func init() { "ContainerHealth", "SystemdFailed", "CPUIOWait", - "CPUSteal" + "CPUSteal", + "NetworkMonitorLoss" ] }, { @@ -119,6 +120,16 @@ func init() { "system": false, "type": "bool" }, + { + "hidden": true, + "id": "json4000656575", + "maxSize": 0, + "name": "state", + "presentable": false, + "required": false, + "system": false, + "type": "json" + }, { "hidden": true, "id": "date1302749137", @@ -152,6 +163,7 @@ func init() { } ], "indexes": [ + "CREATE INDEX idx_alerts_system_name ON alerts (system, name)", "CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)" ], "system": false @@ -234,6 +246,20 @@ func init() { "system": false, "type": "text" }, + { + "autogeneratePattern": "", + "hidden": false, + "id": "text3888135399", + "max": 0, + "min": 0, + "name": "monitor_name", + "pattern": "", + "presentable": false, + "primaryKey": false, + "required": false, + "system": false, + "type": "text" + }, { "hidden": false, "id": "number494360628", @@ -1721,6 +1747,7 @@ func init() { "fields": [ { "autogeneratePattern": "[a-z0-9]{15}", + "help": "", "hidden": false, "id": "text3208210256", "max": 15, @@ -1736,6 +1763,7 @@ func init() { { "cascadeDelete": true, "collectionId": "2hz5ncl8tizk5nx", + "help": "", "hidden": false, "id": "relation1204987316", "maxSelect": 1, @@ -1748,6 +1776,7 @@ func init() { }, { "autogeneratePattern": "", + "help": "", "hidden": false, "id": "text7739291048", "max": 0, @@ -1762,6 +1791,7 @@ func init() { }, { "autogeneratePattern": "", + "help": "", "hidden": false, "id": "text5528164482", "max": 0, @@ -1775,6 +1805,7 @@ func init() { "type": "text" }, { + "help": "", "hidden": false, "id": "number8862034195", "max": null, @@ -1787,6 +1818,7 @@ func init() { "type": "number" }, { + "help": "", "hidden": false, "id": "number4418907321", "max": null, @@ -1799,6 +1831,7 @@ func init() { "type": "number" }, { + "help": "", "hidden": false, "id": "number2904183765", "max": null, @@ -1811,6 +1844,7 @@ func init() { "type": "number" }, { + "help": "", "hidden": false, "id": "json4466109723", "maxSize": 0, @@ -1821,6 +1855,7 @@ func init() { "type": "json" }, { + "help": "", "hidden": false, "id": "json9012873456", "maxSize": 0, @@ -1831,6 +1866,7 @@ func init() { "type": "json" }, { + "help": "", "hidden": false, "id": "json7182045639", "maxSize": 0, @@ -1841,6 +1877,7 @@ func init() { "type": "json" }, { + "help": "", "hidden": false, "id": "date9274163058", "max": "", @@ -1860,19 +1897,401 @@ func init() { "presentable": false, "system": false, "type": "autodate" + }, + { + "autogeneratePattern": "", + "help": "", + "hidden": false, + "id": "text3578368839", + "max": 0, + "min": 0, + "name": "display_name", + "pattern": "", + "presentable": false, + "primaryKey": false, + "required": false, + "system": false, + "type": "text" + }, + { + "help": "", + "hidden": false, + "id": "bool447994709", + "name": "raw", + "presentable": false, + "required": false, + "system": false, + "type": "bool" } ], "id": "pbc_8441057391", "indexes": [ "CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)" ], - "listRule": null, + "listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id", "name": "zfs_pools", "system": false, "type": "base", "updateRule": null, + "viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id" + }, + { + "createRule": null, + "deleteRule": null, + "fields": [ + { + "autogeneratePattern": "[a-z0-9]{10}", + "help": "", + "hidden": false, + "id": "text3208210256", + "max": 10, + "min": 6, + "name": "id", + "pattern": "^[a-z0-9]+$", + "presentable": false, + "primaryKey": true, + "required": true, + "system": true, + "type": "text" + }, + { + "cascadeDelete": true, + "collectionId": "2hz5ncl8tizk5nx", + "help": "", + "hidden": false, + "id": "nm_system", + "maxSelect": 1, + "minSelect": 0, + "name": "system", + "presentable": false, + "required": true, + "system": false, + "type": "relation" + }, + { + "autogeneratePattern": "", + "help": "", + "hidden": false, + "id": "nm_target", + "max": 500, + "min": 1, + "name": "target", + "pattern": "", + "presentable": false, + "primaryKey": false, + "required": true, + "system": false, + "type": "text" + }, + { + "help": "", + "hidden": false, + "id": "nm_protocol", + "maxSelect": 1, + "name": "protocol", + "presentable": false, + "required": true, + "system": false, + "type": "select", + "values": [ + "icmp", + "tcp", + "http", + "dns" + ] + }, + { + "help": "", + "hidden": false, + "id": "nm_port", + "max": 65535, + "min": 0, + "name": "port", + "onlyInt": true, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "nm_interval", + "max": 3600, + "min": 1, + "name": "interval", + "onlyInt": true, + "presentable": false, + "required": true, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "number926446584", + "max": null, + "min": null, + "name": "res", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "number1006954605", + "max": null, + "min": null, + "name": "resAvg1h", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "number4267669802", + "max": null, + "min": null, + "name": "resMin1h", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "number591433223", + "max": null, + "min": null, + "name": "resMax1h", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "number3726709001", + "max": null, + "min": null, + "name": "loss1h", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "nm_enabled", + "name": "enabled", + "presentable": false, + "required": false, + "system": false, + "type": "bool" + }, + { + "hidden": false, + "id": "autodate2990389176", + "name": "created", + "onCreate": true, + "onUpdate": false, + "presentable": false, + "system": false, + "type": "autodate" + }, + { + "help": "", + "hidden": false, + "id": "date3332085495", + "max": "", + "min": "", + "name": "updated", + "presentable": false, + "required": false, + "system": false, + "type": "date" + } + ], + "id": "nm_monitors_001", + "indexes": [ + "CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)" + ], + "listRule": null, + "name": "network_monitors", + "system": false, + "type": "base", + "updateRule": null, "viewRule": null - } + }, + { + "createRule": null, + "deleteRule": null, + "fields": [ + { + "autogeneratePattern": "[a-z0-9]{10}", + "help": "", + "hidden": false, + "id": "text3208210256", + "max": 10, + "min": 10, + "name": "id", + "pattern": "^[a-z0-9]+$", + "presentable": false, + "primaryKey": true, + "required": true, + "system": true, + "type": "text" + }, + { + "cascadeDelete": true, + "collectionId": "2hz5ncl8tizk5nx", + "help": "", + "hidden": false, + "id": "nms_system", + "maxSelect": 1, + "minSelect": 0, + "name": "system", + "presentable": false, + "required": true, + "system": false, + "type": "relation" + }, + { + "cascadeDelete": true, + "collectionId": "nm_monitors_001", + "help": "", + "hidden": false, + "id": "nms_monitor", + "maxSelect": 1, + "minSelect": 0, + "name": "monitor", + "presentable": false, + "required": false, + "system": false, + "type": "relation" + }, + { + "help": "Number of probe attempts", + "hidden": false, + "id": "nms_total_count", + "max": null, + "min": 0, + "name": "total_count", + "onlyInt": true, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "Number of successful probe attempts", + "hidden": false, + "id": "nms_success_count", + "max": null, + "min": 0, + "name": "success_count", + "onlyInt": true, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "Sum of successful response times in microseconds", + "hidden": false, + "id": "nms_res_sum", + "max": null, + "min": 0, + "name": "res_sum", + "onlyInt": true, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "Response time in microseconds", + "hidden": false, + "id": "nms_res_min", + "max": null, + "min": 0, + "name": "res_min", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "Response time in microseconds", + "hidden": false, + "id": "nms_res_max", + "max": null, + "min": 0, + "name": "res_max", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + }, + { + "help": "", + "hidden": false, + "id": "nms_type", + "maxSelect": 1, + "name": "type", + "presentable": false, + "required": true, + "system": false, + "type": "select", + "values": [ + "1m", + "10m", + "20m", + "120m", + "480m" + ] + }, + { + "help": "", + "hidden": false, + "id": "number2990389176", + "max": null, + "min": null, + "name": "created", + "onlyInt": false, + "presentable": false, + "required": false, + "system": false, + "type": "number" + } + ], + "id": "nm_stats_001", + "indexes": [ + "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)", + "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)", + "CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)" + ], + "listRule": null, + "name": "network_monitor_stats", + "system": false, + "type": "base", + "updateRule": null, + "viewRule": null + } ]` err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false) diff --git a/internal/records/monitor_averaging_test.go b/internal/records/monitor_averaging_test.go new file mode 100644 index 00000000..da00330f --- /dev/null +++ b/internal/records/monitor_averaging_test.go @@ -0,0 +1,226 @@ +//go:build testing + +package records_test + +import ( + "testing" + "time" + + monitorEntity "github.com/henrygd/beszel/internal/entities/monitor" + "github.com/henrygd/beszel/internal/records" + "github.com/henrygd/beszel/internal/tests" + + "github.com/pocketbase/dbx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAverageMonitorStats(t *testing.T) { + hub, err := tests.NewTestHub(t.TempDir()) + require.NoError(t, err) + defer hub.Cleanup() + + collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats") + require.NoError(t, err) + assert.Nil(t, collection.Fields.GetByName("res_avg")) + assert.Nil(t, collection.Fields.GetByName("loss")) + + rm := records.NewRecordManager(hub) + user, err := tests.CreateUser(hub, "monitor-avg@example.com", "testtesttest") + require.NoError(t, err) + sys, err := tests.CreateRecord(hub, "systems", map[string]any{ + "name": "monitor-avg-system", + "host": "localhost", + "port": "45876", + "status": "up", + "users": []string{user.Id}, + }) + require.NoError(t, err) + monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": sys.Id, + "name": "cloudflare", + "target": "1.1.1.1", + "protocol": "icmp", + "interval": 30, + "enabled": true, + }) + require.NoError(t, err) + + created := time.Now().UnixMilli() + // Unequal probe counts must weight both latency and loss. + recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{ + "system": sys.Id, + "monitor": monitor.Id, + "type": "1m", + "created": created, + "res_min": 5, + "res_max": 20, + "total_count": 6, "success_count": 6, "res_sum": 60, + }) + require.NoError(t, err) + recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{ + "system": sys.Id, + "monitor": monitor.Id, + "type": "1m", + "created": created, + "res_min": 10, + "res_max": 60, + "total_count": 1, "success_count": 1, "res_sum": 22, + }) + require.NoError(t, err) + + result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1) + require.NoError(t, err) + assert.Equal(t, 2, count) + assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result) + + for _, tc := range []struct { + name, monitor, recordType string + after int64 + }{ + {"other monitor", "missing", "1m", created - 1}, + {"other type", monitor.Id, "10m", created - 1}, + {"exclusive cutoff", monitor.Id, "1m", created}, + } { + t.Run(tc.name, func(t *testing.T) { + stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after) + require.NoError(t, err) + assert.Zero(t, count) + assert.Equal(t, monitorEntity.Stats{}, stats) + }) + } + + // A failure-only bucket counts toward loss but must not lower latency. + recordB.Set("res_min", 0) + recordB.Set("res_max", 0) + recordB.Set("success_count", 0) + recordB.Set("res_sum", 0) + require.NoError(t, hub.Save(recordB)) + result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1) + require.NoError(t, err) + assert.Equal(t, 2, count) + assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result) + // Sparse monitor records must propagate through every rollup level. + rm.CreateLongerRecords() + for _, recordType := range []string{"10m", "20m", "120m", "480m"} { + rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType}) + require.NoError(t, err) + require.Len(t, rollups, 1, recordType) + assert.Equal(t, 5.0, rollups[0].GetFloat("res_min")) + assert.Equal(t, 20.0, rollups[0].GetFloat("res_max")) + assert.Equal(t, 7, rollups[0].GetInt("total_count")) + assert.Equal(t, 6, rollups[0].GetInt("success_count")) + assert.Equal(t, 60, rollups[0].GetInt("res_sum")) + // A sibling with a different number of probes must retain its actual + // weight when the next tier combines their underlying counts. + _, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{ + "system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created, + "res_min": 100, "res_max": 100, + "total_count": 3, "success_count": 1, "res_sum": 100, + }) + require.NoError(t, err) + merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1) + require.NoError(t, err) + assert.Equal(t, 2, count) + assert.Equal(t, monitorEntity.Stats{ + ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30, + TotalCount: 10, SuccessCount: 7, ResponseSum: 160, + }, merged) + } + // All failures produce zero latency, while a genuine zero-microsecond + // success remains a valid minimum (it must not be filtered out as a sentinel). + recordA.Set("success_count", 0) + recordA.Set("res_sum", 0) + require.NoError(t, hub.Save(recordA)) + result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1) + require.NoError(t, err) + assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result) + recordA.Set("success_count", 1) + recordA.Set("res_min", 0) + recordA.Set("res_max", 0) + require.NoError(t, hub.Save(recordA)) + result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1) + require.NoError(t, err) + assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result) +} + +func TestSparseMonitorRollups(t *testing.T) { + for _, tc := range []struct { + name string + interval int + samples int + disabled bool + }{ + {"five minute interval", 300, 2, false}, + {"ten minute interval", 600, 1, false}, + {"fifteen minute interval", 900, 1, false}, + {"empty window", 900, 0, false}, + {"disabled with pending history", 300, 2, true}, + {"disabled empty window", 900, 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + hub, err := tests.NewTestHub(t.TempDir()) + require.NoError(t, err) + defer hub.Cleanup() + user, err := tests.CreateUser(hub, "sparse-monitor@example.com", "testtesttest") + require.NoError(t, err) + sys, err := tests.CreateRecord(hub, "systems", map[string]any{ + "name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up", + "users": []string{user.Id}, + }) + require.NoError(t, err) + monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp", + "interval": tc.interval, "enabled": true, + }) + require.NoError(t, err) + now := time.Now() + for i := range tc.samples { + _, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{ + "system": sys.Id, "monitor": monitor.Id, "type": "1m", + "created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(), + "res_min": 8, "res_max": 20, + "total_count": 4, "success_count": 3, "res_sum": 36, + }) + require.NoError(t, err) + } + // Other collections must still reject fewer than nine minute records. + for _, collection := range []string{"system_stats", "container_stats"} { + stats := `{"cpu":10}` + if collection == "container_stats" { + stats = `[{"name":"test","cpu":10}]` + } + for range 8 { + _, err := tests.CreateRecord(hub, collection, map[string]any{ + "system": sys.Id, "type": "1m", "stats": stats, + }) + require.NoError(t, err) + } + } + if tc.disabled { + monitor.Set("enabled", false) + require.NoError(t, hub.Save(monitor)) + } + records.NewRecordManager(hub).CreateLongerRecords() + for _, recordType := range []string{"10m", "20m", "120m", "480m"} { + rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType}) + require.NoError(t, err) + if tc.samples == 0 { + assert.Empty(t, rollups, recordType) + } else { + require.Len(t, rollups, 1, recordType) + assert.Equal(t, 8.0, rollups[0].GetFloat("res_min")) + assert.Equal(t, 20.0, rollups[0].GetFloat("res_max")) + assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count")) + assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count")) + assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum")) + } + for _, collection := range []string{"system_stats", "container_stats"} { + count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType}) + require.NoError(t, err) + assert.Zero(t, count, "%s %s", collection, recordType) + } + } + }) + } +} diff --git a/internal/records/records.go b/internal/records/records.go index 4ef1b191..7c4bf281 100644 --- a/internal/records/records.go +++ b/internal/records/records.go @@ -3,15 +3,16 @@ package records import ( "encoding/json" - "log/slog" "math" "time" "github.com/henrygd/beszel/internal/entities/container" + "github.com/henrygd/beszel/internal/entities/monitor" "github.com/henrygd/beszel/internal/entities/system" "github.com/pocketbase/dbx" "github.com/pocketbase/pocketbase/core" + "github.com/pocketbase/pocketbase/tools/types" ) type RecordManager struct { @@ -39,7 +40,7 @@ type StatsRecord struct { // Create longer records by averaging shorter records func (rm *RecordManager) CreateLongerRecords() { - // start := time.Now() + now := time.Now().UTC() longerRecordData := []LongerRecordData{ { shorterType: "1m", @@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() { } // wrap the operations in a transaction // Pocketbase cron does not handle errors, log them here. - rm.app.RunInTransaction(func(txApp core.App) error { + err := rm.app.RunInTransaction(func(txApp core.App) error { var err error + collections := [2]*core.Collection{} collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats") if err != nil { - slog.Error("Error finding cached collection using system stats:", "err", err) return err } collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats") if err != nil { - slog.Error("Error finding cached collection using container stats:", "err", err) + return err + } + monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats") + if err != nil { return err } var systems RecordIds db := txApp.DB() - db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems) + if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil { + return err + } // loop through all active systems, time periods, and collections for _, system := range systems { @@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() { recordData := longerRecordData[i] // log.Println("processing longer record type", recordData.longerType) // add one minute padding for longer records because they are created slightly later than the job start time - longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute) + longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute) // shorter records are created independently of longer records, so we shouldn't need to add padding - shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration) - // loop through both collections + shorterRecordPeriod := now.Add(recordData.longerTimeDuration) for _, collection := range collections { // check creation time of last longer record if not 10m, since 10m is created every run if recordData.longerType != "10m" { - count, err := txApp.CountRecords( - collection.Id, - dbx.NewExp( - "system = {:system} AND type = {:type} AND created > {:created}", - dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod}, - ), - ) + count, err := txApp.CountRecords(collection.Id, dbx.NewExp( + "system = {:system} AND type = {:type} AND created > {:created}", + dbx.Params{ + "type": recordData.longerType, + "system": system.Id, + "created": longerRecordPeriod.Format(types.DefaultDateLayout), + }, + )) + if err != nil { + return err + } // continue if longer record exists - if err != nil || count > 0 { + if count > 0 { continue } } // get shorter records from the past x minutes var recordIds RecordIds - err := txApp.DB(). + params := dbx.Params{ + "type": recordData.shorterType, + "system": system.Id, + "created": shorterRecordPeriod.Format(types.DefaultDateLayout), + } + + err := db. Select("id"). From(collection.Name). - AndWhere(dbx.NewExp( + Where(dbx.NewExp( "system={:system} AND type={:type} AND created > {:created}", - dbx.Params{ - "type": recordData.shorterType, - "system": system.Id, - "created": shorterRecordPeriod, - }, + params, )). OrderBy("created"). All(&recordIds) + if err != nil { + return err + } // continue if not enough shorter records - if err != nil || len(recordIds) < recordData.minShorterRecords { + if len(recordIds) < recordData.minShorterRecords { continue } // average the shorter records and create longer record @@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() { case "system_stats": longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds)) case "container_stats": - longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds)) } if err := txApp.SaveNoValidate(longerRecord); err != nil { - slog.Error("failed to save longer record", "err", err) + txApp.Logger().Error("failed to save longer record", "err", err) } } } } + // network_monitor_stats is aggregated per monitor (not per system) + var monitors []struct { + Id string `db:"id"` + System string `db:"system"` + } + // Disabled monitors still have history that must advance through retention tiers. + if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil { + return err + } + + for _, monitorRec := range monitors { + for i := range longerRecordData { + recordData := longerRecordData[i] + longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute) + shorterRecordPeriod := now.Add(recordData.longerTimeDuration) + + if recordData.longerType != "10m" { + count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp( + "monitor={:monitor} AND type={:type} AND created>{:created}", + dbx.Params{ + "monitor": monitorRec.Id, + "type": recordData.longerType, + "created": longerRecordPeriod.UnixMilli(), + }, + )) + if err != nil { + return err + } + if count > 0 { + continue + } + } + + stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli()) + if err != nil { + txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err) + continue + } + // Monitor intervals can exceed the aggregation window, so average + // any available records at every level and skip only empty windows. + if count == 0 { + continue + } + + longerRecord := core.NewRecord(monitorStatsColl) + longerRecord.Set("system", monitorRec.System) + longerRecord.Set("monitor", monitorRec.Id) + longerRecord.Set("type", recordData.longerType) + longerRecord.Set("created", now.UnixMilli()) + longerRecord.Set("res_min", stats.ResMin) + longerRecord.Set("res_max", stats.ResMax) + longerRecord.Set("total_count", stats.TotalCount) + longerRecord.Set("success_count", stats.SuccessCount) + longerRecord.Set("res_sum", stats.ResponseSum) + if err := txApp.SaveNoValidate(longerRecord); err != nil { + txApp.Logger().Error("failed to save monitor longer record", "err", err) + } + } + } + return nil }) + if err != nil { + rm.app.Logger().Error("failed to create longer records", "err", err) + } +} - // log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds()) +func getCreatedTimeField(collectionName string, period time.Time) any { + // network_monitor_stats stores created as unix timestamp in ms, not as a date string + if collectionName == "network_monitor_stats" { + return period.UnixMilli() + } + return period.Format(types.DefaultDateLayout) } // Calculate the average stats of a list of system_stats records without reflect @@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats { return result } +// AverageMonitorStats merges probe counts and response sums, preserving their +// weights through every retention tier. Failed probes do not contribute latency. +func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) { + var result struct { + monitor.Stats + Count int `db:"count"` + } + err := db.Select( + "COUNT(*) AS count", + "COALESCE(SUM(total_count), 0) AS total_count", + "COALESCE(SUM(success_count), 0) AS success_count", + "COALESCE(SUM(res_sum), 0) AS res_sum", + "COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min", + "COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max", + ).From("network_monitor_stats").Where(dbx.NewExp( + "monitor={:monitor} AND type={:type} AND created>{:created}", + dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter}, + )).One(&result) + if err != nil { + return monitor.Stats{}, 0, err + } + if result.SuccessCount > 0 { + result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount)) + } + if result.TotalCount > 0 { + result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount)) + } + return result.Stats, result.Count, nil +} + /* Round float to two decimals */ func twoDecimals(value float64) float64 { return math.Round(value*100) / 100 diff --git a/internal/records/records_deletion.go b/internal/records/records_deletion.go index 6632ded9..94bd0aae 100644 --- a/internal/records/records_deletion.go +++ b/internal/records/records_deletion.go @@ -3,7 +3,6 @@ package records import ( "fmt" "log/slog" - "strings" "time" "github.com/pocketbase/dbx" @@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int) // Deletes system_stats records older than what is displayed in the UI func deleteOldSystemStats(app core.App) error { // Collections to process - collections := [2]string{"system_stats", "container_stats"} + collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"} // Record types and their retention periods type RecordDeletionData struct { @@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error { } now := time.Now().UTC() + db := app.DB() for _, collection := range collections { - // Build the WHERE clause - var conditionParts []string - var params dbx.Params = make(map[string]any) - for i := range recordData { - rd := recordData[i] - // Create parameterized condition for this record type - dateParam := fmt.Sprintf("date%d", i) - conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam)) - params[dateParam] = now.Add(-rd.retention) - } - // Combine conditions with OR - conditionStr := strings.Join(conditionParts, " OR ") - // Construct and execute the full raw query - rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr) - if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil { - return fmt.Errorf("failed to delete from %s: %v", collection, err) + query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}")) + for _, rd := range recordData { + if _, err := query.Bind(dbx.Params{ + "type": rd.recordType, + "created": getCreatedTimeField(collection, now.Add(-rd.retention)), + }).Execute(); err != nil { + return fmt.Errorf("failed to delete from %s: %v", collection, err) + } } } return nil diff --git a/internal/records/records_rollup_test.go b/internal/records/records_rollup_test.go new file mode 100644 index 00000000..16f0076a --- /dev/null +++ b/internal/records/records_rollup_test.go @@ -0,0 +1,86 @@ +//go:build testing + +package records_test + +import ( + "testing" + "time" + + "github.com/henrygd/beszel/internal/records" + "github.com/henrygd/beszel/internal/tests" + "github.com/pocketbase/dbx" + "github.com/pocketbase/pocketbase/tools/types" + "github.com/stretchr/testify/require" +) + +func TestLongerRecordsPreventDuplicates(t *testing.T) { + for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} { + for _, tier := range []struct { + shorter, longer string + count int + }{ + {"10m", "20m", 2}, + {"20m", "120m", 6}, + {"120m", "480m", 4}, + } { + t.Run(collection+"/"+tier.longer, func(t *testing.T) { + hub, err := tests.NewTestHub(t.TempDir()) + require.NoError(t, err) + defer hub.Cleanup() + + user, err := tests.CreateUser(hub, "rollup@example.com", "testtesttest") + require.NoError(t, err) + sys, err := tests.CreateRecord(hub, "systems", map[string]any{ + "name": "rollup-system", "host": "localhost", "port": "45876", + "status": "up", "users": []string{user.Id}, + }) + require.NoError(t, err) + + created := time.Now().UTC().Add(-time.Minute) + data := map[string]any{ + "system": sys.Id, "type": tier.shorter, + "created": created.Format(types.DefaultDateLayout), + } + filter := dbx.HashExp{"system": sys.Id, "type": tier.longer} + switch collection { + case "system_stats": + data["stats"] = `{"cpu":10}` + case "container_stats": + data["stats"] = `[{"name":"test","cpu":10}]` + case "network_monitor_stats": + monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{ + "system": sys.Id, "target": "1.1.1.1", "protocol": "icmp", + "interval": 30, "enabled": true, + }) + require.NoError(t, err) + data["monitor"] = monitor.Id + data["created"] = created.UnixMilli() + data["total_count"] = 1 + data["success_count"] = 1 + data["res_sum"] = 10 + data["res_min"] = 10 + data["res_max"] = 10 + filter["monitor"] = monitor.Id + } + for range tier.count { + _, err := tests.CreateRecord(hub, collection, data) + require.NoError(t, err) + } + + rm := records.NewRecordManager(hub) + rm.CreateLongerRecords() + first, err := hub.FindAllRecords(collection, filter) + require.NoError(t, err) + require.Len(t, first, 1) + + // The shorter records remain eligible, but the existing longer + // record must prevent another rollup on a subsequent invocation. + rm.CreateLongerRecords() + second, err := hub.FindAllRecords(collection, filter) + require.NoError(t, err) + require.Len(t, second, 1) + require.Equal(t, first[0].Id, second[0].Id) + }) + } + } +} diff --git a/internal/site/package.json b/internal/site/package.json index a69c345c..b30e1461 100644 --- a/internal/site/package.json +++ b/internal/site/package.json @@ -1,7 +1,7 @@ { "name": "beszel", "private": true, - "version": "0.19.0", + "version": "0.20.0", "type": "module", "scripts": { "dev": "vite --host", @@ -74,4 +74,4 @@ "optionalDependencies": { "@esbuild/linux-arm64": "^0.21.5" } -} +} \ No newline at end of file diff --git a/internal/site/src/components/active-alerts.tsx b/internal/site/src/components/active-alerts.tsx index 2c5019d9..1e71762f 100644 --- a/internal/site/src/components/active-alerts.tsx +++ b/internal/site/src/components/active-alerts.tsx @@ -61,6 +61,8 @@ export const ActiveAlerts = () => { {info.triggeredDesc ? ( info.triggeredDesc() + ) : alert.name === "NetworkMonitorLoss" ? ( + One or more monitors exceed {alert.value}% loss ) : alert.name === "Status" ? ( Connection is down ) : info.invert ? ( diff --git a/internal/site/src/components/alerts-history-columns.tsx b/internal/site/src/components/alerts-history-columns.tsx index 887ae0b2..2ae9bd97 100644 --- a/internal/site/src/components/alerts-history-columns.tsx +++ b/internal/site/src/components/alerts-history-columns.tsx @@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef[] = [ accessorFn: (record) => { const name = record.name const info = alertInfo[name] - return info?.name().replace("cpu", "CPU") || name + const label = info?.name().replace("cpu", "CPU") || name + return record.monitor_name ? `${label}: ${record.monitor_name}` : label }, header: ({ column }) => ( + + { + if (event.key === "Tab") { + event.preventDefault() + searchRef.current?.focus() + } + }} + align="start" + className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden" + > +
+
+
+
+
+ { + event.preventDefault() + selectFiltered(true) + }} + > + {query ? Select matches : Select all} + + + { + event.preventDefault() + selectFiltered(false) + }} + > + {query ? Clear matches : Clear all} + +
+ + {t`${selectedSystemIds.size} selected`} + +
+
+
+ {filteredSystems.length === 0 && ( + + No systems found. + + )} + {filteredSystems.map((sys) => ( + event.preventDefault()} + onCheckedChange={(checked) => { + const next = new Set(selectedSystemIds) + if (checked) next.add(sys.id) + else next.delete(sys.id) + onChange(next) + }} + className="group min-w-0 gap-2.5 py-2 ps-2.5" + indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3" + > + {sys.name} + + ))} +
+ + + ) +} + +export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) { + const [open, setOpen] = useState(false) + const [bulkOpen, setBulkOpen] = useState(false) + const [bulkInput, setBulkInput] = useState("") + const [bulkLoading, setBulkLoading] = useState(false) + const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState>(new Set()) + const bulkFormRef = useRef(null) + const { toast } = useToast() + const { t } = useLingui() + + const resetBulkForm = () => { + setBulkInput("") + } + + const openBulkAdd = (selectedSystemIds?: Set) => { + if (!systemId && selectedSystemIds) { + setBulkSelectedSystemIds(new Set(selectedSystemIds)) + } + setOpen(false) + setBulkOpen(true) + } + + const openAdd = () => { + setBulkOpen(false) + setOpen(true) + } + + async function handleBulkSubmit(e: React.FormEvent) { + e.preventDefault() + setBulkLoading(true) + let closedForSubmit = false + + try { + const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds) + if (!targetSystems.length) { + throw new Error("Select at least one system.") + } + const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim()) + if (!rawLines.length) { + throw new Error("Enter at least one monitor.") + } + + let totalCreated = 0 + closedForSubmit = true + + for (const system of targetSystems) { + const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system)) + const existingMonitorKeys = new Set( + monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor)) + ) + const newPayloads: typeof payloads = [] + + for (const payload of payloads) { + const monitorKey = getMonitorIdentityKey(payload) + if (existingMonitorKeys.has(monitorKey)) { + continue + } + existingMonitorKeys.add(monitorKey) + newPayloads.push(payload) + } + + if (!newPayloads.length) continue + + let batch = pb.createBatch() + let inBatch = 0 + for (const payload of newPayloads) { + batch.collection("network_monitors").create(payload) + inBatch++ + if (inBatch > 20) { + await batch.send() + batch = pb.createBatch() + inBatch = 0 + } + } + if (inBatch) { + await batch.send() + } + totalCreated += newPayloads.length + } + + if (!totalCreated) { + throw new Error("No new monitors. All entries already exist.") + } + + resetBulkForm() + toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` }) + } catch (err: unknown) { + if (closedForSubmit) { + setBulkOpen(true) + } + toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message }) + } finally { + setBulkLoading(false) + } + } + + return ( + <> +
+ +
+ + + + + + openBulkAdd()}> + + Bulk Add + + + +
+ { + setOpen(nextOpen) + }} + > + + + + { + setBulkOpen(nextOpen) + if (!nextOpen) { + resetBulkForm() + } + }} + > + + + + Bulk Add {{ foo: t`Network Monitors` }} + + target[,protocol[,port[,interval]]] + +
+
+ {!systemId && ( +
+ + +
+ )} +
+ +