Compare commits

...
1 Commits
Author SHA1 Message Date
90ed9a504d feat: add network monitors (ICMP/TCP/HTTP/DNS) (#2266)
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>
Co-authored-by: henrygd <hank@henrygd.me>
2026-09-18 13:10:18 -04:00
89 changed files with 8699 additions and 457 deletions
+2 -2
View File
@@ -1,6 +1,6 @@
# Node.js dependencies
node_modules
internalsite/node_modules
node_modules/
**/node_modules/
# Go build artifacts and binaries
build
+9
View File
@@ -48,6 +48,7 @@ type Agent struct {
keys []gossh.PublicKey // SSH public keys
smartManager *SmartManager // Manages SMART data
systemdManager *systemdManager // Manages systemd services
monitorManager *MonitorManager // Manages network monitors
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
}
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
// initialize handler registry
agent.handlerRegistry = NewHandlerRegistry()
// initialize monitor manager
agent.monitorManager = newMonitorManager()
agent.storagePoolManager = newStoragePoolManager()
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
}
}
if a.monitorManager != nil {
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
slog.Debug("Monitors", "data", data.Monitors)
}
// skip updating systemd services if cache time is not the default 60sec interval
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
totalCount := uint16(a.systemdManager.getServiceStatsCount())
+1
View File
@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
// }
func (c *ConnectionManager) stop() error {
_ = c.agent.StopServer()
c.agent.monitorManager.Stop()
c.closeWebSocket()
return health.CleanUp()
}
+1 -2
View File
@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
}
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
// Save original PATH
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
tests := []struct {
@@ -1213,7 +1212,7 @@ echo "[]"`
{
name: "no gpu tools available",
setupCommands: func(_ string) error {
t.Setenv("PATH", "")
// The subtest already restricts PATH to its empty temporary directory.
return nil
},
wantErr: true,
+20
View File
@@ -7,6 +7,7 @@ import (
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
"log/slog"
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
return registry
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
return hctx.SendResponse(details, hctx.RequestID)
}
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
type SyncNetworkMonitorsHandler struct{}
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
var req monitor.SyncRequest
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
return err
}
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
if err != nil {
return err
}
return hctx.SendResponse(resp, hctx.RequestID)
}
+176
View File
@@ -0,0 +1,176 @@
package agent
import (
"errors"
"fmt"
"net/http"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// MonitorManager manages network monitor configurations and task lifetimes.
type MonitorManager struct {
mu sync.RWMutex
monitors map[string]*monitorTask // keyed by monitor ID
probe monitorProbe
resumeGuard monitorResumeGuard
}
func newMonitorManager() *MonitorManager {
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
}
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
}
// SyncMonitors replaces all monitor tasks with the given configs.
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
pm.mu.Lock()
defer pm.mu.Unlock()
// Build set of new keys
newKeys := make(map[string]monitor.Config, len(configs))
for _, cfg := range configs {
if cfg.ID == "" {
continue
}
newKeys[cfg.ID] = cfg
}
// Stop removed monitors
for key, task := range pm.monitors {
if _, exists := newKeys[key]; !exists {
task.cancel()
delete(pm.monitors, key)
}
}
// Start new monitors and restart tasks whose config changed.
for key, cfg := range newKeys {
task, exists := pm.monitors[key]
if exists && task.config == cfg {
continue
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(cfg, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[key] = task
pm.startMonitor(task)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// HandleSyncRequest applies a full or incremental monitor sync request.
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
switch req.Action {
case monitor.SyncActionReplace:
pm.SyncMonitors(req.Configs)
return monitor.SyncResponse{}, nil
case monitor.SyncActionUpsert:
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
if err != nil {
return monitor.SyncResponse{}, err
}
if result == nil {
return monitor.SyncResponse{}, nil
}
return monitor.SyncResponse{Result: *result}, nil
case monitor.SyncActionDelete:
if req.Config.ID == "" {
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
}
pm.DeleteMonitor(req.Config.ID)
return monitor.SyncResponse{}, nil
default:
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
}
}
// UpsertMonitor creates or replaces a single monitor task.
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
if config.ID == "" {
return nil, errors.New("missing monitor ID")
}
pm.mu.Lock()
task, exists := pm.monitors[config.ID]
if exists && task.config == config {
pm.mu.Unlock()
if !runNow {
return nil, nil
}
return task.runProbe(pm.probe), nil
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(config, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[config.ID] = task
pm.mu.Unlock()
if runNow {
result := task.runProbe(pm.probe)
pm.startMonitor(task)
return result, nil
}
pm.startMonitor(task)
return nil, nil
}
// DeleteMonitor stops and removes a single monitor task.
func (pm *MonitorManager) DeleteMonitor(id string) {
if id == "" {
return
}
pm.mu.Lock()
defer pm.mu.Unlock()
if task, exists := pm.monitors[id]; exists {
task.cancel()
delete(pm.monitors, id)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
pm.mu.RLock()
defer pm.mu.RUnlock()
results := make(map[string]monitor.Result, len(pm.monitors))
now := time.Now()
duration := time.Duration(durationMs) * time.Millisecond
for _, task := range pm.monitors {
result, ok := task.history.result(duration, now)
if !ok {
continue
}
results[task.config.ID] = result
}
return results
}
// Stop stops all monitor tasks.
func (pm *MonitorManager) Stop() {
pm.mu.Lock()
defer pm.mu.Unlock()
for key, task := range pm.monitors {
task.cancel()
delete(pm.monitors, key)
}
pm.resumeGuard.shutdown()
}
+274
View File
@@ -0,0 +1,274 @@
package agent
import (
"math"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// Monitors run at user-defined intervals (e.g., every 10s).
// To keep memory usage low and constant, data is stored in two layers:
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
// wall-clock minute. Samples collected within the same minute are aggregated
// (sum, min, max, count) into a single bucket.
//
// Short-term requests (<= 61s) use raw samples.
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
// of individual data points.
const (
// monitorRawRetention is the duration to keep individual samples
monitorRawRetention = 61 * time.Second
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
monitorMinuteBucketLen int32 = 61
)
// monitorHistory owns retention and aggregation, independently of probe execution.
type monitorHistory struct {
mu sync.Mutex
sampleCount int64
samples []monitorSample
buckets [monitorMinuteBucketLen]monitorBucket
}
func newMonitorHistory() *monitorHistory {
// Start small for typical intervals; append grows the buffer for faster probes.
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
}
func (h *monitorHistory) clone() *monitorHistory {
h.mu.Lock()
defer h.mu.Unlock()
cloned := newMonitorHistory()
cloned.samples = append(cloned.samples, h.samples...)
cloned.buckets = h.buckets
cloned.sampleCount = h.sampleCount
return cloned
}
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
h.mu.Lock()
defer h.mu.Unlock()
return h.resultLocked(duration, now)
}
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
h.mu.Lock()
defer h.mu.Unlock()
h.addSampleLocked(sample)
result, _ := h.resultLocked(time.Minute, sample.timestamp)
return result
}
// monitorSample stores one monitor attempt and its collection time.
type monitorSample struct {
responseUs int64 // -1 means loss
timestamp time.Time
}
// monitorBucket stores one minute of aggregated monitor data.
type monitorBucket struct {
minute int32
filled bool
stats monitorAggregate
}
// monitorAggregate accumulates successful response stats and total sample counts.
type monitorAggregate struct {
sumUs int64
minUs int64
maxUs int64
totalCount int64
successCount int64
}
// newMonitorAggregate initializes an aggregate with an unset minimum value.
func newMonitorAggregate() monitorAggregate {
return monitorAggregate{minUs: math.MaxInt64}
}
// addResponse folds a single monitor sample into the aggregate.
func (agg *monitorAggregate) addResponse(responseUs int64) {
agg.totalCount++
if responseUs < 0 {
return
}
agg.successCount++
agg.sumUs += responseUs
if responseUs < agg.minUs {
agg.minUs = responseUs
}
if responseUs > agg.maxUs {
agg.maxUs = responseUs
}
}
// addAggregate merges another aggregate into this one.
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
if other.totalCount == 0 {
return
}
agg.totalCount += other.totalCount
agg.successCount += other.successCount
agg.sumUs += other.sumUs
if other.successCount == 0 {
return
}
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
agg.minUs = other.minUs
}
if other.maxUs > agg.maxUs {
agg.maxUs = other.maxUs
}
}
// hasData reports whether the aggregate contains any samples.
func (agg monitorAggregate) hasData() bool {
return agg.totalCount > 0
}
// result converts the aggregate into the monitor result format.
func (agg monitorAggregate) result() monitor.Result {
avg := agg.avgResponse()
result := monitor.Result{
AvgResponse: avg,
MinResponse: agg.minUs,
MaxResponse: agg.maxUs,
PacketLoss: agg.lossPercentage(),
TotalCount: agg.totalCount,
SuccessCount: agg.successCount,
ResponseSum: agg.sumUs,
}
if agg.successCount == 0 {
result.MinResponse, result.MaxResponse = 0, 0
}
return result
}
// avgResponse returns the rounded average of successful samples.
func (agg monitorAggregate) avgResponse() int64 {
if agg.successCount == 0 {
return 0
}
return agg.sumUs / agg.successCount
}
// lossPercentage returns the rounded failure rate for the aggregate.
func (agg monitorAggregate) lossPercentage() float64 {
if agg.totalCount == 0 {
return 0
}
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
}
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
agg := h.aggregateLocked(duration, now)
if !agg.hasData() {
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
// the most recent sample so realtime requests still report current status.
agg = h.latestSampleAggregateLocked()
}
hourAgg := h.aggregateLocked(time.Hour, now)
if !agg.hasData() {
return monitor.Result{}, false
}
result := agg.result()
if len(h.samples) > 0 {
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
}
result.AvgResponse1h = hourAgg.avgResponse()
result.MinResponse1h = hourAgg.minUs
result.MaxResponse1h = hourAgg.maxUs
result.PacketLoss1h = hourAgg.lossPercentage()
result.SampleCount = h.sampleCount
if hourAgg.successCount == 0 {
result.MinResponse1h, result.MaxResponse1h = 0, 0
}
return result, true
}
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
agg := newMonitorAggregate()
if len(h.samples) == 0 {
return agg
}
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
return agg
}
// aggregateLocked collects monitor data for the requested time window.
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
cutoff := now.Add(-duration)
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
if duration <= monitorRawRetention {
return aggregateSamplesSince(h.samples, cutoff)
}
return aggregateBucketsSince(h.buckets[:], cutoff, now)
}
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
agg := newMonitorAggregate()
for _, sample := range samples {
if sample.timestamp.Before(cutoff) {
continue
}
agg.addResponse(sample.responseUs)
}
return agg
}
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
agg := newMonitorAggregate()
startMinute := int32(cutoff.Unix() / 60)
endMinute := int32(now.Unix() / 60)
for _, bucket := range buckets {
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
continue
}
agg.addAggregate(bucket.stats)
}
return agg
}
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
h.sampleCount++
cutoff := sample.timestamp.Add(-monitorRawRetention)
start := 0
for i := range h.samples {
if !h.samples[i].timestamp.Before(cutoff) {
start = i
break
}
if i == len(h.samples)-1 {
start = len(h.samples)
}
}
if start > 0 {
size := copy(h.samples, h.samples[start:])
h.samples = h.samples[:size]
}
h.samples = append(h.samples, sample)
minute := int32(sample.timestamp.Unix() / 60)
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
bucket := &h.buckets[minute%monitorMinuteBucketLen]
if !bucket.filled || bucket.minute != minute {
bucket.minute = minute
bucket.filled = true
bucket.stats = newMonitorAggregate()
}
bucket.stats.addResponse(sample.responseUs)
}
+154
View File
@@ -0,0 +1,154 @@
package agent
import (
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorHistoryWindowCounts(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// This older success counts toward lifetime warm-up, but not this window.
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
result, ok := history.result(time.Minute, now)
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
assert.EqualValues(t, 3, result.TotalCount)
assert.EqualValues(t, 2, result.SuccessCount)
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
assert.EqualValues(t, 15, result.AvgResponse)
assert.Equal(t, 33.33, result.PacketLoss)
encoded, err := cbor.Marshal(result)
require.NoError(t, err)
var decoded monitor.Result
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
assert.Equal(t, result, decoded)
stats := monitor.Stats{}.FromResult(decoded)
assert.Equal(t, result.TotalCount, stats.TotalCount)
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
// Reads do not consume samples. A short window's latest-sample fallback
// carries the count for that single failure, not the minute or lifetime count.
repeated, _ := history.result(time.Minute, now)
assert.Equal(t, result, repeated)
fallback, ok := history.result(time.Second, now)
require.True(t, ok)
assert.EqualValues(t, 1, fallback.TotalCount)
assert.Zero(t, fallback.SuccessCount)
assert.Zero(t, fallback.ResponseSum)
assert.Equal(t, 100.0, fallback.PacketLoss)
assert.EqualValues(t, 4, fallback.SampleCount)
}
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
agg := history.aggregateLocked(time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(1), agg.successCount)
result := agg.result()
assert.Equal(t, int64(20), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 50.0, result.PacketLoss)
}
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(4), agg.totalCount)
assert.Equal(t, int64(3), agg.successCount)
result := agg.result()
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(40), result.MaxResponse)
assert.Equal(t, 25.0, result.PacketLoss)
}
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
require.Len(t, history.samples, 1)
assert.Equal(t, int64(20), history.samples[0].responseUs)
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
result := agg.result()
assert.Equal(t, int64(15), result.AvgResponse)
assert.Equal(t, int64(10), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 0.0, result.PacketLoss)
}
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
history := newMonitorHistory()
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
_, ok := history.result(time.Minute, start)
require.False(t, ok)
first := history.record(monitorSample{responseUs: 20, timestamp: start})
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
for minute := 0; minute < 5; minute++ {
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
// Realtime reads must not consume freshness for the persistence request.
for _, window := range []time.Duration{time.Second, time.Minute} {
result, ok := history.result(window, now)
require.True(t, ok)
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
assert.Equal(t, int64(20), result.AvgResponse)
}
}
next := start.Add(5 * time.Minute)
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
assert.Equal(t, float64(100), failed.PacketLoss)
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
require.True(t, ok)
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
assert.Equal(t, float64(100), repeated.PacketLoss)
}
func TestMonitorHistorySampleCount(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// Both failed and successful probes count, including older samples so
// monitors with hourly intervals can finish warming up.
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
for i, response := range []int64{10, -1, 20} {
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
assert.EqualValues(t, i+2, result.SampleCount)
}
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
}
+312
View File
@@ -0,0 +1,312 @@
package agent
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"math"
"net"
"os"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/net/icmp"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
"log/slog"
)
// Match the numeric RTT independently of the localized label used by Windows.
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
var icmpSequence atomic.Uint32
type icmpPacketConn interface {
Close() error
}
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
// all native methods fail, the choice is cached so subsequent monitors skip
// the trial-and-error overhead.
type icmpMethod uint8
const (
icmpUntried icmpMethod = iota // haven't tried yet
icmpRaw // privileged raw socket
icmpDatagram // unprivileged datagram socket
icmpExecFallback // shell out to system ping command
)
// icmpFamily holds the network parameters and cached detection result for one address family.
type icmpFamily struct {
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
dgramNetwork string // e.g. "udp4" or "udp6"
listenAddr string // "0.0.0.0" or "::"
echoType icmp.Type // outgoing echo request type
replyType icmp.Type // expected echo reply type
proto int // IANA protocol number for parsing replies
isIPv6 bool
mode icmpMethod // cached detection result (guarded by icmpModeMu)
}
var (
icmpV4 = icmpFamily{
rawNetwork: "ip4:icmp",
dgramNetwork: "udp4",
listenAddr: "0.0.0.0",
echoType: ipv4.ICMPTypeEcho,
replyType: ipv4.ICMPTypeEchoReply,
proto: 1,
}
icmpV6 = icmpFamily{
rawNetwork: "ip6:ipv6-icmp",
dgramNetwork: "udp6",
listenAddr: "::",
echoType: ipv6.ICMPTypeEchoRequest,
replyType: ipv6.ICMPTypeEchoReply,
proto: 58,
isIPv6: true,
}
icmpModeMu sync.Mutex
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
return icmp.ListenPacket(network, listenAddr)
}
)
// monitorICMP sends an ICMP echo request and measures round-trip response.
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
// unprivileged datagram, or exec fallback) is detected once per address
// family and cached for subsequent monitors.
// Returns response in microseconds, or -1 and an error on failure.
func monitorICMP(ctx context.Context, target string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
family, ip, err := resolveICMPTarget(ctx, target)
if err != nil {
return -1, err
}
icmpModeMu.Lock()
if family.mode == icmpUntried {
family.mode = detectICMPMode(family, icmpListen)
}
mode := family.mode
icmpModeMu.Unlock()
switch mode {
case icmpRaw:
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
case icmpDatagram:
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
case icmpExecFallback:
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
default:
return -1, errors.New("unsupported ICMP mode")
}
}
// resolveICMPTarget resolves a target hostname or IP to determine the address
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
if ip := net.ParseIP(target); ip != nil {
if ip.To4() != nil {
return &icmpV4, ip.To4(), nil
}
return &icmpV6, ip, nil
}
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
if err != nil || len(ips) == 0 {
return nil, nil, err
}
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return &icmpV4, v4, nil
}
}
return &icmpV6, ips[0], nil
}
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
label := "IPv4"
if family.isIPv6 {
label = "IPv6"
}
conn, err := listen(family.rawNetwork, family.listenAddr)
slog.Debug("ICMP raw socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpRaw
}
conn, err = listen(family.dgramNetwork, family.listenAddr)
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpDatagram
}
return icmpExecFallback
}
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
return -1, err
}
defer conn.Close()
return monitorICMPPacket(ctx, conn, family, dst)
}
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
if err := ctx.Err(); err != nil {
return -1, err
}
// Closing the socket interrupts both reads and writes on cancellation.
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
// Prepare correlation data before starting the round-trip timer. The token
// also distinguishes delayed replies after the 16-bit sequence wraps.
token := make([]byte, 16)
if _, err := rand.Read(token); err != nil {
return -1, err
}
echo := &icmp.Echo{
ID: os.Getpid() & 0xffff,
Seq: int(icmpSequence.Add(1) & 0xffff),
Data: token,
}
// Linux ping sockets replace the Echo ID with their bound port. Darwin
// datagram sockets and raw sockets preserve the supplied ID.
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
echo.ID = local.Port
}
targetIP := icmpAddrIP(dst)
msg := &icmp.Message{
Type: family.echoType,
Code: 0,
Body: echo,
}
msgBytes, err := msg.Marshal(nil)
if err != nil {
return -1, err
}
// Set deadline before sending
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
return -1, err
}
buf := make([]byte, 1500)
start := time.Now()
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
return -1, err
}
// Read reply
for {
n, peer, err := conn.ReadFrom(buf)
received := time.Now()
if err != nil {
return -1, err
}
if !targetIP.Equal(icmpAddrIP(peer)) {
continue
}
reply, err := icmp.ParseMessage(family.proto, buf[:n])
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
continue
}
body, ok := reply.Body.(*icmp.Echo)
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
return received.Sub(start).Microseconds(), nil
}
// Keep waiting for our reply without extending the original deadline.
}
}
func icmpAddrIP(addr net.Addr) net.IP {
switch addr := addr.(type) {
case *net.IPAddr:
return addr.IP
case *net.UDPAddr:
return addr.IP
default:
return nil
}
}
// pingCommand selects the executable and arguments for the supported agent platforms.
// The context deadline enforces the timeout: -W has incompatible meanings across
// Linux, BSD IPv4 ping, and macOS ping6.
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
family := "-4"
if isIPv6 {
family = "-6"
}
switch goos {
case "windows":
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
case "linux":
return "ping", []string{family, "-n", "-c", "1", target}, nil
case "darwin", "freebsd", "openbsd":
command := "ping"
if isIPv6 {
command = "ping6"
}
return command, []string{"-n", "-c", "1", target}, nil
default:
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
}
}
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
if err != nil {
return -1, err
}
cmd := exec.CommandContext(ctx, name, args...)
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
cmd.Env = append(os.Environ(), "LC_ALL=C")
output, err := cmd.Output()
if ctx.Err() != nil {
return -1, ctx.Err()
}
if err != nil {
return -1, fmt.Errorf("%s failed: %w", name, err)
}
return parsePingResponse(output)
}
// parsePingResponse returns the reported RTT, never subprocess execution time.
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
func parsePingResponse(output []byte) (int64, error) {
matches := pingTimeRegex.FindSubmatch(output)
if len(matches) < 2 {
return -1, errors.New("ping output contains no round-trip time")
}
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
return -1, errors.New("invalid round-trip time in ping output")
}
return int64(math.Round(ms * 1000)), nil
}
+433
View File
@@ -0,0 +1,433 @@
//go:build testing
package agent
import (
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/icmp"
)
type testICMPPacketConn struct{}
func (testICMPPacketConn) Close() error { return nil }
type blockingICMPConn struct {
net.PacketConn
reading chan struct{}
}
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
return len(p), nil
}
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
close(c.reading)
return c.PacketConn.ReadFrom(p)
}
func TestMonitorICMPPacketCancellation(t *testing.T) {
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
require.NoError(t, err)
defer conn.Close()
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
done <- err
}()
select {
case <-blocking.reading:
case <-time.After(time.Second):
t.Fatal("probe did not begin reading")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("cancellation did not interrupt the socket read")
}
}
func TestMonitorICMPExecCancellation(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
done <- err
}()
select {
case err := <-done:
require.ErrorIs(t, err, context.DeadlineExceeded)
case <-time.After(time.Second):
t.Fatal("cancellation did not terminate ping")
}
}
func TestPingCommand(t *testing.T) {
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
for _, ipv6 := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
target, family := "192.0.2.1", "-4"
if ipv6 {
target, family = "2001:db8::1", "-6"
}
name, args, err := pingCommand(goos, target, ipv6)
require.NoError(t, err)
wantName := "ping"
wantArgs := []string{"-n", "-c", "1", target}
switch goos {
case "windows":
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
case "linux":
wantArgs = append([]string{family}, wantArgs...)
default:
if ipv6 {
wantName = "ping6"
}
}
assert.Equal(t, wantName, name)
assert.Equal(t, wantArgs, args)
})
}
}
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
require.Error(t, err)
}
func TestParsePingResponse(t *testing.T) {
for _, tc := range []struct {
name string
output string
wantUs int64
}{
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
{"rounding", "time=0.1236 ms", 124},
{"empty", "", -1},
{"timeout", "Request timed out.", -1},
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
{"malformed", "time=oops ms", -1},
{"negative", "time=-1 ms", -1},
{"overflow", "time=999999999999999999999 ms", -1},
} {
t.Run(tc.name, func(t *testing.T) {
responseUs, err := parsePingResponse([]byte(tc.output))
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
func TestMonitorICMPExecOutput(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
for _, tc := range []struct {
name string
output string
exit int
wantUs int64
}{
{"success", "time=1.234 ms", 0, 1234},
{"missing RTT", "unrecognized output", 0, -1},
{"failed command with RTT", "time=1.234 ms", 1, -1},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
// Also verify an inherited locale cannot override the C locale.
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("LC_ALL", "de_DE.UTF-8")
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
type icmpTestReply struct {
data []byte
peer net.Addr
}
type scriptedICMPConn struct {
net.PacketConn
local net.Addr
onWrite func([]byte, net.Addr)
replies []icmpTestReply
reads int
deadlineSets int
}
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
c.deadlineSets++
return nil
}
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
c.onWrite(data, dst)
return len(data), nil
}
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
c.reads++
if len(c.replies) == 0 {
return 0, nil, os.ErrDeadlineExceeded
}
reply := c.replies[0]
c.replies = c.replies[1:]
return copy(buf, reply.data), reply.peer, nil
}
func TestMonitorICMPReplyCorrelation(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, datagram := range []bool{false, true} {
network := family.rawNetwork
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
if family.isIPv6 {
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
}
var dst net.Addr = &net.IPAddr{IP: ip}
var wrongPeer net.Addr = &net.IPAddr{IP: other}
if datagram {
network = family.dgramNetwork
dst = &net.UDPAddr{IP: ip}
wrongPeer = &net.UDPAddr{IP: other}
}
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
for _, eventuallyMatches := range []bool{false, true} {
ending := "timeout"
if eventuallyMatches {
ending = "success"
}
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
if datagram {
conn.local = &net.UDPAddr{Port: 12345}
if runtime.GOOS == "linux" {
// Deliberately differ from the process ID.
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
}
}
conn.onWrite = func(data []byte, target net.Addr) {
require.Equal(t, dst, target)
request, err := icmp.ParseMessage(family.proto, data)
require.NoError(t, err)
echo := request.Body.(*icmp.Echo)
expectedID := os.Getpid() & 0xffff
if datagram && runtime.GOOS == "linux" {
expectedID = conn.local.(*net.UDPAddr).Port
}
require.Equal(t, expectedID, echo.ID)
reply := &icmp.Message{Type: family.replyType, Body: echo}
valid, err := reply.Marshal(nil)
require.NoError(t, err)
peer := dst
switch mismatch {
case "source":
peer = wrongPeer
case "id":
echo.ID ^= 1
case "sequence":
echo.Seq ^= 1
case "payload":
echo.Data[0] ^= 1
case "type":
reply.Type = family.echoType
case "code":
reply.Code = 1
}
invalid, err := reply.Marshal(nil)
require.NoError(t, err)
if mismatch == "malformed" {
invalid = invalid[:2]
}
conn.replies = []icmpTestReply{{invalid, peer}}
if eventuallyMatches {
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
if eventuallyMatches {
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
} else {
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
assert.Equal(t, int64(-1), elapsed)
}
assert.Equal(t, 2, conn.reads)
assert.Equal(t, 1, conn.deadlineSets)
})
}
}
}
}
}
func TestMonitorICMPLoopback(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
t.Run(network, func(t *testing.T) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
t.Skipf("ICMP socket unavailable: %v", err)
}
defer conn.Close()
ip := net.ParseIP("127.0.0.1")
if family.isIPv6 {
ip = net.ParseIP("::1")
}
var dst net.Addr = &net.IPAddr{IP: ip}
if network == family.dgramNetwork {
dst = &net.UDPAddr{IP: ip}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
})
}
}
}
func TestDetectICMPMode(t *testing.T) {
tests := []struct {
name string
family *icmpFamily
rawErr error
udpErr error
want icmpMethod
wantNetworks []string
}{
{
name: "IPv4 prefers raw socket when available",
family: &icmpV4,
want: icmpRaw,
wantNetworks: []string{"ip4:icmp"},
},
{
name: "IPv4 uses datagram when raw unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv4 falls back to exec when both unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv6 prefers raw socket when available",
family: &icmpV6,
want: icmpRaw,
wantNetworks: []string{"ip6:ipv6-icmp"},
},
{
name: "IPv6 uses datagram when raw unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
{
name: "IPv6 falls back to exec when both unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
calls := make([]string, 0, 2)
listen := func(network, listenAddr string) (icmpPacketConn, error) {
require.Equal(t, tt.family.listenAddr, listenAddr)
calls = append(calls, network)
switch network {
case tt.family.rawNetwork:
if tt.rawErr != nil {
return nil, tt.rawErr
}
case tt.family.dgramNetwork:
if tt.udpErr != nil {
return nil, tt.udpErr
}
default:
t.Fatalf("unexpected network %q", network)
}
return testICMPPacketConn{}, nil
}
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
assert.Equal(t, tt.wantNetworks, calls)
})
}
}
func TestResolveICMPTarget(t *testing.T) {
t.Run("IPv4 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
t.Run("IPv6 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::1")
require.NoError(t, err)
require.NotNil(t, family)
assert.True(t, family.isIPv6)
assert.Equal(t, "::1", ip.String())
})
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
}
+105
View File
@@ -0,0 +1,105 @@
package agent
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
// Implementations must honor cancellation and bound their execution time.
type monitorProbe func(context.Context, monitor.Config) (int64, error)
func networkMonitorProbe(client *http.Client) monitorProbe {
return func(ctx context.Context, config monitor.Config) (int64, error) {
switch config.Protocol {
case "icmp":
return monitorICMP(ctx, config.Target)
case "tcp":
return monitorTCP(ctx, config.Target, config.Port)
case "http":
return monitorHTTP(ctx, client, config.Target)
case "dns":
return monitorDNS(ctx, config.Target)
default:
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
}
}
}
// monitorTCP measures connection establishment time, including address fallback
// but excluding DNS resolution.
// Returns -1 and an error on failure.
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
// Resolve DNS first, outside the timing window but within the probe deadline.
ips, err := net.DefaultResolver.LookupHost(ctx, target)
if err != nil {
return -1, err
}
if len(ips) == 0 {
return -1, errors.New("no addresses resolved for TCP monitor")
}
portString := fmt.Sprintf("%d", port)
deadline, _ := ctx.Deadline()
// Share the remaining probe budget across addresses so an unresponsive
// first address cannot consume all the time available for alternatives.
start := time.Now()
for i, ip := range ips {
if err := ctx.Err(); err != nil {
return -1, err
}
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
var conn net.Conn
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
if err != nil {
continue
}
responseUs := time.Since(start).Microseconds()
conn.Close()
return responseUs, nil
}
return -1, err
}
// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
func monitorDNS(ctx context.Context, target string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
start := time.Now()
ips, err := net.DefaultResolver.LookupHost(ctx, target)
if err != nil || len(ips) == 0 {
return -1, err
}
return time.Since(start).Microseconds(), nil
}
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
if client == nil {
client = http.DefaultClient
}
start := time.Now()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return -1, err
}
resp, err := client.Do(req)
if err != nil {
return -1, err
}
resp.Body.Close()
if resp.StatusCode >= 400 {
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
}
return time.Since(start).Microseconds(), nil
}
+88
View File
@@ -0,0 +1,88 @@
package agent
import (
"sync"
"time"
)
const (
monitorResumeHeartbeat = 10 * time.Second
// Allow scheduling jitter without mistaking an ordinary tick for resume.
monitorResumeGap = 2 * monitorResumeHeartbeat
monitorResumePause = 10 * time.Second
)
// monitorResumeGuard detects likely suspend/resume using wall time. A long
// process stall or forward clock adjustment can also trigger the bounded pause.
// One heartbeat is shared by all configured monitors.
type monitorResumeGuard struct {
mu sync.Mutex
stop chan struct{}
lastTick time.Time
pauseUntil time.Time
generation uint32
}
func (g *monitorResumeGuard) start() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
return
}
stop := make(chan struct{})
g.stop = stop
g.lastTick = time.Now().Round(0)
g.pauseUntil = time.Time{}
go func() {
ticker := time.NewTicker(monitorResumeHeartbeat)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
g.mu.Lock()
if g.stop == stop {
g.observe(time.Now())
}
g.mu.Unlock()
}
}
}()
}
func (g *monitorResumeGuard) shutdown() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
close(g.stop)
g.stop = nil
g.generation++
}
}
// observe requires mu. Strip the monotonic component because it can stop during
// suspend. Read the current time rather than the ticker's queued timestamp.
func (g *monitorResumeGuard) observe(now time.Time) {
now = now.Round(0)
if now.Sub(g.lastTick) > monitorResumeGap {
g.pauseUntil = now.Add(monitorResumePause)
g.generation++
}
g.lastTick = now
}
// snapshot also observes time so a probe waking before the heartbeat detects
// resume itself. A changed generation invalidates probes spanning suspend.
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
if g == nil {
return 0, true
}
g.mu.Lock()
defer g.mu.Unlock()
if g.stop == nil {
return g.generation, true
}
g.observe(time.Now())
return g.generation, !g.lastTick.Before(g.pauseUntil)
}
+121
View File
@@ -0,0 +1,121 @@
//go:build testing
package agent
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func simulateMonitorSleep(g *monitorResumeGuard) {
g.mu.Lock()
g.lastTick = time.Now().Add(-time.Hour).Round(0)
g.mu.Unlock()
}
func TestMonitorResumePause(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
generation, allowed := g.snapshot()
require.True(t, allowed)
// Heartbeats alone must keep the guard current between infrequent probes.
time.Sleep(time.Minute)
synctest.Wait()
steadyGeneration, allowed := g.snapshot()
require.True(t, allowed)
require.Equal(t, generation, steadyGeneration)
// The probe, rather than the heartbeat, must detect this gap.
simulateMonitorSleep(&g)
next, allowed := g.snapshot()
assert.False(t, allowed)
assert.NotEqual(t, generation, next)
time.Sleep(9 * time.Second)
_, allowed = g.snapshot()
assert.False(t, allowed)
time.Sleep(time.Second)
end, allowed := g.snapshot()
assert.True(t, allowed)
assert.Equal(t, next, end)
})
}
func TestMonitorResumeGuardLifecycle(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
defer pm.Stop()
assert.Nil(t, pm.resumeGuard.stop)
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
stop := pm.resumeGuard.stop
require.NotNil(t, stop)
pm.DeleteMonitor("a")
assert.Equal(t, stop, pm.resumeGuard.stop)
pm.DeleteMonitor("b")
assert.Nil(t, pm.resumeGuard.stop)
select {
case <-stop:
default:
t.Fatal("heartbeat was not stopped")
}
time.Sleep(time.Hour)
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
require.NoError(t, err)
_, allowed := pm.resumeGuard.snapshot()
assert.True(t, allowed, "idle time must not trigger a resume pause")
pm.SyncMonitors(nil)
assert.Nil(t, pm.resumeGuard.stop)
})
}
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
task := newMonitorTask(monitor.Config{ID: "test"})
defer task.cancel()
task.resumeGuard = &g
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
simulateMonitorSleep(&g)
return 0, errors.New("network not ready")
})
assert.Nil(t, result)
assert.Empty(t, task.history.samples)
// Explicit requests may still run during the pause and record real failures.
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
return 0, errors.New("unreachable")
})
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
})
}
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
calls.Add(1)
return 1, nil
})
defer pm.Stop()
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
simulateMonitorSleep(&pm.resumeGuard)
pm.resumeGuard.snapshot()
time.Sleep(9 * time.Second)
synctest.Wait()
assert.Zero(t, calls.Load())
assert.Empty(t, pm.GetResults(1000))
time.Sleep(2 * time.Second)
synctest.Wait()
assert.Positive(t, calls.Load())
})
}
+60
View File
@@ -0,0 +1,60 @@
package agent
import (
"context"
"log/slog"
"math/rand"
"time"
)
func (pm *MonitorManager) startMonitor(task *monitorTask) {
interval := time.Duration(task.config.Interval) * time.Second
if interval < time.Second {
interval = 30 * time.Second
}
delay := getStagger(interval.Milliseconds())
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
go runMonitorSchedule(task.ctx, interval, delay, func() {
if _, allowed := task.resumeGuard.snapshot(); allowed {
task.runProbe(pm.probe)
}
})
}
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
// naturally drop missed ticks rather than building an execution backlog.
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return
case <-timer.C:
}
if ctx.Err() != nil {
return
}
run()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if ctx.Err() != nil {
return
}
run()
}
}
}
// getStagger returns an initial delay between half an interval and one interval.
func getStagger(intervalMilli int64) time.Duration {
delay := rand.Intn(int(intervalMilli))
if delay < int(intervalMilli)/2 {
delay += int(intervalMilli) / 2
}
return time.Duration(delay) * time.Millisecond
}
+167
View File
@@ -0,0 +1,167 @@
//go:build testing
package agent
import (
"context"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorScheduleTiming(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
synctest.Wait()
time.Sleep(4 * time.Second)
synctest.Wait()
assert.Equal(t, 0, int(calls.Load()))
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(10 * time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
})
}
func TestMonitorScheduleSlowProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
release := make(chan struct{})
go runMonitorSchedule(ctx, time.Second, 0, func() {
calls.Add(1)
select {
case <-release:
case <-ctx.Done():
}
})
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
close(release)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
})
}
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
release := make(chan struct{})
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
assert.Equal(t, cfg, config)
calls.Add(1)
<-release
return 42, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
pm.monitors[cfg.ID] = task
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
synctest.Wait()
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
close(release)
synctest.Wait()
first, second := <-results, <-results
require.NotNil(t, first)
require.NotNil(t, second)
assert.Equal(t, int64(42), first.AvgResponse)
assert.Equal(t, first, second)
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
assert.Len(t, task.history.samples, 1)
// A later explicit request must still perform a fresh probe.
_, err := pm.UpsertMonitor(cfg, true)
require.NoError(t, err)
assert.Equal(t, 2, int(calls.Load()))
assert.Len(t, task.history.samples, 2)
})
}
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
if config.Interval == 10 {
<-ctx.Done()
return 0, ctx.Err()
}
return 30, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
updated := cfg
updated.Interval = 20
result, err := pm.UpsertMonitor(updated, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, int64(20), result.AvgResponse)
assert.Zero(t, result.PacketLoss)
synctest.Wait()
assert.Nil(t, <-results)
assert.Nil(t, <-results)
assert.Len(t, task.history.samples, 1)
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
})
}
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
<-ctx.Done()
return 0, ctx.Err()
})
defer pm.Stop()
start := time.Now()
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, 3*time.Second, time.Since(start))
assert.Equal(t, 100.0, result.PacketLoss)
assert.NoError(t, pm.monitors["test"].ctx.Err())
})
}
+116
View File
@@ -0,0 +1,116 @@
package agent
import (
"context"
"log/slog"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
const monitorFailureLogInterval = 5 * time.Minute
// monitorTask coordinates a probe and its history for one immutable configuration.
type monitorTask struct {
config monitor.Config
ctx context.Context
cancel context.CancelFunc
history *monitorHistory
resumeGuard *monitorResumeGuard
runMu sync.Mutex
inflight *monitorRun
lastFailureLog int64 // Unix nanoseconds
}
type monitorRun struct {
done chan struct{}
result *monitor.Result // published by closing done; never mutated afterwards
}
func newMonitorTask(config monitor.Config) *monitorTask {
ctx, cancel := context.WithCancel(context.Background())
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
// Serialize cancellation with publication, so canceled probes cannot enter
// history copied into a replacement task.
task.cancel = func() {
task.runMu.Lock()
cancel()
task.runMu.Unlock()
}
return task
}
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
task := newMonitorTask(config)
if existing != nil {
task.history = existing.history.clone()
}
return task
}
// runProbe shares an in-flight check between scheduled and immediate requests.
// Every completed check contributes exactly one sample, regardless of how many
// callers were waiting for it. No task or history lock is held during network I/O.
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
task.runMu.Lock()
if task.ctx.Err() != nil {
task.runMu.Unlock()
return nil
}
if run := task.inflight; run != nil {
task.runMu.Unlock()
select {
case <-task.ctx.Done():
return nil
case <-run.done:
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
}
run := &monitorRun{done: make(chan struct{})}
task.inflight = run
task.runMu.Unlock()
generation, _ := task.resumeGuard.snapshot()
responseUs, err := probe(task.ctx, task.config)
var logFailure bool
task.runMu.Lock()
currentGeneration, _ := task.resumeGuard.snapshot()
if task.ctx.Err() == nil && generation == currentGeneration {
now := time.Now()
if err != nil {
responseUs = -1
logAt := now.UnixNano()
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
logFailure = true
task.lastFailureLog = logAt
}
} else {
task.lastFailureLog = 0
}
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
run.result = &result
}
task.inflight = nil
close(run.done)
task.runMu.Unlock()
if logFailure {
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
}
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
func copyMonitorResult(result *monitor.Result) *monitor.Result {
if result == nil {
return nil
}
copy := *result
return &copy
}
+79
View File
@@ -0,0 +1,79 @@
//go:build testing
package agent
import (
"bytes"
"context"
"errors"
"log/slog"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorFailureLogCooldown(t *testing.T) {
var logs bytes.Buffer
previous := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
t.Cleanup(func() { slog.SetDefault(previous) })
synctest.Test(t, func(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
defer task.cancel()
failure := errors.New("connection refused")
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
var samples int64
check := func(wantLog bool) {
t.Helper()
logs.Reset()
result := task.runProbe(probe)
require.NotNil(t, result)
samples++
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
if !wantLog {
assert.Empty(t, logs.String())
} else {
assert.Contains(t, logs.String(), `msg="monitor failed"`)
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
}
}
check(true)
check(false)
time.Sleep(5*time.Minute - time.Nanosecond)
check(false)
time.Sleep(time.Nanosecond)
check(true)
check(false)
time.Sleep(5 * time.Minute)
check(true)
check(false)
// Recovery clears the cooldown.
failure = nil
check(false)
failure = errors.New("connection refused again")
check(true)
// Another monitor has its own cooldown.
other := newMonitorTask(task.config)
defer other.cancel()
logs.Reset()
require.NotNil(t, other.runProbe(probe))
assert.Contains(t, logs.String(), `msg="monitor failed"`)
// A canceled probe must not publish a failure or emit a warning.
logs.Reset()
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
other.cancel()
return -1, context.Canceled
})
assert.Nil(t, result)
assert.Empty(t, logs.String())
})
}
+524
View File
@@ -0,0 +1,524 @@
package agent
import (
"context"
"encoding/binary"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/dns/dnsmessage"
)
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(25), result.AvgResponse1h)
assert.Equal(t, int64(30), result.MinResponse)
assert.Equal(t, int64(10), result.MinResponse1h)
assert.Equal(t, int64(30), result.MaxResponse)
assert.Equal(t, int64(40), result.MaxResponse1h)
assert.Equal(t, 50.0, result.PacketLoss)
assert.Equal(t, 20.0, result.PacketLoss1h)
}
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(0), result.AvgResponse)
assert.Equal(t, int64(0), result.AvgResponse1h)
assert.Equal(t, int64(0), result.MinResponse)
assert.Equal(t, int64(0), result.MinResponse1h)
assert.Equal(t, int64(0), result.MaxResponse)
assert.Equal(t, int64(0), result.MaxResponse1h)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
}
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
assert.Equal(t, "monitor-1", cfg.ID)
}
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
pm := newMonitorManager()
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
defer pm.Stop()
_, validExists := pm.monitors[validCfg.ID]
_, invalidExists := pm.monitors[invalidCfg.ID]
assert.True(t, validExists)
assert.False(t, invalidExists)
}
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
keptTask := newMonitorTask(keepCfg)
removedTask := newMonitorTask(removeCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
keepCfg.ID: keptTask,
removeCfg.ID: removedTask,
}
pm.SyncMonitors([]monitor.Config{keepCfg})
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
_, exists := pm.monitors[removeCfg.ID]
assert.False(t, exists)
select {
case <-removedTask.ctx.Done():
default:
t.Fatal("expected removed monitor task to be cancelled")
}
select {
case <-keptTask.ctx.Done():
t.Fatal("expected existing monitor task to remain active")
default:
}
}
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
originalTask := newMonitorTask(originalCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
originalCfg.ID: originalTask,
}
pm.SyncMonitors([]monitor.Config{updatedCfg})
defer pm.Stop()
restartedTask := pm.monitors[updatedCfg.ID]
assert.NotSame(t, originalTask, restartedTask)
assert.Equal(t, updatedCfg, restartedTask.config)
select {
case <-originalTask.ctx.Done():
default:
t.Fatal("expected changed monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
pm := &MonitorManager{
monitors: make(map[string]*monitorTask),
probe: networkMonitorProbe(server.Client()),
}
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
RunNow: true,
})
defer pm.Stop()
require.NoError(t, err)
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
assert.Equal(t, 0.0, resp.Result.PacketLoss)
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
task := pm.monitors["monitor-1"]
require.NotNil(t, task)
task.history.mu.Lock()
defer task.history.mu.Unlock()
require.Len(t, task.history.samples, 1)
}
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
now := time.Now().UTC()
existingTask := newMonitorTask(originalCfg)
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
result, err := pm.UpsertMonitor(updatedCfg, false)
defer pm.Stop()
require.NoError(t, err)
assert.Nil(t, result)
updatedTask := pm.monitors[updatedCfg.ID]
require.NotNil(t, updatedTask)
assert.NotSame(t, existingTask, updatedTask)
assert.Equal(t, updatedCfg, updatedTask.config)
updatedTask.history.mu.Lock()
defer updatedTask.history.mu.Unlock()
require.Len(t, updatedTask.history.samples, 1)
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
agg := updatedTask.history.aggregateLocked(time.Hour, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
assert.Equal(t, int64(18), agg.avgResponse())
select {
case <-existingTask.ctx.Done():
default:
t.Fatal("expected original monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
task := newMonitorTask(config)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{config.ID: task}
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: config.ID},
})
require.NoError(t, err)
_, exists := pm.monitors[config.ID]
assert.False(t, exists)
select {
case <-task.ctx.Done():
default:
t.Fatal("expected deleted monitor task to be cancelled")
}
}
func TestMonitorManagerGetRandomDelay(t *testing.T) {
for i := 1000; i < 360_000; i += 1000 {
delay := getStagger(int64(i))
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
}
}
func TestMonitorHTTP(t *testing.T) {
t.Run("success", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("server error", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "boom", http.StatusInternalServerError)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCP(t *testing.T) {
t.Run("success", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
accepted := make(chan struct{})
go func() {
defer close(accepted)
conn, err := listener.Accept()
if err == nil {
_ = conn.Close()
}
}()
port := uint16(listener.Addr().(*net.TCPAddr).Port)
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
<-accepted
})
t.Run("connection failure", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := uint16(listener.Addr().(*net.TCPAddr).Port)
require.NoError(t, listener.Close())
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCPAddressFallback(t *testing.T) {
for _, tc := range []struct {
name string
ips []string
loss bool
}{
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp4", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
original := net.DefaultResolver
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
defer func() { net.DefaultResolver = original }()
// Verify the resolver preserves the intended order, so success cannot
// accidentally bypass the failed first address in the regression case.
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
require.NoError(t, err)
require.Equal(t, tc.ips, ips)
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
if tc.loss {
require.Error(t, err)
assert.Equal(t, int64(-1), responseUs)
} else {
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
}
})
}
}
// tcpMonitorTestResolver supplies multiple A records without external DNS.
func tcpMonitorTestResolver(ips []string) *net.Resolver {
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
client, server := net.Pipe()
go func() {
defer server.Close()
// net.Resolver uses TCP framing when its connection is not a PacketConn.
var size uint16
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
return
}
packet := make([]byte, size)
if _, err := io.ReadFull(server, packet); err != nil {
return
}
var msg dnsmessage.Message
if err := msg.Unpack(packet); err != nil {
return
}
msg.Header.Response = true
msg.Header.RecursionAvailable = true
for _, question := range msg.Questions {
if question.Type != dnsmessage.TypeA {
continue
}
for _, ip := range ips {
msg.Answers = append(msg.Answers, dnsmessage.Resource{
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
})
}
}
packet, err := msg.Pack()
if err != nil {
return
}
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
_, _ = server.Write(append(response, packet...))
}()
return client, nil
}}
}
func TestMonitorDNS(t *testing.T) {
t.Run("success", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "localhost")
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("lookup failure", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "")
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
t.Run(action, func(t *testing.T) {
started := make(chan struct{})
canceled := make(chan struct{})
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
select {
case <-r.Context().Done():
close(canceled)
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
defer pm.Stop()
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
task := newMonitorTask(cfg)
// Seed history to ensure a canceled RunNow does not return an old result.
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
done := make(chan *monitor.Result, 1)
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
done <- result
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("probe did not start")
}
updated := cfg
updated.Interval--
switch action {
case "stop":
pm.Stop()
case "delete":
pm.DeleteMonitor(cfg.ID)
case "upsert":
_, err := pm.UpsertMonitor(updated, false)
require.NoError(t, err)
case "sync replace":
pm.SyncMonitors([]monitor.Config{updated})
case "sync remove":
pm.SyncMonitors(nil)
}
select {
case <-canceled:
case <-time.After(time.Second):
t.Fatal("active HTTP request was not canceled")
}
select {
case result := <-done:
assert.Nil(t, result)
case <-time.After(time.Second):
t.Fatal("RunNow did not return after cancellation")
}
task.history.mu.Lock()
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
task.history.mu.Unlock()
})
}
}
func TestMonitorResolutionCancellation(t *testing.T) {
for _, protocol := range []string{"tcp", "dns", "icmp"} {
t.Run(protocol, func(t *testing.T) {
started := make(chan struct{}, 1)
original := net.DefaultResolver
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
select {
case started <- struct{}{}:
default:
}
<-ctx.Done()
return nil, ctx.Err()
}}
defer func() { net.DefaultResolver = original }()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
var err error
switch protocol {
case "tcp":
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
case "dns":
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
case "icmp":
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
}
done <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("lookup did not start")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("lookup did not cancel")
}
})
}
}
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
defer task.cancel()
result := task.runProbe(pm.probe)
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
require.Len(t, task.history.samples, 1)
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
}
+4 -1
View File
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
const (
// Version is the current version of the application.
Version = "0.19.0"
Version = "0.20.0"
// AppName is the name of the application.
AppName = "beszel"
)
@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
var MinVersionZfsData = semver.MustParse("0.18.9")
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
+9 -6
View File
@@ -20,10 +20,11 @@ type hubLike interface {
}
type AlertManager struct {
hub hubLike
stopOnce sync.Once
pendingAlerts sync.Map
alertsCache *AlertsCache
hub hubLike
stopOnce sync.Once
pendingAlerts sync.Map
alertsCache *AlertsCache
networkMonitors *networkMonitorCache
}
type AlertMessageData struct {
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
// NewAlertManager creates a new AlertManager instance.
func NewAlertManager(app hubLike) *AlertManager {
am := &AlertManager{
hub: app,
alertsCache: NewAlertsCache(app),
hub: app,
alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
}
am.bindEvents()
return am
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
// Bind events to the alerts collection lifecycle
func (am *AlertManager) bindEvents() {
am.bindNetworkMonitorAlertEvents()
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
+7
View File
@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err)
}
if reqData.Name == alertNameNetworkMonitorLoss {
if reqData.Value < 0 || reqData.Value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
reqData.Min = 0
}
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
if err != nil {
return err
+38 -12
View File
@@ -1,6 +1,7 @@
package alerts
import (
"sync"
"time"
"github.com/pocketbase/dbx"
@@ -18,6 +19,9 @@ type CachedAlertData struct {
Triggered bool
Min uint8
PendingSince time.Time
// Immutable after publication; decoded only when the alert record changes.
MonitorStates map[string]string
MonitorStatesValid bool
// Created types.DateTime
}
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time()
if a.Name == alertNameNetworkMonitorLoss {
var state networkMonitorAlertState
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
a.MonitorStates = state.Monitors
}
// a.Created = record.GetDateTime("created")
}
// AlertsCache provides an in-memory cache for system alerts.
type AlertsCache struct {
// Serialize lazy loads with updates so a late load cannot replace newer state.
loadMu sync.Mutex
app core.App
store *store.Store[string, *store.Store[string, CachedAlertData]]
populated bool
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
func (c *AlertsCache) PopulateFromDB(force bool) error {
c.loadMu.Lock()
defer c.loadMu.Unlock()
if !force && c.populated {
return nil
}
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
}
c.store.RemoveAll()
for _, record := range records {
c.Update(record)
c.update(record)
}
c.populated = true
return nil
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
// Update adds or updates an alert record in the cache.
func (c *AlertsCache) Update(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
c.update(record)
}
func (c *AlertsCache) update(record *core.Record) {
systemID := record.GetString("system")
if systemID == "" {
return
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
// Delete removes an alert record from the cache.
func (c *AlertsCache) Delete(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
systemID := record.GetString("system")
if systemID == "" {
return
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
systemStore, ok := c.store.GetOk(systemID)
if !ok {
// Populate cache for this system
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
if err != nil {
return nil
c.loadMu.Lock()
defer c.loadMu.Unlock()
systemStore, ok = c.store.GetOk(systemID)
if !ok {
// Populate cache for this system
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
if err != nil {
return nil
}
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
}
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
}
all := systemStore.GetAll()
alerts := make([]CachedAlertData, 0, len(all))
+10
View File
@@ -9,6 +9,12 @@ import (
// On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
return err
}
return e.Next()
}
if !e.Record.GetBool("triggered") {
return e.Next()
}
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
// Network monitor incidents have separate history entries per monitor.
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
return e.Next()
}
original := e.Record.Original()
new := e.Record
+269
View File
@@ -0,0 +1,269 @@
package alerts
import (
"database/sql"
"errors"
"fmt"
"math"
"net"
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
// networkMonitorAlertState is this alert type's persisted runtime state.
// Monitor IDs map to their open history entries independently of history retention.
type networkMonitorAlertState struct {
Monitors map[string]string `json:"monitors"`
}
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// Hidden fields are still writable through the record API unless protected.
protectState := func(e *core.RecordRequestEvent) error {
e.Record.Set("state", e.Record.Original().Get("state"))
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
}
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
return e.ForbiddenError("You do not have access to this system", nil)
}
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
value := e.Record.GetFloat("value")
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
e.Record.Set("min", 0)
}
return e.Next()
}
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
cleanup := func(e *core.RecordEvent) error {
if err := e.Next(); err != nil {
return err
}
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
}
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
return e.Next()
}
return cleanup(e)
})
}
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
// using its exact payload (dashboard requests can replace the cached payload).
// Omitted results and disconnected systems never imply recovery.
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
if systemRecord.GetString("status") != "up" {
return nil
}
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
if len(alerts) == 0 {
return nil
}
monitors, err := am.networkMonitors.get(systemRecord.Id)
if err != nil {
return err
}
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
return nil
}
// The cache only predicts a transition. Reload and recheck under the DB
// transaction before persisting, including current system/monitor status.
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
}
// networkMonitorTransitionPending does no IO and never mutates cached maps.
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
for _, alert := range alerts {
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
return true
}
for id := range alert.MonitorStates {
if _, enabled := monitors[id]; !enabled {
return true
}
}
for id, result := range results {
interval, enabled := monitors[id]
if !enabled || !monitorResultReady(result, interval, now) {
continue
}
_, active := alert.MonitorStates[id]
if (result.PacketLoss1h > alert.Value) != active {
return true
}
}
}
return false
}
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
var messages []AlertMessageData
err := app.RunInTransaction(func(tx core.App) error {
// Read configuration inside the transaction so concurrent threshold changes,
// disabling, and evaluations cannot overwrite each other's incident state.
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
if err != nil || len(alerts) == 0 {
return err
}
system, err := tx.FindRecordById("systems", systemID)
if errors.Is(err, sql.ErrNoRows) {
// System deletion cascades to its alerts.
return nil
}
if err != nil {
return err
}
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
if err != nil {
return err
}
enabled := make(map[string]*core.Record, len(monitors))
for _, m := range monitors {
enabled[m.Id] = m
}
now := time.Now()
for _, alert := range alerts {
var state networkMonitorAlertState
if err := alert.UnmarshalJSONField("state", &state); err != nil {
return err
}
states := state.Monitors
if states == nil {
states = map[string]string{}
}
changed := false
// Removing or disabling a monitor closes its incident silently.
for id, historyID := range states {
if _, ok := enabled[id]; !ok {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, id)
changed = true
}
}
if system.GetString("status") == "up" {
for _, m := range monitors {
result, ok := results[m.Id]
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
continue
}
historyID, active := states[m.Id]
triggered := result.PacketLoss1h > alert.GetFloat("value")
if triggered == active {
continue
}
label := m.GetString("target")
if m.GetString("protocol") == "tcp" {
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
}
if triggered {
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
history := core.NewRecord(collection)
history.Load(map[string]any{
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
})
if err := tx.Save(history); err != nil {
return err
}
states[m.Id] = history.Id
} else {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, m.Id)
}
changed = true
state, comparison := "loss", "exceeds"
if !triggered {
state, comparison = "recovered", "is at or below"
}
messages = append(messages, AlertMessageData{
UserID: alert.GetString("user"), SystemID: systemID,
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
})
}
}
if changed || alert.GetBool("triggered") != (len(states) > 0) {
alert.Set("state", networkMonitorAlertState{Monitors: states})
alert.Set("triggered", len(states) > 0)
if err := tx.Save(alert); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
// Match other alert types: persist transitions before delivery, and respect
// the user's existing notification destinations and quiet hours.
for _, message := range messages {
if err := am.SendAlert(message); err != nil {
app.Logger().Error("Failed to send network monitor alert", "err", err)
}
}
return nil
}
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
// Three completed attempts provide a short warm-up, including after an agent
// restart.
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
return false
}
// Never interpret an empty one-hour window as zero loss.
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
age := now.Sub(time.UnixMilli(result.LastProbeAt))
return age >= -time.Minute && age <= maxAge
}
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
record, err := app.FindRecordById("alerts_history", id)
if errors.Is(err, sql.ErrNoRows) {
// History can be purged independently.
return nil
}
if err != nil {
return err
}
if !record.GetDateTime("resolved").IsZero() {
return nil
}
record.Set("resolved", now.UTC())
return app.Save(record)
}
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
if err != nil {
return err
}
for _, record := range records {
record.Set("resolved", time.Now().UTC())
if err := app.Save(record); err != nil {
return err
}
}
return nil
}
@@ -0,0 +1,513 @@
//go:build testing
package alerts_test
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/henrygd/beszel/internal/alerts"
"github.com/henrygd/beszel/internal/entities/monitor"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
pbTests "github.com/pocketbase/pocketbase/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
t.Helper()
hub, system, alert := systemdTestSetup(t, false)
t.Cleanup(hub.Cleanup)
alert.Set("name", "NetworkMonitorLoss")
alert.Set("value", 5)
require.NoError(t, hub.Save(alert))
var monitors []*core.Record
for _, name := range []string{"gateway", "website"} {
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
monitors = append(monitors, record)
}
// Avoid starting a system update worker in tests.
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
system.Set("status", "up")
return hub, system, alert, monitors
}
func monitorResult(loss float64) monitor.Result {
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
}
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
check := func(active bool, open, sent int) {
t.Helper()
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.Equal(t, active, record.GetBool("triggered"))
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
assert.EqualValues(t, open, total)
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
message := hub.TestMailer.Messages()[count]
assert.Contains(t, message.Text, "gateway.example.com")
assert.Contains(t, message.Text, "10.00%")
assert.Contains(t, message.Text, "5.00%")
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
// Persisted monitor state prevents duplicate notifications after a hub restart.
am = alerts.NewTestAlertManagerWithoutWorker(hub)
results[monitors[1].Id] = monitorResult(20)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 2, 2)
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 3)
results[monitors[1].Id] = monitorResult(0)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(false, 0, 4)
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 2)
for _, history := range histories {
assert.NotEmpty(t, history.GetString("monitor_name"))
}
}
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
for _, tc := range []struct {
protocol, target, label string
port int
}{
{"icmp", "gateway.example.com", "gateway.example.com", 0},
{"http", "https://example.com/health", "https://example.com/health", 0},
{"tcp", "example.com", "example.com:8443", 8443},
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
} {
t.Run(tc.label, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
m := monitors[0]
m.Set("protocol", tc.protocol)
m.Set("target", tc.target)
m.Set("port", tc.port)
require.NoError(t, hub.Save(m))
am := alerts.NewTestAlertManagerWithoutWorker(hub)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 1)
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
})
}
}
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
apply := func(loss float64) {
result := monitorResult(loss)
results := map[string]monitor.Result{monitors[0].Id: result}
switch scenario {
case "missing":
results = nil
case "stale":
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
case "expired hourly window":
monitors[0].Set("interval", 3600)
require.NoError(t, hub.Save(monitors[0]))
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "future":
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "warmup":
result.SampleCount = 2
results[monitors[0].Id] = result
case "no probes":
result.SampleCount = 0
results[monitors[0].Id] = result
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
case "paused":
record, err := hub.FindRecordById("systems", system.Id)
require.NoError(t, err)
record.Set("status", "paused")
require.NoError(t, hub.Save(record))
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
}
count := hub.TestMailer.TotalSend()
apply(100)
assert.Equal(t, count, hub.TestMailer.TotalSend())
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
apply(0)
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
})
}
}
func TestNetworkMonitorAlertCleanup(t *testing.T) {
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
switch scenario {
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "delete monitor":
require.NoError(t, hub.Delete(monitors[0]))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "delete system":
require.NoError(t, hub.Delete(system))
case "purge history":
history, err := hub.FindAllRecords("alerts_history")
require.NoError(t, err)
for _, record := range history {
require.NoError(t, hub.Delete(record))
}
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
if scenario == "disable monitor" || scenario == "delete monitor" {
assert.EqualValues(t, 1, open)
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
require.NoError(t, hub.Delete(monitors[1]))
record, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, record.GetBool("triggered"))
} else {
assert.Zero(t, open)
}
})
}
}
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
require.NoError(t, err)
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
require.NoError(t, err)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
other, err = hub.FindRecordById("alerts", other.Id)
require.NoError(t, err)
assert.False(t, other.GetBool("triggered"))
// Editing the threshold re-evaluates on the next batch, without losing state.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, alert.GetBool("triggered"))
}
func TestNetworkMonitorAlertAPI(t *testing.T) {
for _, tc := range []struct {
name string
value float64
direct, denied, patch bool
status int
}{
{name: "zero threshold", value: 0, status: 200},
{name: "fractional threshold", value: 5.5, status: 200},
{name: "negative threshold", value: -1, status: 400},
{name: "unreachable threshold", value: 100, status: 400},
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
{name: "direct private state", value: 5, direct: true, status: 200},
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
} {
t.Run(tc.name, func(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
owner := user.Id
if tc.denied {
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
require.NoError(t, err)
owner = other.Id
}
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
url, method := "/api/beszel/user-alerts", "POST"
if tc.direct {
url = "/api/collections/alerts/records"
body["system"], body["user"] = systems[0].Id, user.Id
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
}
if tc.patch {
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
require.NoError(t, err)
url += "/" + alert.Id
method = "PATCH"
body["triggered"] = false
}
content := `"success":true`
if tc.direct {
content = `"name":"NetworkMonitorLoss"`
}
if tc.status == 400 {
content = `"status":400`
}
if tc.status == 403 {
content = `"status":403`
}
scenario := beszelTests.ApiScenario{
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
}
scenario.Test(t)
records, err := hub.FindAllRecords("alerts")
require.NoError(t, err)
if tc.status != 200 || tc.denied {
assert.Empty(t, records)
return
}
require.Len(t, records, 1)
assert.Equal(t, tc.value, records[0].GetFloat("value"))
assert.Zero(t, records[0].GetInt("min"))
state := struct {
Monitors map[string]string `json:"monitors"`
}{}
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
states := state.Monitors
if tc.patch {
assert.Equal(t, map[string]string{"real": "history"}, states)
assert.True(t, records[0].GetBool("triggered"))
} else {
assert.Empty(t, states)
assert.False(t, records[0].GetBool("triggered"))
}
})
}
}
type monitorCountingHub struct {
*beszelTests.TestHub
transactions atomic.Int64
beforeTransaction func()
}
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
h.transactions.Add(1)
if h.beforeTransaction != nil {
h.beforeTransaction()
}
return h.App.RunInTransaction(fn)
}
// Count actual SQL on both DB connections, including queries through record APIs.
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
t.Helper()
count := &atomic.Int64{}
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
db := builder.(*dbx.DB)
old := db.LogFunc
db.LogFunc = func(string, ...any) { count.Add(1) }
t.Cleanup(func() { db.LogFunc = old })
}
return count
}
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
sql := monitorSQLCounter(t, hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
noWork := func() {
t.Helper()
sql.Store(0)
counted.transactions.Store(0)
for range 100 {
evaluate()
}
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
}
// One-time lazy loads are permitted, including on hub restart.
evaluate()
assert.Positive(t, sql.Load())
noWork()
// Realtime metric saves invoke record hooks but must not invalidate config.
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
require.NoError(t, err)
monitors[0] = fresh
monitors[0].Set("loss1h", 0)
monitors[0].Set("res", 100)
require.NoError(t, hub.Save(monitors[0]))
noWork()
results[monitors[0].Id] = monitorResult(10)
evaluate()
assert.Positive(t, sql.Load(), "transitions must still be persisted")
assert.EqualValues(t, 1, counted.transactions.Load())
noWork()
// Missing and stale observations must not enter the transaction either.
results = nil
noWork()
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
noWork()
results[monitors[0].Id] = monitorResult(0)
evaluate()
noWork()
require.NoError(t, hub.Delete(alert))
noWork()
}
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Widening the interval makes this observation fresh. A stale interval cache
// would miss the failure indefinitely, even though results keep arriving.
result := monitorResult(10)
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
monitors[0].Set("interval", 120)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
// Disable, then re-enable the same ID: its new failure must be detected.
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
monitors[0].Set("enabled", true)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
// A new monitor must also become eligible without restarting the hub.
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
results[created.Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
// Threshold changes refresh cached config and preserve the active incidents.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
t.Run(change, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Change the DB after the cache predicts a transition, before its transaction.
counted.beforeTransaction = func() {
counted.beforeTransaction = nil
switch change {
case "threshold":
alert.Set("value", 20)
require.NoError(t, hub.Save(alert))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
}
}
results[monitors[0].Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.EqualValues(t, 1, counted.transactions.Load())
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
})
}
}
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
count := hub.TestMailer.TotalSend()
var wg sync.WaitGroup
errs := make(chan error, 8)
for range 8 {
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
}
wg.Wait()
close(errs)
for err := range errs {
require.NoError(t, err)
}
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
count := hub.TestMailer.TotalSend()
results[monitors[0].Id] = monitorResult(10)
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
require.NoError(t, err)
// A failed transition must not be published to the cache and mask the retry.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}
+1 -1
View File
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
if len(alerts) == 0 {
return nil
}
+3 -2
View File
@@ -11,8 +11,9 @@ import (
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
return &AlertManager{
hub: app,
alertsCache: NewAlertsCache(app),
hub: app,
alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
}
}
+79
View File
@@ -0,0 +1,79 @@
package alerts
import (
"sync"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
// needed for the alert fast path. Names and targets are read only on transitions.
// Returned maps are immutable; configuration changes invalidate the whole entry.
type networkMonitorCache struct {
app core.App
mu sync.RWMutex
systems map[string]map[string]int
}
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
invalidate := func(e *core.RecordEvent) error {
c.invalidate(e.Record.GetString("system"))
return e.Next()
}
app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
old := e.Record.Original()
// Realtime metric saves also invoke this hook. They must not evict config.
if old.GetString("system") != e.Record.GetString("system") ||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
old.GetInt("interval") != e.Record.GetInt("interval") {
c.invalidate(old.GetString("system"))
c.invalidate(e.Record.GetString("system"))
}
return e.Next()
})
app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
c.invalidate(e.Record.Id)
return e.Next()
})
return c
}
func (c *networkMonitorCache) invalidate(systemID string) {
c.mu.Lock()
delete(c.systems, systemID)
c.mu.Unlock()
}
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
c.mu.RLock()
monitors, ok := c.systems[systemID]
c.mu.RUnlock()
if ok {
return monitors, nil
}
c.mu.Lock()
defer c.mu.Unlock()
if monitors, ok := c.systems[systemID]; ok {
return monitors, nil
}
// Keep the lock through the load so a concurrent config change cannot be
// invalidated first and then overwritten by the older query result.
var rows []struct {
ID string `db:"id"`
Interval int `db:"interval"`
}
if err := c.app.DB().Select("id", "interval").From("network_monitors").
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
return nil, err
}
monitors = make(map[string]int, len(rows))
for _, row := range rows {
monitors[row.ID] = row.Interval
}
c.systems[systemID] = monitors
return monitors, nil
}
+2
View File
@@ -24,6 +24,8 @@ const (
GetSystemdInfo
// Request ZFS detail data from agent
GetZfsData
// Sync network monitor configuration to agent
SyncNetworkMonitors
// Add new actions here...
)
+102
View File
@@ -0,0 +1,102 @@
package monitor
import "time"
// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
// Hub requests that run a probe must allow this time in addition to transport overhead.
const MaxProbeTimeout = 10 * time.Second
type SyncAction uint8
const (
// SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
SyncActionReplace SyncAction = iota
// SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
SyncActionUpsert
// SyncActionDelete indicates an incremental sync where the provided config should be removed.
SyncActionDelete
)
// Config defines a network monitor task sent from hub to agent.
type Config struct {
// ID is the stable network_monitors record ID generated by the hub.
ID string `cbor:"0,keyasint"`
Target string `cbor:"1,keyasint"`
Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
Port uint16 `cbor:"3,keyasint,omitempty"`
Interval uint16 `cbor:"4,keyasint"` // seconds
}
// SyncRequest defines an incremental or full monitor sync request sent to the agent.
type SyncRequest struct {
Action SyncAction `cbor:"0,keyasint"`
Config Config `cbor:"1,keyasint,omitempty"`
Configs []Config `cbor:"2,keyasint,omitempty"`
RunNow bool `cbor:"3,keyasint,omitempty"`
}
// SyncResponse returns the immediate result for an upsert when requested.
type SyncResponse struct {
Result Result `cbor:"0,keyasint,omitempty"`
}
// Result holds aggregated monitor results for a single target.
//
// 0: avg response in microseconds
//
// 1: 1h average response in microseconds
//
// 2: min response in microseconds
//
// 3: 1h min response in microseconds
//
// 4: max response in microseconds
//
// 5: 1h max response in microseconds
//
// 6: packet loss percentage (0-100)
//
// 7: 1h packet loss percentage (0-100)
type Result struct {
AvgResponse int64 `cbor:"0,keyasint,omitempty"`
AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
MinResponse int64 `cbor:"2,keyasint,omitempty"`
MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
MaxResponse int64 `cbor:"4,keyasint,omitempty"`
MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
PacketLoss float64 `cbor:"6,keyasint,omitempty"`
PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
// LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
LastProbeAt int64 `cbor:"8,keyasint"`
// SampleCount includes all completed probes since this monitor started.
// Used for alert warm-up even when the interval is longer than 20 minutes.
SampleCount int64 `cbor:"9,keyasint,omitempty"`
// Counts and sum cover the current response window (or latest-sample
// fallback), not the hourly window or lifetime SampleCount.
TotalCount int64 `cbor:"10,keyasint"`
SuccessCount int64 `cbor:"11,keyasint"`
ResponseSum int64 `cbor:"12,keyasint"`
}
// Stats holds response times in microseconds and packet loss percentage (0-100).
type Stats struct {
ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
ResMin float64 `json:"res_min" db:"res_min"`
ResMax float64 `json:"res_max" db:"res_max"`
Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
TotalCount int64 `json:"-" db:"total_count"`
SuccessCount int64 `json:"-" db:"success_count"`
ResponseSum int64 `json:"-" db:"res_sum"`
}
func (s Stats) FromResult(result Result) Stats {
return Stats{
ResAvg: float64(result.AvgResponse),
ResMin: float64(result.MinResponse),
ResMax: float64(result.MaxResponse),
Loss: result.PacketLoss,
TotalCount: result.TotalCount,
SuccessCount: result.SuccessCount,
ResponseSum: result.ResponseSum,
}
}
+3 -1
View File
@@ -7,6 +7,7 @@ import (
"time"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/systemd"
)
@@ -210,5 +211,6 @@ type CombinedData struct {
Details *Details `cbor:"4,keyasint,omitempty"`
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
// that omitted systemd data (for example, a short-cache dashboard request).
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
Monitors map[string]monitor.Result `cbor:"6,keyasint"`
}
+11 -1
View File
@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
return err
}
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
list: &systemScopedReadRule,
}); err != nil {
return err
@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {
return err
}
if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
create: &systemScopedWriteRule,
update: &systemScopedWriteRule,
delete: &systemScopedWriteRule,
}); err != nil {
return err
}
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
+2
View File
@@ -109,6 +109,8 @@ func (h *Hub) StartHub() error {
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
bindNetworkMonitorsEvents(h)
pb, ok := h.App.(*pocketbase.PocketBase)
if !ok {
return errors.New("not a pocketbase app")
+158
View File
@@ -0,0 +1,158 @@
package hub
import (
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/hub/systems"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
)
// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
func generateMonitorID(systemId string, config monitor.Config) string {
args := []string{systemId, config.Target, config.Protocol}
// only use port for TCP monitors, since for other protocols it's not relevant as standalone value
if config.Protocol == "tcp" {
args = append(args, strconv.FormatUint(uint64(config.Port), 10))
}
return systems.MakeStableHashId(args...)
}
// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
func bindNetworkMonitorsEvents(hub *Hub) {
// on create, make sure the id is set to a stable hash
hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
systemID := e.Record.GetString("system")
config := monitorConfigFromRecord(e.Record)
id := generateMonitorID(systemID, *config)
e.Record.Set("id", id)
return e.Next()
})
// sync monitor to agent on creation and persist the first result immediately when available
hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
err := e.Next()
if err != nil {
return err
}
if !e.Record.GetBool("enabled") {
return nil
}
// If connected, run the monitor immediately. Paused systems may be absent
// from the manager; their monitors will sync when they reconnect.
system, err := hub.sm.GetSystem(e.Record.GetString("system"))
if err == nil && system.Status == "up" {
go hub.upsertNetworkMonitor(e.Record, true)
}
return nil
})
// On API update requests, if the monitor config changed in a way that requires a new ID, create a new
// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
systemID := e.Record.GetString("system")
// only tcp uses port - set other protocols port to zero
if e.Record.GetString("protocol") != "tcp" {
e.Record.Set("port", 0)
}
ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
if ID != e.Record.Id {
newRecord := copyMonitorToNewRecord(e.Record, ID)
if err := e.App.Save(newRecord); err != nil {
return err
}
if err := e.App.Delete(e.Record); err != nil {
return err
}
return nil
}
err := e.Next()
if err != nil {
return err
}
if e.Record.GetBool("enabled") {
// if the monitor is enabled, sync the updated config to the agent now
runNow := !e.Record.Original().GetBool("enabled")
err = hub.upsertNetworkMonitor(e.Record, runNow)
} else {
// if the monitor is paused, remove it from the agent
err = hub.deleteNetworkMonitor(e.Record)
}
if err != nil {
hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
}
return nil
})
// sync monitor to agent on delete
hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if err := hub.deleteNetworkMonitor(e.Record); err != nil {
hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
}
return e.Next()
})
}
// monitorConfigFromRecord builds a monitor config from a network_monitors record.
func monitorConfigFromRecord(record *core.Record) *monitor.Config {
return &monitor.Config{
ID: record.Id,
Target: record.GetString("target"),
Protocol: record.GetString("protocol"),
Port: uint16(record.GetInt("port")),
Interval: uint16(record.GetInt("interval")),
}
}
// setMonitorResultFields stores the latest monitor result values on the record.
func setMonitorResultFields(record *core.Record, result monitor.Result) {
nowString := time.Now().UTC().Format(types.DefaultDateLayout)
record.Set("res", result.AvgResponse)
record.Set("resAvg1h", result.AvgResponse1h)
record.Set("resMin1h", result.MinResponse1h)
record.Set("resMax1h", result.MaxResponse1h)
record.Set("loss1h", result.PacketLoss1h)
record.Set("updated", nowString)
}
// copyMonitorToNewRecord creates a new record with the same field values as the old one.
// This is used when the monitor config changes in a way that requires a new ID, so we need
// to create a new record with the new ID and delete the old one.
func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
collection := oldRecord.Collection()
newRecord := core.NewRecord(collection)
newRecord.Id = newID
fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}
for _, field := range fields {
newRecord.Set(field, oldRecord.Get(field))
}
return newRecord
}
// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
// is true, it will also trigger an immediate monitor run and update the record with the result.
func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
if err != nil || result == nil {
return err
}
setMonitorResultFields(record, *result)
return h.App.SaveNoValidate(record)
}
// deleteNetworkMonitor removes the record's monitor from the target system.
func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
return system.DeleteNetworkMonitor(record.Id)
}
+225
View File
@@ -0,0 +1,225 @@
package hub
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/pocketbase/apis"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
for _, batch := range []bool{false, true} {
name := "single"
if batch {
name = "batch"
}
t.Run(name, func(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
bindNetworkMonitorsEvents(hub)
user, err := createTestUser(hub)
require.NoError(t, err)
system, err := createTestRecord(hub, "systems", map[string]any{
"name": "Paused", "host": "localhost", "port": "45876",
"status": "paused", "users": []string{user.Id},
})
require.NoError(t, err)
// Paused systems are not loaded into the manager at startup.
_, err = hub.sm.GetSystem(system.Id)
require.Error(t, err)
payload := func(target string) map[string]any {
return map[string]any{
"system": system.Id, "target": target, "protocol": "icmp",
"interval": 60, "enabled": true,
}
}
url := "/api/collections/network_monitors/records"
var body any = payload("1.1.1.1")
count := 1
if batch {
body = map[string]any{"requests": []map[string]any{
{"method": "POST", "url": url, "body": payload("1.1.1.1")},
{"method": "POST", "url": url, "body": payload("8.8.8.8")},
}}
url = "/api/batch"
count = 2
}
data, err := json.Marshal(body)
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
router, err := apis.NewRouter(hub)
require.NoError(t, err)
handler, err := router.BuildMux()
require.NoError(t, err)
request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", token)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
records, err := hub.FindAllRecords("network_monitors")
require.NoError(t, err)
require.Len(t, records, count)
for _, record := range records {
assert.Equal(t, system.Id, record.GetString("system"))
assert.True(t, record.GetBool("enabled"))
}
})
}
}
func TestGenerateMonitorID(t *testing.T) {
tests := []struct {
name string
systemID string
config monitor.Config
expected string
}{
{
name: "HTTP monitor on example.com",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 0,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different port",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 8080,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different system ID",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 60,
},
expected: "ab602ae7",
},
{
name: "Same monitor, different interval",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 120,
},
expected: "ab602ae7",
},
{
name: "ICMP monitor on 1.1.1.1",
systemID: "sys456",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "6d13a4a4",
}, {
name: "ICMP monitor on 1.1.1.1 with different system ID",
systemID: "sys4567",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "ddd6c81",
},
{
name: "TCP monitor on example.com with port 443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 443,
Interval: 30,
},
expected: "677b991",
},
{
name: "TCP monitor on example.com with port 8443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 8443,
Interval: 30,
},
expected: "84167969",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := generateMonitorID(tt.systemID, tt.config)
assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
})
}
}
func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
assert.Nil(t, collection.Fields.GetByName("name"))
oldRecord := core.NewRecord(collection)
oldRecord.Load(map[string]any{
"system": "sys123",
"target": "https://example.com",
"protocol": "http",
"port": 443,
"interval": 60,
"enabled": true,
"res": 1200,
"resAvg1h": 1300,
"resMin1h": 900,
"resMax1h": 1600,
"loss1h": 5,
"updated": "2026-04-29 12:00:00.000Z",
})
newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
assert.Equal(t, "next12345", newRecord.Id)
assert.Equal(t, "https://example.com", newRecord.GetString("target"))
assert.Equal(t, "http", newRecord.GetString("protocol"))
assert.Equal(t, 443, newRecord.GetInt("port"))
assert.True(t, newRecord.GetBool("enabled"))
assert.Zero(t, newRecord.GetFloat("res"))
assert.Zero(t, newRecord.GetFloat("resAvg1h"))
assert.Zero(t, newRecord.GetFloat("resMin1h"))
assert.Zero(t, newRecord.GetFloat("resMax1h"))
assert.Zero(t, newRecord.GetFloat("loss1h"))
assert.Equal(t, "", newRecord.GetString("updated"))
}
@@ -0,0 +1,224 @@
//go:build testing
package systems
import (
"fmt"
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/subscriptions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNetworkMonitorProbePruning(t *testing.T) {
for _, tc := range []struct {
name string
monitors map[string]monitor.Result
fail bool
want map[string]int64
}{
{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
} {
t.Run(tc.name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
// Preserve the distinction between nil and empty across the agent transport.
encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
require.NoError(t, err)
var data system.CombinedData
require.NoError(t, cbor.Unmarshal(encoded, &data))
if tc.fail {
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
}
_, err = sys.createRecords(&data)
if tc.fail {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
})
}
}
func TestNetworkMonitorStatsFreshness(t *testing.T) {
for _, realtime := range []bool{false, true} {
name := "sql"
if realtime {
name = "realtime"
}
t.Run(name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
}
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
for _, id := range []string{"monitor1", "monitor2"} {
record := core.NewRecord(col)
record.Id = id
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
}
data := &system.CombinedData{Monitors: map[string]monitor.Result{
"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
}}
count := func(want int64) {
t.Helper()
got, err := app.CountRecords("network_monitor_stats")
require.NoError(t, err)
assert.Equal(t, want, got)
}
save := func() {
t.Helper()
_, err := sys.createRecords(data)
require.NoError(t, err)
}
save()
count(2)
stored, err := app.FindAllRecords("network_monitor_stats")
require.NoError(t, err)
for _, record := range stored {
result := data.Monitors[record.GetString("monitor")]
assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
}
// A resume can overlap the scheduled update with the same probe.
errs := make(chan error, 4)
for range 4 {
go func() {
_, err := sys.createRecords(data)
errs <- err
}()
}
for range 4 {
require.NoError(t, <-errs)
}
count(2)
// A rolling hourly value can change without a new probe.
result := data.Monitors["monitor1"]
result.AvgResponse1h = 42
data.Monitors["monitor1"] = result
save()
count(2)
record, err := app.FindRecordById("network_monitors", "monitor1")
require.NoError(t, err)
assert.Equal(t, 42, record.GetInt("resAvg1h"))
// Identical response values and failed probes still count as new measurements.
for id, result := range data.Monitors {
result.LastProbeAt = 301000
data.Monitors[id] = result
}
save()
count(4)
// A failed individual insert must remain retryable, even if others commit.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
require.NoError(t, err)
for id, result := range data.Monitors {
result.LastProbeAt = 601000
data.Monitors[id] = result
}
save()
count(5)
assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
require.NoError(t, err)
save()
count(6)
// Failure after inserting stats rolls back the whole transaction and its markers.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
result = data.Monitors["monitor1"]
result.LastProbeAt = 901000
data.Monitors["monitor1"] = result
_, err = sys.createRecords(data)
require.Error(t, err)
count(6)
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
require.NoError(t, err)
save()
count(7)
// Clock rollback is a new probe identity, not a reason to stall writes.
result.LastProbeAt = 500
data.Monitors["monitor1"] = result
save()
count(8)
// Recreated systems intentionally accept the first result without restoring state.
sys = &System{Id: sys.Id, manager: sys.manager}
save()
count(10)
})
}
}
// Observes the committed DB through the hub, not the transaction's app.
type monitorAlertHub struct {
stubHub
handle func(*core.Record, map[string]monitor.Result) error
}
func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
return h.handle(record, results)
}
func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
for _, realtime := range []bool{false, true} {
t.Run(fmt.Sprint(realtime), func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
}
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
record := core.NewRecord(collection)
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
called := 0
result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
called++
assert.Equal(t, sys.Id, systemRecord.Id)
assert.Equal(t, result, results[record.Id])
saved, err := app.FindRecordById("network_monitors", record.Id)
require.NoError(t, err)
assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
return nil
}}
data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
_, err = sys.createRecords(data)
require.NoError(t, err)
assert.Equal(t, 1, called)
// A transaction that fails after writing monitor stats must not notify.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
_, err = sys.createRecords(data)
require.Error(t, err)
assert.Equal(t, 1, called)
})
}
}
@@ -0,0 +1,144 @@
//go:build testing
package systems
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/blang/semver"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/lxzan/gws"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/require"
)
type monitorSyncClient struct {
gws.BuiltinEventHandler
requests chan common.HubRequest[monitor.SyncRequest]
}
func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
defer message.Close()
var req common.HubRequest[monitor.SyncRequest]
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
return
}
c.requests <- req
data, _ := cbor.Marshal(monitor.SyncResponse{})
response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data})
_ = conn.WriteMessage(gws.OpcodeBinary, response)
}
// Avoid the production delayed disconnect notification; these tests explicitly
// remove each connection from the manager before reconnecting.
type monitorSyncServer struct{ ws.Handler }
func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
t.Run(version, func(t *testing.T) {
// No transport: attempting to send any request would fail.
sys := &System{agentVersion: semver.MustParse(version)}
require.NoError(t, sys.SyncNetworkMonitors(nil))
result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
require.NoError(t, err)
require.Nil(t, result)
require.NoError(t, sys.DeleteNetworkMonitor("test"))
})
}
}
func TestNetworkMonitorReconnectSync(t *testing.T) {
for _, change := range []string{"delete", "disable"} {
t.Run(change, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
record, err := app.FindRecordById("systems", sys.Id)
require.NoError(t, err)
// Suppress unrelated system-stat requests while exercising reconnects.
record.Set("status", paused)
require.NoError(t, app.SaveNoValidate(record))
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
probe := core.NewRecord(collection)
probe.Load(map[string]any{
"system": sys.Id, "target": "localhost", "protocol": "tcp",
"port": 80, "interval": 60, "enabled": true,
})
require.NoError(t, app.SaveNoValidate(probe))
sm := NewSystemManager(stubHub{app})
t.Cleanup(func() {
sm.cancel()
_ = sm.RemoveSystem(sys.Id)
sm.smartFetchMap.StopCleaner()
sm.zfsFetchMap.StopCleaner()
})
version := semver.MustParse("0.20.0")
connections := make(chan *ws.WsConn, 1)
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r)
if err != nil {
t.Error(err)
return
}
wsConn := ws.NewWsConnection(conn, version)
conn.Session().Store("wsConn", wsConn)
connections <- wsConn
conn.ReadLoop()
}))
t.Cleanup(server.Close)
client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
connect := func() monitor.SyncRequest {
t.Helper()
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.NetConn().Close() })
go conn.ReadLoop()
select {
case wsConn := <-connections:
require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
case <-time.After(3 * time.Second):
t.Fatal("websocket connection was not established")
}
select {
case req := <-client.requests:
require.Equal(t, common.SyncNetworkMonitors, req.Action)
require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
return req.Data
case <-time.After(3 * time.Second):
t.Fatal("reconnected agent did not receive a monitor replacement")
return monitor.SyncRequest{}
}
}
initial := connect()
require.Len(t, initial.Configs, 1)
require.Equal(t, probe.Id, initial.Configs[0].ID)
require.NoError(t, sm.RemoveSystem(sys.Id))
if change == "delete" {
require.NoError(t, app.Delete(probe))
} else {
probe.Set("enabled", false)
require.NoError(t, app.SaveNoValidate(probe))
}
require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
})
}
}
func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
sys, app := newTestSystemWithHub(t)
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
require.NoError(t, err)
_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
require.Error(t, err, "a failed query must not be treated as an empty monitor set")
}
+58
View File
@@ -0,0 +1,58 @@
package systems
import (
"context"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// SyncNetworkMonitors sends monitor configurations to the agent.
func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
return err
}
// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: config,
RunNow: runNow,
})
if err != nil {
return nil, err
}
if resp.Result == (monitor.Result{}) {
return nil, nil
}
result := resp.Result
return &result, nil
}
// DeleteNetworkMonitor removes a single monitor task from the agent.
func (sys *System) DeleteNetworkMonitor(id string) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: id},
})
return err
}
func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
return monitor.SyncResponse{}, nil
}
timeout := 5 * time.Second
if req.Action == monitor.SyncActionUpsert && req.RunNow {
// Allow the probe to finish, including a timeout result, while preserving
// the normal request budget for transport and response handling.
timeout += monitor.MaxProbeTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
var result monitor.SyncResponse
return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
}
+143 -2
View File
@@ -9,6 +9,7 @@ import (
"math/rand"
"net"
"strings"
"sync"
"sync/atomic"
"time"
@@ -18,6 +19,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/systemd"
@@ -30,6 +32,8 @@ import (
"github.com/lxzan/gws"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/security"
"github.com/pocketbase/pocketbase/tools/types"
"golang.org/x/crypto/ssh"
)
@@ -52,6 +56,10 @@ type System struct {
smartInterval time.Duration // Interval for periodic SMART data updates
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
// Serialize persistence from scheduled updates and resumes through commit.
recordsMu sync.Mutex
// Protected by recordsMu; realtime reads don't consume probes.
lastSavedMonitorProbe map[string]int64
}
func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -211,11 +219,15 @@ func (sys *System) handlePaused() {
// createRecords updates the system record and adds system_stats and container_stats records
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
sys.recordsMu.Lock()
defer sys.recordsMu.Unlock()
systemRecord, err := sys.getRecord(sys.manager.hub)
if err != nil {
return nil, err
}
hub := sys.manager.hub
savedMonitorProbes := make(map[string]int64)
err = hub.RunInTransaction(func(txApp core.App) error {
// add system_stats record
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
@@ -266,6 +278,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
}
}
if data.Monitors != nil {
if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
return err
}
}
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
return err
}
@@ -287,6 +305,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
return nil
})
// Publish only successful inserts after the entire transaction commits.
if err == nil && len(savedMonitorProbes) > 0 {
if sys.lastSavedMonitorProbe == nil {
sys.lastSavedMonitorProbe = savedMonitorProbes
} else {
for id, timestamp := range savedMonitorProbes {
sys.lastSavedMonitorProbe[id] = timestamp
}
}
}
// A non-nil report includes cached results for all remaining monitors.
if err == nil && data.Monitors != nil {
for id := range sys.lastSavedMonitorProbe {
if _, exists := data.Monitors[id]; !exists {
delete(sys.lastSavedMonitorProbe, id)
}
}
}
if err == nil {
if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
}
}
return systemRecord, err
}
@@ -337,7 +378,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
}
suffix := fmt.Sprintf("%d", i)
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
params["id"+suffix] = makeStableHashId(systemId, service.Name)
params["id"+suffix] = MakeStableHashId(systemId, service.Name)
params["name"+suffix] = service.Name
params["state"+suffix] = service.State
params["sub"+suffix] = service.Sub
@@ -363,6 +404,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
return err
}
func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
if len(monitorResults) == 0 {
return nil
}
var err error
systemId := sys.Id
const monitorCollectionName = "network_monitors"
// If realtime updates are active, we save via PocketBase records to trigger realtime events.
// Otherwise we can do a more efficient direct update via SQL
realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
})
now := time.Now().UTC()
nowMilli := now.UnixMilli()
nowString := now.Format(types.DefaultDateLayout)
var db dbx.Builder
var updateQuery *dbx.Query
if !realtimeActive {
db = app.DB()
monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
setClauses := make([]string, len(monitorFields))
for i, f := range monitorFields {
setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
}
queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
updateQuery = db.NewQuery(queryString)
}
// update network_monitors records
for id, result := range monitorResults {
monitorData := map[string]any{
"id": id,
"res": result.AvgResponse,
"resAvg1h": result.AvgResponse1h,
"resMin1h": result.MinResponse1h,
"resMax1h": result.MaxResponse1h,
"loss1h": result.PacketLoss1h,
"updated": nowString,
}
switch realtimeActive {
case true:
var record *core.Record
record, err = app.FindRecordById(monitorCollectionName, id)
if err == nil {
record.Load(monitorData)
err = app.SaveNoValidate(record)
}
default:
_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
}
if err != nil {
app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
}
}
// handle stats collection — one record per monitor
const statsCollectionName = "network_monitor_stats"
var statsCollection *core.Collection
if realtimeActive {
statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
}
for monitorId, result := range monitorResults {
// Compare identity, not ordering, so agent clock changes don't stall writes.
if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
continue
}
statsRecordData := map[string]any{
"system": systemId,
"monitor": monitorId,
"type": "1m",
"created": nowMilli,
"res_min": result.MinResponse,
"res_max": result.MaxResponse,
"total_count": result.TotalCount,
"success_count": result.SuccessCount,
"res_sum": result.ResponseSum,
}
switch realtimeActive {
case true:
record := core.NewRecord(statsCollection)
record.Load(statsRecordData)
err = app.SaveNoValidate(record)
default:
statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
}
if err != nil {
app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
} else {
savedProbes[monitorId] = result.LastProbeAt
}
}
return nil
}
// createContainerRecords creates container records
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
if len(data) == 0 {
@@ -622,7 +763,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
return &result, err
}
func makeStableHashId(strings ...string) string {
func MakeStableHashId(strings ...string) string {
hash := fnv.New32a()
for _, str := range strings {
hash.Write([]byte(str))
+29 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
@@ -17,6 +18,7 @@ import (
"github.com/henrygd/beszel"
"github.com/blang/semver"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/store"
"golang.org/x/crypto/ssh"
@@ -62,6 +64,7 @@ type hubLike interface {
core.App
GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
HandleStatusAlerts(status string, systemRecord *core.Record) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
CancelPendingStatusAlerts(systemID string)
@@ -350,6 +353,20 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
if err := sm.AddRecord(systemRecord, system); err != nil {
return err
}
// Sync network monitors to the newly connected agent
go func() {
configs, err := sm.GetMonitorConfigsForSystem(systemId)
if err != nil {
sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err)
return
}
// An empty set must also replace any probes retained across a disconnect.
if err := system.SyncNetworkMonitors(configs); err != nil {
sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err)
}
}()
return nil
}
@@ -362,6 +379,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
}
}
// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
var configs []monitor.Config
err := sm.hub.DB().
NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
Bind(dbx.Params{"system": systemID}).
All(&configs)
return configs, err
}
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
// agent reconnect retries ZFS discovery immediately after configuration changes.
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
@@ -397,11 +424,12 @@ func (sm *SystemManager) createSSHClientConfig() error {
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
// This is called when a system is paused or goes offline to prevent continued alerts.
// Monitor incidents remain open: a missing observation does not establish recovery.
func deactivateAlerts(app core.App, systemID string) error {
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
if err != nil {
return err
}
+19 -1
View File
@@ -6,6 +6,8 @@ import (
"time"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/utils"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/apis"
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
if err != nil {
return
}
bytes, err := json.Marshal(data)
bytes, err := marshalRealtimeData(data)
if err == nil {
notify(sm.hub, system, fetch.subscription, bytes)
}
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
}
}
// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
// the per-monitor results into the derived metric fields the frontend charts expect.
func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
if len(data.Monitors) == 0 {
return json.Marshal(data)
}
monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
for id, result := range data.Monitors {
monitorStats[id] = monitor.Stats{}.FromResult(result)
}
return json.Marshal(struct {
*system.CombinedData
Monitors map[string]monitor.Stats `json:"Monitors"`
}{data, monitorStats})
}
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
// Custom topics bypass collection rules, so check current access for every
// recipient, including clients whose authentication or membership was revoked.
+2 -2
View File
@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
currentIDs := make(map[string]struct{}, len(smartData))
for deviceKey := range smartData {
currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}
currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
}
err = hub.RunInTransaction(func(txApp core.App) error {
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
}
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
recordID := makeStableHashId(sys.Id, deviceKey)
recordID := MakeStableHashId(sys.Id, deviceKey)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
+4 -2
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
return nil
}
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
return nil
}
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
}, false))
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))
recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
require.NoError(t, err)
assert.EqualValues(t, 42, recordA.GetInt("temp"))
}
+9 -9
View File
@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName := "nginx.service"
// Call multiple times and ensure same result
id1 := makeStableHashId(systemId, serviceName)
id2 := makeStableHashId(systemId, serviceName)
id3 := makeStableHashId(systemId, serviceName)
id1 := MakeStableHashId(systemId, serviceName)
id2 := MakeStableHashId(systemId, serviceName)
id3 := MakeStableHashId(systemId, serviceName)
assert.Equal(t, id1, id2)
assert.Equal(t, id2, id3)
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName1 := "nginx.service"
serviceName2 := "apache.service"
id1 := makeStableHashId(systemId1, serviceName1)
id2 := makeStableHashId(systemId2, serviceName1)
id3 := makeStableHashId(systemId1, serviceName2)
id4 := makeStableHashId(systemId2, serviceName2)
id1 := MakeStableHashId(systemId1, serviceName1)
id2 := MakeStableHashId(systemId2, serviceName1)
id3 := MakeStableHashId(systemId1, serviceName2)
id4 := MakeStableHashId(systemId2, serviceName2)
// All IDs should be different
assert.NotEqual(t, id1, id2)
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
}
for _, tc := range testCases {
id := makeStableHashId(tc.systemId, tc.serviceName)
id := MakeStableHashId(tc.systemId, tc.serviceName)
// FNV-32 produces 8 hex characters
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
}
})
t.Run("hexadecimal output", func(t *testing.T) {
id := makeStableHashId("test-system", "test-service")
id := MakeStableHashId("test-system", "test-service")
assert.NotEmpty(t, id)
// Should only contain hexadecimal characters
+2 -2
View File
@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
}
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
recordID := makeStableHashId(sys.Id, pool.Name)
recordID := MakeStableHashId(sys.Id, pool.Name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
if pool == nil {
continue
}
recordID := makeStableHashId(sys.Id, name)
recordID := MakeStableHashId(sys.Id, name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
if !errors.Is(err, sql.ErrNoRows) {
+6 -6
View File
@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
}}
require.NoError(t, sys.saveZfsPools(initial))
failedID := makeStableHashId(sys.Id, failedKey)
failedID := MakeStableHashId(sys.Id, failedKey)
before, err := app.FindRecordById("zfs_pools", failedID)
require.NoError(t, err)
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
}}
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey))
fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
require.NoError(t, err)
assert.EqualValues(t, 20, fresh.GetInt("alloc"))
cached, err := app.FindRecordById("zfs_pools", failedID)
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
}))
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err)
firstUpdated := record.GetDateTime("updated")
assert.Equal(t, "ONLINE", record.GetString("health"))
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
func TestZfsRawCapacityPersistence(t *testing.T) {
sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs"))
record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
require.NoError(t, err)
require.True(t, record.GetBool("raw"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
key: {DisplayName: "tank", Health: "ONLINE"},
"tank": {Health: "ONLINE"},
}))
id := makeStableHashId(sys.Id, key)
id := MakeStableHashId(sys.Id, key)
record, err := app.FindRecordById("zfs_pools", id)
require.NoError(t, err)
assert.Equal(t, "tank", record.GetString("display_name"))
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
record, err = app.FindRecordById("zfs_pools", id)
require.NoError(t, err)
assert.Equal(t, "detail name", record.GetString("display_name"))
_, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank"))
_, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err)
}
+28 -1
View File
@@ -1,7 +1,11 @@
// Package utils provides utility functions for the hub.
package utils
import "os"
import (
"os"
"github.com/pocketbase/pocketbase/core"
)
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
func GetEnv(key string) (value string, exists bool) {
@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {
}
return os.LookupEnv(key)
}
// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
broker := app.SubscriptionsBroker()
if broker.TotalClients() == 0 {
return false
}
for _, client := range broker.Clients() {
subs := client.Subscriptions(collectionName)
if len(subs) > 0 {
if validateFn == nil {
return true
}
for k := range subs {
filter := subs[k].Query["filter"]
if validateFn(filter) {
return true
}
}
}
}
return false
}
@@ -83,7 +83,8 @@ func init() {
"ContainerHealth",
"SystemdFailed",
"CPUIOWait",
"CPUSteal"
"CPUSteal",
"NetworkMonitorLoss"
]
},
{
@@ -119,6 +120,16 @@ func init() {
"system": false,
"type": "bool"
},
{
"hidden": true,
"id": "json4000656575",
"maxSize": 0,
"name": "state",
"presentable": false,
"required": false,
"system": false,
"type": "json"
},
{
"hidden": true,
"id": "date1302749137",
@@ -152,6 +163,7 @@ func init() {
}
],
"indexes": [
"CREATE INDEX idx_alerts_system_name ON alerts (system, name)",
"CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"
],
"system": false
@@ -234,6 +246,20 @@ func init() {
"system": false,
"type": "text"
},
{
"autogeneratePattern": "",
"hidden": false,
"id": "text3888135399",
"max": 0,
"min": 0,
"name": "monitor_name",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{
"hidden": false,
"id": "number494360628",
@@ -1721,6 +1747,7 @@ func init() {
"fields": [
{
"autogeneratePattern": "[a-z0-9]{15}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 15,
@@ -1736,6 +1763,7 @@ func init() {
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false,
"id": "relation1204987316",
"maxSelect": 1,
@@ -1748,6 +1776,7 @@ func init() {
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text7739291048",
"max": 0,
@@ -1762,6 +1791,7 @@ func init() {
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text5528164482",
"max": 0,
@@ -1775,6 +1805,7 @@ func init() {
"type": "text"
},
{
"help": "",
"hidden": false,
"id": "number8862034195",
"max": null,
@@ -1787,6 +1818,7 @@ func init() {
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number4418907321",
"max": null,
@@ -1799,6 +1831,7 @@ func init() {
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number2904183765",
"max": null,
@@ -1811,6 +1844,7 @@ func init() {
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "json4466109723",
"maxSize": 0,
@@ -1821,6 +1855,7 @@ func init() {
"type": "json"
},
{
"help": "",
"hidden": false,
"id": "json9012873456",
"maxSize": 0,
@@ -1831,6 +1866,7 @@ func init() {
"type": "json"
},
{
"help": "",
"hidden": false,
"id": "json7182045639",
"maxSize": 0,
@@ -1841,6 +1877,7 @@ func init() {
"type": "json"
},
{
"help": "",
"hidden": false,
"id": "date9274163058",
"max": "",
@@ -1860,19 +1897,401 @@ func init() {
"presentable": false,
"system": false,
"type": "autodate"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text3578368839",
"max": 0,
"min": 0,
"name": "display_name",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{
"help": "",
"hidden": false,
"id": "bool447994709",
"name": "raw",
"presentable": false,
"required": false,
"system": false,
"type": "bool"
}
],
"id": "pbc_8441057391",
"indexes": [
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
],
"listRule": null,
"listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id",
"name": "zfs_pools",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id"
},
{
"createRule": null,
"deleteRule": null,
"fields": [
{
"autogeneratePattern": "[a-z0-9]{10}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 10,
"min": 6,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false,
"id": "nm_system",
"maxSelect": 1,
"minSelect": 0,
"name": "system",
"presentable": false,
"required": true,
"system": false,
"type": "relation"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "nm_target",
"max": 500,
"min": 1,
"name": "target",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": true,
"system": false,
"type": "text"
},
{
"help": "",
"hidden": false,
"id": "nm_protocol",
"maxSelect": 1,
"name": "protocol",
"presentable": false,
"required": true,
"system": false,
"type": "select",
"values": [
"icmp",
"tcp",
"http",
"dns"
]
},
{
"help": "",
"hidden": false,
"id": "nm_port",
"max": 65535,
"min": 0,
"name": "port",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nm_interval",
"max": 3600,
"min": 1,
"name": "interval",
"onlyInt": true,
"presentable": false,
"required": true,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number926446584",
"max": null,
"min": null,
"name": "res",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number1006954605",
"max": null,
"min": null,
"name": "resAvg1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number4267669802",
"max": null,
"min": null,
"name": "resMin1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number591433223",
"max": null,
"min": null,
"name": "resMax1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number3726709001",
"max": null,
"min": null,
"name": "loss1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nm_enabled",
"name": "enabled",
"presentable": false,
"required": false,
"system": false,
"type": "bool"
},
{
"hidden": false,
"id": "autodate2990389176",
"name": "created",
"onCreate": true,
"onUpdate": false,
"presentable": false,
"system": false,
"type": "autodate"
},
{
"help": "",
"hidden": false,
"id": "date3332085495",
"max": "",
"min": "",
"name": "updated",
"presentable": false,
"required": false,
"system": false,
"type": "date"
}
],
"id": "nm_monitors_001",
"indexes": [
"CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)"
],
"listRule": null,
"name": "network_monitors",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": null
}
},
{
"createRule": null,
"deleteRule": null,
"fields": [
{
"autogeneratePattern": "[a-z0-9]{10}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 10,
"min": 10,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false,
"id": "nms_system",
"maxSelect": 1,
"minSelect": 0,
"name": "system",
"presentable": false,
"required": true,
"system": false,
"type": "relation"
},
{
"cascadeDelete": true,
"collectionId": "nm_monitors_001",
"help": "",
"hidden": false,
"id": "nms_monitor",
"maxSelect": 1,
"minSelect": 0,
"name": "monitor",
"presentable": false,
"required": false,
"system": false,
"type": "relation"
},
{
"help": "Number of probe attempts",
"hidden": false,
"id": "nms_total_count",
"max": null,
"min": 0,
"name": "total_count",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Number of successful probe attempts",
"hidden": false,
"id": "nms_success_count",
"max": null,
"min": 0,
"name": "success_count",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Sum of successful response times in microseconds",
"hidden": false,
"id": "nms_res_sum",
"max": null,
"min": 0,
"name": "res_sum",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Response time in microseconds",
"hidden": false,
"id": "nms_res_min",
"max": null,
"min": 0,
"name": "res_min",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Response time in microseconds",
"hidden": false,
"id": "nms_res_max",
"max": null,
"min": 0,
"name": "res_max",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nms_type",
"maxSelect": 1,
"name": "type",
"presentable": false,
"required": true,
"system": false,
"type": "select",
"values": [
"1m",
"10m",
"20m",
"120m",
"480m"
]
},
{
"help": "",
"hidden": false,
"id": "number2990389176",
"max": null,
"min": null,
"name": "created",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
}
],
"id": "nm_stats_001",
"indexes": [
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)"
],
"listRule": null,
"name": "network_monitor_stats",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": null
}
]`
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
+226
View File
@@ -0,0 +1,226 @@
//go:build testing
package records_test
import (
"testing"
"time"
monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/records"
"github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAverageMonitorStats(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")
require.NoError(t, err)
assert.Nil(t, collection.Fields.GetByName("res_avg"))
assert.Nil(t, collection.Fields.GetByName("loss"))
rm := records.NewRecordManager(hub)
user, err := tests.CreateUser(hub, "monitor-avg@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "monitor-avg-system",
"host": "localhost",
"port": "45876",
"status": "up",
"users": []string{user.Id},
})
require.NoError(t, err)
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id,
"name": "cloudflare",
"target": "1.1.1.1",
"protocol": "icmp",
"interval": 30,
"enabled": true,
})
require.NoError(t, err)
created := time.Now().UnixMilli()
// Unequal probe counts must weight both latency and loss.
recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id,
"monitor": monitor.Id,
"type": "1m",
"created": created,
"res_min": 5,
"res_max": 20,
"total_count": 6, "success_count": 6, "res_sum": 60,
})
require.NoError(t, err)
recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id,
"monitor": monitor.Id,
"type": "1m",
"created": created,
"res_min": 10,
"res_max": 60,
"total_count": 1, "success_count": 1, "res_sum": 22,
})
require.NoError(t, err)
result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)
for _, tc := range []struct {
name, monitor, recordType string
after int64
}{
{"other monitor", "missing", "1m", created - 1},
{"other type", monitor.Id, "10m", created - 1},
{"exclusive cutoff", monitor.Id, "1m", created},
} {
t.Run(tc.name, func(t *testing.T) {
stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)
require.NoError(t, err)
assert.Zero(t, count)
assert.Equal(t, monitorEntity.Stats{}, stats)
})
}
// A failure-only bucket counts toward loss but must not lower latency.
recordB.Set("res_min", 0)
recordB.Set("res_max", 0)
recordB.Set("success_count", 0)
recordB.Set("res_sum", 0)
require.NoError(t, hub.Save(recordB))
result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)
// Sparse monitor records must propagate through every rollup level.
rm.CreateLongerRecords()
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
require.NoError(t, err)
require.Len(t, rollups, 1, recordType)
assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
assert.Equal(t, 7, rollups[0].GetInt("total_count"))
assert.Equal(t, 6, rollups[0].GetInt("success_count"))
assert.Equal(t, 60, rollups[0].GetInt("res_sum"))
// A sibling with a different number of probes must retain its actual
// weight when the next tier combines their underlying counts.
_, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,
"res_min": 100, "res_max": 100,
"total_count": 3, "success_count": 1, "res_sum": 100,
})
require.NoError(t, err)
merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{
ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,
TotalCount: 10, SuccessCount: 7, ResponseSum: 160,
}, merged)
}
// All failures produce zero latency, while a genuine zero-microsecond
// success remains a valid minimum (it must not be filtered out as a sentinel).
recordA.Set("success_count", 0)
recordA.Set("res_sum", 0)
require.NoError(t, hub.Save(recordA))
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)
recordA.Set("success_count", 1)
recordA.Set("res_min", 0)
recordA.Set("res_max", 0)
require.NoError(t, hub.Save(recordA))
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)
}
func TestSparseMonitorRollups(t *testing.T) {
for _, tc := range []struct {
name string
interval int
samples int
disabled bool
}{
{"five minute interval", 300, 2, false},
{"ten minute interval", 600, 1, false},
{"fifteen minute interval", 900, 1, false},
{"empty window", 900, 0, false},
{"disabled with pending history", 300, 2, true},
{"disabled empty window", 900, 0, true},
} {
t.Run(tc.name, func(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
user, err := tests.CreateUser(hub, "sparse-monitor@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",
"users": []string{user.Id},
})
require.NoError(t, err)
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
"interval": tc.interval, "enabled": true,
})
require.NoError(t, err)
now := time.Now()
for i := range tc.samples {
_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id, "monitor": monitor.Id, "type": "1m",
"created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),
"res_min": 8, "res_max": 20,
"total_count": 4, "success_count": 3, "res_sum": 36,
})
require.NoError(t, err)
}
// Other collections must still reject fewer than nine minute records.
for _, collection := range []string{"system_stats", "container_stats"} {
stats := `{"cpu":10}`
if collection == "container_stats" {
stats = `[{"name":"test","cpu":10}]`
}
for range 8 {
_, err := tests.CreateRecord(hub, collection, map[string]any{
"system": sys.Id, "type": "1m", "stats": stats,
})
require.NoError(t, err)
}
}
if tc.disabled {
monitor.Set("enabled", false)
require.NoError(t, hub.Save(monitor))
}
records.NewRecordManager(hub).CreateLongerRecords()
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
require.NoError(t, err)
if tc.samples == 0 {
assert.Empty(t, rollups, recordType)
} else {
require.Len(t, rollups, 1, recordType)
assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))
assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))
assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))
}
for _, collection := range []string{"system_stats", "container_stats"} {
count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})
require.NoError(t, err)
assert.Zero(t, count, "%s %s", collection, recordType)
}
}
})
}
}
+140 -28
View File
@@ -3,15 +3,16 @@ package records
import (
"encoding/json"
"log/slog"
"math"
"time"
"github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
)
type RecordManager struct {
@@ -39,7 +40,7 @@ type StatsRecord struct {
// Create longer records by averaging shorter records
func (rm *RecordManager) CreateLongerRecords() {
// start := time.Now()
now := time.Now().UTC()
longerRecordData := []LongerRecordData{
{
shorterType: "1m",
@@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() {
}
// wrap the operations in a transaction
// Pocketbase cron does not handle errors, log them here.
rm.app.RunInTransaction(func(txApp core.App) error {
err := rm.app.RunInTransaction(func(txApp core.App) error {
var err error
collections := [2]*core.Collection{}
collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")
if err != nil {
slog.Error("Error finding cached collection using system stats:", "err", err)
return err
}
collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")
if err != nil {
slog.Error("Error finding cached collection using container stats:", "err", err)
return err
}
monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats")
if err != nil {
return err
}
var systems RecordIds
db := txApp.DB()
db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems)
if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil {
return err
}
// loop through all active systems, time periods, and collections
for _, system := range systems {
@@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() {
recordData := longerRecordData[i]
// log.Println("processing longer record type", recordData.longerType)
// add one minute padding for longer records because they are created slightly later than the job start time
longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute)
longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
// shorter records are created independently of longer records, so we shouldn't need to add padding
shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration)
// loop through both collections
shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
for _, collection := range collections {
// check creation time of last longer record if not 10m, since 10m is created every run
if recordData.longerType != "10m" {
count, err := txApp.CountRecords(
collection.Id,
dbx.NewExp(
"system = {:system} AND type = {:type} AND created > {:created}",
dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod},
),
)
count, err := txApp.CountRecords(collection.Id, dbx.NewExp(
"system = {:system} AND type = {:type} AND created > {:created}",
dbx.Params{
"type": recordData.longerType,
"system": system.Id,
"created": longerRecordPeriod.Format(types.DefaultDateLayout),
},
))
if err != nil {
return err
}
// continue if longer record exists
if err != nil || count > 0 {
if count > 0 {
continue
}
}
// get shorter records from the past x minutes
var recordIds RecordIds
err := txApp.DB().
params := dbx.Params{
"type": recordData.shorterType,
"system": system.Id,
"created": shorterRecordPeriod.Format(types.DefaultDateLayout),
}
err := db.
Select("id").
From(collection.Name).
AndWhere(dbx.NewExp(
Where(dbx.NewExp(
"system={:system} AND type={:type} AND created > {:created}",
dbx.Params{
"type": recordData.shorterType,
"system": system.Id,
"created": shorterRecordPeriod,
},
params,
)).
OrderBy("created").
All(&recordIds)
if err != nil {
return err
}
// continue if not enough shorter records
if err != nil || len(recordIds) < recordData.minShorterRecords {
if len(recordIds) < recordData.minShorterRecords {
continue
}
// average the shorter records and create longer record
@@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() {
case "system_stats":
longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))
case "container_stats":
longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))
}
if err := txApp.SaveNoValidate(longerRecord); err != nil {
slog.Error("failed to save longer record", "err", err)
txApp.Logger().Error("failed to save longer record", "err", err)
}
}
}
}
// network_monitor_stats is aggregated per monitor (not per system)
var monitors []struct {
Id string `db:"id"`
System string `db:"system"`
}
// Disabled monitors still have history that must advance through retention tiers.
if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil {
return err
}
for _, monitorRec := range monitors {
for i := range longerRecordData {
recordData := longerRecordData[i]
longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
if recordData.longerType != "10m" {
count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp(
"monitor={:monitor} AND type={:type} AND created>{:created}",
dbx.Params{
"monitor": monitorRec.Id,
"type": recordData.longerType,
"created": longerRecordPeriod.UnixMilli(),
},
))
if err != nil {
return err
}
if count > 0 {
continue
}
}
stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli())
if err != nil {
txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err)
continue
}
// Monitor intervals can exceed the aggregation window, so average
// any available records at every level and skip only empty windows.
if count == 0 {
continue
}
longerRecord := core.NewRecord(monitorStatsColl)
longerRecord.Set("system", monitorRec.System)
longerRecord.Set("monitor", monitorRec.Id)
longerRecord.Set("type", recordData.longerType)
longerRecord.Set("created", now.UnixMilli())
longerRecord.Set("res_min", stats.ResMin)
longerRecord.Set("res_max", stats.ResMax)
longerRecord.Set("total_count", stats.TotalCount)
longerRecord.Set("success_count", stats.SuccessCount)
longerRecord.Set("res_sum", stats.ResponseSum)
if err := txApp.SaveNoValidate(longerRecord); err != nil {
txApp.Logger().Error("failed to save monitor longer record", "err", err)
}
}
}
return nil
})
if err != nil {
rm.app.Logger().Error("failed to create longer records", "err", err)
}
}
// log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds())
func getCreatedTimeField(collectionName string, period time.Time) any {
// network_monitor_stats stores created as unix timestamp in ms, not as a date string
if collectionName == "network_monitor_stats" {
return period.UnixMilli()
}
return period.Format(types.DefaultDateLayout)
}
// Calculate the average stats of a list of system_stats records without reflect
@@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats {
return result
}
// AverageMonitorStats merges probe counts and response sums, preserving their
// weights through every retention tier. Failed probes do not contribute latency.
func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) {
var result struct {
monitor.Stats
Count int `db:"count"`
}
err := db.Select(
"COUNT(*) AS count",
"COALESCE(SUM(total_count), 0) AS total_count",
"COALESCE(SUM(success_count), 0) AS success_count",
"COALESCE(SUM(res_sum), 0) AS res_sum",
"COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min",
"COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max",
).From("network_monitor_stats").Where(dbx.NewExp(
"monitor={:monitor} AND type={:type} AND created>{:created}",
dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter},
)).One(&result)
if err != nil {
return monitor.Stats{}, 0, err
}
if result.SuccessCount > 0 {
result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount))
}
if result.TotalCount > 0 {
result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount))
}
return result.Stats, result.Count, nil
}
/* Round float to two decimals */
func twoDecimals(value float64) float64 {
return math.Round(value*100) / 100
+10 -18
View File
@@ -3,7 +3,6 @@ package records
import (
"fmt"
"log/slog"
"strings"
"time"
"github.com/pocketbase/dbx"
@@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int)
// Deletes system_stats records older than what is displayed in the UI
func deleteOldSystemStats(app core.App) error {
// Collections to process
collections := [2]string{"system_stats", "container_stats"}
collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"}
// Record types and their retention periods
type RecordDeletionData struct {
@@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error {
}
now := time.Now().UTC()
db := app.DB()
for _, collection := range collections {
// Build the WHERE clause
var conditionParts []string
var params dbx.Params = make(map[string]any)
for i := range recordData {
rd := recordData[i]
// Create parameterized condition for this record type
dateParam := fmt.Sprintf("date%d", i)
conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam))
params[dateParam] = now.Add(-rd.retention)
}
// Combine conditions with OR
conditionStr := strings.Join(conditionParts, " OR ")
// Construct and execute the full raw query
rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr)
if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil {
return fmt.Errorf("failed to delete from %s: %v", collection, err)
query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}"))
for _, rd := range recordData {
if _, err := query.Bind(dbx.Params{
"type": rd.recordType,
"created": getCreatedTimeField(collection, now.Add(-rd.retention)),
}).Execute(); err != nil {
return fmt.Errorf("failed to delete from %s: %v", collection, err)
}
}
}
return nil
+86
View File
@@ -0,0 +1,86 @@
//go:build testing
package records_test
import (
"testing"
"time"
"github.com/henrygd/beszel/internal/records"
"github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/tools/types"
"github.com/stretchr/testify/require"
)
func TestLongerRecordsPreventDuplicates(t *testing.T) {
for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} {
for _, tier := range []struct {
shorter, longer string
count int
}{
{"10m", "20m", 2},
{"20m", "120m", 6},
{"120m", "480m", 4},
} {
t.Run(collection+"/"+tier.longer, func(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
user, err := tests.CreateUser(hub, "rollup@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "rollup-system", "host": "localhost", "port": "45876",
"status": "up", "users": []string{user.Id},
})
require.NoError(t, err)
created := time.Now().UTC().Add(-time.Minute)
data := map[string]any{
"system": sys.Id, "type": tier.shorter,
"created": created.Format(types.DefaultDateLayout),
}
filter := dbx.HashExp{"system": sys.Id, "type": tier.longer}
switch collection {
case "system_stats":
data["stats"] = `{"cpu":10}`
case "container_stats":
data["stats"] = `[{"name":"test","cpu":10}]`
case "network_monitor_stats":
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
"interval": 30, "enabled": true,
})
require.NoError(t, err)
data["monitor"] = monitor.Id
data["created"] = created.UnixMilli()
data["total_count"] = 1
data["success_count"] = 1
data["res_sum"] = 10
data["res_min"] = 10
data["res_max"] = 10
filter["monitor"] = monitor.Id
}
for range tier.count {
_, err := tests.CreateRecord(hub, collection, data)
require.NoError(t, err)
}
rm := records.NewRecordManager(hub)
rm.CreateLongerRecords()
first, err := hub.FindAllRecords(collection, filter)
require.NoError(t, err)
require.Len(t, first, 1)
// The shorter records remain eligible, but the existing longer
// record must prevent another rollup on a subsequent invocation.
rm.CreateLongerRecords()
second, err := hub.FindAllRecords(collection, filter)
require.NoError(t, err)
require.Len(t, second, 1)
require.Equal(t, first[0].Id, second[0].Id)
})
}
}
}
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "beszel",
"private": true,
"version": "0.19.0",
"version": "0.20.0",
"type": "module",
"scripts": {
"dev": "vite --host",
@@ -74,4 +74,4 @@
"optionalDependencies": {
"@esbuild/linux-arm64": "^0.21.5"
}
}
}
@@ -61,6 +61,8 @@ export const ActiveAlerts = () => {
<AlertDescription>
{info.triggeredDesc ? (
info.triggeredDesc()
) : alert.name === "NetworkMonitorLoss" ? (
<Trans>One or more monitors exceed {alert.value}% loss</Trans>
) : alert.name === "Status" ? (
<Trans>Connection is down</Trans>
) : info.invert ? (
@@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
accessorFn: (record) => {
const name = record.name
const info = alertInfo[name]
return info?.name().replace("cpu", "CPU") || name
const label = info?.name().replace("cpu", "CPU") || name
return record.monitor_name ? `${label}: ${record.monitor_name}` : label
},
header: ({ column }) => (
<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
@@ -239,13 +239,13 @@ export function AlertContent({
/** Alerts that fire on first observation have no duration to configure */
const noDuration = alertData.noDuration === true
/** Binary alerts have no threshold to configure */
const noThreshold = !!singleDescription || noDuration
const noThreshold = !!singleDescription || alertData.noThreshold === true
/** Whether enabling the alert reveals anything to configure */
const hasControls = !(noThreshold && noDuration)
const [checked, setChecked] = useState(global ? false : !!alert)
const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))
const [value, setValue] = useState(alert?.value || (noThreshold ? 0 : (alertData.start ?? 80)))
const [value, setValue] = useState(alert?.value ?? (noThreshold ? 0 : (alertData.start ?? 80)))
const Icon = alertData.icon
@@ -319,7 +319,7 @@ export function AlertContent({
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
<Suspense fallback={<div className="h-10" />}>
{!noThreshold && (
<div>
<div className={cn(noDuration && "col-span-full")}>
<p id={`v${name}`} className="text-sm block h-6">
{alertData.invert ? (
<Trans>
@@ -66,7 +66,7 @@ export default function AreaChartDefault({
}) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
const sourceData = customData ?? chartData.systemStats
const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -111,6 +111,8 @@ export default function AreaChartDefault({
})
}, [areasKey, displayMaxToggled])
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
return useMemo(() => {
if (displayData.length === 0) {
return null
@@ -146,7 +148,7 @@ export default function AreaChartDefault({
axisLine={false}
/>
)}
{xAxis(chartData)}
{XAxis}
<ChartTooltip
animationEasing="ease-out"
animationDuration={150}
@@ -167,5 +169,5 @@ export default function AreaChartDefault({
</AreaChart>
</ChartContainer>
)
}, [displayData, yAxisWidth, filter, Areas])
}, [displayData, yAxisWidth, filter, Areas, XAxis])
}
@@ -9,14 +9,21 @@ import { memo } from "react"
export default memo(function ChartTimeSelect({
className,
agentVersion,
chartTimeStore = $chartTime,
allowRealtime = true,
}: {
className?: string
agentVersion: SemVer
chartTimeStore?: typeof $chartTime
allowRealtime?: boolean
}) {
const chartTime = useStore($chartTime)
const chartTime = useStore(chartTimeStore)
// remove chart times that are not supported by the system agent version
const availableChartTimes = Object.entries(chartTimeData).filter(([_, { minVersion }]) => {
const availableChartTimes = Object.entries(chartTimeData).filter(([value, { minVersion }]) => {
if (value === "1m" && !allowRealtime) {
return false
}
if (!minVersion) {
return true
}
@@ -24,7 +31,7 @@ export default memo(function ChartTimeSelect({
})
return (
<Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => $chartTime.set(value)}>
<Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => chartTimeStore.set(value)}>
<SelectTrigger className={cn(className, "relative ps-10 pe-5")}>
<HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
<SelectValue />
@@ -22,6 +22,10 @@ export type DataPoint<T = SystemStatsRecord> = {
order?: number
strokeOpacity?: number
activeDot?: boolean
dot?: boolean
/** Which Y axis this series plots against. Defaults to "left". */
yAxisId?: "left" | "right"
strokeDasharray?: string
}
export default function LineChartDefault({
@@ -30,9 +34,12 @@ export default function LineChartDefault({
max,
maxToggled,
tickFormatter,
tickFormatter2,
contentFormatter,
dataPoints,
domain,
domain2,
max2,
legend,
itemSorter,
showTotal = false,
@@ -41,18 +48,24 @@ export default function LineChartDefault({
filter,
truncate = false,
chartProps,
connectNulls,
}: {
chartData: ChartData
// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)
customData?: any[]
max?: number
max2?: number
maxToggled?: boolean
tickFormatter: (value: number, index: number) => string
/** Tick formatter for the right ("right"-yAxisId) axis, when any dataPoint uses it. */
tickFormatter2?: (value: number, index: number) => string
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
contentFormatter: (item: any, key: string) => ReactNode
// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types
dataPoints?: DataPoint<any>[]
domain?: AxisDomain
/** Domain for the right axis, when any dataPoint uses it. */
domain2?: AxisDomain
legend?: boolean
showTotal?: boolean
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
@@ -62,10 +75,15 @@ export default function LineChartDefault({
filter?: string
truncate?: boolean
chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">
connectNulls?: boolean
}) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const hasRightAxis = !!dataPoints?.some((dp) => dp.yAxisId === "right")
// fixed width for the secondary axis rather than measured, since its labels (e.g. loss %) are short
// and predictable, and this avoids depending on a second async width-measurement pass to settle
const rightAxisWidth = 38
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
const sourceData = customData ?? chartData.systemStats
const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -83,7 +101,9 @@ export default function LineChartDefault({
}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])
// Use a stable key derived from data point identities and visual properties
const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity ?? ""}`).join("\0")
const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity}${d.dot}${d.yAxisId}${d.strokeDasharray}`).join("\0")
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
const Lines = useMemo(() => {
return dataPoints?.map((dataPoint, i) => {
@@ -94,17 +114,20 @@ export default function LineChartDefault({
return (
<Line
key={dataPoint.label}
yAxisId={dataPoint.yAxisId ?? "left"}
dataKey={dataPoint.dataKey}
name={dataPoint.label}
type="monotoneX"
dot={false}
dot={dataPoint.dot || false}
strokeWidth={1.5}
stroke={color}
strokeOpacity={dataPoint.strokeOpacity}
strokeDasharray={dataPoint.strokeDasharray}
isAnimationActive={false}
// stackId={dataPoint.stackId}
order={dataPoint.order || i}
activeDot={dataPoint.activeDot ?? true}
connectNulls={connectNulls}
/>
)
})
@@ -135,6 +158,7 @@ export default function LineChartDefault({
<CartesianGrid vertical={false} />
{!hideYAxis && (
<YAxis
yAxisId="left"
direction="ltr"
orientation={chartData.orientation}
className="tracking-tighter"
@@ -145,7 +169,20 @@ export default function LineChartDefault({
axisLine={false}
/>
)}
{xAxis(chartData)}
{!hideYAxis && hasRightAxis && (
<YAxis
yAxisId="right"
direction="ltr"
orientation={chartData.orientation === "left" ? "right" : "left"}
className="tracking-tighter"
width={rightAxisWidth}
domain={domain2 ?? [0, max2 ?? "auto"]}
tickFormatter={tickFormatter2 ?? tickFormatter}
tickLine={false}
axisLine={false}
/>
)}
{XAxis}
<ChartTooltip
animationEasing="ease-out"
animationDuration={150}
@@ -166,5 +203,5 @@ export default function LineChartDefault({
</LineChart>
</ChartContainer>
)
}, [displayData, yAxisWidth, filter, Lines])
}, [displayData, yAxisWidth, hasRightAxis, filter, Lines, XAxis])
}
@@ -11,6 +11,7 @@ import {
HardDriveIcon,
LogsIcon,
MailIcon,
NetworkIcon,
Server,
ServerIcon,
SettingsIcon,
@@ -122,6 +123,20 @@ export default memo(function CommandPalette({ open, setOpen }: { open: boolean;
<Trans>Page</Trans>
</CommandShortcut>
</CommandItem>
<CommandItem
onSelect={() => {
navigate(getPagePath($router, "monitors"))
setOpen(false)
}}
>
<NetworkIcon className="me-2 size-4" />
<span>
<Trans>Network Monitors</Trans>
</span>
<CommandShortcut>
<Trans>Page</Trans>
</CommandShortcut>
</CommandItem>
<CommandItem
onSelect={() => {
navigate(getPagePath($router, "settings", { name: "general" }))
@@ -16,7 +16,7 @@ import {
import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"
import { Badge } from "../ui/badge"
import { t } from "@lingui/core/macro"
import { $allSystemsById, $longestSystemNameLen } from "@/lib/stores"
import { $allSystemsById, $longestSystemName } from "@/lib/stores"
import { useStore } from "@nanostores/react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
@@ -64,10 +64,13 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById)
const longestName = useStore($longestSystemNameLen)
const longestName = useStore($longestSystemName)
return (
<div className="ms-1 max-w-40 truncate" style={{ width: `${longestName / 1.05}ch` }}>
{allSystems[getValue() as string]?.name ?? ""}
<div className="ms-1 relative w-fit max-w-40">
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div>
)
},
+23
View File
@@ -8,6 +8,7 @@ import {
LogOutIcon,
LogsIcon,
MenuIcon,
NetworkIcon,
PlusIcon,
SearchIcon,
ServerIcon,
@@ -108,6 +109,13 @@ export default function Navbar() {
<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
<span>S.M.A.R.T.</span>
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => navigate(getPagePath($router, "monitors"))}
className="flex items-center"
>
<NetworkIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
<Trans>Network Monitors</Trans>
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}
className="flex items-center"
@@ -179,6 +187,21 @@ export default function Navbar() {
</TooltipTrigger>
<TooltipContent>S.M.A.R.T.</TooltipContent>
</Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Link
href={getPagePath($router, "monitors")}
className={cn("hidden md:grid", buttonVariants({ variant: "ghost", size: "icon" }))}
aria-label="Network Monitors"
onMouseEnter={() => import("@/components/routes/monitors")}
>
<NetworkIcon className="h-[1.2rem] w-[1.2rem]" strokeWidth={1.5} />
</Link>
</TooltipTrigger>
<TooltipContent>
<Trans>Network Monitors</Trans>
</TooltipContent>
</Tooltip>
<ModeToggle />
<Tooltip>
<TooltipTrigger asChild>
@@ -0,0 +1,778 @@
import { useCallback, useEffect, useRef, useState } from "react"
import { Trans, useLingui } from "@lingui/react/macro"
import { useStore } from "@nanostores/react"
import { pb } from "@/lib/api"
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import {
DropdownMenu,
DropdownMenuCheckboxItem,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"
import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
import { Textarea } from "@/components/ui/textarea"
import { ChevronDownIcon, ListIcon, SearchIcon, ServerIcon } from "lucide-react"
import { useToast } from "@/components/ui/use-toast"
import { $systems } from "@/lib/stores"
import { cn, supportsNetworkMonitors } from "@/lib/utils"
import type { NetworkMonitorRecord } from "@/types"
import * as v from "valibot"
type MonitorProtocol = "icmp" | "tcp" | "http" | "dns"
type MonitorValues = {
system: string
target: string
protocol: MonitorProtocol
port: number
interval: string
}
type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {
interval: number
}
type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">
const defaultInterval = 30
const MonitorProtocolSchema = v.picklist(["icmp", "tcp", "http", "dns"])
const MonitorIntervalSchema = v.pipe(v.string(), v.toNumber(), v.minValue(1), v.maxValue(3600))
// Both the single-monitor form and the bulk importer flow through this schema so
// defaults and HTTP target normalization stay in one place.
const NormalizedMonitorValuesSchema = v.pipe(
v.object({
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
protocol: MonitorProtocolSchema,
port: v.number(),
interval: MonitorIntervalSchema,
}),
v.transform((input): NormalizedMonitorValues => {
let { protocol, port } = input
let httpTarget = input.target
if (protocol === "icmp" || protocol === "http" || protocol === "dns") {
if (protocol === "http") {
httpTarget = normalizeHttpTarget(input.target, port)
}
port = 0
} else if (protocol === "tcp" && !port) {
port = 443
}
return {
// HTTP monitors may be entered as bare hostnames, so normalize them to a
// scheme-bearing URL before the payload is sent to PocketBase.
target: protocol === "http" ? httpTarget : input.target,
protocol,
port,
interval: input.interval,
}
}),
v.forward(
v.check((input) => {
if (input.protocol === "icmp" || input.protocol === "http" || input.protocol === "dns") {
return input.port === 0
}
return Number.isInteger(input.port) && input.port >= 1 && input.port <= 65535
}, "Port must be between 1 and 65535"),
["port"]
)
)
// Bulk parsing only trims raw CSV fields. Inference, defaults, and protocol-
// specific validation still go through the shared normalization schema above.
const BulkMonitorSchema = v.object({
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
protocol: v.optional(v.pipe(v.string(), v.trim())),
port: v.optional(v.pipe(v.string(), v.trim())),
interval: v.optional(v.pipe(v.string(), v.trim())),
})
function normalizeHttpTarget(target: string, port = 0) {
const useExplicitPort = port > 0 && port !== 80 && port !== 443
const hasOriginOnlyTarget = /^https?:\/\/[^/?#]+$/i.test(target)
if (!/^https?:\/\//i.test(target)) {
const scheme = port === 80 ? "http" : "https"
return `${scheme}://${target}${useExplicitPort ? `:${port}` : ""}`
}
let parsedUrl: URL
try {
parsedUrl = new URL(target)
} catch {
return target
}
if (!parsedUrl.port && useExplicitPort) {
parsedUrl.port = `${port}`
}
// avoid converting "http://localhost:8090" to "http://localhost:8090/" - keep the original formatting if the URL is just an origin
if (hasOriginOnlyTarget && parsedUrl.pathname === "/" && !parsedUrl.search && !parsedUrl.hash) {
return parsedUrl.origin
}
return parsedUrl.toString()
}
function trimTrailingEmptyFields(fields: string[]) {
let lastValueIndex = fields.length - 1
while (lastValueIndex > 0 && !fields[lastValueIndex]) {
lastValueIndex--
}
return fields.slice(0, lastValueIndex + 1)
}
function buildMonitorPayload(values: MonitorValues, enabled = true) {
const normalizedValues = v.safeParse(NormalizedMonitorValuesSchema, values)
if (!normalizedValues.success) {
throw new Error(normalizedValues.issues[0]?.message || "Invalid monitor")
}
const payload = {
system: values.system,
enabled,
...normalizedValues.output,
}
return payload
}
type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">
function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {
return `${system}${target}${protocol}${port}`
}
function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {
const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")
const parsed = v.safeParse(BulkMonitorSchema, {
target: rawTarget,
protocol: rawProtocol,
port: rawPort,
interval: rawInterval,
})
if (!parsed.success) {
throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)
}
const protocol = (parsed.output.protocol?.toLowerCase() ||
(/^https?:\/\//i.test(parsed.output.target) ? "http" : "icmp")) as MonitorProtocol
return buildMonitorPayload({
system,
target: parsed.output.target,
protocol,
port: parsed.output.port ? Number(parsed.output.port) : 0,
interval: parsed.output.interval || `${defaultInterval}`,
})
}
export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {
const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`
const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`
return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")
}
function SystemMultiSelect({
id,
selectedSystemIds,
onChange,
disabled,
className,
}: {
id: string
selectedSystemIds: Set<string>
onChange: (ids: Set<string>) => void
disabled?: boolean
className?: string
}) {
const systems = useStore($systems)
const { t } = useLingui()
const [search, setSearch] = useState("")
const searchRef = useRef<HTMLInputElement>(null)
const focusSearchOnMount = useCallback((node: HTMLInputElement | null) => {
searchRef.current = node
if (!node) return
// Focus after the menu has completed its own initial focus handling.
const frame = requestAnimationFrame(() => node.focus())
return () => cancelAnimationFrame(frame)
}, [])
const contentRef = useRef<HTMLDivElement>(null)
const query = search.trim().toLocaleLowerCase()
const filteredSystems = systems.filter(
(system) => supportsNetworkMonitors(system) && system.name.toLocaleLowerCase().includes(query)
)
const allSelected = filteredSystems.every((system) => selectedSystemIds.has(system.id))
const anySelected = filteredSystems.some((system) => selectedSystemIds.has(system.id))
const selectFiltered = (selected: boolean) => {
const next = new Set(selectedSystemIds)
for (const system of filteredSystems) {
if (selected) next.add(system.id)
else next.delete(system.id)
}
onChange(next)
}
return (
<DropdownMenu onOpenChange={() => setSearch("")}>
<DropdownMenuTrigger asChild>
<Button
id={id}
disabled={disabled}
type="button"
variant="outline"
className={cn("relative w-full min-w-0 ps-10 pe-10 justify-start font-normal text-start", className)}
>
<ServerIcon className="size-3.5 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
<span className="truncate">
{selectedSystemIds.size === 0
? t`Select systems`
: selectedSystemIds.size === 1
? systems.find((s) => selectedSystemIds.has(s.id))?.name
: t`${selectedSystemIds.size} systems selected`}
</span>
<ChevronDownIcon className="size-4 absolute end-4 top-1/2 -translate-y-1/2 opacity-50" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent
ref={contentRef}
onKeyDown={(event) => {
if (event.key === "Tab") {
event.preventDefault()
searchRef.current?.focus()
}
}}
align="start"
className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden"
>
<div className="shrink-0 border-b mb-1">
<div className="flex items-center gap-2 px-2.5">
<SearchIcon aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />
<Input
ref={focusSearchOnMount}
value={search}
onChange={(event) => setSearch(event.target.value)}
placeholder={t`Search systems`}
aria-label={t`Search systems`}
className="h-10 min-w-0 rounded-none border-0 bg-transparent px-0 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0"
onKeyDown={(event) => {
if (event.key === "Escape") return
// Keep menu typeahead and form submission from consuming search input.
event.stopPropagation()
if (event.key === "Enter") event.preventDefault()
if (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Tab") {
event.preventDefault()
const items = contentRef.current?.querySelectorAll<HTMLElement>(
'[role^="menuitem"]:not([data-disabled])'
)
const index = event.key === "ArrowUp" || event.shiftKey ? (items?.length ?? 1) - 1 : 0
items?.[index]?.focus()
}
}}
/>
</div>
<div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 px-1 pb-1">
<div className="flex items-center">
<DropdownMenuItem
className="px-1.5 py-1 text-xs text-muted-foreground"
disabled={!filteredSystems.length || allSelected}
onSelect={(event) => {
event.preventDefault()
selectFiltered(true)
}}
>
{query ? <Trans>Select matches</Trans> : <Trans>Select all</Trans>}
</DropdownMenuItem>
<span aria-hidden="true" className="text-xs text-muted-foreground/50">
·
</span>
<DropdownMenuItem
className="px-1.5 py-1 text-xs text-muted-foreground"
disabled={!anySelected}
onSelect={(event) => {
event.preventDefault()
selectFiltered(false)
}}
>
{query ? <Trans>Clear matches</Trans> : <Trans>Clear all</Trans>}
</DropdownMenuItem>
</div>
<span className="px-1.5 text-xs tabular-nums text-muted-foreground">
{t`${selectedSystemIds.size} selected`}
</span>
</div>
</div>
<div className="min-h-0 overflow-y-auto">
{filteredSystems.length === 0 && (
<output className="block px-2.5 py-3 text-sm text-muted-foreground">
<Trans>No systems found.</Trans>
</output>
)}
{filteredSystems.map((sys) => (
<DropdownMenuCheckboxItem
key={sys.id}
checked={selectedSystemIds.has(sys.id)}
onSelect={(event) => event.preventDefault()}
onCheckedChange={(checked) => {
const next = new Set(selectedSystemIds)
if (checked) next.add(sys.id)
else next.delete(sys.id)
onChange(next)
}}
className="group min-w-0 gap-2.5 py-2 ps-2.5"
indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3"
>
<span className="truncate">{sys.name}</span>
</DropdownMenuCheckboxItem>
))}
</div>
</DropdownMenuContent>
</DropdownMenu>
)
}
export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) {
const [open, setOpen] = useState(false)
const [bulkOpen, setBulkOpen] = useState(false)
const [bulkInput, setBulkInput] = useState("")
const [bulkLoading, setBulkLoading] = useState(false)
const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState<Set<string>>(new Set())
const bulkFormRef = useRef<HTMLFormElement>(null)
const { toast } = useToast()
const { t } = useLingui()
const resetBulkForm = () => {
setBulkInput("")
}
const openBulkAdd = (selectedSystemIds?: Set<string>) => {
if (!systemId && selectedSystemIds) {
setBulkSelectedSystemIds(new Set(selectedSystemIds))
}
setOpen(false)
setBulkOpen(true)
}
const openAdd = () => {
setBulkOpen(false)
setOpen(true)
}
async function handleBulkSubmit(e: React.FormEvent) {
e.preventDefault()
setBulkLoading(true)
let closedForSubmit = false
try {
const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds)
if (!targetSystems.length) {
throw new Error("Select at least one system.")
}
const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim())
if (!rawLines.length) {
throw new Error("Enter at least one monitor.")
}
let totalCreated = 0
closedForSubmit = true
for (const system of targetSystems) {
const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system))
const existingMonitorKeys = new Set(
monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor))
)
const newPayloads: typeof payloads = []
for (const payload of payloads) {
const monitorKey = getMonitorIdentityKey(payload)
if (existingMonitorKeys.has(monitorKey)) {
continue
}
existingMonitorKeys.add(monitorKey)
newPayloads.push(payload)
}
if (!newPayloads.length) continue
let batch = pb.createBatch()
let inBatch = 0
for (const payload of newPayloads) {
batch.collection("network_monitors").create(payload)
inBatch++
if (inBatch > 20) {
await batch.send()
batch = pb.createBatch()
inBatch = 0
}
}
if (inBatch) {
await batch.send()
}
totalCreated += newPayloads.length
}
if (!totalCreated) {
throw new Error("No new monitors. All entries already exist.")
}
resetBulkForm()
toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` })
} catch (err: unknown) {
if (closedForSubmit) {
setBulkOpen(true)
}
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
} finally {
setBulkLoading(false)
}
}
return (
<>
<div className="flex gap-0 rounded-lg">
<Button variant="outline" onClick={openAdd} className="rounded-e-none grow">
{/* <PlusIcon className="size-4 me-1" /> */}
<Trans>Add {{ foo: t`Monitor` }}</Trans>
</Button>
<div className="w-px h-full bg-muted"></div>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="outline" className="px-2 rounded-s-none border-s-0" aria-label={t`More monitor actions`}>
<ChevronDownIcon className="size-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={() => openBulkAdd()}>
<ListIcon className="size-4 me-2" />
<Trans>Bulk Add</Trans>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
<Dialog
open={open}
onOpenChange={(nextOpen) => {
setOpen(nextOpen)
}}
>
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} onOpenBulkAdd={openBulkAdd} />
</Dialog>
<Sheet
open={bulkOpen}
onOpenChange={(nextOpen) => {
setBulkOpen(nextOpen)
if (!nextOpen) {
resetBulkForm()
}
}}
>
<SheetContent className="w-full sm:max-w-xl gap-0">
<SheetHeader className="border-b">
<SheetTitle>
<Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans>
</SheetTitle>
<SheetDescription>target[,protocol[,port[,interval]]]</SheetDescription>
</SheetHeader>
<form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">
<div className="flex-1 flex flex-col space-y-4 overflow-auto p-4">
{!systemId && (
<div className="grid gap-2">
<Label htmlFor="bulk-monitor-systems" className="sr-only">
<Trans>Systems</Trans>
</Label>
<SystemMultiSelect
id="bulk-monitor-systems"
selectedSystemIds={bulkSelectedSystemIds}
onChange={setBulkSelectedSystemIds}
disabled={bulkLoading}
className="bg-card"
/>
</div>
)}
<div className="grow flex flex-col gap-2">
<Label htmlFor="bulk-monitors" className="sr-only">
Entries
</Label>
<Textarea
id="bulk-monitors"
value={bulkInput}
onChange={(e) => setBulkInput(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
e.preventDefault()
bulkFormRef.current?.requestSubmit()
}
}}
className="font-mono grow text-sm bg-card"
placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}
required
/>
<p className="text-xs text-muted-foreground">target[,protocol[,port[,interval]]]</p>
</div>
</div>
<SheetFooter className="border-t">
<Button type="submit" disabled={bulkLoading || (!systemId && !bulkSelectedSystemIds.size)}>
<Trans>Add {{ foo: t`Network Monitors` }}</Trans>
</Button>
</SheetFooter>
</form>
</SheetContent>
</Sheet>
</>
)
}
export function EditMonitorDialog({
open,
setOpen,
systemId,
monitor,
}: {
open: boolean
setOpen: (open: boolean) => void
systemId?: string
monitor?: NetworkMonitorRecord
}) {
const hasOpened = useRef(false)
if (!monitor && !hasOpened.current) {
return null
}
hasOpened.current = true
return (
<Dialog open={open} onOpenChange={setOpen}>
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} monitor={monitor} />
</Dialog>
)
}
function MonitorDialogContent({
open,
setOpen,
systemId,
monitor,
onOpenBulkAdd,
}: {
open: boolean
setOpen: (open: boolean) => void
systemId?: string
monitor?: NetworkMonitorRecord
onOpenBulkAdd?: (selectedSystemIds: Set<string>) => void
}) {
const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")
const [target, setTarget] = useState(monitor?.target ?? "")
const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))
const [loading, setLoading] = useState(false)
const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")
const [selectedSystemIds, setSelectedSystemIds] = useState<Set<string>>(new Set())
const systems = useStore($systems)
const { toast } = useToast()
const { t } = useLingui()
const isEditing = !!monitor
// When the dialog is opened, initialize form fields with monitor values (if editing) or defaults (if adding).
useEffect(() => {
if (!open) {
return
}
setProtocol(monitor?.protocol ?? "icmp")
setTarget(monitor?.target ?? "")
setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
setMonitorInterval(String(monitor?.interval ?? defaultInterval))
setSelectedSystemId(monitor?.system ?? "")
setSelectedSystemIds(new Set())
setLoading(false)
}, [open, monitor])
async function handleSubmit(e: React.FormEvent) {
e.preventDefault()
setLoading(true)
const targetSystems = systemId ? [systemId] : monitor ? [selectedSystemId] : Array.from(selectedSystemIds)
const remainingSystemIds = new Set(targetSystems)
try {
if (!targetSystems.length || !targetSystems[0]) throw new Error("Select at least one system.")
const payload = buildMonitorPayload(
{
system: targetSystems[0],
target,
protocol,
port: protocol === "tcp" ? Number(port) : 0,
interval: monitorInterval,
},
monitor ? monitor.enabled : true
)
if (monitor) {
await pb.collection("network_monitors").update(monitor.id, payload)
} else {
for (const system of targetSystems) {
await pb.collection("network_monitors").create({ ...payload, system })
remainingSystemIds.delete(system)
}
}
setOpen(false)
} catch (err: unknown) {
if (!monitor && !systemId) {
// Retain only unfinished systems so retrying cannot duplicate successful creates.
setSelectedSystemIds(remainingSystemIds)
}
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
} finally {
setLoading(false)
}
}
return (
<DialogContent className="max-w-md">
<DialogHeader>
<DialogTitle>
{isEditing ? (
<Trans>Edit {{ foo: t`Network Monitor` }}</Trans>
) : (
<Trans>Add {{ foo: t`Network Monitor` }}</Trans>
)}
</DialogTitle>
<DialogDescription>
<Trans>Configure response monitoring from this agent.</Trans>
</DialogDescription>
</DialogHeader>
<form onSubmit={handleSubmit} className="grid gap-4 tabular-nums">
{!systemId && !isEditing && (
<div className="grid gap-2">
<Label htmlFor="monitor-systems">
<Trans>Systems</Trans>
</Label>
<SystemMultiSelect
id="monitor-systems"
selectedSystemIds={selectedSystemIds}
onChange={setSelectedSystemIds}
disabled={loading}
/>
</div>
)}
{!systemId && isEditing && (
<div className="grid gap-2">
<Label>
<Trans>System</Trans>
</Label>
<Select value={selectedSystemId} onValueChange={setSelectedSystemId} required>
<SelectTrigger>
<SelectValue placeholder={t`Select a system`} />
</SelectTrigger>
<SelectContent>
{systems
.filter((sys) => sys.id === monitor?.system || supportsNetworkMonitors(sys))
.map((sys) => (
<SelectItem key={sys.id} value={sys.id}>
{sys.name}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
<div className="grid gap-2">
<Label>
<Trans>Target</Trans>
</Label>
<Input
value={target}
onChange={(e) => setTarget(e.target.value)}
placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}
required
/>
</div>
<div className="grid gap-2">
<Label>
<Trans>Protocol</Trans>
</Label>
<Select value={protocol} onValueChange={(value) => setProtocol(value as MonitorProtocol)}>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="icmp">ICMP</SelectItem>
<SelectItem value="tcp">TCP</SelectItem>
<SelectItem value="http">HTTP</SelectItem>
<SelectItem value="dns">DNS</SelectItem>
</SelectContent>
</Select>
</div>
{protocol === "tcp" && (
<div className="grid gap-2">
<Label>
<Trans>Port</Trans>
</Label>
<Input
type="number"
value={port}
onChange={(e) => setPort(e.target.value)}
placeholder="443"
min={1}
max={65535}
/>
</div>
)}
<div className="grid gap-2">
<Label>
<Trans>Interval (seconds)</Trans>
</Label>
<Input
type="number"
value={monitorInterval}
onChange={(e) => setMonitorInterval(e.target.value)}
min={1}
max={3600}
required
/>
</div>
<DialogFooter>
{!isEditing && onOpenBulkAdd && (
<Button
type="button"
variant="outline"
onClick={() => onOpenBulkAdd(selectedSystemIds)}
disabled={loading}
className="me-auto"
>
<ListIcon className="size-4 me-2" />
<Trans>Bulk Add</Trans>
</Button>
)}
<Button
type="submit"
disabled={loading || (!systemId && (isEditing ? !selectedSystemId : !selectedSystemIds.size))}
>
{loading ? (
isEditing ? (
<Trans>Saving...</Trans>
) : (
<Trans>Creating...</Trans>
)
) : isEditing ? (
<Trans>Save {{ foo: t`Monitor` }}</Trans>
) : (
<Trans>Add {{ foo: t`Monitor` }}</Trans>
)}
</Button>
</DialogFooter>
</form>
</DialogContent>
)
}
@@ -0,0 +1,411 @@
import type { CellContext, Column, ColumnDef } from "@tanstack/react-table"
import { Button } from "@/components/ui/button"
import { cn, copyToClipboard, decimalString, formatMicroseconds, hourWithSeconds } from "@/lib/utils"
import {
GlobeIcon,
TimerIcon,
WifiOffIcon,
Trash2Icon,
ArrowLeftRightIcon,
MoreHorizontalIcon,
ServerIcon,
ClockIcon,
RefreshCwIcon,
PenBoxIcon,
PauseCircleIcon,
PlayCircleIcon,
CopyIcon,
CopyPlusIcon,
} from "lucide-react"
import { t } from "@lingui/core/macro"
import type { NetworkMonitorRecord, SystemRecord } from "@/types"
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuSeparator,
DropdownMenuSub,
DropdownMenuSubContent,
DropdownMenuSubTrigger,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"
import { Trans } from "@lingui/react/macro"
import { $allSystemsById, $longestSystemName } from "@/lib/stores"
import { useStore } from "@nanostores/react"
import { SystemStatus } from "@/lib/enums"
import { Checkbox } from "@/components/ui/checkbox"
import { useMemo } from "react"
import { formatBulkMonitorLine } from "@/components/network-monitors-table/monitor-dialog"
import { Badge } from "../ui/badge"
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import { pb } from "@/lib/api"
const protocolColors: Record<string, string> = {
icmp: "bg-blue-500/15! text-blue-600 dark:text-blue-400",
tcp: "bg-purple-500/15! text-purple-600 dark:text-purple-400",
http: "bg-green-500/15! text-green-700 dark:text-green-400",
dns: "bg-amber-500/15! text-amber-600 dark:text-amber-400",
}
const SYSTEM_STATUS_COLORS = {
[SystemStatus.Up]: "bg-green-500",
[SystemStatus.Down]: "bg-red-500",
[SystemStatus.Paused]: "bg-primary/40",
[SystemStatus.Pending]: "bg-yellow-500",
} as const
/**
* A monitor is considered muted if it's disabled or if its associated system is not up.
*/
const isMuted = (record: NetworkMonitorRecord, systemRecord: SystemRecord | undefined) =>
!record.enabled || systemRecord?.status !== SystemStatus.Up
export function getMonitorColumns(
longestTarget = "",
{
onEdit,
onDelete,
onSetEnabled,
}: {
onEdit?: (monitor: NetworkMonitorRecord) => void
onDelete?: (monitors: NetworkMonitorRecord[]) => void | Promise<void>
onSetEnabled?: (monitors: NetworkMonitorRecord[], enabled: boolean) => void | Promise<void>
} = {}
): ColumnDef<NetworkMonitorRecord>[] {
return [
{
id: "select",
header: ({ table }) => (
<Checkbox
className="ms-2"
checked={table.getIsAllRowsSelected() || (table.getIsSomeRowsSelected() && "indeterminate")}
onClick={(event) => event.stopPropagation()}
onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}
aria-label={t`Select all`}
/>
),
cell: ({ row }) => (
<Checkbox
checked={row.getIsSelected()}
onClick={(event) => event.stopPropagation()}
onCheckedChange={(value) => row.toggleSelected(!!value)}
aria-label={t`Select row`}
/>
),
enableSorting: false,
enableHiding: false,
size: 44,
},
{
id: "system",
accessorFn: (record) => record.system,
sortingFn: (a, b) => {
const allSystems = $allSystemsById.get()
const systemNameA = allSystems[a.original.system]?.name ?? ""
const systemNameB = allSystems[b.original.system]?.name ?? ""
const primary = systemNameA.localeCompare(systemNameB)
if (primary !== 0) {
return primary
}
return a.original.target.localeCompare(b.original.target)
},
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => {
const system = useStore($allSystemsById)[getValue() as string] as SystemRecord | undefined
const longestSystemName = useStore($longestSystemName)
const name = system?.name
const status = system?.status as SystemStatus // undefined val is fine but makes lsp mad
return useMemo(
() => (
<div className="ms-1.5 max-w-44 flex gap-2 items-center tabular-nums">
<span className={cn("shrink-0 size-2 rounded-full", SYSTEM_STATUS_COLORS[status])} />
<div className="relative w-fit min-w-0 max-w-full">
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestSystemName}
</span>
<span className="absolute inset-0 truncate">{name}</span>
</div>
</div>
),
[status, name]
)
},
},
{
id: "target",
sortingFn: (a, b) => a.original.target.localeCompare(b.original.target),
accessorFn: (record) => getMonitorTarget(record),
header: ({ column }) => <HeaderButton column={column} name={t`Target`} Icon={GlobeIcon} />,
cell: ({ row, getValue }) => {
const monitor = row.original
const { status } = useStore($allSystemsById)[monitor.system] || {}
let color = "bg-green-500"
if (!monitor.enabled || status === SystemStatus.Paused) {
color = "bg-primary/40"
} else if (status === SystemStatus.Down || status === SystemStatus.Pending) {
color = "bg-yellow-500"
}
return (
<div className="ms-1.5 max-w-64 flex gap-2 items-center tabular-nums">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
<div className="relative w-fit min-w-0 max-w-full">
<span className="invisible block overflow-hidden whitespace-nowrap" aria-hidden="true">
{longestTarget}
</span>
<span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
</div>
)
},
},
{
id: "protocol",
accessorFn: (record) => record.protocol,
header: ({ column }) => <HeaderButton column={column} name={t`Protocol`} Icon={ArrowLeftRightIcon} />,
cell: ({ getValue }) => {
const protocol = getValue() as string
return <Badge className={cn("uppercase", protocolColors[protocol])}>{protocol}</Badge>
},
},
{
id: "interval",
accessorFn: (record) => record.interval,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Interval`} Icon={RefreshCwIcon} />,
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{getValue() as number}s</span>,
},
{
id: "res",
accessorFn: (record) => record.res,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Response`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "res1h",
accessorFn: (record) => record.resAvg1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Avg 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "max1h",
accessorFn: (record) => record.resMax1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Max 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "min1h",
accessorFn: (record) => record.resMin1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Min 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "loss",
accessorFn: (record) => record.loss1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Loss 1h`} Icon={WifiOffIcon} />,
cell: ({ row }) => {
const { loss1h, res, system } = row.original
const systemRecord = useStore($allSystemsById)[system]
if (loss1h === undefined || (!res && !loss1h)) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
const muted = isMuted(row.original, systemRecord)
let color = "bg-green-500"
if (muted) {
color = "bg-muted-foreground/50"
} else if (loss1h) {
color = loss1h > 20 ? "bg-red-500" : "bg-yellow-500"
}
return (
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
{loss1h === 100 ? loss1h : decimalString(loss1h, loss1h >= 10 ? 1 : 2)}%
</span>
)
},
},
{
id: "updated",
invertSorting: true,
accessorFn: (record) => record.updated,
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
cell: ({ getValue }) => {
const timestamp = getValue() as number
if (!timestamp) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
return <span className="ms-1.5 tabular-nums">{hourWithSeconds(timestamp)}</span>
},
},
{
id: "actions",
enableSorting: false,
enableHiding: false,
header: () => null,
size: 40,
cell: ({ row, table }) => {
const selectedRows = table.getSelectedRowModel().rows
const actionRows =
row.getIsSelected() && selectedRows.length > 1
? selectedRows.map((selectedRow) => selectedRow.original)
: [row.original]
const isBulkAction = actionRows.length > 1
const shouldPause = actionRows.some((monitor) => monitor.enabled)
const bulkCopyContent = actionRows.map((monitor) => formatBulkMonitorLine(monitor)).join("\n")
const allSystems = useStore($allSystemsById)
const otherSystems = useMemo(
() => Object.values(allSystems).filter((s) => !isBulkAction && s.id !== row.original.system),
[allSystems, isBulkAction]
)
return (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="icon" className="size-10">
<span className="sr-only">
<Trans>Open menu</Trans>
</span>
<MoreHorizontalIcon className="w-5" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
{!isBulkAction && (
<DropdownMenuItem
onClick={() => {
onEdit?.(row.original)
}}
>
<PenBoxIcon className="me-2.5 size-4" />
<Trans>Edit</Trans>
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={() => {
onSetEnabled?.(actionRows, !shouldPause)
}}
>
{shouldPause ? (
<>
<PauseCircleIcon className="me-2.5 size-4" />
<Trans>Pause</Trans>
</>
) : (
<>
<PlayCircleIcon className="me-2.5 size-4" />
<Trans>Resume</Trans>
</>
)}
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => {
copyToClipboard(bulkCopyContent)
}}
>
<CopyIcon className="me-2.5 size-4" />
<Trans>Bulk copy</Trans>
</DropdownMenuItem>
{!isBulkAction && otherSystems.length > 0 && (
<DropdownMenuSub>
<DropdownMenuSubTrigger>
<CopyPlusIcon className="me-2.5 size-4" />
<Trans>Copy to system</Trans>
</DropdownMenuSubTrigger>
<DropdownMenuSubContent className="max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">
{otherSystems.map((sys) => (
<DropdownMenuItem
key={sys.id}
onClick={() => {
const { id: _id, system: _system, ...rest } = row.original
pb.collection("network_monitors")
.create({ ...rest, system: sys.id })
.catch(() => {})
}}
>
{sys.name}
</DropdownMenuItem>
))}
</DropdownMenuSubContent>
</DropdownMenuSub>
)}
<DropdownMenuSeparator />
<DropdownMenuItem
onClick={() => {
onDelete?.(actionRows)
}}
>
<Trash2Icon className="me-2.5 size-4" />
<Trans>Delete</Trans>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)
},
},
]
}
const responseTimeThresholds = {
http: { warning: 800_000, critical: 3_000_000 },
tcp: { warning: 500_000, critical: 2_000_000 },
icmp: { warning: 100_000, critical: 500_000 },
dns: { warning: 150_000, critical: 800_000 },
}
function responseTimeCell(cell: CellContext<NetworkMonitorRecord, unknown>) {
const monitor = cell.row.original
const systemRecord = useStore($allSystemsById)[monitor.system]
const responseTime = cell.getValue() as number | undefined
if (!responseTime) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
const muted = isMuted(monitor, systemRecord)
let color = "bg-green-500"
if (muted) {
color = "bg-muted-foreground/50"
} else if (responseTime > responseTimeThresholds[monitor.protocol].warning) {
color = "bg-yellow-500"
}
if (!muted && responseTime > responseTimeThresholds[monitor.protocol].critical) {
color = "bg-red-500"
}
return (
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
{formatMicroseconds(responseTime)}
</span>
)
}
function HeaderButton({
column,
name,
Icon,
}: {
column: Column<NetworkMonitorRecord>
name: string
Icon: React.ElementType
}) {
const isSorted = column.getIsSorted()
return (
<Button
className={cn(
"h-9 px-3 flex items-center gap-2 duration-50",
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
)}
variant="ghost"
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
>
{Icon && <Icon className="size-4" />}
{name}
</Button>
)
}
@@ -0,0 +1,575 @@
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import { t } from "@lingui/core/macro"
import { Trans } from "@lingui/react/macro"
import {
type ColumnFiltersState,
flexRender,
getCoreRowModel,
getFilteredRowModel,
getSortedRowModel,
type Row,
type RowSelectionState,
type SortingState,
type Table as TableType,
useReactTable,
type VisibilityState,
} from "@tanstack/react-table"
import { useVirtualizer, type VirtualItem } from "@tanstack/react-virtual"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import { Button, buttonVariants } from "@/components/ui/button"
import { memo, useCallback, useMemo, useRef, useState } from "react"
import { getMonitorColumns } from "@/components/network-monitors-table/network-monitors-columns"
import { Card, CardHeader, CardTitle } from "@/components/ui/card"
import { Input } from "@/components/ui/input"
import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { useToast } from "@/components/ui/use-toast"
import { isReadOnlyUser } from "@/lib/api"
import { pb } from "@/lib/api"
import { $allSystemsById, $direction, $userSettings } from "@/lib/stores"
import {
cn,
isVisuallyLonger,
matchesFilterGroups,
parseFilterGroups,
parseSemVer,
useBrowserStorage,
} from "@/lib/utils"
import type { ChartData, NetworkMonitorRecord } from "@/types"
import { AddMonitorDialog, EditMonitorDialog } from "./monitor-dialog"
import { ArrowLeftRightIcon, EthernetPortIcon, LoaderCircleIcon, ServerIcon, XIcon } from "lucide-react"
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
import ChartTimeSelect from "@/components/charts/chart-time-select"
import { LossChart, AvgMinMaxResponseChart } from "@/components/routes/system/charts/monitors-charts"
import { useNetworkMonitorStats } from "@/lib/use-network-monitors"
import { useStore } from "@nanostores/react"
import { atom } from "nanostores"
import { Separator } from "../ui/separator"
import { $router, Link } from "../router"
import { getPagePath } from "@nanostores/router"
export default function NetworkMonitorsTableNew({
systemId,
monitors,
isLoading,
}: {
systemId?: string
monitors: NetworkMonitorRecord[]
isLoading: boolean
}) {
const [sorting, setSorting] = useBrowserStorage<SortingState>(
`sort-np-target-${systemId ? 1 : 0}`,
[{ id: systemId ? "target" : "system", desc: false }],
sessionStorage
)
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
const [rowSelection, setRowSelection] = useState<RowSelectionState>({})
const [globalFilter, setGlobalFilter] = useState("")
const [deleteOpen, setDeleteOpen] = useState(false)
const [pendingDeleteIds, setPendingDeleteIds] = useState<string[]>([])
const [editingMonitor, setEditingMonitor] = useState<NetworkMonitorRecord>()
const { toast } = useToast()
const canManageMonitors = !isReadOnlyUser()
const longestTarget = useMemo(() => {
let longestTarget = ""
for (const p of monitors) {
if (isVisuallyLonger(getMonitorTarget(p), longestTarget)) {
longestTarget = getMonitorTarget(p)
}
}
return longestTarget
}, [monitors])
const runMonitorBatch = useCallback(
async (ids: string[], enqueue: (batch: ReturnType<typeof pb.createBatch>, id: string) => void) => {
let batch = pb.createBatch()
let inBatch = 0
for (const id of ids) {
enqueue(batch, id)
if (++inBatch >= 20) {
await batch.send()
batch = pb.createBatch()
inBatch = 0
}
}
if (inBatch) {
await batch.send()
}
},
[]
)
const handleDeleteRequest = useCallback(
async (monitorsToDelete: NetworkMonitorRecord[]) => {
if (!monitorsToDelete.length) {
return
}
const ids = monitorsToDelete.map((monitor) => monitor.id)
if (ids.length === 1) {
try {
await pb.collection("network_monitors").delete(ids[0])
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to delete monitors.`,
})
}
return
}
setPendingDeleteIds(ids)
setDeleteOpen(true)
},
[toast]
)
const handleBulkDelete = async () => {
setDeleteOpen(false)
if (!pendingDeleteIds.length) {
return
}
try {
await runMonitorBatch(pendingDeleteIds, (batch, id) => batch.collection("network_monitors").delete(id))
setPendingDeleteIds([])
setRowSelection({})
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to delete monitors.`,
})
}
}
const handleSetEnabled = useCallback(
async (monitorsToUpdate: NetworkMonitorRecord[], enabled: boolean) => {
if (!monitorsToUpdate.length) {
return
}
const pendingUpdates = monitorsToUpdate.filter((monitor) => monitor.enabled !== enabled)
if (!pendingUpdates.length) {
return
}
try {
if (pendingUpdates.length === 1) {
await pb.collection("network_monitors").update(pendingUpdates[0].id, { enabled })
return
}
await runMonitorBatch(
pendingUpdates.map((monitor) => monitor.id),
(batch, id) => batch.collection("network_monitors").update(id, { enabled })
)
if (monitorsToUpdate.length > 1) {
setRowSelection({})
}
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to update monitors.`,
})
}
},
[runMonitorBatch, toast]
)
const columns = useMemo(() => {
let columns = getMonitorColumns(longestTarget, {
onEdit: setEditingMonitor,
onDelete: handleDeleteRequest,
onSetEnabled: handleSetEnabled,
})
columns = systemId ? columns.filter((col) => col.id !== "system") : columns
columns = canManageMonitors ? columns : columns.filter((col) => col.id !== "actions")
return columns
}, [canManageMonitors, handleDeleteRequest, handleSetEnabled, systemId, longestTarget])
const table = useReactTable({
data: monitors,
columns,
getRowId: (row) => row.id,
getCoreRowModel: getCoreRowModel(),
getSortedRowModel: getSortedRowModel(),
getFilteredRowModel: getFilteredRowModel(),
onSortingChange: setSorting,
onColumnFiltersChange: setColumnFilters,
onColumnVisibilityChange: setColumnVisibility,
onRowSelectionChange: setRowSelection,
defaultColumn: {
sortUndefined: "last",
size: 900,
minSize: 0,
},
state: {
sorting,
columnFilters,
columnVisibility,
rowSelection,
globalFilter,
},
onGlobalFilterChange: setGlobalFilter,
globalFilterFn: (row, _columnId, filterValue) => {
const value = (filterValue as string).trim()
if (!value) return true
const monitor = row.original
const systemName = $allSystemsById.get()[monitor.system]?.name ?? ""
const searchString = `${getMonitorTarget(monitor)}${monitor.protocol}${systemName}`.toLocaleLowerCase()
return matchesFilterGroups(searchString, parseFilterGroups(value))
},
})
const rows = table.getRowModel().rows
const visibleColumns = table.getVisibleLeafColumns()
return (
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
<CardHeader className="p-0 mb-3 sm:mb-4">
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
<div className="px-2 sm:px-1">
<CardTitle className="mb-2">
<Trans>Network Monitors</Trans>
</CardTitle>
<div className="text-sm text-muted-foreground flex items-center flex-wrap">
<Trans>Response time monitoring from agents.</Trans>
</div>
</div>
<div className="md:ms-auto flex items-center gap-2">
{monitors.length > 0 && (
<div className="relative">
<Input
placeholder={t`Filter...`}
title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}
value={globalFilter}
onChange={(e) => setGlobalFilter(e.target.value)}
className="ms-auto px-4 w-full max-w-full md:w-50"
/>
{globalFilter && (
<Button
type="button"
variant="ghost"
size="icon"
aria-label={t`Clear`}
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
onClick={() => setGlobalFilter("")}
>
<XIcon className="h-4 w-4" />
</Button>
)}
</div>
)}
{canManageMonitors ? <AddMonitorDialog systemId={systemId} monitors={monitors} /> : null}
{canManageMonitors ? (
<EditMonitorDialog
systemId={systemId}
monitor={editingMonitor}
open={!!editingMonitor}
setOpen={(open) => {
if (!open) {
setEditingMonitor(undefined)
}
}}
/>
) : null}
<AlertDialog
open={deleteOpen}
onOpenChange={(open) => {
setDeleteOpen(open)
if (!open) {
setPendingDeleteIds([])
}
}}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
<Trans>Are you sure?</Trans>
</AlertDialogTitle>
<AlertDialogDescription>
<Trans>This will permanently delete all selected records from the database.</Trans>
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>
<Trans>Cancel</Trans>
</AlertDialogCancel>
<AlertDialogAction
className={cn(buttonVariants({ variant: "destructive" }))}
onClick={handleBulkDelete}
>
<Trans>Continue</Trans>
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
</div>
</CardHeader>
<div className="rounded-md">
<NetworkMonitorsTable
table={table}
rows={rows}
colLength={visibleColumns.length}
rowSelection={rowSelection}
isLoading={isLoading}
/>
</div>
</Card>
)
}
const NetworkMonitorsTable = memo(function NetworkMonitorTable({
table,
rows,
colLength,
rowSelection,
isLoading,
}: {
table: TableType<NetworkMonitorRecord>
rows: Row<NetworkMonitorRecord>[]
colLength: number
rowSelection: RowSelectionState
isLoading: boolean
}) {
const scrollRef = useRef<HTMLDivElement>(null)
const [sheetOpen, setSheetOpen] = useState(false)
const [activeMonitorId, setActiveMonitorId] = useState<string | null>(null)
const activeMonitor = activeMonitorId
? table.options.data.find((monitor) => monitor.id === activeMonitorId)
: undefined
const openSheet = useCallback((monitor: NetworkMonitorRecord) => {
setActiveMonitorId(monitor.id)
setSheetOpen(true)
}, [])
const virtualizer = useVirtualizer<HTMLDivElement, HTMLTableRowElement>({
count: rows.length,
estimateSize: () => 54,
getScrollElement: () => scrollRef.current,
overscan: 5,
})
const virtualRows = virtualizer.getVirtualItems()
const paddingTop = Math.max(0, virtualRows[0]?.start ?? 0 - virtualizer.options.scrollMargin)
const paddingBottom = Math.max(0, virtualizer.getTotalSize() - (virtualRows[virtualRows.length - 1]?.end ?? 0))
return (
<div
className={cn(
"h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto border rounded-md",
(!rows.length || rows.length > 2) && "min-h-50"
)}
ref={scrollRef}
>
<div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>
<table className="text-sm w-full h-full text-nowrap">
<NetworkMonitorTableHead table={table} />
<TableBody>
{rows.length ? (
virtualRows.map((virtualRow) => {
const row = rows[virtualRow.index]
return (
<NetworkMonitorTableRow
key={row.id}
row={row}
virtualRow={virtualRow}
isSelected={row.getIsSelected()}
rowSelection={rowSelection}
openSheet={openSheet}
/>
)
})
) : (
<TableRow>
<TableCell colSpan={colLength} className="h-37 text-center pointer-events-none">
{isLoading ? (
<LoaderCircleIcon className="animate-spin size-10 opacity-60 mx-auto" />
) : (
<Trans>No results.</Trans>
)}
</TableCell>
</TableRow>
)}
</TableBody>
</table>
</div>
<NetworkMonitorSheet
open={sheetOpen}
onOpenChange={(nextOpen) => {
setSheetOpen(nextOpen)
}}
monitor={activeMonitor}
/>
</div>
)
})
function NetworkMonitorTableHead({ table }: { table: TableType<NetworkMonitorRecord> }) {
return (
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
{table.getHeaderGroups().map((headerGroup) => (
<tr key={headerGroup.id}>
{headerGroup.headers.map((header) => {
return (
<TableHead className="px-2" key={header.id}>
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
</TableHead>
)
})}
</tr>
))}
</TableHeader>
)
}
const NetworkMonitorTableRow = memo(function NetworkMonitorTableRow({
row,
virtualRow,
isSelected,
rowSelection: _rowSelection,
openSheet,
}: {
row: Row<NetworkMonitorRecord>
virtualRow: VirtualItem
isSelected: boolean
// Menus depend on the entire selection, including changes to other rows.
rowSelection: RowSelectionState
openSheet: (monitor: NetworkMonitorRecord) => void
}) {
return (
<TableRow
data-state={isSelected && "selected"}
className="cursor-pointer transition-opacity"
onClick={() => openSheet(row.original)}
>
{row.getVisibleCells().map((cell) => (
<TableCell
key={cell.id}
className="py-0"
style={{
width: `${cell.column.getSize()}px`,
height: virtualRow.size,
}}
>
{flexRender(cell.column.columnDef.cell, cell.getContext())}
</TableCell>
))}
</TableRow>
)
})
function NetworkMonitorSheet({
open,
onOpenChange,
monitor,
}: {
open: boolean
onOpenChange: (open: boolean) => void
monitor?: NetworkMonitorRecord
}) {
if (!monitor) {
return null
}
return <NetworkMonitorSheetContent key={monitor.system} open={open} onOpenChange={onOpenChange} monitor={monitor} />
}
function NetworkMonitorSheetContent({
open,
onOpenChange,
monitor,
}: {
open: boolean
onOpenChange: (open: boolean) => void
monitor: NetworkMonitorRecord
}) {
// Keep monitor exploration independent of the system charts' time range.
const [chartTimeStore] = useState(() => {
const defaultTime = $userSettings.get().chartTime
return atom(defaultTime === "1m" ? "1h" : defaultTime)
})
const chartTime = useStore(chartTimeStore)
const direction = useStore($direction)
const system = useStore($allSystemsById)[monitor.system]
const monitorStats = useNetworkMonitorStats({
systemId: monitor.system,
monitorId: monitor.id,
chartTime,
enabled: open,
})
const chartData = useMemo<ChartData>(
() => ({
agentVersion: parseSemVer(system?.info?.v),
orientation: direction === "rtl" ? "right" : "left",
chartTime,
}),
[system?.info?.v, direction, chartTime]
)
const hasMonitorStats = monitorStats.some((record) => record.stats?.[monitor.id] != null)
const monitorLabel = getMonitorTarget(monitor)
return (
<Sheet open={open} onOpenChange={onOpenChange}>
<SheetContent className="w-full sm:max-w-220 overflow-auto p-4 sm:p-6">
<SheetHeader className="mb-0 border-b p-0 pb-4">
<SheetTitle>{monitorLabel}</SheetTitle>
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
<ServerIcon className="size-3.5 text-muted-foreground" />
<Link className="hover:underline" href={getPagePath($router, "system", { id: system?.id ?? "" })}>
{system?.name ?? ""}
</Link>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<ArrowLeftRightIcon className="size-3.5 text-muted-foreground" />
{monitor.protocol.toUpperCase()}
{monitor.protocol === "tcp" && monitor.port > 0 && (
<>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<EthernetPortIcon className="size-3.5 text-muted-foreground" />
<span>{monitor.port}</span>
</>
)}
</SheetDescription>
</SheetHeader>
<div className="grid gap-4">
<ChartTimeSelect
className="bg-card"
agentVersion={chartData.agentVersion}
chartTimeStore={chartTimeStore}
allowRealtime={false}
/>
<AvgMinMaxResponseChart
monitorStats={monitorStats}
monitor={monitor}
chartData={chartData}
empty={!hasMonitorStats}
/>
<LossChart
monitorStats={monitorStats}
grid={false}
monitors={[monitor]}
chartData={chartData}
empty={!hasMonitorStats}
showFilter={false}
/>
</div>
</SheetContent>
</Sheet>
)
}
+1
View File
@@ -4,6 +4,7 @@ const routes = {
home: "/",
containers: "/containers",
smart: "/smart",
monitors: "/monitors",
system: `/system/:id`,
settings: `/settings/:name?`,
forgot_password: `/forgot-password`,
@@ -0,0 +1,33 @@
import { useLingui } from "@lingui/react/macro"
import { memo, useEffect } from "react"
import NetworkMonitorsTableNew from "@/components/network-monitors-table/network-monitors-table"
import { ActiveAlerts } from "@/components/active-alerts"
import { FooterRepoLink } from "@/components/footer-repo-link"
import { useNetworkMonitors } from "@/lib/use-network-monitors"
import { $allSystemsById } from "@/lib/stores"
import { supportsNetworkMonitors } from "@/lib/utils"
import { useStore } from "@nanostores/react"
export default memo(() => {
const { t } = useLingui()
const { monitors, isLoading } = useNetworkMonitors({})
const systems = useStore($allSystemsById)
const visibleMonitors = monitors.filter((monitor) => {
const system = systems[monitor.system]
return !system || supportsNetworkMonitors(system)
})
useEffect(() => {
document.title = `${t`Network Monitors`} / Beszel`
}, [t])
return (
<>
<div className="grid gap-4">
<ActiveAlerts />
<NetworkMonitorsTableNew monitors={visibleMonitors} isLoading={isLoading} />
</div>
<FooterRepoLink />
</>
)
})
@@ -78,7 +78,7 @@ export default function AlertsHistoryDataTable() {
let unsubscribe: (() => void) | undefined
const pbOptions = {
expand: "system",
fields: "id,name,value,state,created,resolved,expand.system.name",
fields: "id,name,monitor_name,value,state,created,resolved,expand.system.name",
}
// Initial load
pb.collection<AlertsHistoryRecord>("alerts_history")
@@ -199,7 +199,7 @@ export default function AlertsHistoryDataTable() {
if (!selectedRows.length) return
const cells: Record<string, (record: AlertsHistoryRecord) => string> = {
system: (record) => record.expand?.system?.name || record.system,
name: (record) => alertInfo[record.name]?.name() || record.name,
name: (record) => [alertInfo[record.name]?.name() || record.name, record.monitor_name].filter(Boolean).join(": "),
value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),
state: (record) => (record.resolved ? t`Resolved` : t`Active`),
created: (record) => formatShortDate(record.created),
+30 -6
View File
@@ -1,6 +1,6 @@
import { memo, useState } from "react"
import { Trans } from "@lingui/react/macro"
import { compareSemVer, parseSemVer } from "@/lib/utils"
import { compareSemVer, parseSemVer, supportsNetworkMonitors } from "@/lib/utils"
import type { GPUData } from "@/types"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import InfoBar from "./system/info-bar"
@@ -12,9 +12,15 @@ import { ZfsCharts } from "./system/charts/storage-pool-charts"
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
import {
LazyContainersTable,
LazyNetworkMonitorsTable,
LazySmartTable,
LazySystemdTable,
LazyZfsTable,
} from "./system/lazy-tables"
import { LoadAverageChart } from "./system/charts/load-average-chart"
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
import { ContainerIcon, CpuIcon, HardDriveIcon, NetworkIcon, TerminalSquareIcon } from "lucide-react"
import { GpuIcon } from "../ui/icons"
import SystemdTable from "../systemd-table/systemd-table"
import ContainersTable from "../containers-table/containers-table"
@@ -65,9 +71,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
const hasSystemd = system.info.sv
const hasGpu = hasGpuData || hasGpuPowerData
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
const hasNetworkMonitors = supportsNetworkMonitors(system)
// keep tabsRef in sync for keyboard navigation
const tabs = ["core", "disk"]
const tabs = ["core", "network", "disk"]
if (hasGpu) tabs.push("gpu")
if (hasContainers) tabs.push("containers")
if (hasSystemd) tabs.push("services")
@@ -153,6 +160,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
{hasSystemd && <LazySystemdTable systemId={system.id} />}
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
</>
)
}
@@ -165,6 +174,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<CpuIcon className="size-3.5" />
<Trans context="Core system metrics">Core</Trans>
</TabsTrigger>
<TabsTrigger value="network" className="w-full flex items-center gap-1.5">
<NetworkIcon className="size-3.5" />
<Trans>Network</Trans>
</TabsTrigger>
<TabsTrigger value="disk" className="w-full flex items-center gap-1.5">
<HardDriveIcon className="size-3.5" />
<Trans>Disk</Trans>
@@ -192,9 +205,9 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}>
<div className="grid xl:grid-cols-2 gap-4">
<CpuChart {...coreProps} />
<MemoryChart {...coreProps} />
<LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />
<BandwidthChart {...coreProps} systemStats={systemStats} />
<MemoryChart {...coreProps} />
<SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
<TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} />
<FanChart {...coreProps} />
<BatteryChart system={system} {...coreProps} />
@@ -203,6 +216,17 @@ export default memo(function SystemDetail({ id }: { id: string }) {
</div>
</TabsContent>
<TabsContent value="network" forceMount className={activeTab === "network" ? "contents" : "hidden"}>
{mountedTabs.has("network") && (
<>
<div className="grid xl:grid-cols-2 gap-4">
<BandwidthChart {...coreProps} systemStats={systemStats} />
</div>
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
</>
)}
</TabsContent>
<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>
{mountedTabs.has("disk") && (
<>
@@ -44,6 +44,7 @@ export function FilterBar({ store = $containerFilter }: { store?: typeof $contai
<>
<Input
placeholder={t`Filter...`}
title={t`Use commas to match any of multiple terms, e.g. "system1, system2"`}
className="ps-4 pe-8 w-full sm:w-44"
onChange={handleChange}
value={inputValue}
@@ -1,7 +1,13 @@
import { timeTicks } from "d3-time"
import { getPbTimestamp, pb } from "@/lib/api"
import { chartTimeData } from "@/lib/utils"
import type { ChartData, ChartTimes, ContainerStatsRecord, SystemStatsRecord } from "@/types"
import type {
ChartData,
ChartDataContainer,
ChartTimes,
ContainerStatsRecord,
NetworkMonitorStatsRecord,
SystemStatsRecord,
} from "@/types"
type ChartTimeData = {
time: number
@@ -17,31 +23,10 @@ export const cache = new Map<
ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]
>()
// create ticks and domain for charts
export function getTimeData(chartTime: ChartTimes, lastCreated: number) {
const cached = cache.get("td") as ChartTimeData | undefined
if (cached && cached.chartTime === chartTime) {
if (!lastCreated || cached.time >= lastCreated) {
return cached.data
}
}
// const buffer = chartTime === "1m" ? 400 : 20_000
const now = new Date(Date.now())
const startTime = chartTimeData[chartTime].getOffset(now)
const ticks = timeTicks(startTime, now, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
const data = {
ticks,
domain: [chartTimeData[chartTime].getOffset(now).getTime(), now.getTime()],
}
cache.set("td", { time: now.getTime(), data, chartTime })
return data
}
/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.
* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */
export function appendData<T extends { created: string | number | null }>(
prev: T[],
prev: T[] = [],
newRecords: T[],
expectedInterval: number,
maxLen?: number
@@ -66,17 +51,18 @@ export function appendData<T extends { created: string | number | null }>(
return result
}
export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>(
export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | NetworkMonitorStatsRecord>(
collection: string,
systemId: string,
chartTime: ChartTimes
chartTime: ChartTimes,
cachedStats?: { created: string | number | null }[],
createdIsNumber?: boolean
): Promise<T[]> {
const cachedStats = cache.get(`${systemId}_${chartTime}_${collection}`) as T[] | undefined
const lastCached = cachedStats?.at(-1)?.created as number
return await pb.collection<T>(collection).getFullList({
filter: pb.filter("system={:id} && created > {:created} && type={:type}", {
id: systemId,
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined),
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, createdIsNumber),
type: chartTimeData[chartTime].type,
}),
fields: "created,stats",
@@ -84,11 +70,11 @@ export async function getStats<T extends SystemStatsRecord | ContainerStatsRecor
})
}
export function makeContainerData(containers: ContainerStatsRecord[]): ChartData["containerData"] {
const result = [] as ChartData["containerData"]
export function makeContainerData(containers: ContainerStatsRecord[]): ChartDataContainer[] {
const result = [] as ChartDataContainer[]
for (const { created, stats } of containers) {
if (!created) {
result.push({ created: null } as ChartData["containerData"][0])
result.push({ created: null } as ChartDataContainer)
continue
}
result.push(makeContainerPoint(new Date(created).getTime(), stats))
@@ -97,11 +83,8 @@ export function makeContainerData(containers: ContainerStatsRecord[]): ChartData
}
/** Transform a single realtime container stats message into a ChartDataContainer point. */
export function makeContainerPoint(
created: number,
stats: ContainerStatsRecord["stats"]
): ChartData["containerData"][0] {
const point: ChartData["containerData"][0] = { created } as ChartData["containerData"][0]
export function makeContainerPoint(created: number, stats: ContainerStatsRecord["stats"]): ChartDataContainer {
const point: ChartDataContainer = { created } as ChartDataContainer
for (const container of stats) {
;(point as Record<string, unknown>)[container.n] = container
}
@@ -0,0 +1,212 @@
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import LineChartDefault from "@/components/charts/line-chart"
import type { DataPoint } from "@/components/charts/line-chart"
import { decimalString, formatMicroseconds, matchesFilterGroups, parseFilterGroups, toFixedFloat } from "@/lib/utils"
import { $monitorFilter } from "@/lib/stores"
import { useLingui } from "@lingui/react/macro"
import { ChartCard, FilterBar } from "../chart-card"
import type { ChartData, MonitorStats, NetworkMonitorRecord, NetworkMonitorStatsRecord } from "@/types"
import { useMemo } from "react"
import { useStore } from "@nanostores/react"
type MonitorChartProps = {
monitorStats: NetworkMonitorStatsRecord[]
grid?: boolean
monitors: NetworkMonitorRecord[]
chartData: ChartData
empty: boolean
showFilter?: boolean
/** Prepended to the chart title, e.g. a target/system name (rendered as "{titlePrefix} — Response"). */
titlePrefix?: string
}
type MonitorChartBaseProps = MonitorChartProps & {
metric: keyof MonitorStats
title: string
description: string
tickFormatter: (value: number) => string
contentFormatter: ({ value }: { value: number | string }) => string | number
domain?: [number | "auto", number | "auto"]
}
function MonitorChart({
monitorStats,
grid,
monitors,
chartData,
empty,
metric,
title,
description,
tickFormatter,
contentFormatter,
domain,
showFilter = monitors.length > 1,
}: MonitorChartBaseProps) {
const storedFilter = useStore($monitorFilter)
const filter = showFilter ? storedFilter : ""
const { dataPoints, visibleKeys } = useMemo(() => {
const sortedMonitors = [...monitors].sort((a, b) => b.resAvg1h - a.resAvg1h)
const count = sortedMonitors.length
const points: DataPoint<NetworkMonitorStatsRecord>[] = []
const visibleIDs: string[] = []
const filterGroups = parseFilterGroups(filter)
const dot = chartData.chartTime === "1m"
for (let i = 0; i < count; i++) {
const p = sortedMonitors[i]
const label = getMonitorTarget(p)
const labelLower = label.toLowerCase()
const filtered = filterGroups.length > 0 && !matchesFilterGroups(labelLower, filterGroups)
if (filtered) {
continue
}
visibleIDs.push(p.id)
points.push({
order: i,
label,
dataKey: (record: NetworkMonitorStatsRecord) => record.stats?.[p.id]?.[metric] ?? null,
dot,
color: count <= 5 ? i + 1 : `hsl(${(i * 360) / count}, var(--chart-saturation), var(--chart-lightness))`,
})
}
return { dataPoints: points, visibleKeys: visibleIDs }
}, [monitors, filter, metric, chartData.chartTime])
const filteredMonitorStats = useMemo(() => {
if (!visibleKeys.length) return monitorStats
return monitorStats.filter((record) => visibleKeys.some((id) => record.stats?.[id] != null))
}, [monitorStats, visibleKeys])
const legend = dataPoints.length < 10 && showFilter
return (
<ChartCard
legend={legend || !showFilter}
cornerEl={showFilter ? <FilterBar store={$monitorFilter} /> : undefined}
empty={empty}
title={title}
description={description}
grid={grid}
>
<LineChartDefault
truncate
chartData={chartData}
customData={filteredMonitorStats}
dataPoints={dataPoints}
domain={domain ?? ["auto", "auto"]}
connectNulls
tickFormatter={tickFormatter}
contentFormatter={contentFormatter}
legend={legend}
filter={filter}
/>
</ChartCard>
)
}
interface AvgMinMaxResponseChartProps {
monitorStats: NetworkMonitorStatsRecord[]
monitor: NetworkMonitorRecord | null
chartData: ChartData
empty: boolean
}
export function AvgMinMaxResponseChart({ monitorStats, monitor, chartData, empty }: AvgMinMaxResponseChartProps) {
const { t } = useLingui()
const { chartTime } = chartData
const hasLongInterval = (monitor?.interval ?? 61) > 60
// only one monitor is relevant for this chart
const dataPoints: DataPoint<NetworkMonitorStatsRecord>[] = useMemo(() => {
const dataFn = (metric: keyof MonitorStats) => (record: NetworkMonitorStatsRecord) =>
record.stats?.[monitor?.id ?? ""]?.[metric] ?? "-"
const avgPoint = {
label: "Avg",
dataKey: dataFn("res_avg"),
color: 1,
order: 0,
}
if (chartTime === "1m" || (hasLongInterval && chartTime === "1h")) {
// avg, min, max are all the same for 1m interval, so just show avg
return [avgPoint]
}
return [
{
label: "Max",
dataKey: dataFn("res_max"),
color: 3,
order: 0,
},
avgPoint,
{
label: "Min",
dataKey: dataFn("res_min"),
color: 2,
order: 2,
},
]
}, [chartTime, hasLongInterval, monitor?.id])
const data = useMemo(() => {
if (!monitor) return []
return monitorStats.filter((record) => record.stats && monitor.id in record.stats)
}, [monitor, monitorStats])
const legend = dataPoints.length > 1
return (
<ChartCard
legend={true}
empty={empty}
title={t`Response`}
description={t`Average, minimum, and maximum response time`}
grid={false}
>
<LineChartDefault
truncate
chartData={chartData}
customData={data}
dataPoints={dataPoints}
domain={["auto", "auto"]}
connectNulls
legend={legend}
tickFormatter={(value) => formatMicroseconds(value, false)}
contentFormatter={({ value }) => {
if (typeof value !== "number") {
return value
}
return formatMicroseconds(value)
}}
/>
</ChartCard>
)
}
export function LossChart({ monitorStats, grid, monitors, chartData, empty, titlePrefix }: MonitorChartProps) {
const { t } = useLingui()
const lossTitle = t`Loss`
const title = titlePrefix ? `${titlePrefix} — ${lossTitle}` : lossTitle
return (
<MonitorChart
monitorStats={monitorStats}
grid={grid}
monitors={monitors}
chartData={chartData}
empty={empty}
metric="loss"
title={title}
description={t`Packet loss (%)`}
domain={[0, 100]}
tickFormatter={(value) => `${toFixedFloat(value, value >= 10 ? 0 : 1)}%`}
contentFormatter={({ value }) => {
if (typeof value !== "number") {
return value
}
return `${decimalString(value, 2)}%`
}}
/>
)
}
@@ -1,6 +1,7 @@
import { lazy } from "react"
import { useIntersectionObserver } from "@/lib/use-intersection-observer"
import { cn } from "@/lib/utils"
import { useNetworkMonitors } from "@/lib/use-network-monitors"
const ContainersTable = lazy(() => import("../../containers-table/containers-table"))
@@ -45,3 +46,19 @@ export function LazySystemdTable({ systemId }: { systemId: string }) {
</div>
)
}
const NetworkMonitorsTable = lazy(() => import("../../network-monitors-table/network-monitors-table"))
export function LazyNetworkMonitorsTable({ systemId }: { systemId: string }) {
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
return (
<div ref={ref} className={cn(isIntersecting && "contents")}>
{isIntersecting && <SystemNetworkMonitorsTable systemId={systemId} />}
</div>
)
}
function SystemNetworkMonitorsTable({ systemId }: { systemId: string }) {
const { monitors, isLoading } = useNetworkMonitors({ systemId })
return <NetworkMonitorsTable systemId={systemId} monitors={monitors} isLoading={isLoading} />
}
@@ -43,7 +43,6 @@ import {
toFixedFloat,
formatTemperature,
cn,
getVisualStringWidth,
secondsToString,
hourWithSeconds,
formatShortDate,
@@ -117,9 +116,9 @@ function formatDataUnits(units: number): string {
const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"
export const createColumns = (
longestName: number,
longestModel: number,
longestDevice: number
longestName: string,
longestModel: string,
longestDevice: string
): ColumnDef<SmartDeviceRecord>[] => [
{
id: "system",
@@ -134,8 +133,11 @@ export const createColumns = (
cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById)
return (
<div className="ms-1.5 max-w-40 block truncate" style={{ width: `${longestName / 1.05}ch` }}>
{allSystems[getValue() as string]?.name ?? ""}
<div className="ms-1.5 relative w-fit max-w-44">
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div>
)
},
@@ -145,12 +147,11 @@ export const createColumns = (
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,
cell: ({ getValue }) => (
<div
className="font-medium max-w-40 truncate ms-1"
title={getValue() as string}
style={{ width: `${longestDevice / 1.05}ch` }}
>
{getValue() as string}
<div className="font-medium ms-1 relative w-fit max-w-44" title={getValue() as string}>
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestDevice}
</span>
<span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
),
},
@@ -161,12 +162,11 @@ export const createColumns = (
<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />
),
cell: ({ getValue }) => (
<div
className="max-w-48 truncate ms-1"
title={getValue() as string}
style={{ width: `${longestModel / 1.05}ch` }}
>
{getValue() as string}
<div className="ms-1 relative w-fit max-w-44" title={getValue() as string}>
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestModel}
</span>
<span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
),
},
@@ -320,7 +320,7 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
// Calculate the right width for the columns based on the longest strings among the displayed devices
const { longestName, longestModel, longestDevice } = useMemo(() => {
const result = { longestName: 0, longestModel: 0, longestDevice: 0 }
const result = { longestName: "", longestModel: "", longestDevice: "" }
if (!smartDevices || Object.keys(allSystems).length === 0) {
return result
}
@@ -329,10 +329,16 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
if (!systemId && !seenSystems.has(device.system)) {
seenSystems.add(device.system)
const name = allSystems[device.system]?.name ?? ""
result.longestName = Math.max(result.longestName, getVisualStringWidth(name))
if (name.length > result.longestName.length) {
result.longestName = name
}
}
if ((device.model ?? "").length > result.longestModel.length) {
result.longestModel = device.model ?? ""
}
if ((device.name ?? "").length > result.longestDevice.length) {
result.longestDevice = device.name ?? ""
}
result.longestModel = Math.max(result.longestModel, getVisualStringWidth(device.model ?? ""))
result.longestDevice = Math.max(result.longestDevice, getVisualStringWidth(device.name ?? ""))
}
return result
}, [smartDevices, systemId, allSystems])
@@ -26,7 +26,7 @@ import type {
SystemStatsRecord,
} from "@/types"
import { $router, navigate } from "../../router"
import { appendData, cache, getStats, getTimeData, makeContainerData, makeContainerPoint } from "./chart-data"
import { appendData, cache, getStats, makeContainerData, makeContainerPoint } from "./chart-data"
export type SystemData = ReturnType<typeof useSystemData>
@@ -185,16 +185,11 @@ export function useSystemData(id: string) {
const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])
const chartData: ChartData = useMemo(() => {
const lastCreated = Math.max(
(systemStats.at(-1)?.created as number) ?? 0,
(containerData.at(-1)?.created as number) ?? 0
)
return {
systemStats,
containerData,
chartTime,
orientation: direction === "rtl" ? "right" : "left",
...getTimeData(chartTime, lastCreated),
agentVersion,
}
}, [systemStats, containerData, direction])
@@ -234,8 +229,8 @@ export function useSystemData(id: string) {
}
Promise.allSettled([
getStats<SystemStatsRecord>("system_stats", systemId, chartTime),
getStats<ContainerStatsRecord>("container_stats", systemId, chartTime),
getStats<SystemStatsRecord>("system_stats", systemId, chartTime, cachedSystemStats),
getStats<ContainerStatsRecord>("container_stats", systemId, chartTime, cachedContainerData),
]).then(([systemStats, containerStats]) => {
// Ignore responses for a previous system or chart time
if (requestId !== statsRequestId.current) {
@@ -327,7 +322,7 @@ export function useSystemData(id: string) {
// derived values
const isLongerChart = !["1m", "1h"].includes(chartTime)
const showMax = maxValues && isLongerChart
const dataEmpty = !chartLoading && chartData.systemStats.length === 0
const dataEmpty = !chartLoading && chartData.systemStats?.length === 0
const lastGpus = systemStats.at(-1)?.stats?.g
const isPodman = details?.podman ?? system.info?.p ?? false
@@ -26,7 +26,7 @@ import { memo, useMemo, useRef, useState } from "react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
import { isReadOnlyUser, pb } from "@/lib/api"
import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"
import { $longestSystemNameLen, $userSettings } from "@/lib/stores"
import { $longestSystemName, $userSettings } from "@/lib/stores"
import {
cn,
copyToClipboard,
@@ -135,7 +135,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
Icon: ServerIcon,
cell: (info) => {
const { name, id } = info.row.original
const longestName = useStore($longestSystemNameLen)
const longestName = useStore($longestSystemName)
const linkUrl = getPagePath($router, "system", { id })
return (
@@ -145,8 +145,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
<Link
href={linkUrl}
tabIndex={-1}
className="truncate z-10 relative"
style={{ width: `${longestName / 1.05}ch` }}
className="relative w-fit max-w-48 z-10"
onMouseEnter={(e) => {
// set title on hover if text is truncated to show full name
const a = e.currentTarget
@@ -157,7 +156,10 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
}
}}
>
{name}
<span className="invisible block" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{name}</span>
</Link>
</span>
<Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>
+66 -24
View File
@@ -1,11 +1,11 @@
import type { JSX } from "react"
import { useLingui } from "@lingui/react/macro"
import * as React from "react"
import * as RechartsPrimitive from "recharts"
import { chartTimeData, cn } from "@/lib/utils"
import type { ChartData } from "@/types"
import type { ChartTimes } from "@/types"
import { Separator } from "./separator"
import { AxisDomain } from "recharts/types/util/types"
import type { AxisDomain } from "recharts/types/util/types"
import { timeTicks } from "d3-time"
// Format: { THEME_NAME: CSS_SELECTOR }
const THEMES = { light: "", dark: ".dark" } as const
@@ -101,7 +101,7 @@ const ChartTooltipContent = React.forwardRef<
labelKey?: string
unit?: string
filter?: string
contentFormatter?: (item: any, key: string) => React.ReactNode | string
contentFormatter?: (item: unknown, key: string) => React.ReactNode | string
truncate?: boolean
showTotal?: boolean
totalLabel?: React.ReactNode
@@ -175,7 +175,13 @@ const ChartTooltipContent = React.forwardRef<
}
const totalKey = "__total__"
const totalItem: any = {
const totalItem: {
value: number
name: string
dataKey: string
color: string | undefined
payload?: unknown
} = {
value: totalValue,
name: totalName,
dataKey: totalKey,
@@ -222,6 +228,11 @@ const ChartTooltipContent = React.forwardRef<
return null
}
payload = payload.filter((item) => item.value != null)
if (!payload.length) {
return null
}
// const nestLabel = payload.length === 1 && indicator !== 'dot'
const nestLabel = false
@@ -400,26 +411,57 @@ function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key:
return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]
}
let cachedAxis: JSX.Element
const xAxis = ({ domain, ticks, chartTime }: ChartData) => {
if (cachedAxis && domain[0] === cachedAxis.props.domain[0]) {
return cachedAxis
interface XAxisData {
el: React.ReactElement
domain: [number, number]
}
const xAxisCache = new Map<ChartTimes, XAxisData>()
function createXAxisData(chartTime: ChartTimes): XAxisData {
// console.log("Creating XAxis for", chartTime, new Date())
const axisEndTime = Date.now() + 500
const axisEndDate = new Date(axisEndTime)
const startTime = chartTimeData[chartTime].getOffset(axisEndDate)
const ticks = timeTicks(startTime, axisEndDate, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
const domain: [number, number] = [startTime.getTime(), axisEndTime]
return {
domain,
el: (
<RechartsPrimitive.XAxis
dataKey="created"
domain={domain}
ticks={ticks}
allowDataOverflow
type="number"
scale="time"
minTickGap={12}
tickMargin={8}
axisLine={false}
tickFormatter={chartTimeData[chartTime].format}
/>
),
}
cachedAxis = (
<RechartsPrimitive.XAxis
dataKey="created"
domain={domain}
ticks={ticks}
allowDataOverflow
type="number"
scale="time"
minTickGap={12}
tickMargin={8}
axisLine={false}
tickFormatter={chartTimeData[chartTime].format}
/>
)
return cachedAxis
}
function xAxis(chartTime: ChartTimes, lastCreated: number) {
if (!lastCreated) {
return null
}
const cachedAxis = xAxisCache.get(chartTime)
const expectedInterval = chartTimeData[chartTime].expectedInterval
const conservativeEndTime = Date.now() - expectedInterval / 2
const axisEndTime = Math.max(lastCreated, conservativeEndTime)
if (cachedAxis && axisEndTime < cachedAxis.domain[1]) {
return cachedAxis.el
}
const axisData = createXAxisData(chartTime)
xAxisCache.set(chartTime, axisData)
return axisData.el
}
export {
@@ -90,8 +90,10 @@ DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName
const DropdownMenuCheckboxItem = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>
>(({ className, children, checked, ...props }, ref) => (
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> & {
indicatorClassName?: string
}
>(({ className, children, checked, indicatorClassName, ...props }, ref) => (
<DropdownMenuPrimitive.CheckboxItem
ref={ref}
className={cn(
@@ -101,7 +103,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<
checked={checked}
{...props}
>
<span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
<span className={cn("absolute left-2 flex h-3.5 w-3.5 items-center justify-center", indicatorClassName)}>
<DropdownMenuPrimitive.ItemIndicator>
<Check className="h-4 w-4" />
</DropdownMenuPrimitive.ItemIndicator>
+1 -1
View File
@@ -41,7 +41,7 @@ const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTML
<tr
ref={ref}
className={cn(
"border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted!",
"border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted/40!",
className
)}
{...props}
+204 -191
View File
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro"
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase"
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
import { $alerts } from "@/lib/stores"
@@ -9,210 +9,223 @@ import { ThermometerIcon, BatteryMediumIcon, HourglassIcon } from "@/components/
/** Alert info for each alert type */
export const alertInfo: Record<string, AlertInfo> = {
Status: {
name: () => t`Status`,
unit: "",
icon: ServerIcon,
desc: () => t`Triggers when status switches between up and down`,
/** "for x minutes" is appended to desc when only one value */
singleDesc: () => `${t`System`} ${t`Down`}`,
},
CPU: {
name: () => t`CPU Usage`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU usage exceeds a threshold`,
},
CPUIOWait: {
name: () => t`CPU I/O Wait`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
},
CPUSteal: {
name: () => t`CPU Steal Time`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU steal time exceeds a threshold`,
},
Memory: {
name: () => t`Memory Usage`,
unit: "%",
icon: MemoryStickIcon,
desc: () => t`Triggers when memory usage exceeds a threshold`,
},
Disk: {
name: () => t`Disk Usage`,
unit: "%",
icon: HardDriveIcon,
desc: () => t`Triggers when usage of any disk exceeds a threshold`,
},
Bandwidth: {
name: () => t`Bandwidth`,
unit: " MB/s",
icon: EthernetIcon,
desc: () => t`Triggers when combined up/down exceeds a threshold`,
max: 250,
},
GPU: {
name: () => t`GPU Usage`,
unit: "%",
icon: GpuIcon,
desc: () => t`Triggers when GPU usage exceeds a threshold`,
},
Temperature: {
name: () => t`Temperature`,
unit: "°C",
icon: ThermometerIcon,
desc: () => t`Triggers when any sensor exceeds a threshold`,
},
LoadAvg1: {
name: () => t`Load Average 1m`,
unit: "",
icon: HourglassIcon,
max: 100,
min: 0.1,
start: 10,
step: 0.1,
desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
},
LoadAvg5: {
name: () => t`Load Average 5m`,
unit: "",
icon: HourglassIcon,
max: 100,
min: 0.1,
start: 10,
step: 0.1,
desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
},
LoadAvg15: {
name: () => t`Load Average 15m`,
unit: "",
icon: HourglassIcon,
min: 0.1,
max: 100,
start: 10,
step: 0.1,
desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
},
Battery: {
name: () => t`Battery`,
unit: "%",
icon: BatteryMediumIcon,
desc: () => t`Triggers when battery charge drops below a threshold`,
start: 20,
invert: true,
},
ContainerHealth: {
name: () => t`Container Health`,
unit: "",
icon: ContainerIcon,
desc: () => t`Triggers when a container's health check reports unhealthy`,
note: () =>
t`Notifications may include recent container log excerpts.`,
triggeredDesc: () => t`One or more containers are unhealthy`,
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
Status: {
name: () => t`Status`,
unit: "",
icon: ServerIcon,
desc: () => t`Triggers when status switches between up and down`,
/** "for x minutes" is appended to desc when only one value */
singleDesc: () => `${t`System`} ${t`Down`}`,
},
CPU: {
name: () => t`CPU Usage`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU usage exceeds a threshold`,
},
CPUIOWait: {
name: () => t`CPU I/O Wait`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
},
CPUSteal: {
name: () => t`CPU Steal Time`,
unit: "%",
icon: CpuIcon,
desc: () => t`Triggers when CPU steal time exceeds a threshold`,
},
Memory: {
name: () => t`Memory Usage`,
unit: "%",
icon: MemoryStickIcon,
desc: () => t`Triggers when memory usage exceeds a threshold`,
},
Disk: {
name: () => t`Disk Usage`,
unit: "%",
icon: HardDriveIcon,
desc: () => t`Triggers when usage of any disk exceeds a threshold`,
},
Bandwidth: {
name: () => t`Bandwidth`,
unit: " MB/s",
icon: EthernetIcon,
desc: () => t`Triggers when combined up/down exceeds a threshold`,
max: 250,
},
NetworkMonitorLoss: {
name: () => t`Network Monitor Loss`,
unit: "%",
icon: NetworkIcon,
desc: () => t`Triggers when one hour loss exceeds a threshold`,
// note: () => t`Uses available history after three probes.`,
noDuration: true,
min: 0,
max: 99.9,
step: 0.1,
start: 5,
},
GPU: {
name: () => t`GPU Usage`,
unit: "%",
icon: GpuIcon,
desc: () => t`Triggers when GPU usage exceeds a threshold`,
},
Temperature: {
name: () => t`Temperature`,
unit: "°C",
icon: ThermometerIcon,
desc: () => t`Triggers when any sensor exceeds a threshold`,
},
LoadAvg1: {
name: () => t`Load Average 1m`,
unit: "",
icon: HourglassIcon,
max: 100,
min: 0.1,
start: 10,
step: 0.1,
desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
},
LoadAvg5: {
name: () => t`Load Average 5m`,
unit: "",
icon: HourglassIcon,
max: 100,
min: 0.1,
start: 10,
step: 0.1,
desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
},
LoadAvg15: {
name: () => t`Load Average 15m`,
unit: "",
icon: HourglassIcon,
min: 0.1,
max: 100,
start: 10,
step: 0.1,
desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
},
Battery: {
name: () => t`Battery`,
unit: "%",
icon: BatteryMediumIcon,
desc: () => t`Triggers when battery charge drops below a threshold`,
start: 20,
invert: true,
},
ContainerHealth: {
name: () => t`Container Health`,
unit: "",
icon: ContainerIcon,
desc: () => t`Triggers when a container's health check reports unhealthy`,
note: () =>
t`Notifications may include recent container log excerpts.`,
triggeredDesc: () => t`One or more containers are unhealthy`,
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
},
SystemdFailed: {
name: () => t`Failed Services`,
unit: "",
icon: ServerCrashIcon,
desc: () => t`Triggers when any systemd service enters the failed state`,
triggeredDesc: () => t`One or more services are in a failed state`,
/** Fires on first observation - the agent only polls systemd every 10 minutes */
noDuration: true,
},
},
SystemdFailed: {
name: () => t`Failed Services`,
unit: "",
icon: ServerCrashIcon,
desc: () => t`Triggers when any systemd service enters the failed state`,
triggeredDesc: () => t`One or more services are in a failed state`,
/** Fires on first observation - the agent only polls systemd every 10 minutes */
noDuration: true,
noThreshold: true,
},
} as const
/** Helper to manage user alerts */
export const alertManager = (() => {
const collection = pb.collection<AlertRecord>("alerts")
let unsub: () => void
const collection = pb.collection<AlertRecord>("alerts")
let unsub: () => void
/** Fields to fetch from alerts collection */
const fields = "id,name,system,value,min,triggered"
/** Fields to fetch from alerts collection */
const fields = "id,name,system,value,min,triggered"
/** Fetch alerts from collection */
async function fetchAlerts(): Promise<AlertRecord[]> {
return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
}
/** Fetch alerts from collection */
async function fetchAlerts(): Promise<AlertRecord[]> {
return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
}
/** Format alerts into a map of system id to alert name to alert record */
function add(alerts: AlertRecord[]) {
for (const alert of alerts) {
const systemId = alert.system
const systemAlerts = $alerts.get()[systemId] ?? new Map()
const newAlerts = new Map(systemAlerts)
newAlerts.set(alert.name, alert)
$alerts.setKey(systemId, newAlerts)
}
}
/** Format alerts into a map of system id to alert name to alert record */
function add(alerts: AlertRecord[]) {
for (const alert of alerts) {
const systemId = alert.system
const systemAlerts = $alerts.get()[systemId] ?? new Map()
const newAlerts = new Map(systemAlerts)
newAlerts.set(alert.name, alert)
$alerts.setKey(systemId, newAlerts)
}
}
function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
for (const alert of alerts) {
const systemId = alert.system
const systemAlerts = $alerts.get()[systemId]
const newAlerts = new Map(systemAlerts)
newAlerts.delete(alert.name)
$alerts.setKey(systemId, newAlerts)
}
}
function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
for (const alert of alerts) {
const systemId = alert.system
const systemAlerts = $alerts.get()[systemId]
const newAlerts = new Map(systemAlerts)
newAlerts.delete(alert.name)
$alerts.setKey(systemId, newAlerts)
}
}
const actionFns = {
create: add,
update: add,
delete: remove,
}
const actionFns = {
create: add,
update: add,
delete: remove,
}
// batch alert updates to prevent unnecessary re-renders when adding many alerts at once
const batchUpdate = (() => {
const batch = new Map<string, RecordSubscription<AlertRecord>>()
let timeout: ReturnType<typeof setTimeout>
// batch alert updates to prevent unnecessary re-renders when adding many alerts at once
const batchUpdate = (() => {
const batch = new Map<string, RecordSubscription<AlertRecord>>()
let timeout: ReturnType<typeof setTimeout>
return (data: RecordSubscription<AlertRecord>) => {
const { record } = data
batch.set(`${record.system}${record.name}`, data)
clearTimeout(timeout)
timeout = setTimeout(() => {
const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
for (const { action, record } of batch.values()) {
groups[action]?.push(record)
}
for (const key in groups) {
if (groups[key].length) {
actionFns[key as keyof typeof actionFns]?.(groups[key])
}
}
batch.clear()
}, 50)
}
})()
return (data: RecordSubscription<AlertRecord>) => {
const { record } = data
batch.set(`${record.system}${record.name}`, data)
clearTimeout(timeout)
timeout = setTimeout(() => {
const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
for (const { action, record } of batch.values()) {
groups[action]?.push(record)
}
for (const key in groups) {
if (groups[key].length) {
actionFns[key as keyof typeof actionFns]?.(groups[key])
}
}
batch.clear()
}, 50)
}
})()
async function subscribe() {
unsub = await collection.subscribe("*", batchUpdate, { fields })
}
async function subscribe() {
unsub = await collection.subscribe("*", batchUpdate, { fields })
}
function unsubscribe() {
unsub?.()
}
function unsubscribe() {
unsub?.()
}
async function refresh() {
const records = await fetchAlerts()
add(records)
}
async function refresh() {
const records = await fetchAlerts()
add(records)
}
return {
/** Add alerts to store */
add,
/** Remove alerts from store */
remove,
/** Subscribe to alerts */
subscribe,
/** Unsubscribe from alerts */
unsubscribe,
/** Refresh alerts with latest data from hub */
refresh,
}
return {
/** Add alerts to store */
add,
/** Remove alerts from store */
remove,
/** Subscribe to alerts */
subscribe,
/** Unsubscribe from alerts */
unsubscribe,
/** Refresh alerts with latest data from hub */
refresh,
}
})()
+4 -1
View File
@@ -106,8 +106,11 @@ export async function updateUserSettings() {
}
}
export function getPbTimestamp(timeString: ChartTimes, d?: Date) {
export function getPbTimestamp(timeString: ChartTimes, d?: Date, createdIsNumber?: boolean) {
d ||= chartTimeData[timeString].getOffset(new Date())
if (createdIsNumber) {
return d.getTime()
}
const year = d.getUTCFullYear()
const month = String(d.getUTCMonth() + 1).padStart(2, "0")
const day = String(d.getUTCDate()).padStart(2, "0")
@@ -0,0 +1,17 @@
import type { MonitorStats, NetworkMonitorRecord, RawMonitorStatsRecord } from "@/types"
/** Derive chart metrics from the counts and response sum stored at every retention tier. */
export function getMonitorStats(record: RawMonitorStatsRecord): MonitorStats {
return {
res_avg: record.success_count > 0 ? record.res_sum / record.success_count : 0,
res_min: record.res_min,
res_max: record.res_max,
loss: record.total_count > 0 ? ((record.total_count - record.success_count) / record.total_count) * 100 : 0,
}
}
export function getMonitorTarget(monitor: Pick<NetworkMonitorRecord, "target" | "protocol" | "port">) {
if (monitor.protocol !== "tcp") return monitor.target
const host = monitor.target.includes(":") && !monitor.target.startsWith("[") ? `[${monitor.target}]` : monitor.target
return `${host}:${monitor.port}`
}
+5 -4
View File
@@ -64,6 +64,9 @@ export const $containerFilter = atom("")
/** Temperature chart filter */
export const $temperatureFilter = atom("")
/** Filter for network monitor charts (compare page and per-system monitor charts) */
export const $monitorFilter = atom("")
/** Fan-speed chart filter */
export const $fanFilter = atom("")
@@ -73,7 +76,5 @@ export const $copyContent = atom("")
/** Direction for localization */
export const $direction = atom<"ltr" | "rtl">("ltr")
/** Longest system name length. Used to set table column width. I know this
* is stupid but the table is virtualized and I know this will work.
*/
export const $longestSystemNameLen = atom(8)
/** Longest system name string. Used to reserve width in virtualized tables. */
export const $longestSystemName = atom("")
+19 -14
View File
@@ -5,20 +5,17 @@ import {
$allSystemsById,
$allSystemsByName,
$downSystems,
$longestSystemNameLen,
$longestSystemName,
$pausedSystems,
$upSystems,
} from "@/lib/stores"
import { getVisualStringWidth, updateFavicon } from "@/lib/utils"
import { isVisuallyLonger, updateFavicon } from "@/lib/utils"
import type { SystemRecord } from "@/types"
import { SystemStatus } from "./enums"
const COLLECTION = pb.collection<SystemRecord>("systems")
const FIELDS_DEFAULT = "id,name,host,port,info,status"
/** Maximum system name length for display purposes */
const MAX_SYSTEM_NAME_LENGTH = 22
let initialized = false
// biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks
let unsub: (() => void) | undefined | void
@@ -44,7 +41,7 @@ export function init() {
}
if (!newSystem) {
onSystemsChanged(newSystems, undefined)
onSystemsChanged(newSystems, newSystem, oldSystem)
return
}
@@ -68,20 +65,28 @@ export function init() {
}
// run things that need to be done when systems change
onSystemsChanged(newSystems, newSystem)
onSystemsChanged(newSystems, newSystem, oldSystem)
})
}
/** Update the longest system name length and favicon based on system status */
function onSystemsChanged(_: Record<string, SystemRecord>, changedSystem: SystemRecord | undefined) {
/** Update the longest system name string and favicon based on system status */
function onSystemsChanged(systems: Record<string, SystemRecord>, newSystem?: SystemRecord, oldSystem?: SystemRecord) {
const downSystemsStore = $downSystems.get()
const downSystems = Object.values(downSystemsStore)
// Update longest system name length
const longestName = $longestSystemNameLen.get()
const nameLen = Math.min(MAX_SYSTEM_NAME_LENGTH, getVisualStringWidth(changedSystem?.name || ""))
if (nameLen > longestName) {
$longestSystemNameLen.set(nameLen)
// if the old system's old name was the longest, we need to find the new longest name
// otherwise, if the changed system's new name is longer than the current longest, update it
const longestName = $longestSystemName.get()
if (oldSystem?.name === longestName && oldSystem.name !== newSystem?.name) {
let newLongest = ""
for (const id in systems) {
if (isVisuallyLonger(systems[id].name, newLongest)) {
newLongest = systems[id].name
}
}
$longestSystemName.set(newLongest)
} else if (newSystem && newSystem.name !== longestName && isVisuallyLonger(newSystem.name, longestName)) {
$longestSystemName.set(newSystem.name)
}
updateFavicon(downSystems.length)
@@ -0,0 +1,351 @@
import { chartTimeData } from "@/lib/utils"
import { getMonitorStats } from "@/lib/network-monitor-utils"
import type {
ChartTimes,
MonitorStats,
NetworkMonitorRecord,
NetworkMonitorStatsRecord,
RawMonitorStatsRecord,
} from "@/types"
import { useEffect, useRef, useState } from "react"
import { appendData } from "@/components/routes/system/chart-data"
import { pb, getPbTimestamp } from "@/lib/api"
import { toast } from "@/components/ui/use-toast"
import type { RecordListOptions, RecordSubscription } from "pocketbase"
const cache = new Map<string, NetworkMonitorStatsRecord[]>()
function getCacheValue(monitorId: string, chartTime: ChartTimes | "rt") {
return cache.get(`${monitorId}:${chartTime}`) || []
}
function appendCacheValue(
monitorId: string,
chartTime: ChartTimes | "rt",
newStats: NetworkMonitorStatsRecord[],
maxPoints = 100
) {
const cache_key = `${monitorId}:${chartTime}`
const existingStats = getCacheValue(monitorId, chartTime)
if (existingStats) {
const { expectedInterval } = chartTimeData[chartTime]
const updatedStats = appendData(existingStats, newStats, expectedInterval, maxPoints)
cache.set(cache_key, updatedStats)
return updatedStats
} else {
cache.set(cache_key, newStats)
return newStats
}
}
/** Merge an array of per-monitor raw records into the map-keyed format expected by chart components. */
export function mergeMonitorStats(rawRecords: RawMonitorStatsRecord[]): NetworkMonitorStatsRecord[] {
const byTimestamp = new Map<number, Record<string, MonitorStats>>()
for (const rec of rawRecords) {
let statsMap = byTimestamp.get(rec.created)
if (!statsMap) {
statsMap = {}
byTimestamp.set(rec.created, statsMap)
}
statsMap[rec.monitor] = getMonitorStats(rec)
}
return Array.from(byTimestamp.entries())
.sort(([a], [b]) => a - b)
.map(([created, stats]) => ({ created, stats }))
}
/** Fetch stats for one monitor and time range, returning merged chart records. */
async function fetchMonitorStats(
monitorId: string,
chartTime: ChartTimes,
cached?: NetworkMonitorStatsRecord[]
): Promise<NetworkMonitorStatsRecord[]> {
const lastCached = cached?.at(-1)?.created as number | undefined
const rawRecords = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").getFullList({
filter: pb.filter("monitor={:id} && created>{:created} && type={:type}", {
id: monitorId,
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, true),
type: chartTimeData[chartTime].type,
}),
fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created",
sort: "created",
})
return mergeMonitorStats(rawRecords)
}
const NETWORK_MONITOR_FIELDS =
"id,system,target,protocol,port,interval,res,resMin1h,resMax1h,resAvg1h,loss1h,enabled,updated"
interface UseNetworkMonitorsProps {
systemId?: string
}
export function useNetworkMonitors(props: UseNetworkMonitorsProps) {
const { systemId } = props
const [monitors, setMonitors] = useState<NetworkMonitorRecord[]>([])
const [isLoading, setIsLoading] = useState(true)
const pendingMonitorEvents = useRef(new Map<string, RecordSubscription<NetworkMonitorRecord>>())
const monitorBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)
// initial load
useEffect(() => {
let cancelled = false
setIsLoading(true)
setMonitors([])
fetchMonitors(systemId).then((monitors) => {
if (cancelled) return
setMonitors(monitors)
setIsLoading(false)
})
return () => {
cancelled = true
}
}, [systemId])
// subscribe to updates
useEffect(() => {
let unsubscribe: (() => void) | undefined
function flushPendingMonitorEvents() {
monitorBatchTimeout.current = null
if (!pendingMonitorEvents.current.size) {
return
}
const events = pendingMonitorEvents.current
pendingMonitorEvents.current = new Map()
setMonitors((currentMonitors) => {
return applyMonitorEvents(currentMonitors ?? [], events.values(), systemId)
})
}
const pbOptions: RecordListOptions = { fields: NETWORK_MONITOR_FIELDS }
if (systemId) {
pbOptions.filter = pb.filter("system = {:system}", { system: systemId })
}
;(async () => {
try {
unsubscribe = await pb.collection<NetworkMonitorRecord>("network_monitors").subscribe(
"*",
(event) => {
pendingMonitorEvents.current.set(event.record.id, event)
if (!monitorBatchTimeout.current) {
monitorBatchTimeout.current = setTimeout(flushPendingMonitorEvents, 50)
}
},
pbOptions
)
} catch (error) {
console.error("Failed to subscribe to monitors", error)
}
})()
return () => {
if (monitorBatchTimeout.current !== null) {
clearTimeout(monitorBatchTimeout.current)
monitorBatchTimeout.current = null
}
pendingMonitorEvents.current.clear()
unsubscribe?.()
}
}, [systemId])
return { monitors, isLoading }
}
interface UseNetworkMonitorStatsProps {
systemId: string
monitorId: string
chartTime: ChartTimes
enabled?: boolean
}
export function useNetworkMonitorStats(props: UseNetworkMonitorStatsProps) {
const { systemId, monitorId, chartTime, enabled = true } = props
const [monitorStats, setMonitorStats] = useState<NetworkMonitorStatsRecord[]>([])
// pending raw events to be merged (keyed by monitor+created)
const pendingRaw = useRef(new Map<string, RawMonitorStatsRecord>())
const mergeBatchTimeout = useRef<ReturnType<typeof setTimeout> | null>(null)
useEffect(() => {
setMonitorStats(getCacheValue(monitorId, chartTime === "1m" ? "rt" : chartTime))
}, [monitorId, chartTime])
// Fetch only the selected monitor's missing history.
useEffect(() => {
if (!enabled || chartTime === "1m") {
return
}
let cancelled = false
const { expectedInterval } = chartTimeData[chartTime]
const cachedMonitorStats = getCacheValue(monitorId, chartTime)
if (cachedMonitorStats.length) {
setMonitorStats(cachedMonitorStats)
const lastCreated = cachedMonitorStats.at(-1)?.created
if (lastCreated && Date.now() - lastCreated < expectedInterval * 0.9) {
return
}
}
fetchMonitorStats(monitorId, chartTime, cachedMonitorStats)
.then((newMonitorStats) => {
if (cancelled) return
setMonitorStats(appendCacheValue(monitorId, chartTime, newMonitorStats))
})
.catch((error) => {
if (!cancelled) console.error("Failed to fetch monitor stats:", error)
})
return () => {
cancelled = true
}
}, [monitorId, chartTime, enabled])
// subscribe to new per-monitor stats records; batch them into merged chart records
useEffect(() => {
if (!enabled || chartTime === "1m") {
return
}
let cancelled = false
let unsubscribe: (() => void) | undefined
const pbOptions = {
fields: "monitor,res_min,res_max,total_count,success_count,res_sum,created,type",
filter: pb.filter("monitor={:monitor} && type={:type}", {
monitor: monitorId,
type: chartTimeData[chartTime].type,
}),
}
function flushPending() {
mergeBatchTimeout.current = null
const pending = pendingRaw.current
pendingRaw.current = new Map()
const merged = mergeMonitorStats(Array.from(pending.values()))
if (merged.length > 0) {
const newStats = appendCacheValue(monitorId, chartTime, merged)
setMonitorStats(newStats)
}
}
;(async () => {
try {
unsubscribe = await pb.collection<RawMonitorStatsRecord>("network_monitor_stats").subscribe(
"*",
(event) => {
if (cancelled || event.action !== "create") {
return
}
const rec = event.record
pendingRaw.current.set(`${rec.monitor}:${rec.created}`, rec)
if (!mergeBatchTimeout.current) {
mergeBatchTimeout.current = setTimeout(flushPending, 200)
}
},
pbOptions
)
if (cancelled) unsubscribe()
} catch (error) {
console.error("Failed to subscribe to monitor stats:", error)
}
})()
return () => {
cancelled = true
if (mergeBatchTimeout.current) {
clearTimeout(mergeBatchTimeout.current)
mergeBatchTimeout.current = null
}
pendingRaw.current.clear()
unsubscribe?.()
}
}, [monitorId, chartTime, enabled])
// subscribe to realtime metrics if chart time is 1m
useEffect(() => {
if (!enabled || chartTime !== "1m") {
return
}
let cancelled = false
let unsubscribe: (() => void) | undefined
pb.realtime
.subscribe(
`rt_metrics`,
(data: { Monitors: NetworkMonitorStatsRecord["stats"] }) => {
const monitorStats = data.Monitors?.[monitorId]
if (cancelled || !monitorStats) return
const stats = { created: Date.now(), stats: { [monitorId]: monitorStats } }
const newStats = appendCacheValue(monitorId, "rt", [stats], 120)
setMonitorStats(newStats)
},
{ query: { system: systemId } }
)
.then((us) => {
unsubscribe = us
if (cancelled) unsubscribe()
})
return () => {
cancelled = true
unsubscribe?.()
}
}, [chartTime, systemId, monitorId, enabled])
return monitorStats
}
async function fetchMonitors(system?: string) {
try {
return await pb.collection<NetworkMonitorRecord>("network_monitors").getFullList({
fields: NETWORK_MONITOR_FIELDS,
filter: system ? pb.filter("system={:system}", { system }) : undefined,
})
} catch (error) {
toast({
title: "Error",
description: (error as Error)?.message,
variant: "destructive",
})
return []
}
}
function applyMonitorEvents(
monitors: NetworkMonitorRecord[],
events: Iterable<RecordSubscription<NetworkMonitorRecord>>,
systemId?: string
) {
const monitorById = new Map(monitors.map((monitor) => [monitor.id, monitor]))
const createdMonitors: NetworkMonitorRecord[] = []
for (const { action, record } of events) {
const matchesSystemScope = !systemId || record.system === systemId
if (action === "delete" || !matchesSystemScope) {
monitorById.delete(record.id)
continue
}
if (!monitorById.has(record.id)) {
createdMonitors.push(record)
}
monitorById.set(record.id, record)
}
const nextMonitors: NetworkMonitorRecord[] = []
for (let index = createdMonitors.length - 1; index >= 0; index -= 1) {
nextMonitors.push(createdMonitors[index])
}
for (const monitor of monitors) {
const nextMonitor = monitorById.get(monitor.id)
if (!nextMonitor) {
continue
}
nextMonitors.push(nextMonitor)
monitorById.delete(monitor.id)
}
return nextMonitors
}
+75 -11
View File
@@ -72,7 +72,7 @@ export const formatShortDate = (timestamp: string) => {
return shortDateFormatter.format(new Date(timestamp))
}
export const hourWithSeconds = (timestamp: string) => {
export const hourWithSeconds = (timestamp: string | number) => {
return hourWithSecondsFormatter.format(new Date(timestamp))
}
@@ -111,17 +111,18 @@ export const updateFavicon = (() => {
</linearGradient>
</defs>
<path fill="url(#gradient)" d="M35 70H0V0h35q4.4 0 8.2 1.7a21.4 21.4 0 0 1 6.6 4.5q2.9 2.8 4.5 6.6Q56 16.7 56 21a15.4 15.4 0 0 1-.3 3.2 17.6 17.6 0 0 1-.2.8 19.4 19.4 0 0 1-1.5 4 17 17 0 0 1-2.4 3.4 13.5 13.5 0 0 1-2.6 2.3 12.5 12.5 0 0 1-.4.3q1.7 1 3 2.5Q53 39.1 54 41a18.3 18.3 0 0 1 1.5 4 17.4 17.4 0 0 1 .5 3 15.3 15.3 0 0 1 0 1q0 4.4-1.7 8.2a21.4 21.4 0 0 1-4.5 6.6q-2.8 2.9-6.6 4.6Q39.4 70 35 70ZM14 14v14h21a7 7 0 0 0 2.3-.3 6.6 6.6 0 0 0 .4-.2Q39 27 40 26a6.9 6.9 0 0 0 1.5-2.2q.5-1.3.5-2.8a7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 17 40 16a7 7 0 0 0-2.3-1.4 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Zm0 28v14h21a7 7 0 0 0 2.3-.4 6.6 6.6 0 0 0 .4-.1Q39 54.9 40 54a7 7 0 0 0 1.5-2.2 6.9 6.9 0 0 0 .5-2.6 7.9 7.9 0 0 0 0-.2 7 7 0 0 0-.4-2.3 6.6 6.6 0 0 0-.1-.4Q40.9 45 40 44a7 7 0 0 0-2.3-1.5 6.9 6.9 0 0 0-2.5-.6 7.9 7.9 0 0 0-.2 0H14Z"/>
${downCount > 0 &&
`
${
downCount > 0 &&
`
<circle cx="40" cy="50" r="22" fill="#f00"/>
<text x="40" y="60" font-size="34" text-anchor="middle" fill="#fff" font-family="Arial" font-weight="bold">${downCount}</text>
`
}
}
</svg>
`
const blob = new Blob([svg], { type: "image/svg+xml" })
const url = URL.createObjectURL(blob)
; (document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url
;(document.querySelector("link[rel='icon']") as HTMLLinkElement).href = url
}
})()
@@ -198,6 +199,26 @@ export function decimalString(num: number, digits = 2) {
return formatter.format(num)
}
export function formatMicroseconds(microseconds: number, showDigits = true): string {
if (!Number.isFinite(microseconds)) {
return "-"
}
if (microseconds < 1000) {
return `${microseconds}μs`
}
if (microseconds < 1_000_000) {
const milliseconds = microseconds / 1000
const digits = milliseconds >= 10 ? 1 : 2
return `${decimalString(milliseconds, showDigits ? digits : 0)}ms`
}
const seconds = microseconds / 1_000_000
const digits = seconds >= 10 ? 1 : 2
return `${decimalString(seconds, showDigits ? digits : 0)}s`
}
/** Get value from local or session storage */
function getStorageValue(key: string, defaultValue: unknown, storageInterface: Storage = localStorage) {
const saved = storageInterface?.getItem(key)
@@ -365,12 +386,12 @@ export function formatDuration(
.join(" ")
}
/** Parse semver string into major, minor, and patch numbers
/** Parse semver string into major, minor, and patch numbers
* @example
* const semVer = "1.2.3"
* const { major, minor, patch } = parseSemVer(semVer)
* console.log(major, minor, patch) // 1, 2, 3
*/
*/
export const parseSemVer = (semVer = ""): SemVer => {
// if (semVer.startsWith("v")) {
// semVer = semVer.slice(1)
@@ -393,6 +414,12 @@ export function compareSemVer(a: SemVer, b: SemVer) {
return a.patch - b.patch
}
const MIN_NETWORK_MONITOR_AGENT_VERSION = parseSemVer("0.20.0")
export function supportsNetworkMonitors(system: Pick<SystemRecord, "info">) {
return compareSemVer(parseSemVer(system.info?.v), MIN_NETWORK_MONITOR_AGENT_VERSION) >= 0
}
// biome-ignore lint/suspicious/noExplicitAny: any is used to allow any function to be passed in
export function debounce<T extends (...args: any[]) => any>(func: T, wait: number): (...args: Parameters<T>) => void {
let timeout: ReturnType<typeof setTimeout>
@@ -422,10 +449,22 @@ export function runOnce<T extends (...args: any[]) => any>(fn: T): T {
}) as T
}
/** Get the visual width of a string, accounting for full-width characters */
export function getVisualStringWidth(str: string): number {
const visualWidthCache = new Map<string, number>()
/** Get the visual width of a string, accounting for full-width and narrow punctuation characters.
* Don't use for monospaced fonts, use .length instead
*/
function getVisualStringWidth(str: string): number {
const cached = visualWidthCache.get(str)
if (cached !== undefined) {
return cached
}
let width = 0
for (const char of str) {
if (char === ".") {
width += 0.7
continue
}
const code = char.codePointAt(0) || 0
// Hangul Jamo and Syllables are often slightly thinner than Hanzi/Kanji
if ((code >= 0x1100 && code <= 0x115f) || (code >= 0xac00 && code <= 0xd7af)) {
@@ -443,16 +482,41 @@ export function getVisualStringWidth(str: string): number {
code > 0xffff // Emojis and other supplementary plane characters
width += isFullWidth ? 2 : 1
}
visualWidthCache.set(str, width)
return width
}
/** Compare the visual width of two strings imprecisely */
export function isVisuallyLonger(str1: string, str2: string): boolean {
return getVisualStringWidth(str1) > getVisualStringWidth(str2)
}
/** Parses a filter string into OR'd groups of AND'd terms: "a b, c" -> [["a","b"], ["c"]] */
export function parseFilterGroups(value: string): string[][] {
return value
.toLowerCase()
.split(",")
.map((group) => group.trim().split(" ").filter((term) => term.length > 0))
.filter((terms) => terms.length > 0)
}
/** True if every term in at least one OR'd group is found in searchString. */
export function matchesFilterGroups(searchString: string, groups: string[][]): boolean {
return groups.some((terms) => terms.every((term) => searchString.includes(term)))
}
/** Format seconds to hours, minutes, or seconds */
export function secondsToString(seconds: number, unit: "hour" | "minute" | "day"): string {
const count = Math.floor(seconds / (unit === "hour" ? 3600 : unit === "minute" ? 60 : 86400))
const countString = count.toLocaleString()
switch (unit) {
case "minute":
return plural(count, { one: `${countString} minute`, few: `${countString} minutes`, many: `${countString} minutes`, other: `${countString} minutes` })
return plural(count, {
one: `${countString} minute`,
few: `${countString} minutes`,
many: `${countString} minutes`,
other: `${countString} minutes`,
})
case "hour":
return plural(count, { one: `${countString} hour`, other: `${countString} hours` })
case "day":
@@ -469,4 +533,4 @@ export function secondsToUptimeString(seconds: number): string {
} else {
return secondsToString(seconds, "day")
}
}
}
+3
View File
@@ -30,6 +30,7 @@ const LoginPage = lazy(() => import("@/components/login/login.tsx"))
const Home = lazy(() => import("@/components/routes/home.tsx"))
const Containers = lazy(() => import("@/components/routes/containers.tsx"))
const Smart = lazy(() => import("@/components/routes/smart.tsx"))
const Monitors = lazy(() => import("@/components/routes/monitors.tsx"))
const SystemDetail = lazy(() => import("@/components/routes/system.tsx"))
const CopyToClipboardDialog = lazy(() => import("@/components/copy-to-clipboard.tsx"))
@@ -79,6 +80,8 @@ const App = memo(() => {
return <Containers />
} else if (page.route === "smart") {
return <Smart />
} else if (page.route === "monitors") {
return <Monitors />
} else if (page.route === "settings") {
return <Settings />
}
+54 -4
View File
@@ -307,6 +307,7 @@ export interface AlertRecord extends RecordModel {
}
export interface AlertsHistoryRecord extends RecordModel {
monitor_name?: string
alert: string
user: string
system: string
@@ -393,11 +394,9 @@ export interface SemVer {
export interface ChartData {
agentVersion: SemVer
systemStats: SystemStatsRecord[]
containerData: ChartDataContainer[]
systemStats?: SystemStatsRecord[]
containerData?: ChartDataContainer[]
orientation: "right" | "left"
ticks: number[]
domain: number[]
chartTime: ChartTimes
}
@@ -414,6 +413,8 @@ export interface AlertInfo {
singleDesc?: () => string
/** Hides the duration slider for alerts that fire on first observation */
noDuration?: boolean
/** Hides the threshold control for binary alerts */
noThreshold?: boolean
/** Description shown instead of numeric threshold and duration values */
triggeredDesc?: () => string
/** Additional information that remains visible while the alert is enabled */
@@ -632,3 +633,52 @@ export interface UpdateInfo {
v: string // new version
url: string // url to new version
}
export interface NetworkMonitorRecord {
id: string
system: string
target: string
protocol: "icmp" | "tcp" | "http" | "dns"
port: number
res: number
resMin1h: number
resMax1h: number
resAvg1h: number
loss: number
loss1h: number
interval: number
enabled: boolean
updated: string
}
/** Response times in microseconds and packet loss percentage (0-100). */
export interface MonitorStats {
res_avg: number
res_min: number
res_max: number
loss: number
}
/** Raw per-monitor record stored in the DB. */
export interface RawMonitorStatsRecord {
res_min: number
res_max: number
total_count: number
success_count: number
res_sum: number
id?: string
type?: string
monitor: string
created: number // unix timestamp (ms)
}
/**
* Merged stats record keyed by monitor ID, used by chart components.
* Constructed from multiple RawMonitorStatsRecord entries sharing the same timestamp.
*/
export interface NetworkMonitorStatsRecord {
id?: string
type?: string
stats: Record<string, MonitorStats>
created: number // unix timestamp (ms) for Recharts xAxis
}
@@ -0,0 +1,36 @@
import { describe, expect, test } from "bun:test"
import { getMonitorStats } from "../src/lib/network-monitor-utils"
describe("monitor stats derived from stored counts", () => {
test("retains probe weights and response precision", () => {
const stats = getMonitorStats({
monitor: "monitor1",
created: 1000,
res_min: 5,
res_max: 20,
total_count: 7,
success_count: 6,
res_sum: 61,
})
expect(stats.res_avg).toBeCloseTo(10.1666667)
expect(stats.loss).toBeCloseTo(14.2857143)
expect(stats.res_min).toBe(5)
expect(stats.res_max).toBe(20)
})
test.each([
{ total_count: 3, success_count: 0, loss: 100 },
{ total_count: 0, success_count: 0, loss: 0 },
{ total_count: 1, success_count: 1, loss: 0 },
])("handles zero sums with $total_count attempts and $success_count successes", ({ loss, ...counts }) => {
const stats = getMonitorStats({
monitor: "monitor1",
created: 1000,
res_min: 0,
res_max: 0,
res_sum: 0,
...counts,
})
expect(stats).toEqual({ res_avg: 0, res_min: 0, res_max: 0, loss })
})
})