Compare commits

...
22 Commits
Author SHA1 Message Date
henrygd 97e6f64bdc fix(ui): swap usage chart shown twice in tabs view (#2356) 2026-09-19 13:27:00 -04:00
henrygd 4a5915b141 add SMART to readme features 2026-09-19 12:11:10 -04:00
henrygd e68372dce4 update readme features 2026-09-19 12:09:26 -04:00
henrygd c52f3acb94 fix(docker): enable Debian contrib for ZFS utilities in slim NVIDIA image 2026-09-19 12:03:05 -04:00
hankGitHubgithub-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
c09eb8c6df chore(helm): update app version to 0.20.0 (#2355)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2026-09-19 11:39:13 -04:00
henrygd a0dc19eacf fix(ui): sort containers alphabetically within systems 2026-09-19 11:18:17 -04:00
henrygd 912bc50874 docker: update alpine version and remove unnecessary GOGC 2026-09-19 00:19:02 -04:00
henrygd c54dbfba7c test: wait for hub setup in universal token integration test 2026-09-19 00:14:19 -04:00
henrygd dd3f7d58b5 test: use virtual time to eliminate MQTT notification timeout waits 2026-09-19 00:10:02 -04:00
henrygd 0509053a69 test: wait for hub system status in agent WebSocket integration test 2026-09-19 00:02:29 -04:00
henrygd 187dc886a9 fix: resync network monitors after SSH reconnect and retry failed syncs 2026-09-18 23:49:54 -04:00
henrygd 2784460621 fix(agent): avoid RAID health warnings during healthy scrubs (#2109) 2026-09-18 21:05:49 -04:00
henrygd b0bc727411 update migration files for 0.20.0 2026-09-18 18:51:42 -04:00
henrygd 6937453282 chore: update changelog 2026-09-18 18:46:45 -04:00
henrygd 97de1471d7 fix: prevent variant Docker images from overwriting latest tags 2026-09-18 18:38:18 -04:00
hankandGitHub bd7e359dcd New Crowdin updates (#2352) 2026-09-18 18:28:28 -04:00
henrygd d2352e8882 upgrade go packages 2026-09-18 17:10:56 -04:00
henrygd f7fd3ef403 force oauth created users to start with user role 2026-09-18 17:10:37 -04:00
henrygd db3afeabd9 i18n: update translations 2026-09-18 14:34:16 -04:00
henrygd b347599928 update translation strings 2026-09-18 14:00:20 -04:00
hankandGitHub 6e5440c21f New Crowdin updates (#2323) 2026-09-18 13:40:00 -04:00
bb1b39928e feat: network monitoring from agents (#2266, #1911)
Co-authored-by: Sven van Ginkel <svenvanginkel@icloud.com>
Co-authored-by: xiaomiku01 <xiaomiku01@outlook.com>
2026-09-18 13:22:50 -04:00
149 changed files with 16763 additions and 1059 deletions
+2 -2
View File
@@ -1,6 +1,6 @@
# Node.js dependencies # Node.js dependencies
node_modules node_modules/
internalsite/node_modules **/node_modules/
# Go build artifacts and binaries # Go build artifacts and binaries
build build
+7 -1
View File
@@ -29,6 +29,7 @@ jobs:
# henrygd/beszel-agent:alpine # henrygd/beszel-agent:alpine
- image: henrygd/beszel-agent - image: henrygd/beszel-agent
dockerfile: ./internal/dockerfile_agent_alpine dockerfile: ./internal/dockerfile_agent_alpine
flavor: latest=false
registry: docker.io registry: docker.io
username_secret: DOCKERHUB_USERNAME username_secret: DOCKERHUB_USERNAME
password_secret: DOCKERHUB_TOKEN password_secret: DOCKERHUB_TOKEN
@@ -55,6 +56,7 @@ jobs:
# henrygd/beszel-agent-nvidia:slim # henrygd/beszel-agent-nvidia:slim
- image: henrygd/beszel-agent-nvidia - image: henrygd/beszel-agent-nvidia
dockerfile: ./internal/dockerfile_agent_nvidia_slim dockerfile: ./internal/dockerfile_agent_nvidia_slim
flavor: latest=false
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
registry: docker.io registry: docker.io
username_secret: DOCKERHUB_USERNAME username_secret: DOCKERHUB_USERNAME
@@ -123,6 +125,7 @@ jobs:
# ghcr.io/henrygd/beszel-agent-nvidia:slim # ghcr.io/henrygd/beszel-agent-nvidia:slim
- image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia - image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia
dockerfile: ./internal/dockerfile_agent_nvidia_slim dockerfile: ./internal/dockerfile_agent_nvidia_slim
flavor: latest=false
platforms: linux/amd64,linux/arm64 platforms: linux/amd64,linux/arm64
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
@@ -150,6 +153,7 @@ jobs:
# ghcr.io/henrygd/beszel-agent:alpine # ghcr.io/henrygd/beszel-agent:alpine
- image: ghcr.io/${{ github.repository }}/beszel-agent - image: ghcr.io/${{ github.repository }}/beszel-agent
dockerfile: ./internal/dockerfile_agent_alpine dockerfile: ./internal/dockerfile_agent_alpine
flavor: latest=false
registry: ghcr.io registry: ghcr.io
username: ${{ github.actor }} username: ${{ github.actor }}
password_secret: GITHUB_TOKEN password_secret: GITHUB_TOKEN
@@ -159,7 +163,7 @@ jobs:
type=semver,pattern={{major}}.{{minor}}-alpine type=semver,pattern={{major}}.{{minor}}-alpine
type=semver,pattern={{major}}-alpine type=semver,pattern={{major}}-alpine
# henrygd/beszel-agent (keep at bottom so it gets built after :alpine and gets the latest tag) # henrygd/beszel-agent
- image: henrygd/beszel-agent - image: henrygd/beszel-agent
dockerfile: ./internal/dockerfile_agent dockerfile: ./internal/dockerfile_agent
registry: docker.io registry: docker.io
@@ -200,6 +204,8 @@ jobs:
uses: docker/metadata-action@v6 uses: docker/metadata-action@v6
with: with:
images: ${{ matrix.image }} images: ${{ matrix.image }}
# Variant images must not overwrite the standard image's latest tag.
flavor: ${{ matrix.flavor || 'latest=auto' }}
tags: ${{ matrix.tags }} tags: ${{ matrix.tags }}
# https://github.com/docker/login-action # https://github.com/docker/login-action
+9
View File
@@ -48,6 +48,7 @@ type Agent struct {
keys []gossh.PublicKey // SSH public keys keys []gossh.PublicKey // SSH public keys
smartManager *SmartManager // Manages SMART data smartManager *SmartManager // Manages SMART data
systemdManager *systemdManager // Manages systemd services systemdManager *systemdManager // Manages systemd services
monitorManager *MonitorManager // Manages network monitors
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
} }
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
// initialize handler registry // initialize handler registry
agent.handlerRegistry = NewHandlerRegistry() agent.handlerRegistry = NewHandlerRegistry()
// initialize monitor manager
agent.monitorManager = newMonitorManager()
agent.storagePoolManager = newStoragePoolManager() agent.storagePoolManager = newStoragePoolManager()
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval. // Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
} }
} }
if a.monitorManager != nil {
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
slog.Debug("Monitors", "data", data.Monitors)
}
// skip updating systemd services if cache time is not the default 60sec interval // skip updating systemd services if cache time is not the default 60sec interval
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs { if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
totalCount := uint16(a.systemdManager.getServiceStatsCount()) totalCount := uint16(a.systemdManager.getServiceStatsCount())
+1
View File
@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
// } // }
func (c *ConnectionManager) stop() error { func (c *ConnectionManager) stop() error {
_ = c.agent.StopServer() _ = c.agent.StopServer()
c.agent.monitorManager.Stop()
c.closeWebSocket() c.closeWebSocket()
return health.CleanUp() return health.CleanUp()
} }
+1 -2
View File
@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
} }
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) { func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
// Save original PATH
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs() hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
tests := []struct { tests := []struct {
@@ -1213,7 +1212,7 @@ echo "[]"`
{ {
name: "no gpu tools available", name: "no gpu tools available",
setupCommands: func(_ string) error { setupCommands: func(_ string) error {
t.Setenv("PATH", "") // The subtest already restricts PATH to its empty temporary directory.
return nil return nil
}, },
wantErr: true, wantErr: true,
+20
View File
@@ -7,6 +7,7 @@ import (
"github.com/fxamacker/cbor/v2" "github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common" "github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
"log/slog" "log/slog"
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{}) registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
registry.Register(common.GetSmartData, &GetSmartDataHandler{}) registry.Register(common.GetSmartData, &GetSmartDataHandler{})
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{}) registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
registry.Register(common.GetZfsData, &GetZfsDataHandler{}) registry.Register(common.GetZfsData, &GetZfsDataHandler{})
return registry return registry
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
return hctx.SendResponse(details, hctx.RequestID) return hctx.SendResponse(details, hctx.RequestID)
} }
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
type SyncNetworkMonitorsHandler struct{}
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
var req monitor.SyncRequest
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
return err
}
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
if err != nil {
return err
}
return hctx.SendResponse(resp, hctx.RequestID)
}
+3 -6
View File
@@ -201,12 +201,9 @@ func mdraidSmartStatus(health mdraidHealth) string {
if health.mismatchCnt > 0 { if health.mismatchCnt > 0 {
return "WARNING" return "WARNING"
} }
// "check" scans for consistency problems without repairing mismatches. // "check" and "repair" are requested consistency scans, not evidence of
// With no mismatches, keep it green while reporting progress attributes. // array failure. With no health issues above, keep scrubbing green while
switch syncAction { // reporting the sync action and progress attributes.
case "repair":
return "WARNING"
}
switch state { switch state {
case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly": case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly":
return "PASSED" return "PASSED"
+19 -2
View File
@@ -174,8 +174,25 @@ func TestMdraidSmartStatus(t *testing.T) {
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" { if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" {
t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got) t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got)
} }
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", syncAction: "repair"}); got != "WARNING" { for _, tc := range []struct {
t.Fatalf("mdraidSmartStatus(repair) = %q, want WARNING", got) name string
health mdraidHealth
want string
}{
{"clean", mdraidHealth{arrayState: "clean"}, "PASSED"},
{"active", mdraidHealth{arrayState: "active"}, "PASSED"},
{"mismatch", mdraidHealth{arrayState: "active", mismatchCnt: 1}, "WARNING"},
{"degraded", mdraidHealth{arrayState: "active", degraded: 1}, "FAILED"},
{"faulty member", mdraidHealth{arrayState: "active", faultyDisks: 1}, "FAILED"},
{"inactive", mdraidHealth{arrayState: "inactive"}, "FAILED"},
{"unknown", mdraidHealth{arrayState: "unknown"}, "UNKNOWN"},
} {
t.Run("repair/"+tc.name, func(t *testing.T) {
tc.health.syncAction = "repair"
if got := mdraidSmartStatus(tc.health); got != tc.want {
t.Fatalf("mdraidSmartStatus(%+v) = %q, want %s", tc.health, got, tc.want)
}
})
} }
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" { if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" {
t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got) t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got)
+176
View File
@@ -0,0 +1,176 @@
package agent
import (
"errors"
"fmt"
"net/http"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// MonitorManager manages network monitor configurations and task lifetimes.
type MonitorManager struct {
mu sync.RWMutex
monitors map[string]*monitorTask // keyed by monitor ID
probe monitorProbe
resumeGuard monitorResumeGuard
}
func newMonitorManager() *MonitorManager {
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
}
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
}
// SyncMonitors replaces all monitor tasks with the given configs.
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
pm.mu.Lock()
defer pm.mu.Unlock()
// Build set of new keys
newKeys := make(map[string]monitor.Config, len(configs))
for _, cfg := range configs {
if cfg.ID == "" {
continue
}
newKeys[cfg.ID] = cfg
}
// Stop removed monitors
for key, task := range pm.monitors {
if _, exists := newKeys[key]; !exists {
task.cancel()
delete(pm.monitors, key)
}
}
// Start new monitors and restart tasks whose config changed.
for key, cfg := range newKeys {
task, exists := pm.monitors[key]
if exists && task.config == cfg {
continue
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(cfg, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[key] = task
pm.startMonitor(task)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// HandleSyncRequest applies a full or incremental monitor sync request.
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
switch req.Action {
case monitor.SyncActionReplace:
pm.SyncMonitors(req.Configs)
return monitor.SyncResponse{}, nil
case monitor.SyncActionUpsert:
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
if err != nil {
return monitor.SyncResponse{}, err
}
if result == nil {
return monitor.SyncResponse{}, nil
}
return monitor.SyncResponse{Result: *result}, nil
case monitor.SyncActionDelete:
if req.Config.ID == "" {
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
}
pm.DeleteMonitor(req.Config.ID)
return monitor.SyncResponse{}, nil
default:
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
}
}
// UpsertMonitor creates or replaces a single monitor task.
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
if config.ID == "" {
return nil, errors.New("missing monitor ID")
}
pm.mu.Lock()
task, exists := pm.monitors[config.ID]
if exists && task.config == config {
pm.mu.Unlock()
if !runNow {
return nil, nil
}
return task.runProbe(pm.probe), nil
}
if exists {
task.cancel()
}
task = newMonitorTaskFromExisting(config, task)
task.resumeGuard = &pm.resumeGuard
pm.resumeGuard.start()
pm.monitors[config.ID] = task
pm.mu.Unlock()
if runNow {
result := task.runProbe(pm.probe)
pm.startMonitor(task)
return result, nil
}
pm.startMonitor(task)
return nil, nil
}
// DeleteMonitor stops and removes a single monitor task.
func (pm *MonitorManager) DeleteMonitor(id string) {
if id == "" {
return
}
pm.mu.Lock()
defer pm.mu.Unlock()
if task, exists := pm.monitors[id]; exists {
task.cancel()
delete(pm.monitors, id)
}
if len(pm.monitors) == 0 {
pm.resumeGuard.shutdown()
}
}
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
pm.mu.RLock()
defer pm.mu.RUnlock()
results := make(map[string]monitor.Result, len(pm.monitors))
now := time.Now()
duration := time.Duration(durationMs) * time.Millisecond
for _, task := range pm.monitors {
result, ok := task.history.result(duration, now)
if !ok {
continue
}
results[task.config.ID] = result
}
return results
}
// Stop stops all monitor tasks.
func (pm *MonitorManager) Stop() {
pm.mu.Lock()
defer pm.mu.Unlock()
for key, task := range pm.monitors {
task.cancel()
delete(pm.monitors, key)
}
pm.resumeGuard.shutdown()
}
+274
View File
@@ -0,0 +1,274 @@
package agent
import (
"math"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// Monitors run at user-defined intervals (e.g., every 10s).
// To keep memory usage low and constant, data is stored in two layers:
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
// wall-clock minute. Samples collected within the same minute are aggregated
// (sum, min, max, count) into a single bucket.
//
// Short-term requests (<= 61s) use raw samples.
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
// of individual data points.
const (
// monitorRawRetention is the duration to keep individual samples
monitorRawRetention = 61 * time.Second
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
monitorMinuteBucketLen int32 = 61
)
// monitorHistory owns retention and aggregation, independently of probe execution.
type monitorHistory struct {
mu sync.Mutex
sampleCount int64
samples []monitorSample
buckets [monitorMinuteBucketLen]monitorBucket
}
func newMonitorHistory() *monitorHistory {
// Start small for typical intervals; append grows the buffer for faster probes.
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
}
func (h *monitorHistory) clone() *monitorHistory {
h.mu.Lock()
defer h.mu.Unlock()
cloned := newMonitorHistory()
cloned.samples = append(cloned.samples, h.samples...)
cloned.buckets = h.buckets
cloned.sampleCount = h.sampleCount
return cloned
}
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
h.mu.Lock()
defer h.mu.Unlock()
return h.resultLocked(duration, now)
}
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
h.mu.Lock()
defer h.mu.Unlock()
h.addSampleLocked(sample)
result, _ := h.resultLocked(time.Minute, sample.timestamp)
return result
}
// monitorSample stores one monitor attempt and its collection time.
type monitorSample struct {
responseUs int64 // -1 means loss
timestamp time.Time
}
// monitorBucket stores one minute of aggregated monitor data.
type monitorBucket struct {
minute int32
filled bool
stats monitorAggregate
}
// monitorAggregate accumulates successful response stats and total sample counts.
type monitorAggregate struct {
sumUs int64
minUs int64
maxUs int64
totalCount int64
successCount int64
}
// newMonitorAggregate initializes an aggregate with an unset minimum value.
func newMonitorAggregate() monitorAggregate {
return monitorAggregate{minUs: math.MaxInt64}
}
// addResponse folds a single monitor sample into the aggregate.
func (agg *monitorAggregate) addResponse(responseUs int64) {
agg.totalCount++
if responseUs < 0 {
return
}
agg.successCount++
agg.sumUs += responseUs
if responseUs < agg.minUs {
agg.minUs = responseUs
}
if responseUs > agg.maxUs {
agg.maxUs = responseUs
}
}
// addAggregate merges another aggregate into this one.
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
if other.totalCount == 0 {
return
}
agg.totalCount += other.totalCount
agg.successCount += other.successCount
agg.sumUs += other.sumUs
if other.successCount == 0 {
return
}
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
agg.minUs = other.minUs
}
if other.maxUs > agg.maxUs {
agg.maxUs = other.maxUs
}
}
// hasData reports whether the aggregate contains any samples.
func (agg monitorAggregate) hasData() bool {
return agg.totalCount > 0
}
// result converts the aggregate into the monitor result format.
func (agg monitorAggregate) result() monitor.Result {
avg := agg.avgResponse()
result := monitor.Result{
AvgResponse: avg,
MinResponse: agg.minUs,
MaxResponse: agg.maxUs,
PacketLoss: agg.lossPercentage(),
TotalCount: agg.totalCount,
SuccessCount: agg.successCount,
ResponseSum: agg.sumUs,
}
if agg.successCount == 0 {
result.MinResponse, result.MaxResponse = 0, 0
}
return result
}
// avgResponse returns the rounded average of successful samples.
func (agg monitorAggregate) avgResponse() int64 {
if agg.successCount == 0 {
return 0
}
return agg.sumUs / agg.successCount
}
// lossPercentage returns the rounded failure rate for the aggregate.
func (agg monitorAggregate) lossPercentage() float64 {
if agg.totalCount == 0 {
return 0
}
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
}
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
agg := h.aggregateLocked(duration, now)
if !agg.hasData() {
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
// the most recent sample so realtime requests still report current status.
agg = h.latestSampleAggregateLocked()
}
hourAgg := h.aggregateLocked(time.Hour, now)
if !agg.hasData() {
return monitor.Result{}, false
}
result := agg.result()
if len(h.samples) > 0 {
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
}
result.AvgResponse1h = hourAgg.avgResponse()
result.MinResponse1h = hourAgg.minUs
result.MaxResponse1h = hourAgg.maxUs
result.PacketLoss1h = hourAgg.lossPercentage()
result.SampleCount = h.sampleCount
if hourAgg.successCount == 0 {
result.MinResponse1h, result.MaxResponse1h = 0, 0
}
return result, true
}
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
agg := newMonitorAggregate()
if len(h.samples) == 0 {
return agg
}
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
return agg
}
// aggregateLocked collects monitor data for the requested time window.
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
cutoff := now.Add(-duration)
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
if duration <= monitorRawRetention {
return aggregateSamplesSince(h.samples, cutoff)
}
return aggregateBucketsSince(h.buckets[:], cutoff, now)
}
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
agg := newMonitorAggregate()
for _, sample := range samples {
if sample.timestamp.Before(cutoff) {
continue
}
agg.addResponse(sample.responseUs)
}
return agg
}
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
agg := newMonitorAggregate()
startMinute := int32(cutoff.Unix() / 60)
endMinute := int32(now.Unix() / 60)
for _, bucket := range buckets {
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
continue
}
agg.addAggregate(bucket.stats)
}
return agg
}
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
h.sampleCount++
cutoff := sample.timestamp.Add(-monitorRawRetention)
start := 0
for i := range h.samples {
if !h.samples[i].timestamp.Before(cutoff) {
start = i
break
}
if i == len(h.samples)-1 {
start = len(h.samples)
}
}
if start > 0 {
size := copy(h.samples, h.samples[start:])
h.samples = h.samples[:size]
}
h.samples = append(h.samples, sample)
minute := int32(sample.timestamp.Unix() / 60)
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
bucket := &h.buckets[minute%monitorMinuteBucketLen]
if !bucket.filled || bucket.minute != minute {
bucket.minute = minute
bucket.filled = true
bucket.stats = newMonitorAggregate()
}
bucket.stats.addResponse(sample.responseUs)
}
+154
View File
@@ -0,0 +1,154 @@
package agent
import (
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorHistoryWindowCounts(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// This older success counts toward lifetime warm-up, but not this window.
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
result, ok := history.result(time.Minute, now)
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
assert.EqualValues(t, 3, result.TotalCount)
assert.EqualValues(t, 2, result.SuccessCount)
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
assert.EqualValues(t, 15, result.AvgResponse)
assert.Equal(t, 33.33, result.PacketLoss)
encoded, err := cbor.Marshal(result)
require.NoError(t, err)
var decoded monitor.Result
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
assert.Equal(t, result, decoded)
stats := monitor.Stats{}.FromResult(decoded)
assert.Equal(t, result.TotalCount, stats.TotalCount)
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
// Reads do not consume samples. A short window's latest-sample fallback
// carries the count for that single failure, not the minute or lifetime count.
repeated, _ := history.result(time.Minute, now)
assert.Equal(t, result, repeated)
fallback, ok := history.result(time.Second, now)
require.True(t, ok)
assert.EqualValues(t, 1, fallback.TotalCount)
assert.Zero(t, fallback.SuccessCount)
assert.Zero(t, fallback.ResponseSum)
assert.Equal(t, 100.0, fallback.PacketLoss)
assert.EqualValues(t, 4, fallback.SampleCount)
}
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
agg := history.aggregateLocked(time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(1), agg.successCount)
result := agg.result()
assert.Equal(t, int64(20), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 50.0, result.PacketLoss)
}
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(4), agg.totalCount)
assert.Equal(t, int64(3), agg.successCount)
result := agg.result()
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(20), result.MinResponse)
assert.Equal(t, int64(40), result.MaxResponse)
assert.Equal(t, 25.0, result.PacketLoss)
}
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
history := newMonitorHistory()
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
require.Len(t, history.samples, 1)
assert.Equal(t, int64(20), history.samples[0].responseUs)
agg := history.aggregateLocked(10*time.Minute, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
result := agg.result()
assert.Equal(t, int64(15), result.AvgResponse)
assert.Equal(t, int64(10), result.MinResponse)
assert.Equal(t, int64(20), result.MaxResponse)
assert.Equal(t, 0.0, result.PacketLoss)
}
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
history := newMonitorHistory()
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
_, ok := history.result(time.Minute, start)
require.False(t, ok)
first := history.record(monitorSample{responseUs: 20, timestamp: start})
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
for minute := 0; minute < 5; minute++ {
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
// Realtime reads must not consume freshness for the persistence request.
for _, window := range []time.Duration{time.Second, time.Minute} {
result, ok := history.result(window, now)
require.True(t, ok)
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
assert.Equal(t, int64(20), result.AvgResponse)
}
}
next := start.Add(5 * time.Minute)
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
assert.Equal(t, float64(100), failed.PacketLoss)
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
require.True(t, ok)
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
assert.Equal(t, float64(100), repeated.PacketLoss)
}
func TestMonitorHistorySampleCount(t *testing.T) {
history := newMonitorHistory()
now := time.Now()
// Both failed and successful probes count, including older samples so
// monitors with hourly intervals can finish warming up.
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
for i, response := range []int64{10, -1, 20} {
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
assert.EqualValues(t, i+2, result.SampleCount)
}
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
require.True(t, ok)
assert.EqualValues(t, 4, result.SampleCount)
}
+312
View File
@@ -0,0 +1,312 @@
package agent
import (
"bytes"
"context"
"crypto/rand"
"errors"
"fmt"
"math"
"net"
"os"
"os/exec"
"regexp"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"golang.org/x/net/icmp"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
"log/slog"
)
// Match the numeric RTT independently of the localized label used by Windows.
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
var icmpSequence atomic.Uint32
type icmpPacketConn interface {
Close() error
}
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
// all native methods fail, the choice is cached so subsequent monitors skip
// the trial-and-error overhead.
type icmpMethod uint8
const (
icmpUntried icmpMethod = iota // haven't tried yet
icmpRaw // privileged raw socket
icmpDatagram // unprivileged datagram socket
icmpExecFallback // shell out to system ping command
)
// icmpFamily holds the network parameters and cached detection result for one address family.
type icmpFamily struct {
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
dgramNetwork string // e.g. "udp4" or "udp6"
listenAddr string // "0.0.0.0" or "::"
echoType icmp.Type // outgoing echo request type
replyType icmp.Type // expected echo reply type
proto int // IANA protocol number for parsing replies
isIPv6 bool
mode icmpMethod // cached detection result (guarded by icmpModeMu)
}
var (
icmpV4 = icmpFamily{
rawNetwork: "ip4:icmp",
dgramNetwork: "udp4",
listenAddr: "0.0.0.0",
echoType: ipv4.ICMPTypeEcho,
replyType: ipv4.ICMPTypeEchoReply,
proto: 1,
}
icmpV6 = icmpFamily{
rawNetwork: "ip6:ipv6-icmp",
dgramNetwork: "udp6",
listenAddr: "::",
echoType: ipv6.ICMPTypeEchoRequest,
replyType: ipv6.ICMPTypeEchoReply,
proto: 58,
isIPv6: true,
}
icmpModeMu sync.Mutex
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
return icmp.ListenPacket(network, listenAddr)
}
)
// monitorICMP sends an ICMP echo request and measures round-trip response.
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
// unprivileged datagram, or exec fallback) is detected once per address
// family and cached for subsequent monitors.
// Returns response in microseconds, or -1 and an error on failure.
func monitorICMP(ctx context.Context, target string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
family, ip, err := resolveICMPTarget(ctx, target)
if err != nil {
return -1, err
}
icmpModeMu.Lock()
if family.mode == icmpUntried {
family.mode = detectICMPMode(family, icmpListen)
}
mode := family.mode
icmpModeMu.Unlock()
switch mode {
case icmpRaw:
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
case icmpDatagram:
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
case icmpExecFallback:
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
default:
return -1, errors.New("unsupported ICMP mode")
}
}
// resolveICMPTarget resolves a target hostname or IP to determine the address
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
if ip := net.ParseIP(target); ip != nil {
if ip.To4() != nil {
return &icmpV4, ip.To4(), nil
}
return &icmpV6, ip, nil
}
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
if err != nil || len(ips) == 0 {
return nil, nil, err
}
for _, ip := range ips {
if v4 := ip.To4(); v4 != nil {
return &icmpV4, v4, nil
}
}
return &icmpV6, ips[0], nil
}
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
label := "IPv4"
if family.isIPv6 {
label = "IPv6"
}
conn, err := listen(family.rawNetwork, family.listenAddr)
slog.Debug("ICMP raw socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpRaw
}
conn, err = listen(family.dgramNetwork, family.listenAddr)
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
if err == nil {
conn.Close()
return icmpDatagram
}
return icmpExecFallback
}
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
return -1, err
}
defer conn.Close()
return monitorICMPPacket(ctx, conn, family, dst)
}
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
if err := ctx.Err(); err != nil {
return -1, err
}
// Closing the socket interrupts both reads and writes on cancellation.
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
// Prepare correlation data before starting the round-trip timer. The token
// also distinguishes delayed replies after the 16-bit sequence wraps.
token := make([]byte, 16)
if _, err := rand.Read(token); err != nil {
return -1, err
}
echo := &icmp.Echo{
ID: os.Getpid() & 0xffff,
Seq: int(icmpSequence.Add(1) & 0xffff),
Data: token,
}
// Linux ping sockets replace the Echo ID with their bound port. Darwin
// datagram sockets and raw sockets preserve the supplied ID.
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
echo.ID = local.Port
}
targetIP := icmpAddrIP(dst)
msg := &icmp.Message{
Type: family.echoType,
Code: 0,
Body: echo,
}
msgBytes, err := msg.Marshal(nil)
if err != nil {
return -1, err
}
// Set deadline before sending
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
return -1, err
}
buf := make([]byte, 1500)
start := time.Now()
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
return -1, err
}
// Read reply
for {
n, peer, err := conn.ReadFrom(buf)
received := time.Now()
if err != nil {
return -1, err
}
if !targetIP.Equal(icmpAddrIP(peer)) {
continue
}
reply, err := icmp.ParseMessage(family.proto, buf[:n])
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
continue
}
body, ok := reply.Body.(*icmp.Echo)
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
return received.Sub(start).Microseconds(), nil
}
// Keep waiting for our reply without extending the original deadline.
}
}
func icmpAddrIP(addr net.Addr) net.IP {
switch addr := addr.(type) {
case *net.IPAddr:
return addr.IP
case *net.UDPAddr:
return addr.IP
default:
return nil
}
}
// pingCommand selects the executable and arguments for the supported agent platforms.
// The context deadline enforces the timeout: -W has incompatible meanings across
// Linux, BSD IPv4 ping, and macOS ping6.
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
family := "-4"
if isIPv6 {
family = "-6"
}
switch goos {
case "windows":
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
case "linux":
return "ping", []string{family, "-n", "-c", "1", target}, nil
case "darwin", "freebsd", "openbsd":
command := "ping"
if isIPv6 {
command = "ping6"
}
return command, []string{"-n", "-c", "1", target}, nil
default:
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
}
}
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
if err != nil {
return -1, err
}
cmd := exec.CommandContext(ctx, name, args...)
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
cmd.Env = append(os.Environ(), "LC_ALL=C")
output, err := cmd.Output()
if ctx.Err() != nil {
return -1, ctx.Err()
}
if err != nil {
return -1, fmt.Errorf("%s failed: %w", name, err)
}
return parsePingResponse(output)
}
// parsePingResponse returns the reported RTT, never subprocess execution time.
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
func parsePingResponse(output []byte) (int64, error) {
matches := pingTimeRegex.FindSubmatch(output)
if len(matches) < 2 {
return -1, errors.New("ping output contains no round-trip time")
}
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
return -1, errors.New("invalid round-trip time in ping output")
}
return int64(math.Round(ms * 1000)), nil
}
+433
View File
@@ -0,0 +1,433 @@
//go:build testing
package agent
import (
"context"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/icmp"
)
type testICMPPacketConn struct{}
func (testICMPPacketConn) Close() error { return nil }
type blockingICMPConn struct {
net.PacketConn
reading chan struct{}
}
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
return len(p), nil
}
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
close(c.reading)
return c.PacketConn.ReadFrom(p)
}
func TestMonitorICMPPacketCancellation(t *testing.T) {
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
require.NoError(t, err)
defer conn.Close()
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
done <- err
}()
select {
case <-blocking.reading:
case <-time.After(time.Second):
t.Fatal("probe did not begin reading")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("cancellation did not interrupt the socket read")
}
}
func TestMonitorICMPExecCancellation(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
dir := t.TempDir()
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
done := make(chan error, 1)
go func() {
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
done <- err
}()
select {
case err := <-done:
require.ErrorIs(t, err, context.DeadlineExceeded)
case <-time.After(time.Second):
t.Fatal("cancellation did not terminate ping")
}
}
func TestPingCommand(t *testing.T) {
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
for _, ipv6 := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
target, family := "192.0.2.1", "-4"
if ipv6 {
target, family = "2001:db8::1", "-6"
}
name, args, err := pingCommand(goos, target, ipv6)
require.NoError(t, err)
wantName := "ping"
wantArgs := []string{"-n", "-c", "1", target}
switch goos {
case "windows":
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
case "linux":
wantArgs = append([]string{family}, wantArgs...)
default:
if ipv6 {
wantName = "ping6"
}
}
assert.Equal(t, wantName, name)
assert.Equal(t, wantArgs, args)
})
}
}
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
require.Error(t, err)
}
func TestParsePingResponse(t *testing.T) {
for _, tc := range []struct {
name string
output string
wantUs int64
}{
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
{"rounding", "time=0.1236 ms", 124},
{"empty", "", -1},
{"timeout", "Request timed out.", -1},
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
{"malformed", "time=oops ms", -1},
{"negative", "time=-1 ms", -1},
{"overflow", "time=999999999999999999999 ms", -1},
} {
t.Run(tc.name, func(t *testing.T) {
responseUs, err := parsePingResponse([]byte(tc.output))
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
func TestMonitorICMPExecOutput(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("test uses a POSIX shell stub for ping")
}
for _, tc := range []struct {
name string
output string
exit int
wantUs int64
}{
{"success", "time=1.234 ms", 0, 1234},
{"missing RTT", "unrecognized output", 0, -1},
{"failed command with RTT", "time=1.234 ms", 1, -1},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
// Also verify an inherited locale cannot override the C locale.
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
t.Setenv("LC_ALL", "de_DE.UTF-8")
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
if tc.wantUs < 0 {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.wantUs, responseUs)
})
}
}
type icmpTestReply struct {
data []byte
peer net.Addr
}
type scriptedICMPConn struct {
net.PacketConn
local net.Addr
onWrite func([]byte, net.Addr)
replies []icmpTestReply
reads int
deadlineSets int
}
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
c.deadlineSets++
return nil
}
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
c.onWrite(data, dst)
return len(data), nil
}
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
c.reads++
if len(c.replies) == 0 {
return 0, nil, os.ErrDeadlineExceeded
}
reply := c.replies[0]
c.replies = c.replies[1:]
return copy(buf, reply.data), reply.peer, nil
}
func TestMonitorICMPReplyCorrelation(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, datagram := range []bool{false, true} {
network := family.rawNetwork
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
if family.isIPv6 {
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
}
var dst net.Addr = &net.IPAddr{IP: ip}
var wrongPeer net.Addr = &net.IPAddr{IP: other}
if datagram {
network = family.dgramNetwork
dst = &net.UDPAddr{IP: ip}
wrongPeer = &net.UDPAddr{IP: other}
}
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
for _, eventuallyMatches := range []bool{false, true} {
ending := "timeout"
if eventuallyMatches {
ending = "success"
}
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
if datagram {
conn.local = &net.UDPAddr{Port: 12345}
if runtime.GOOS == "linux" {
// Deliberately differ from the process ID.
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
}
}
conn.onWrite = func(data []byte, target net.Addr) {
require.Equal(t, dst, target)
request, err := icmp.ParseMessage(family.proto, data)
require.NoError(t, err)
echo := request.Body.(*icmp.Echo)
expectedID := os.Getpid() & 0xffff
if datagram && runtime.GOOS == "linux" {
expectedID = conn.local.(*net.UDPAddr).Port
}
require.Equal(t, expectedID, echo.ID)
reply := &icmp.Message{Type: family.replyType, Body: echo}
valid, err := reply.Marshal(nil)
require.NoError(t, err)
peer := dst
switch mismatch {
case "source":
peer = wrongPeer
case "id":
echo.ID ^= 1
case "sequence":
echo.Seq ^= 1
case "payload":
echo.Data[0] ^= 1
case "type":
reply.Type = family.echoType
case "code":
reply.Code = 1
}
invalid, err := reply.Marshal(nil)
require.NoError(t, err)
if mismatch == "malformed" {
invalid = invalid[:2]
}
conn.replies = []icmpTestReply{{invalid, peer}}
if eventuallyMatches {
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
if eventuallyMatches {
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
} else {
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
assert.Equal(t, int64(-1), elapsed)
}
assert.Equal(t, 2, conn.reads)
assert.Equal(t, 1, conn.deadlineSets)
})
}
}
}
}
}
func TestMonitorICMPLoopback(t *testing.T) {
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
t.Run(network, func(t *testing.T) {
conn, err := icmp.ListenPacket(network, family.listenAddr)
if err != nil {
t.Skipf("ICMP socket unavailable: %v", err)
}
defer conn.Close()
ip := net.ParseIP("127.0.0.1")
if family.isIPv6 {
ip = net.ParseIP("::1")
}
var dst net.Addr = &net.IPAddr{IP: ip}
if network == family.dgramNetwork {
dst = &net.UDPAddr{IP: ip}
}
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, int64(0))
})
}
}
}
func TestDetectICMPMode(t *testing.T) {
tests := []struct {
name string
family *icmpFamily
rawErr error
udpErr error
want icmpMethod
wantNetworks []string
}{
{
name: "IPv4 prefers raw socket when available",
family: &icmpV4,
want: icmpRaw,
wantNetworks: []string{"ip4:icmp"},
},
{
name: "IPv4 uses datagram when raw unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv4 falls back to exec when both unavailable",
family: &icmpV4,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip4:icmp", "udp4"},
},
{
name: "IPv6 prefers raw socket when available",
family: &icmpV6,
want: icmpRaw,
wantNetworks: []string{"ip6:ipv6-icmp"},
},
{
name: "IPv6 uses datagram when raw unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
want: icmpDatagram,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
{
name: "IPv6 falls back to exec when both unavailable",
family: &icmpV6,
rawErr: errors.New("operation not permitted"),
udpErr: errors.New("protocol not supported"),
want: icmpExecFallback,
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
calls := make([]string, 0, 2)
listen := func(network, listenAddr string) (icmpPacketConn, error) {
require.Equal(t, tt.family.listenAddr, listenAddr)
calls = append(calls, network)
switch network {
case tt.family.rawNetwork:
if tt.rawErr != nil {
return nil, tt.rawErr
}
case tt.family.dgramNetwork:
if tt.udpErr != nil {
return nil, tt.udpErr
}
default:
t.Fatalf("unexpected network %q", network)
}
return testICMPPacketConn{}, nil
}
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
assert.Equal(t, tt.wantNetworks, calls)
})
}
}
func TestResolveICMPTarget(t *testing.T) {
t.Run("IPv4 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
t.Run("IPv6 literal", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::1")
require.NoError(t, err)
require.NotNil(t, family)
assert.True(t, family.isIPv6)
assert.Equal(t, "::1", ip.String())
})
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
require.NoError(t, err)
require.NotNil(t, family)
assert.False(t, family.isIPv6)
assert.Equal(t, "127.0.0.1", ip.String())
})
}
+105
View File
@@ -0,0 +1,105 @@
package agent
import (
"context"
"errors"
"fmt"
"net"
"net/http"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
// Implementations must honor cancellation and bound their execution time.
type monitorProbe func(context.Context, monitor.Config) (int64, error)
func networkMonitorProbe(client *http.Client) monitorProbe {
return func(ctx context.Context, config monitor.Config) (int64, error) {
switch config.Protocol {
case "icmp":
return monitorICMP(ctx, config.Target)
case "tcp":
return monitorTCP(ctx, config.Target, config.Port)
case "http":
return monitorHTTP(ctx, client, config.Target)
case "dns":
return monitorDNS(ctx, config.Target)
default:
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
}
}
}
// monitorTCP measures connection establishment time, including address fallback
// but excluding DNS resolution.
// Returns -1 and an error on failure.
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
// Resolve DNS first, outside the timing window but within the probe deadline.
ips, err := net.DefaultResolver.LookupHost(ctx, target)
if err != nil {
return -1, err
}
if len(ips) == 0 {
return -1, errors.New("no addresses resolved for TCP monitor")
}
portString := fmt.Sprintf("%d", port)
deadline, _ := ctx.Deadline()
// Share the remaining probe budget across addresses so an unresponsive
// first address cannot consume all the time available for alternatives.
start := time.Now()
for i, ip := range ips {
if err := ctx.Err(); err != nil {
return -1, err
}
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
var conn net.Conn
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
if err != nil {
continue
}
responseUs := time.Since(start).Microseconds()
conn.Close()
return responseUs, nil
}
return -1, err
}
// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
func monitorDNS(ctx context.Context, target string) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
start := time.Now()
ips, err := net.DefaultResolver.LookupHost(ctx, target)
if err != nil || len(ips) == 0 {
return -1, err
}
return time.Since(start).Microseconds(), nil
}
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
if client == nil {
client = http.DefaultClient
}
start := time.Now()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return -1, err
}
resp, err := client.Do(req)
if err != nil {
return -1, err
}
resp.Body.Close()
if resp.StatusCode >= 400 {
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
}
return time.Since(start).Microseconds(), nil
}
+88
View File
@@ -0,0 +1,88 @@
package agent
import (
"sync"
"time"
)
const (
monitorResumeHeartbeat = 10 * time.Second
// Allow scheduling jitter without mistaking an ordinary tick for resume.
monitorResumeGap = 2 * monitorResumeHeartbeat
monitorResumePause = 10 * time.Second
)
// monitorResumeGuard detects likely suspend/resume using wall time. A long
// process stall or forward clock adjustment can also trigger the bounded pause.
// One heartbeat is shared by all configured monitors.
type monitorResumeGuard struct {
mu sync.Mutex
stop chan struct{}
lastTick time.Time
pauseUntil time.Time
generation uint32
}
func (g *monitorResumeGuard) start() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
return
}
stop := make(chan struct{})
g.stop = stop
g.lastTick = time.Now().Round(0)
g.pauseUntil = time.Time{}
go func() {
ticker := time.NewTicker(monitorResumeHeartbeat)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
g.mu.Lock()
if g.stop == stop {
g.observe(time.Now())
}
g.mu.Unlock()
}
}
}()
}
func (g *monitorResumeGuard) shutdown() {
g.mu.Lock()
defer g.mu.Unlock()
if g.stop != nil {
close(g.stop)
g.stop = nil
g.generation++
}
}
// observe requires mu. Strip the monotonic component because it can stop during
// suspend. Read the current time rather than the ticker's queued timestamp.
func (g *monitorResumeGuard) observe(now time.Time) {
now = now.Round(0)
if now.Sub(g.lastTick) > monitorResumeGap {
g.pauseUntil = now.Add(monitorResumePause)
g.generation++
}
g.lastTick = now
}
// snapshot also observes time so a probe waking before the heartbeat detects
// resume itself. A changed generation invalidates probes spanning suspend.
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
if g == nil {
return 0, true
}
g.mu.Lock()
defer g.mu.Unlock()
if g.stop == nil {
return g.generation, true
}
g.observe(time.Now())
return g.generation, !g.lastTick.Before(g.pauseUntil)
}
+121
View File
@@ -0,0 +1,121 @@
//go:build testing
package agent
import (
"context"
"errors"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func simulateMonitorSleep(g *monitorResumeGuard) {
g.mu.Lock()
g.lastTick = time.Now().Add(-time.Hour).Round(0)
g.mu.Unlock()
}
func TestMonitorResumePause(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
generation, allowed := g.snapshot()
require.True(t, allowed)
// Heartbeats alone must keep the guard current between infrequent probes.
time.Sleep(time.Minute)
synctest.Wait()
steadyGeneration, allowed := g.snapshot()
require.True(t, allowed)
require.Equal(t, generation, steadyGeneration)
// The probe, rather than the heartbeat, must detect this gap.
simulateMonitorSleep(&g)
next, allowed := g.snapshot()
assert.False(t, allowed)
assert.NotEqual(t, generation, next)
time.Sleep(9 * time.Second)
_, allowed = g.snapshot()
assert.False(t, allowed)
time.Sleep(time.Second)
end, allowed := g.snapshot()
assert.True(t, allowed)
assert.Equal(t, next, end)
})
}
func TestMonitorResumeGuardLifecycle(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
defer pm.Stop()
assert.Nil(t, pm.resumeGuard.stop)
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
stop := pm.resumeGuard.stop
require.NotNil(t, stop)
pm.DeleteMonitor("a")
assert.Equal(t, stop, pm.resumeGuard.stop)
pm.DeleteMonitor("b")
assert.Nil(t, pm.resumeGuard.stop)
select {
case <-stop:
default:
t.Fatal("heartbeat was not stopped")
}
time.Sleep(time.Hour)
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
require.NoError(t, err)
_, allowed := pm.resumeGuard.snapshot()
assert.True(t, allowed, "idle time must not trigger a resume pause")
pm.SyncMonitors(nil)
assert.Nil(t, pm.resumeGuard.stop)
})
}
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var g monitorResumeGuard
g.start()
defer g.shutdown()
task := newMonitorTask(monitor.Config{ID: "test"})
defer task.cancel()
task.resumeGuard = &g
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
simulateMonitorSleep(&g)
return 0, errors.New("network not ready")
})
assert.Nil(t, result)
assert.Empty(t, task.history.samples)
// Explicit requests may still run during the pause and record real failures.
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
return 0, errors.New("unreachable")
})
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
})
}
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
calls.Add(1)
return 1, nil
})
defer pm.Stop()
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
simulateMonitorSleep(&pm.resumeGuard)
pm.resumeGuard.snapshot()
time.Sleep(9 * time.Second)
synctest.Wait()
assert.Zero(t, calls.Load())
assert.Empty(t, pm.GetResults(1000))
time.Sleep(2 * time.Second)
synctest.Wait()
assert.Positive(t, calls.Load())
})
}
+60
View File
@@ -0,0 +1,60 @@
package agent
import (
"context"
"log/slog"
"math/rand"
"time"
)
func (pm *MonitorManager) startMonitor(task *monitorTask) {
interval := time.Duration(task.config.Interval) * time.Second
if interval < time.Second {
interval = 30 * time.Second
}
delay := getStagger(interval.Milliseconds())
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
go runMonitorSchedule(task.ctx, interval, delay, func() {
if _, allowed := task.resumeGuard.snapshot(); allowed {
task.runProbe(pm.probe)
}
})
}
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
// naturally drop missed ticks rather than building an execution backlog.
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
return
case <-timer.C:
}
if ctx.Err() != nil {
return
}
run()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if ctx.Err() != nil {
return
}
run()
}
}
}
// getStagger returns an initial delay between half an interval and one interval.
func getStagger(intervalMilli int64) time.Duration {
delay := rand.Intn(int(intervalMilli))
if delay < int(intervalMilli)/2 {
delay += int(intervalMilli) / 2
}
return time.Duration(delay) * time.Millisecond
}
+167
View File
@@ -0,0 +1,167 @@
//go:build testing
package agent
import (
"context"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorScheduleTiming(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
synctest.Wait()
time.Sleep(4 * time.Second)
synctest.Wait()
assert.Equal(t, 0, int(calls.Load()))
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(10 * time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
})
}
func TestMonitorScheduleSlowProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var calls atomic.Int32
release := make(chan struct{})
go runMonitorSchedule(ctx, time.Second, 0, func() {
calls.Add(1)
select {
case <-release:
case <-ctx.Done():
}
})
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
time.Sleep(time.Minute)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
close(release)
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
time.Sleep(time.Second)
synctest.Wait()
assert.Equal(t, 2, int(calls.Load()))
cancel()
synctest.Wait()
})
}
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
var calls atomic.Int32
release := make(chan struct{})
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
assert.Equal(t, cfg, config)
calls.Add(1)
<-release
return 42, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
pm.monitors[cfg.ID] = task
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
synctest.Wait()
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
assert.Equal(t, 1, int(calls.Load()))
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
close(release)
synctest.Wait()
first, second := <-results, <-results
require.NotNil(t, first)
require.NotNil(t, second)
assert.Equal(t, int64(42), first.AvgResponse)
assert.Equal(t, first, second)
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
assert.Len(t, task.history.samples, 1)
// A later explicit request must still perform a fresh probe.
_, err := pm.UpsertMonitor(cfg, true)
require.NoError(t, err)
assert.Equal(t, 2, int(calls.Load()))
assert.Len(t, task.history.samples, 2)
})
}
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
cfg := monitor.Config{ID: "test", Interval: 10}
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
if config.Interval == 10 {
<-ctx.Done()
return 0, ctx.Err()
}
return 30, nil
})
defer pm.Stop()
task := newMonitorTask(cfg)
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
results := make(chan *monitor.Result, 2)
for range 2 {
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
results <- result
}()
}
synctest.Wait()
updated := cfg
updated.Interval = 20
result, err := pm.UpsertMonitor(updated, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, int64(20), result.AvgResponse)
assert.Zero(t, result.PacketLoss)
synctest.Wait()
assert.Nil(t, <-results)
assert.Nil(t, <-results)
assert.Len(t, task.history.samples, 1)
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
})
}
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
defer cancel()
<-ctx.Done()
return 0, ctx.Err()
})
defer pm.Stop()
start := time.Now()
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
require.NoError(t, err)
require.NotNil(t, result)
assert.Equal(t, 3*time.Second, time.Since(start))
assert.Equal(t, 100.0, result.PacketLoss)
assert.NoError(t, pm.monitors["test"].ctx.Err())
})
}
+116
View File
@@ -0,0 +1,116 @@
package agent
import (
"context"
"log/slog"
"sync"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
)
const monitorFailureLogInterval = 5 * time.Minute
// monitorTask coordinates a probe and its history for one immutable configuration.
type monitorTask struct {
config monitor.Config
ctx context.Context
cancel context.CancelFunc
history *monitorHistory
resumeGuard *monitorResumeGuard
runMu sync.Mutex
inflight *monitorRun
lastFailureLog int64 // Unix nanoseconds
}
type monitorRun struct {
done chan struct{}
result *monitor.Result // published by closing done; never mutated afterwards
}
func newMonitorTask(config monitor.Config) *monitorTask {
ctx, cancel := context.WithCancel(context.Background())
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
// Serialize cancellation with publication, so canceled probes cannot enter
// history copied into a replacement task.
task.cancel = func() {
task.runMu.Lock()
cancel()
task.runMu.Unlock()
}
return task
}
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
task := newMonitorTask(config)
if existing != nil {
task.history = existing.history.clone()
}
return task
}
// runProbe shares an in-flight check between scheduled and immediate requests.
// Every completed check contributes exactly one sample, regardless of how many
// callers were waiting for it. No task or history lock is held during network I/O.
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
task.runMu.Lock()
if task.ctx.Err() != nil {
task.runMu.Unlock()
return nil
}
if run := task.inflight; run != nil {
task.runMu.Unlock()
select {
case <-task.ctx.Done():
return nil
case <-run.done:
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
}
run := &monitorRun{done: make(chan struct{})}
task.inflight = run
task.runMu.Unlock()
generation, _ := task.resumeGuard.snapshot()
responseUs, err := probe(task.ctx, task.config)
var logFailure bool
task.runMu.Lock()
currentGeneration, _ := task.resumeGuard.snapshot()
if task.ctx.Err() == nil && generation == currentGeneration {
now := time.Now()
if err != nil {
responseUs = -1
logAt := now.UnixNano()
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
logFailure = true
task.lastFailureLog = logAt
}
} else {
task.lastFailureLog = 0
}
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
run.result = &result
}
task.inflight = nil
close(run.done)
task.runMu.Unlock()
if logFailure {
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
}
if task.ctx.Err() != nil {
return nil
}
return copyMonitorResult(run.result)
}
func copyMonitorResult(result *monitor.Result) *monitor.Result {
if result == nil {
return nil
}
copy := *result
return &copy
}
+79
View File
@@ -0,0 +1,79 @@
//go:build testing
package agent
import (
"bytes"
"context"
"errors"
"log/slog"
"testing"
"testing/synctest"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestMonitorFailureLogCooldown(t *testing.T) {
var logs bytes.Buffer
previous := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
t.Cleanup(func() { slog.SetDefault(previous) })
synctest.Test(t, func(t *testing.T) {
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
defer task.cancel()
failure := errors.New("connection refused")
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
var samples int64
check := func(wantLog bool) {
t.Helper()
logs.Reset()
result := task.runProbe(probe)
require.NotNil(t, result)
samples++
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
if !wantLog {
assert.Empty(t, logs.String())
} else {
assert.Contains(t, logs.String(), `msg="monitor failed"`)
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
}
}
check(true)
check(false)
time.Sleep(5*time.Minute - time.Nanosecond)
check(false)
time.Sleep(time.Nanosecond)
check(true)
check(false)
time.Sleep(5 * time.Minute)
check(true)
check(false)
// Recovery clears the cooldown.
failure = nil
check(false)
failure = errors.New("connection refused again")
check(true)
// Another monitor has its own cooldown.
other := newMonitorTask(task.config)
defer other.cancel()
logs.Reset()
require.NotNil(t, other.runProbe(probe))
assert.Contains(t, logs.String(), `msg="monitor failed"`)
// A canceled probe must not publish a failure or emit a warning.
logs.Reset()
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
other.cancel()
return -1, context.Canceled
})
assert.Nil(t, result)
assert.Empty(t, logs.String())
})
}
+524
View File
@@ -0,0 +1,524 @@
package agent
import (
"context"
"encoding/binary"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/dns/dnsmessage"
)
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(30), result.AvgResponse)
assert.Equal(t, int64(25), result.AvgResponse1h)
assert.Equal(t, int64(30), result.MinResponse)
assert.Equal(t, int64(10), result.MinResponse1h)
assert.Equal(t, int64(30), result.MaxResponse)
assert.Equal(t, int64(40), result.MaxResponse1h)
assert.Equal(t, 50.0, result.PacketLoss)
assert.Equal(t, 20.0, result.PacketLoss1h)
}
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
now := time.Now().UTC()
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
result, ok := results["monitor-1"]
require.True(t, ok)
assert.Equal(t, int64(0), result.AvgResponse)
assert.Equal(t, int64(0), result.AvgResponse1h)
assert.Equal(t, int64(0), result.MinResponse)
assert.Equal(t, int64(0), result.MinResponse1h)
assert.Equal(t, int64(0), result.MaxResponse)
assert.Equal(t, int64(0), result.MaxResponse1h)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
}
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
assert.Equal(t, "monitor-1", cfg.ID)
}
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
pm := newMonitorManager()
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
defer pm.Stop()
_, validExists := pm.monitors[validCfg.ID]
_, invalidExists := pm.monitors[invalidCfg.ID]
assert.True(t, validExists)
assert.False(t, invalidExists)
}
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
keptTask := newMonitorTask(keepCfg)
removedTask := newMonitorTask(removeCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
keepCfg.ID: keptTask,
removeCfg.ID: removedTask,
}
pm.SyncMonitors([]monitor.Config{keepCfg})
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
_, exists := pm.monitors[removeCfg.ID]
assert.False(t, exists)
select {
case <-removedTask.ctx.Done():
default:
t.Fatal("expected removed monitor task to be cancelled")
}
select {
case <-keptTask.ctx.Done():
t.Fatal("expected existing monitor task to remain active")
default:
}
}
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
originalTask := newMonitorTask(originalCfg)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{
originalCfg.ID: originalTask,
}
pm.SyncMonitors([]monitor.Config{updatedCfg})
defer pm.Stop()
restartedTask := pm.monitors[updatedCfg.ID]
assert.NotSame(t, originalTask, restartedTask)
assert.Equal(t, updatedCfg, restartedTask.config)
select {
case <-originalTask.ctx.Done():
default:
t.Fatal("expected changed monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
pm := &MonitorManager{
monitors: make(map[string]*monitorTask),
probe: networkMonitorProbe(server.Client()),
}
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
RunNow: true,
})
defer pm.Stop()
require.NoError(t, err)
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
assert.Equal(t, 0.0, resp.Result.PacketLoss)
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
task := pm.monitors["monitor-1"]
require.NotNil(t, task)
task.history.mu.Lock()
defer task.history.mu.Unlock()
require.Len(t, task.history.samples, 1)
}
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
now := time.Now().UTC()
existingTask := newMonitorTask(originalCfg)
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
result, err := pm.UpsertMonitor(updatedCfg, false)
defer pm.Stop()
require.NoError(t, err)
assert.Nil(t, result)
updatedTask := pm.monitors[updatedCfg.ID]
require.NotNil(t, updatedTask)
assert.NotSame(t, existingTask, updatedTask)
assert.Equal(t, updatedCfg, updatedTask.config)
updatedTask.history.mu.Lock()
defer updatedTask.history.mu.Unlock()
require.Len(t, updatedTask.history.samples, 1)
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
agg := updatedTask.history.aggregateLocked(time.Hour, now)
require.True(t, agg.hasData())
assert.Equal(t, int64(2), agg.totalCount)
assert.Equal(t, int64(2), agg.successCount)
assert.Equal(t, int64(18), agg.avgResponse())
select {
case <-existingTask.ctx.Done():
default:
t.Fatal("expected original monitor task to be cancelled")
}
}
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
task := newMonitorTask(config)
pm := newMonitorManager()
pm.monitors = map[string]*monitorTask{config.ID: task}
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: config.ID},
})
require.NoError(t, err)
_, exists := pm.monitors[config.ID]
assert.False(t, exists)
select {
case <-task.ctx.Done():
default:
t.Fatal("expected deleted monitor task to be cancelled")
}
}
func TestMonitorManagerGetRandomDelay(t *testing.T) {
for i := 1000; i < 360_000; i += 1000 {
delay := getStagger(int64(i))
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
}
}
func TestMonitorHTTP(t *testing.T) {
t.Run("success", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("server error", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "boom", http.StatusInternalServerError)
}))
defer server.Close()
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCP(t *testing.T) {
t.Run("success", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
accepted := make(chan struct{})
go func() {
defer close(accepted)
conn, err := listener.Accept()
if err == nil {
_ = conn.Close()
}
}()
port := uint16(listener.Addr().(*net.TCPAddr).Port)
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
<-accepted
})
t.Run("connection failure", func(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
port := uint16(listener.Addr().(*net.TCPAddr).Port)
require.NoError(t, listener.Close())
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorTCPAddressFallback(t *testing.T) {
for _, tc := range []struct {
name string
ips []string
loss bool
}{
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
} {
t.Run(tc.name, func(t *testing.T) {
listener, err := net.Listen("tcp4", "127.0.0.1:0")
require.NoError(t, err)
defer listener.Close()
original := net.DefaultResolver
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
defer func() { net.DefaultResolver = original }()
// Verify the resolver preserves the intended order, so success cannot
// accidentally bypass the failed first address in the regression case.
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
require.NoError(t, err)
require.Equal(t, tc.ips, ips)
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
if tc.loss {
require.Error(t, err)
assert.Equal(t, int64(-1), responseUs)
} else {
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
}
})
}
}
// tcpMonitorTestResolver supplies multiple A records without external DNS.
func tcpMonitorTestResolver(ips []string) *net.Resolver {
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
client, server := net.Pipe()
go func() {
defer server.Close()
// net.Resolver uses TCP framing when its connection is not a PacketConn.
var size uint16
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
return
}
packet := make([]byte, size)
if _, err := io.ReadFull(server, packet); err != nil {
return
}
var msg dnsmessage.Message
if err := msg.Unpack(packet); err != nil {
return
}
msg.Header.Response = true
msg.Header.RecursionAvailable = true
for _, question := range msg.Questions {
if question.Type != dnsmessage.TypeA {
continue
}
for _, ip := range ips {
msg.Answers = append(msg.Answers, dnsmessage.Resource{
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
})
}
}
packet, err := msg.Pack()
if err != nil {
return
}
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
_, _ = server.Write(append(response, packet...))
}()
return client, nil
}}
}
func TestMonitorDNS(t *testing.T) {
t.Run("success", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "localhost")
require.NoError(t, err)
assert.GreaterOrEqual(t, responseUs, int64(0))
})
t.Run("lookup failure", func(t *testing.T) {
responseUs, err := monitorDNS(context.Background(), "")
assert.Equal(t, int64(-1), responseUs)
require.Error(t, err)
})
}
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
t.Run(action, func(t *testing.T) {
started := make(chan struct{})
canceled := make(chan struct{})
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
close(started)
select {
case <-r.Context().Done():
close(canceled)
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
defer pm.Stop()
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
task := newMonitorTask(cfg)
// Seed history to ensure a canceled RunNow does not return an old result.
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
pm.monitors[cfg.ID] = task
done := make(chan *monitor.Result, 1)
go func() {
result, _ := pm.UpsertMonitor(cfg, true)
done <- result
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("probe did not start")
}
updated := cfg
updated.Interval--
switch action {
case "stop":
pm.Stop()
case "delete":
pm.DeleteMonitor(cfg.ID)
case "upsert":
_, err := pm.UpsertMonitor(updated, false)
require.NoError(t, err)
case "sync replace":
pm.SyncMonitors([]monitor.Config{updated})
case "sync remove":
pm.SyncMonitors(nil)
}
select {
case <-canceled:
case <-time.After(time.Second):
t.Fatal("active HTTP request was not canceled")
}
select {
case result := <-done:
assert.Nil(t, result)
case <-time.After(time.Second):
t.Fatal("RunNow did not return after cancellation")
}
task.history.mu.Lock()
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
task.history.mu.Unlock()
})
}
}
func TestMonitorResolutionCancellation(t *testing.T) {
for _, protocol := range []string{"tcp", "dns", "icmp"} {
t.Run(protocol, func(t *testing.T) {
started := make(chan struct{}, 1)
original := net.DefaultResolver
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
select {
case started <- struct{}{}:
default:
}
<-ctx.Done()
return nil, ctx.Err()
}}
defer func() { net.DefaultResolver = original }()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() {
var err error
switch protocol {
case "tcp":
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
case "dns":
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
case "icmp":
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
}
done <- err
}()
select {
case <-started:
case <-time.After(time.Second):
t.Fatal("lookup did not start")
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(time.Second):
t.Fatal("lookup did not cancel")
}
})
}
}
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
release := make(chan struct{})
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
select {
case <-r.Context().Done():
case <-release:
}
}))
defer server.Close()
defer close(release)
pm := newMonitorManager()
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
defer task.cancel()
result := task.runProbe(pm.probe)
require.NotNil(t, result)
assert.Equal(t, 100.0, result.PacketLoss)
assert.Equal(t, 100.0, result.PacketLoss1h)
require.Len(t, task.history.samples, 1)
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
}
+4 -1
View File
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
const ( const (
// Version is the current version of the application. // Version is the current version of the application.
Version = "0.19.0" Version = "0.20.0"
// AppName is the name of the application. // AppName is the name of the application.
AppName = "beszel" AppName = "beszel"
) )
@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests. // MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
var MinVersionZfsData = semver.MustParse("0.18.9") var MinVersionZfsData = semver.MustParse("0.18.9")
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
+21 -15
View File
@@ -7,22 +7,23 @@ require (
github.com/coreos/go-systemd/v22 v22.7.0 github.com/coreos/go-systemd/v22 v22.7.0
github.com/distribution/reference v0.6.0 github.com/distribution/reference v0.6.0
github.com/ebitengine/purego v0.11.0 github.com/ebitengine/purego v0.11.0
github.com/fxamacker/cbor/v2 v2.9.3 github.com/fxamacker/cbor/v2 v2.9.4
github.com/gliderlabs/ssh v0.3.8 github.com/gliderlabs/ssh v0.3.8
github.com/lxzan/gws v1.10.1 github.com/lxzan/gws v1.10.2
github.com/nicholas-fedor/shoutrrr v0.20.0 github.com/nicholas-fedor/shoutrrr v0.21.0
github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest v1.0.0
github.com/pocketbase/dbx v1.12.0 github.com/pocketbase/dbx v1.12.0
github.com/pocketbase/pocketbase v0.40.2 github.com/pocketbase/pocketbase v0.40.4
github.com/shirou/gopsutil/v4 v4.26.8 github.com/shirou/gopsutil/v4 v4.26.8
github.com/spf13/cast v1.10.0 github.com/spf13/cast v1.10.0
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10 github.com/spf13/pflag v1.0.10
github.com/stretchr/testify v1.12.1 github.com/stretchr/testify v1.12.1
golang.org/x/crypto v0.56.0 golang.org/x/crypto v0.57.0
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
golang.org/x/net v0.58.0 golang.org/x/net v0.59.0
golang.org/x/sys v0.47.0 golang.org/x/oauth2 v0.37.0
golang.org/x/sys v0.48.0
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
howett.net/plist v1.0.1 howett.net/plist v1.0.1
) )
@@ -32,7 +33,7 @@ require (
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/disintegration/imaging v1.6.2 // indirect github.com/disintegration/imaging v1.6.2 // indirect
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.1.0 // indirect
github.com/eclipse/paho.golang v0.23.0 // indirect github.com/eclipse/paho.golang v0.23.0 // indirect
github.com/fatih/color v1.19.0 // indirect github.com/fatih/color v1.19.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect
@@ -51,18 +52,23 @@ require (
github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect
github.com/tklauser/numcpus v0.12.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/image v0.45.0 // indirect golang.org/x/image v0.46.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/mod v0.41.0 // indirect
golang.org/x/sync v0.22.0 // indirect golang.org/x/sync v0.23.0 // indirect
golang.org/x/term v0.45.0 // indirect golang.org/x/term v0.46.0 // indirect
golang.org/x/text v0.41.0 // indirect golang.org/x/text v0.42.0 // indirect
golang.org/x/tools v0.50.0 // indirect
mellium.im/reader v0.1.0 // indirect
mellium.im/sasl v0.3.2 // indirect
mellium.im/xmlstream v0.15.4 // indirect
mellium.im/xmpp v0.23.0 // indirect
modernc.org/libc v1.74.4 // indirect modernc.org/libc v1.74.4 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect modernc.org/memory v1.12.1 // indirect
+44 -36
View File
@@ -19,8 +19,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI= github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk= github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
@@ -31,8 +31,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q= github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo=
github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ= github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus= github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
@@ -77,18 +77,18 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0= github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
github.com/lxzan/gws v1.10.1 h1:1xG+tDOV0lgDeVPf0wNT74u3cn0K3LpcavRrTPTrMwQ= github.com/lxzan/gws v1.10.2 h1:htReTvcY89iMk1ScVtUbk6J96kIZWaafj6r/lasK/NA=
github.com/lxzan/gws v1.10.1/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc= github.com/lxzan/gws v1.10.2/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/nicholas-fedor/shoutrrr v0.20.0 h1:hMAxIYlfAeZ1FcTDgU0kUOvVXUsOirWo8IWlnzGLkac= github.com/nicholas-fedor/shoutrrr v0.21.0 h1:as/mEwdaZMijCVu0FkTUEXashhvC3Y7C5g9dsXMcmQc=
github.com/nicholas-fedor/shoutrrr v0.20.0/go.mod h1:hgde37yNWCXh8+N6WemyDRMNYLOFTf326GsBx8Z7CFA= github.com/nicholas-fedor/shoutrrr v0.21.0/go.mod h1:dgg4kJv9K0tLXBH/1TXiSibNbM2hcd4SK6xb0sglyU4=
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw= github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU= github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -98,10 +98,10 @@ github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs= github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g= github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
github.com/pocketbase/pocketbase v0.40.2 h1:7gTqvt3bmilkphyZZ1QNhX19g3BXHqT7ynDyU81RVT4= github.com/pocketbase/pocketbase v0.40.4 h1:0SvSUreR3NhUMCs9LchE59oEG53efZ3cKiMGyAGBN9U=
github.com/pocketbase/pocketbase v0.40.2/go.mod h1:jc3YuyToy+ZXM4CeO7uSCN/htgR8yv+tjSE3eJZ8eh8= github.com/pocketbase/pocketbase v0.40.4/go.mod h1:2mU+80FLiY1fb13WZRg8Xx/lKg4nTjgKVJMigyRH6k0=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@@ -136,37 +136,37 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0= golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4= golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -176,6 +176,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM= howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI=
mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4=
mellium.im/xmpp v0.23.0 h1:rvKvOvMdIURCLaAWEJN8J0QpO3AJYCDKjxLsqtTPSjY=
mellium.im/xmpp v0.23.0/go.mod h1:GHDKlKKQe0LNmD9YqExyxnFEEBiz84KGqnfiA2VNzb8=
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
+9 -6
View File
@@ -20,10 +20,11 @@ type hubLike interface {
} }
type AlertManager struct { type AlertManager struct {
hub hubLike hub hubLike
stopOnce sync.Once stopOnce sync.Once
pendingAlerts sync.Map pendingAlerts sync.Map
alertsCache *AlertsCache alertsCache *AlertsCache
networkMonitors *networkMonitorCache
} }
type AlertMessageData struct { type AlertMessageData struct {
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
// NewAlertManager creates a new AlertManager instance. // NewAlertManager creates a new AlertManager instance.
func NewAlertManager(app hubLike) *AlertManager { func NewAlertManager(app hubLike) *AlertManager {
am := &AlertManager{ am := &AlertManager{
hub: app, hub: app,
alertsCache: NewAlertsCache(app), alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
} }
am.bindEvents() am.bindEvents()
return am return am
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
// Bind events to the alerts collection lifecycle // Bind events to the alerts collection lifecycle
func (am *AlertManager) bindEvents() { func (am *AlertManager) bindEvents() {
am.bindNetworkMonitorAlertEvents()
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate) am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete) am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert) am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
+7
View File
@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
return e.BadRequestError("Bad data", err) return e.BadRequestError("Bad data", err)
} }
if reqData.Name == alertNameNetworkMonitorLoss {
if reqData.Value < 0 || reqData.Value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
reqData.Min = 0
}
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts") alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
if err != nil { if err != nil {
return err return err
+15
View File
@@ -11,6 +11,7 @@ import (
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/synctest"
beszelTests "github.com/henrygd/beszel/internal/tests" beszelTests "github.com/henrygd/beszel/internal/tests"
pbTests "github.com/pocketbase/pocketbase/tests" pbTests "github.com/pocketbase/pocketbase/tests"
@@ -533,6 +534,20 @@ func TestSendTestNotification(t *testing.T) {
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} { for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
scenarios = append(scenarios, beszelTests.ApiScenario{ scenarios = append(scenarios, beszelTests.ApiScenario{
BeforeTestFunc: func(tb testing.TB, _ *pbTests.TestApp, e *core.ServeEvent) {
if !strings.HasPrefix(url, "mqtt://") {
return
}
// Keep the real MQTT rejection path, but advance its library's
// fixed timeout using virtual time instead of waiting 10 seconds.
e.Router.BindFunc(func(re *core.RequestEvent) error {
var err error
synctest.Test(tb.(*testing.T), func(t *testing.T) {
err = re.Next()
})
return err
})
},
Name: "readonly cannot send to " + url, Name: "readonly cannot send to " + url,
Method: http.MethodPost, Method: http.MethodPost,
URL: "/api/beszel/test-notification", URL: "/api/beszel/test-notification",
+38 -12
View File
@@ -1,6 +1,7 @@
package alerts package alerts
import ( import (
"sync"
"time" "time"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
@@ -18,6 +19,9 @@ type CachedAlertData struct {
Triggered bool Triggered bool
Min uint8 Min uint8
PendingSince time.Time PendingSince time.Time
// Immutable after publication; decoded only when the alert record changes.
MonitorStates map[string]string
MonitorStatesValid bool
// Created types.DateTime // Created types.DateTime
} }
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
a.Triggered = record.GetBool("triggered") a.Triggered = record.GetBool("triggered")
a.Min = uint8(record.GetInt("min")) a.Min = uint8(record.GetInt("min"))
a.PendingSince = record.GetDateTime("pending_since").Time() a.PendingSince = record.GetDateTime("pending_since").Time()
if a.Name == alertNameNetworkMonitorLoss {
var state networkMonitorAlertState
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
a.MonitorStates = state.Monitors
}
// a.Created = record.GetDateTime("created") // a.Created = record.GetDateTime("created")
} }
// AlertsCache provides an in-memory cache for system alerts. // AlertsCache provides an in-memory cache for system alerts.
type AlertsCache struct { type AlertsCache struct {
// Serialize lazy loads with updates so a late load cannot replace newer state.
loadMu sync.Mutex
app core.App app core.App
store *store.Store[string, *store.Store[string, CachedAlertData]] store *store.Store[string, *store.Store[string, CachedAlertData]]
populated bool populated bool
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
// PopulateFromDB clears current entries and loads all alerts from the database into the cache. // PopulateFromDB clears current entries and loads all alerts from the database into the cache.
func (c *AlertsCache) PopulateFromDB(force bool) error { func (c *AlertsCache) PopulateFromDB(force bool) error {
c.loadMu.Lock()
defer c.loadMu.Unlock()
if !force && c.populated { if !force && c.populated {
return nil return nil
} }
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
} }
c.store.RemoveAll() c.store.RemoveAll()
for _, record := range records { for _, record := range records {
c.Update(record) c.update(record)
} }
c.populated = true c.populated = true
return nil return nil
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
// Update adds or updates an alert record in the cache. // Update adds or updates an alert record in the cache.
func (c *AlertsCache) Update(record *core.Record) { func (c *AlertsCache) Update(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
c.update(record)
}
func (c *AlertsCache) update(record *core.Record) {
systemID := record.GetString("system") systemID := record.GetString("system")
if systemID == "" { if systemID == "" {
return return
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
// Delete removes an alert record from the cache. // Delete removes an alert record from the cache.
func (c *AlertsCache) Delete(record *core.Record) { func (c *AlertsCache) Delete(record *core.Record) {
c.loadMu.Lock()
defer c.loadMu.Unlock()
systemID := record.GetString("system") systemID := record.GetString("system")
if systemID == "" { if systemID == "" {
return return
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData { func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
systemStore, ok := c.store.GetOk(systemID) systemStore, ok := c.store.GetOk(systemID)
if !ok { if !ok {
// Populate cache for this system c.loadMu.Lock()
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID})) defer c.loadMu.Unlock()
if err != nil { systemStore, ok = c.store.GetOk(systemID)
return nil if !ok {
// Populate cache for this system
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
if err != nil {
return nil
}
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
} }
systemStore = store.New(map[string]CachedAlertData{})
for _, record := range records {
var ca CachedAlertData
ca.PopulateFromRecord(record)
systemStore.Set(record.Id, ca)
}
c.store.Set(systemID, systemStore)
} }
all := systemStore.GetAll() all := systemStore.GetAll()
alerts := make([]CachedAlertData, 0, len(all)) alerts := make([]CachedAlertData, 0, len(all))
+10
View File
@@ -9,6 +9,12 @@ import (
// On triggered alert record delete, set matching alert history record to resolved // On triggered alert record delete, set matching alert history record to resolved
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error { func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
return err
}
return e.Next()
}
if !e.Record.GetBool("triggered") { if !e.Record.GetBool("triggered") {
return e.Next() return e.Next()
} }
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
// On alert record update, update alert history record // On alert record update, update alert history record
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error { func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
// Network monitor incidents have separate history entries per monitor.
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
return e.Next()
}
original := e.Record.Original() original := e.Record.Original()
new := e.Record new := e.Record
+269
View File
@@ -0,0 +1,269 @@
package alerts
import (
"database/sql"
"errors"
"fmt"
"math"
"net"
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
// networkMonitorAlertState is this alert type's persisted runtime state.
// Monitor IDs map to their open history entries independently of history retention.
type networkMonitorAlertState struct {
Monitors map[string]string `json:"monitors"`
}
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
// Hidden fields are still writable through the record API unless protected.
protectState := func(e *core.RecordRequestEvent) error {
e.Record.Set("state", e.Record.Original().Get("state"))
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
}
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
return e.ForbiddenError("You do not have access to this system", nil)
}
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
value := e.Record.GetFloat("value")
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
}
e.Record.Set("min", 0)
}
return e.Next()
}
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
cleanup := func(e *core.RecordEvent) error {
if err := e.Next(); err != nil {
return err
}
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
}
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
return e.Next()
}
return cleanup(e)
})
}
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
// using its exact payload (dashboard requests can replace the cached payload).
// Omitted results and disconnected systems never imply recovery.
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
if systemRecord.GetString("status") != "up" {
return nil
}
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
if len(alerts) == 0 {
return nil
}
monitors, err := am.networkMonitors.get(systemRecord.Id)
if err != nil {
return err
}
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
return nil
}
// The cache only predicts a transition. Reload and recheck under the DB
// transaction before persisting, including current system/monitor status.
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
}
// networkMonitorTransitionPending does no IO and never mutates cached maps.
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
for _, alert := range alerts {
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
return true
}
for id := range alert.MonitorStates {
if _, enabled := monitors[id]; !enabled {
return true
}
}
for id, result := range results {
interval, enabled := monitors[id]
if !enabled || !monitorResultReady(result, interval, now) {
continue
}
_, active := alert.MonitorStates[id]
if (result.PacketLoss1h > alert.Value) != active {
return true
}
}
}
return false
}
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
var messages []AlertMessageData
err := app.RunInTransaction(func(tx core.App) error {
// Read configuration inside the transaction so concurrent threshold changes,
// disabling, and evaluations cannot overwrite each other's incident state.
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
if err != nil || len(alerts) == 0 {
return err
}
system, err := tx.FindRecordById("systems", systemID)
if errors.Is(err, sql.ErrNoRows) {
// System deletion cascades to its alerts.
return nil
}
if err != nil {
return err
}
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
if err != nil {
return err
}
enabled := make(map[string]*core.Record, len(monitors))
for _, m := range monitors {
enabled[m.Id] = m
}
now := time.Now()
for _, alert := range alerts {
var state networkMonitorAlertState
if err := alert.UnmarshalJSONField("state", &state); err != nil {
return err
}
states := state.Monitors
if states == nil {
states = map[string]string{}
}
changed := false
// Removing or disabling a monitor closes its incident silently.
for id, historyID := range states {
if _, ok := enabled[id]; !ok {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, id)
changed = true
}
}
if system.GetString("status") == "up" {
for _, m := range monitors {
result, ok := results[m.Id]
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
continue
}
historyID, active := states[m.Id]
triggered := result.PacketLoss1h > alert.GetFloat("value")
if triggered == active {
continue
}
label := m.GetString("target")
if m.GetString("protocol") == "tcp" {
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
}
if triggered {
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
history := core.NewRecord(collection)
history.Load(map[string]any{
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
})
if err := tx.Save(history); err != nil {
return err
}
states[m.Id] = history.Id
} else {
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
return err
}
delete(states, m.Id)
}
changed = true
state, comparison := "loss", "exceeds"
if !triggered {
state, comparison = "recovered", "is at or below"
}
messages = append(messages, AlertMessageData{
UserID: alert.GetString("user"), SystemID: systemID,
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
})
}
}
if changed || alert.GetBool("triggered") != (len(states) > 0) {
alert.Set("state", networkMonitorAlertState{Monitors: states})
alert.Set("triggered", len(states) > 0)
if err := tx.Save(alert); err != nil {
return err
}
}
}
return nil
})
if err != nil {
return err
}
// Match other alert types: persist transitions before delivery, and respect
// the user's existing notification destinations and quiet hours.
for _, message := range messages {
if err := am.SendAlert(message); err != nil {
app.Logger().Error("Failed to send network monitor alert", "err", err)
}
}
return nil
}
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
// Three completed attempts provide a short warm-up, including after an agent
// restart.
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
return false
}
// Never interpret an empty one-hour window as zero loss.
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
age := now.Sub(time.UnixMilli(result.LastProbeAt))
return age >= -time.Minute && age <= maxAge
}
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
record, err := app.FindRecordById("alerts_history", id)
if errors.Is(err, sql.ErrNoRows) {
// History can be purged independently.
return nil
}
if err != nil {
return err
}
if !record.GetDateTime("resolved").IsZero() {
return nil
}
record.Set("resolved", now.UTC())
return app.Save(record)
}
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
if err != nil {
return err
}
for _, record := range records {
record.Set("resolved", time.Now().UTC())
if err := app.Save(record); err != nil {
return err
}
}
return nil
}
@@ -0,0 +1,513 @@
//go:build testing
package alerts_test
import (
"sync"
"sync/atomic"
"testing"
"time"
"github.com/henrygd/beszel/internal/alerts"
"github.com/henrygd/beszel/internal/entities/monitor"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
pbTests "github.com/pocketbase/pocketbase/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
t.Helper()
hub, system, alert := systemdTestSetup(t, false)
t.Cleanup(hub.Cleanup)
alert.Set("name", "NetworkMonitorLoss")
alert.Set("value", 5)
require.NoError(t, hub.Save(alert))
var monitors []*core.Record
for _, name := range []string{"gateway", "website"} {
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
monitors = append(monitors, record)
}
// Avoid starting a system update worker in tests.
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
system.Set("status", "up")
return hub, system, alert, monitors
}
func monitorResult(loss float64) monitor.Result {
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
}
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
count := hub.TestMailer.TotalSend()
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
check := func(active bool, open, sent int) {
t.Helper()
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.Equal(t, active, record.GetBool("triggered"))
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
assert.EqualValues(t, open, total)
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
message := hub.TestMailer.Messages()[count]
assert.Contains(t, message.Text, "gateway.example.com")
assert.Contains(t, message.Text, "10.00%")
assert.Contains(t, message.Text, "5.00%")
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 1)
// Persisted monitor state prevents duplicate notifications after a hub restart.
am = alerts.NewTestAlertManagerWithoutWorker(hub)
results[monitors[1].Id] = monitorResult(20)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 2, 2)
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(true, 1, 3)
results[monitors[1].Id] = monitorResult(0)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
check(false, 0, 4)
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 2)
for _, history := range histories {
assert.NotEmpty(t, history.GetString("monitor_name"))
}
}
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
for _, tc := range []struct {
protocol, target, label string
port int
}{
{"icmp", "gateway.example.com", "gateway.example.com", 0},
{"http", "https://example.com/health", "https://example.com/health", 0},
{"tcp", "example.com", "example.com:8443", 8443},
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
} {
t.Run(tc.label, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
m := monitors[0]
m.Set("protocol", tc.protocol)
m.Set("target", tc.target)
m.Set("port", tc.port)
require.NoError(t, hub.Save(m))
am := alerts.NewTestAlertManagerWithoutWorker(hub)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
require.NoError(t, err)
require.Len(t, histories, 1)
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
})
}
}
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
apply := func(loss float64) {
result := monitorResult(loss)
results := map[string]monitor.Result{monitors[0].Id: result}
switch scenario {
case "missing":
results = nil
case "stale":
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
case "expired hourly window":
monitors[0].Set("interval", 3600)
require.NoError(t, hub.Save(monitors[0]))
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "future":
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
results[monitors[0].Id] = result
case "warmup":
result.SampleCount = 2
results[monitors[0].Id] = result
case "no probes":
result.SampleCount = 0
results[monitors[0].Id] = result
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
case "paused":
record, err := hub.FindRecordById("systems", system.Id)
require.NoError(t, err)
record.Set("status", "paused")
require.NoError(t, hub.Save(record))
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
}
count := hub.TestMailer.TotalSend()
apply(100)
assert.Equal(t, count, hub.TestMailer.TotalSend())
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
apply(0)
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
})
}
}
func TestNetworkMonitorAlertCleanup(t *testing.T) {
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
t.Run(scenario, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
switch scenario {
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "delete monitor":
require.NoError(t, hub.Delete(monitors[0]))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "delete system":
require.NoError(t, hub.Delete(system))
case "purge history":
history, err := hub.FindAllRecords("alerts_history")
require.NoError(t, err)
for _, record := range history {
require.NoError(t, hub.Delete(record))
}
}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
require.NoError(t, err)
if scenario == "disable monitor" || scenario == "delete monitor" {
assert.EqualValues(t, 1, open)
record, err := hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.True(t, record.GetBool("triggered"))
require.NoError(t, hub.Delete(monitors[1]))
record, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, record.GetBool("triggered"))
} else {
assert.Zero(t, open)
}
})
}
}
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
require.NoError(t, err)
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
require.NoError(t, err)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
other, err = hub.FindRecordById("alerts", other.Id)
require.NoError(t, err)
assert.False(t, other.GetBool("triggered"))
// Editing the threshold re-evaluates on the next batch, without losing state.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
assert.False(t, alert.GetBool("triggered"))
}
func TestNetworkMonitorAlertAPI(t *testing.T) {
for _, tc := range []struct {
name string
value float64
direct, denied, patch bool
status int
}{
{name: "zero threshold", value: 0, status: 200},
{name: "fractional threshold", value: 5.5, status: 200},
{name: "negative threshold", value: -1, status: 400},
{name: "unreachable threshold", value: 100, status: 400},
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
{name: "direct private state", value: 5, direct: true, status: 200},
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
} {
t.Run(tc.name, func(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
owner := user.Id
if tc.denied {
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
require.NoError(t, err)
owner = other.Id
}
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
url, method := "/api/beszel/user-alerts", "POST"
if tc.direct {
url = "/api/collections/alerts/records"
body["system"], body["user"] = systems[0].Id, user.Id
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
}
if tc.patch {
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
require.NoError(t, err)
url += "/" + alert.Id
method = "PATCH"
body["triggered"] = false
}
content := `"success":true`
if tc.direct {
content = `"name":"NetworkMonitorLoss"`
}
if tc.status == 400 {
content = `"status":400`
}
if tc.status == 403 {
content = `"status":403`
}
scenario := beszelTests.ApiScenario{
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
}
scenario.Test(t)
records, err := hub.FindAllRecords("alerts")
require.NoError(t, err)
if tc.status != 200 || tc.denied {
assert.Empty(t, records)
return
}
require.Len(t, records, 1)
assert.Equal(t, tc.value, records[0].GetFloat("value"))
assert.Zero(t, records[0].GetInt("min"))
state := struct {
Monitors map[string]string `json:"monitors"`
}{}
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
states := state.Monitors
if tc.patch {
assert.Equal(t, map[string]string{"real": "history"}, states)
assert.True(t, records[0].GetBool("triggered"))
} else {
assert.Empty(t, states)
assert.False(t, records[0].GetBool("triggered"))
}
})
}
}
type monitorCountingHub struct {
*beszelTests.TestHub
transactions atomic.Int64
beforeTransaction func()
}
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
h.transactions.Add(1)
if h.beforeTransaction != nil {
h.beforeTransaction()
}
return h.App.RunInTransaction(fn)
}
// Count actual SQL on both DB connections, including queries through record APIs.
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
t.Helper()
count := &atomic.Int64{}
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
db := builder.(*dbx.DB)
old := db.LogFunc
db.LogFunc = func(string, ...any) { count.Add(1) }
t.Cleanup(func() { db.LogFunc = old })
}
return count
}
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
sql := monitorSQLCounter(t, hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
noWork := func() {
t.Helper()
sql.Store(0)
counted.transactions.Store(0)
for range 100 {
evaluate()
}
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
}
// One-time lazy loads are permitted, including on hub restart.
evaluate()
assert.Positive(t, sql.Load())
noWork()
// Realtime metric saves invoke record hooks but must not invalidate config.
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
require.NoError(t, err)
monitors[0] = fresh
monitors[0].Set("loss1h", 0)
monitors[0].Set("res", 100)
require.NoError(t, hub.Save(monitors[0]))
noWork()
results[monitors[0].Id] = monitorResult(10)
evaluate()
assert.Positive(t, sql.Load(), "transitions must still be persisted")
assert.EqualValues(t, 1, counted.transactions.Load())
noWork()
// Missing and stale observations must not enter the transaction either.
results = nil
noWork()
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
noWork()
results[monitors[0].Id] = monitorResult(0)
evaluate()
noWork()
require.NoError(t, hub.Delete(alert))
noWork()
}
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Widening the interval makes this observation fresh. A stale interval cache
// would miss the failure indefinitely, even though results keep arriving.
result := monitorResult(10)
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
results[monitors[0].Id] = result
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
monitors[0].Set("interval", 120)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
// Disable, then re-enable the same ID: its new failure must be detected.
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
monitors[0].Set("enabled", true)
require.NoError(t, hub.Save(monitors[0]))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
// A new monitor must also become eligible without restarting the hub.
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
})
require.NoError(t, err)
results[created.Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
// Threshold changes refresh cached config and preserve the active incidents.
alert, err = hub.FindRecordById("alerts", alert.Id)
require.NoError(t, err)
alert.Set("value", 15)
require.NoError(t, hub.Save(alert))
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
t.Run(change, func(t *testing.T) {
hub, system, alert, monitors := networkAlertSetup(t)
counted := &monitorCountingHub{TestHub: hub}
am := alerts.NewTestAlertManagerWithoutWorker(counted)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
count := hub.TestMailer.TotalSend()
// Change the DB after the cache predicts a transition, before its transaction.
counted.beforeTransaction = func() {
counted.beforeTransaction = nil
switch change {
case "threshold":
alert.Set("value", 20)
require.NoError(t, hub.Save(alert))
case "disable alert":
require.NoError(t, hub.Delete(alert))
case "disable monitor":
monitors[0].Set("enabled", false)
require.NoError(t, hub.Save(monitors[0]))
case "down":
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
require.NoError(t, err)
}
}
results[monitors[0].Id] = monitorResult(10)
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.EqualValues(t, 1, counted.transactions.Load())
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
})
}
}
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
count := hub.TestMailer.TotalSend()
var wg sync.WaitGroup
errs := make(chan error, 8)
for range 8 {
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
}
wg.Wait()
close(errs)
for err := range errs {
require.NoError(t, err)
}
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
hub, system, _, monitors := networkAlertSetup(t)
am := alerts.NewTestAlertManagerWithoutWorker(hub)
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
count := hub.TestMailer.TotalSend()
results[monitors[0].Id] = monitorResult(10)
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count, hub.TestMailer.TotalSend())
histories, err := hub.CountRecords("alerts_history")
require.NoError(t, err)
assert.Zero(t, histories)
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
require.NoError(t, err)
// A failed transition must not be published to the cache and mask the retry.
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
}
+1 -1
View File
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil return nil
} }
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName) alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
if len(alerts) == 0 { if len(alerts) == 0 {
return nil return nil
} }
+3 -2
View File
@@ -11,8 +11,9 @@ import (
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager { func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
return &AlertManager{ return &AlertManager{
hub: app, hub: app,
alertsCache: NewAlertsCache(app), alertsCache: NewAlertsCache(app),
networkMonitors: newNetworkMonitorCache(app),
} }
} }
+79
View File
@@ -0,0 +1,79 @@
package alerts
import (
"sync"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
// needed for the alert fast path. Names and targets are read only on transitions.
// Returned maps are immutable; configuration changes invalidate the whole entry.
type networkMonitorCache struct {
app core.App
mu sync.RWMutex
systems map[string]map[string]int
}
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
invalidate := func(e *core.RecordEvent) error {
c.invalidate(e.Record.GetString("system"))
return e.Next()
}
app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
old := e.Record.Original()
// Realtime metric saves also invoke this hook. They must not evict config.
if old.GetString("system") != e.Record.GetString("system") ||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
old.GetInt("interval") != e.Record.GetInt("interval") {
c.invalidate(old.GetString("system"))
c.invalidate(e.Record.GetString("system"))
}
return e.Next()
})
app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
c.invalidate(e.Record.Id)
return e.Next()
})
return c
}
func (c *networkMonitorCache) invalidate(systemID string) {
c.mu.Lock()
delete(c.systems, systemID)
c.mu.Unlock()
}
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
c.mu.RLock()
monitors, ok := c.systems[systemID]
c.mu.RUnlock()
if ok {
return monitors, nil
}
c.mu.Lock()
defer c.mu.Unlock()
if monitors, ok := c.systems[systemID]; ok {
return monitors, nil
}
// Keep the lock through the load so a concurrent config change cannot be
// invalidated first and then overwritten by the older query result.
var rows []struct {
ID string `db:"id"`
Interval int `db:"interval"`
}
if err := c.app.DB().Select("id", "interval").From("network_monitors").
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
return nil, err
}
monitors = make(map[string]int, len(rows))
for _, row := range rows {
monitors[row.ID] = row.Interval
}
c.systems[systemID] = monitors
return monitors, nil
}
+36 -29
View File
@@ -11,6 +11,7 @@ import (
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"testing/synctest"
"github.com/nicholas-fedor/shoutrrr/pkg/types" "github.com/nicholas-fedor/shoutrrr/pkg/types"
"golang.org/x/net/dns/dnsmessage" "golang.org/x/net/dns/dnsmessage"
@@ -178,39 +179,45 @@ func TestPublicNotificationTCP(t *testing.T) {
} { } {
t.Run(rawURL, func(t *testing.T) { t.Run(rawURL, func(t *testing.T) {
t.Parallel() t.Parallel()
// MQTT waits for a fixed library timeout even after a dial failure.
// Virtual time preserves the full send/cleanup path without that delay.
t.Run("internal destination", func(t *testing.T) { t.Run("internal destination", func(t *testing.T) {
err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test") synctest.Test(t, func(t *testing.T) {
if !errors.Is(err, errInternalDestination) { err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test")
t.Fatalf("expected blocked destination, got %v", err) if !errors.Is(err, errInternalDestination) {
} t.Fatalf("expected blocked destination, got %v", err)
}
})
}) })
t.Run("public destination uses injected dialer", func(t *testing.T) { t.Run("public destination uses injected dialer", func(t *testing.T) {
var calls atomic.Int32 synctest.Test(t, func(t *testing.T) {
stopped := errors.New("test dial stopped") var calls atomic.Int32
service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{ stopped := errors.New("test dial stopped")
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{
calls.Add(1) DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") { calls.Add(1)
t.Errorf("unexpected dial: %s %s", network, address) if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") {
} t.Errorf("unexpected dial: %s %s", network, address)
if err := checkNotificationAddress(address); err != nil { }
t.Error(err) if err := checkNotificationAddress(address); err != nil {
} t.Error(err)
return nil, stopped }
}, return nil, stopped
},
})
if err != nil {
t.Fatal(err)
}
if closer, ok := service.(io.Closer); ok {
defer closer.Close()
}
if err := service.Send("test", &types.Params{}); err == nil {
t.Fatal("expected dial failure")
}
if calls.Load() == 0 {
t.Fatal("custom dialer was not used")
}
}) })
if err != nil {
t.Fatal(err)
}
if closer, ok := service.(io.Closer); ok {
defer closer.Close()
}
if err := service.Send("test", &types.Params{}); err == nil {
t.Fatal("expected dial failure")
}
if calls.Load() == 0 {
t.Fatal("custom dialer was not used")
}
}) })
}) })
} }
+2
View File
@@ -24,6 +24,8 @@ const (
GetSystemdInfo GetSystemdInfo
// Request ZFS detail data from agent // Request ZFS detail data from agent
GetZfsData GetZfsData
// Sync network monitor configuration to agent
SyncNetworkMonitors
// Add new actions here... // Add new actions here...
) )
+1 -1
View File
@@ -12,7 +12,7 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
RUN rm -rf /tmp/* RUN rm -rf /tmp/*
+2 -2
View File
@@ -10,14 +10,14 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
RUN rm -rf /tmp/* RUN rm -rf /tmp/*
# -------------------------- # --------------------------
# Final image: default scratch-based agent # Final image: default scratch-based agent
# -------------------------- # --------------------------
FROM alpine:3.23 FROM alpine:3.24
COPY --from=builder /agent /agent COPY --from=builder /agent /agent
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read) # AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
+2 -2
View File
@@ -10,13 +10,13 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
# Final image # Final image
# Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume # Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume
# -------------------------- # --------------------------
FROM alpine:3.23 FROM alpine:3.24
COPY --from=builder /agent /agent COPY --from=builder /agent /agent
+1 -1
View File
@@ -10,7 +10,7 @@ COPY . ./
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
# Smartmontools builder stage # Smartmontools builder stage
+4 -2
View File
@@ -17,7 +17,7 @@ RUN set -eux; \
if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \ if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \
export GOARM="${TARGETVARIANT#v}"; \ export GOARM="${TARGETVARIANT#v}"; \
fi; \ fi; \
CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
# -------------------------- # --------------------------
@@ -70,7 +70,9 @@ RUN set -eux; \
# -------------------------- # --------------------------
FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder
RUN apt-get update && apt-get install -y --no-install-recommends \ # zfsutils-linux is distributed in Debian's contrib component.
RUN sed -i 's/Components: main/Components: main contrib/' /etc/apt/sources.list.d/debian.sources \
&& apt-get update && apt-get install -y --no-install-recommends \
zfsutils-linux \ zfsutils-linux \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
+1 -1
View File
@@ -17,7 +17,7 @@ RUN update-ca-certificates
# Build # Build
ARG TARGETOS TARGETARCH ARG TARGETOS TARGETARCH
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub
# ? ------------------------- # ? -------------------------
FROM scratch FROM scratch
+102
View File
@@ -0,0 +1,102 @@
package monitor
import "time"
// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
// Hub requests that run a probe must allow this time in addition to transport overhead.
const MaxProbeTimeout = 10 * time.Second
type SyncAction uint8
const (
// SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
SyncActionReplace SyncAction = iota
// SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
SyncActionUpsert
// SyncActionDelete indicates an incremental sync where the provided config should be removed.
SyncActionDelete
)
// Config defines a network monitor task sent from hub to agent.
type Config struct {
// ID is the stable network_monitors record ID generated by the hub.
ID string `cbor:"0,keyasint"`
Target string `cbor:"1,keyasint"`
Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
Port uint16 `cbor:"3,keyasint,omitempty"`
Interval uint16 `cbor:"4,keyasint"` // seconds
}
// SyncRequest defines an incremental or full monitor sync request sent to the agent.
type SyncRequest struct {
Action SyncAction `cbor:"0,keyasint"`
Config Config `cbor:"1,keyasint,omitempty"`
Configs []Config `cbor:"2,keyasint,omitempty"`
RunNow bool `cbor:"3,keyasint,omitempty"`
}
// SyncResponse returns the immediate result for an upsert when requested.
type SyncResponse struct {
Result Result `cbor:"0,keyasint,omitempty"`
}
// Result holds aggregated monitor results for a single target.
//
// 0: avg response in microseconds
//
// 1: 1h average response in microseconds
//
// 2: min response in microseconds
//
// 3: 1h min response in microseconds
//
// 4: max response in microseconds
//
// 5: 1h max response in microseconds
//
// 6: packet loss percentage (0-100)
//
// 7: 1h packet loss percentage (0-100)
type Result struct {
AvgResponse int64 `cbor:"0,keyasint,omitempty"`
AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
MinResponse int64 `cbor:"2,keyasint,omitempty"`
MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
MaxResponse int64 `cbor:"4,keyasint,omitempty"`
MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
PacketLoss float64 `cbor:"6,keyasint,omitempty"`
PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
// LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
LastProbeAt int64 `cbor:"8,keyasint"`
// SampleCount includes all completed probes since this monitor started.
// Used for alert warm-up even when the interval is longer than 20 minutes.
SampleCount int64 `cbor:"9,keyasint,omitempty"`
// Counts and sum cover the current response window (or latest-sample
// fallback), not the hourly window or lifetime SampleCount.
TotalCount int64 `cbor:"10,keyasint"`
SuccessCount int64 `cbor:"11,keyasint"`
ResponseSum int64 `cbor:"12,keyasint"`
}
// Stats holds response times in microseconds and packet loss percentage (0-100).
type Stats struct {
ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
ResMin float64 `json:"res_min" db:"res_min"`
ResMax float64 `json:"res_max" db:"res_max"`
Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
TotalCount int64 `json:"-" db:"total_count"`
SuccessCount int64 `json:"-" db:"success_count"`
ResponseSum int64 `json:"-" db:"res_sum"`
}
func (s Stats) FromResult(result Result) Stats {
return Stats{
ResAvg: float64(result.AvgResponse),
ResMin: float64(result.MinResponse),
ResMax: float64(result.MaxResponse),
Loss: result.PacketLoss,
TotalCount: result.TotalCount,
SuccessCount: result.SuccessCount,
ResponseSum: result.ResponseSum,
}
}
+3 -1
View File
@@ -7,6 +7,7 @@ import (
"time" "time"
"github.com/henrygd/beszel/internal/entities/container" "github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/systemd" "github.com/henrygd/beszel/internal/entities/systemd"
) )
@@ -210,5 +211,6 @@ type CombinedData struct {
Details *Details `cbor:"4,keyasint,omitempty"` Details *Details `cbor:"4,keyasint,omitempty"`
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response // SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
// that omitted systemd data (for example, a short-cache dashboard request). // that omitted systemd data (for example, a short-cache dashboard request).
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"` SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
Monitors map[string]monitor.Result `cbor:"6,keyasint"`
} }
+42 -34
View File
@@ -978,11 +978,18 @@ func TestAgentWebSocketIntegration(t *testing.T) {
} }
} }
// Verify system status // A connected WebSocket does not mean the hub has finished verifying
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id) // the agent and updating the system. Wait for the database state rather
require.NoError(t, err) // than assuming that work completes within a fixed sleep under load.
status := updatedSystemRecord.GetString("status") var status string
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value") require.EventuallyWithT(t, func(c *assert.CollectT) {
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id)
if !assert.NoError(c, err) {
return
}
status = updatedSystemRecord.GetString("status")
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
}, 5*time.Second, 20*time.Millisecond)
t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint) t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint)
}) })
@@ -1142,42 +1149,43 @@ func TestMultipleSystemsWithSameUniversalToken(t *testing.T) {
// Verify system creation/reuse behavior // Verify system creation/reuse behavior
if tc.expectConnection { if tc.expectConnection {
// Count systems after connection expectedSystemsAfter := systemsBeforeCount
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
require.NoError(t, err)
systemsAfterCount := len(systemsAfter)
if tc.expectNewSystem { if tc.expectNewSystem {
// Should have created a new system expectedSystemsAfter++
systemCount++ systemCount++
assert.Equal(t, systemsBeforeCount+1, systemsAfterCount, "Should have created a new system")
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should match expected")
} else {
// Should have reused existing system
assert.Equal(t, systemsBeforeCount, systemsAfterCount, "Should not have created a new system")
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should remain the same")
} }
time.Sleep(20 * time.Millisecond) // WebSocket connection precedes the hub's asynchronous system
// setup. Re-read all database state until setup is complete.
var systemId, status string
require.EventuallyWithT(t, func(c *assert.CollectT) {
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
if !assert.NoError(c, err) {
return
}
assert.Len(c, systemsAfter, expectedSystemsAfter, "System creation/reuse should match expected behavior")
assert.Len(c, systemsAfter, systemCount, "Total system count should match expected")
// Verify that a fingerprint record exists for this fingerprint fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{
fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{ "token": universalToken,
"token": universalToken, "fingerprint": tc.agentFingerprint,
"fingerprint": tc.agentFingerprint, })
}) if !assert.NoError(c, err) || !assert.Len(c, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination") {
require.NoError(t, err) return
require.Len(t, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination") }
fingerprint := fingerprints[0] fingerprint := fingerprints[0]
assert.Equal(t, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token") assert.Equal(c, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token")
assert.Equal(t, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint") assert.Equal(c, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint")
// Verify system status systemId = fingerprint.GetString("system")
systemId := fingerprint.GetString("system") system, err := testApp.FindRecordById("systems", systemId)
system, err := testApp.FindRecordById("systems", systemId) if !assert.NoError(c, err) {
require.NoError(t, err) return
status := system.GetString("status") }
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value") status = system.GetString("status")
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
}, 5*time.Second, 20*time.Millisecond)
t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem) t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem)
} }
+11 -1
View File
@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
return err return err
} }
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{ if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
list: &systemScopedReadRule, list: &systemScopedReadRule,
}); err != nil { }); err != nil {
return err return err
@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {
return err return err
} }
if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
create: &systemScopedWriteRule,
update: &systemScopedWriteRule,
delete: &systemScopedWriteRule,
}); err != nil {
return err
}
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{ if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
list: &systemScopedReadRule, list: &systemScopedReadRule,
view: &systemScopedReadRule, view: &systemScopedReadRule,
+3
View File
@@ -106,9 +106,12 @@ func (h *Hub) StartHub() error {
// TODO: move to users package // TODO: move to users package
// handle default values for user / user_settings creation // handle default values for user / user_settings creation
h.App.OnRecordAuthWithOAuth2Request("users").BindFunc(h.um.InitializeOAuthUserRole)
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole) h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings) h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
bindNetworkMonitorsEvents(h)
pb, ok := h.App.(*pocketbase.PocketBase) pb, ok := h.App.(*pocketbase.PocketBase)
if !ok { if !ok {
return errors.New("not a pocketbase app") return errors.New("not a pocketbase app")
+158
View File
@@ -0,0 +1,158 @@
package hub
import (
"strconv"
"time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/hub/systems"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
)
// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
func generateMonitorID(systemId string, config monitor.Config) string {
args := []string{systemId, config.Target, config.Protocol}
// only use port for TCP monitors, since for other protocols it's not relevant as standalone value
if config.Protocol == "tcp" {
args = append(args, strconv.FormatUint(uint64(config.Port), 10))
}
return systems.MakeStableHashId(args...)
}
// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
func bindNetworkMonitorsEvents(hub *Hub) {
// on create, make sure the id is set to a stable hash
hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
systemID := e.Record.GetString("system")
config := monitorConfigFromRecord(e.Record)
id := generateMonitorID(systemID, *config)
e.Record.Set("id", id)
return e.Next()
})
// sync monitor to agent on creation and persist the first result immediately when available
hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
err := e.Next()
if err != nil {
return err
}
if !e.Record.GetBool("enabled") {
return nil
}
// If connected, run the monitor immediately. Paused systems may be absent
// from the manager; their monitors will sync when they reconnect.
system, err := hub.sm.GetSystem(e.Record.GetString("system"))
if err == nil && system.Status == "up" {
go hub.upsertNetworkMonitor(e.Record, true)
}
return nil
})
// On API update requests, if the monitor config changed in a way that requires a new ID, create a new
// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
systemID := e.Record.GetString("system")
// only tcp uses port - set other protocols port to zero
if e.Record.GetString("protocol") != "tcp" {
e.Record.Set("port", 0)
}
ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
if ID != e.Record.Id {
newRecord := copyMonitorToNewRecord(e.Record, ID)
if err := e.App.Save(newRecord); err != nil {
return err
}
if err := e.App.Delete(e.Record); err != nil {
return err
}
return nil
}
err := e.Next()
if err != nil {
return err
}
if e.Record.GetBool("enabled") {
// if the monitor is enabled, sync the updated config to the agent now
runNow := !e.Record.Original().GetBool("enabled")
err = hub.upsertNetworkMonitor(e.Record, runNow)
} else {
// if the monitor is paused, remove it from the agent
err = hub.deleteNetworkMonitor(e.Record)
}
if err != nil {
hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
}
return nil
})
// sync monitor to agent on delete
hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
if err := hub.deleteNetworkMonitor(e.Record); err != nil {
hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
}
return e.Next()
})
}
// monitorConfigFromRecord builds a monitor config from a network_monitors record.
func monitorConfigFromRecord(record *core.Record) *monitor.Config {
return &monitor.Config{
ID: record.Id,
Target: record.GetString("target"),
Protocol: record.GetString("protocol"),
Port: uint16(record.GetInt("port")),
Interval: uint16(record.GetInt("interval")),
}
}
// setMonitorResultFields stores the latest monitor result values on the record.
func setMonitorResultFields(record *core.Record, result monitor.Result) {
nowString := time.Now().UTC().Format(types.DefaultDateLayout)
record.Set("res", result.AvgResponse)
record.Set("resAvg1h", result.AvgResponse1h)
record.Set("resMin1h", result.MinResponse1h)
record.Set("resMax1h", result.MaxResponse1h)
record.Set("loss1h", result.PacketLoss1h)
record.Set("updated", nowString)
}
// copyMonitorToNewRecord creates a new record with the same field values as the old one.
// This is used when the monitor config changes in a way that requires a new ID, so we need
// to create a new record with the new ID and delete the old one.
func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
collection := oldRecord.Collection()
newRecord := core.NewRecord(collection)
newRecord.Id = newID
fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}
for _, field := range fields {
newRecord.Set(field, oldRecord.Get(field))
}
return newRecord
}
// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
// is true, it will also trigger an immediate monitor run and update the record with the result.
func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
if err != nil || result == nil {
return err
}
setMonitorResultFields(record, *result)
return h.App.SaveNoValidate(record)
}
// deleteNetworkMonitor removes the record's monitor from the target system.
func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
systemID := record.GetString("system")
system, err := h.sm.GetSystem(systemID)
if err != nil {
return err
}
return system.DeleteNetworkMonitor(record.Id)
}
+225
View File
@@ -0,0 +1,225 @@
package hub
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/pocketbase/pocketbase/apis"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
for _, batch := range []bool{false, true} {
name := "single"
if batch {
name = "batch"
}
t.Run(name, func(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
bindNetworkMonitorsEvents(hub)
user, err := createTestUser(hub)
require.NoError(t, err)
system, err := createTestRecord(hub, "systems", map[string]any{
"name": "Paused", "host": "localhost", "port": "45876",
"status": "paused", "users": []string{user.Id},
})
require.NoError(t, err)
// Paused systems are not loaded into the manager at startup.
_, err = hub.sm.GetSystem(system.Id)
require.Error(t, err)
payload := func(target string) map[string]any {
return map[string]any{
"system": system.Id, "target": target, "protocol": "icmp",
"interval": 60, "enabled": true,
}
}
url := "/api/collections/network_monitors/records"
var body any = payload("1.1.1.1")
count := 1
if batch {
body = map[string]any{"requests": []map[string]any{
{"method": "POST", "url": url, "body": payload("1.1.1.1")},
{"method": "POST", "url": url, "body": payload("8.8.8.8")},
}}
url = "/api/batch"
count = 2
}
data, err := json.Marshal(body)
require.NoError(t, err)
token, err := user.NewAuthToken()
require.NoError(t, err)
router, err := apis.NewRouter(hub)
require.NoError(t, err)
handler, err := router.BuildMux()
require.NoError(t, err)
request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Authorization", token)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
records, err := hub.FindAllRecords("network_monitors")
require.NoError(t, err)
require.Len(t, records, count)
for _, record := range records {
assert.Equal(t, system.Id, record.GetString("system"))
assert.True(t, record.GetBool("enabled"))
}
})
}
}
func TestGenerateMonitorID(t *testing.T) {
tests := []struct {
name string
systemID string
config monitor.Config
expected string
}{
{
name: "HTTP monitor on example.com",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 0,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different port",
systemID: "sys123",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 8080,
Interval: 60,
},
expected: "a20a5827",
},
{
name: "HTTP monitor on example.com with different system ID",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 60,
},
expected: "ab602ae7",
},
{
name: "Same monitor, different interval",
systemID: "sys1234",
config: monitor.Config{
Protocol: "http",
Target: "example.com",
Port: 80,
Interval: 120,
},
expected: "ab602ae7",
},
{
name: "ICMP monitor on 1.1.1.1",
systemID: "sys456",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "6d13a4a4",
}, {
name: "ICMP monitor on 1.1.1.1 with different system ID",
systemID: "sys4567",
config: monitor.Config{
Protocol: "icmp",
Target: "1.1.1.1",
Port: 0,
Interval: 10,
},
expected: "ddd6c81",
},
{
name: "TCP monitor on example.com with port 443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 443,
Interval: 30,
},
expected: "677b991",
},
{
name: "TCP monitor on example.com with port 8443",
systemID: "sys789",
config: monitor.Config{
Protocol: "tcp",
Target: "example.com",
Port: 8443,
Interval: 30,
},
expected: "84167969",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := generateMonitorID(tt.systemID, tt.config)
assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
})
}
}
func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
hub, testApp, err := createTestHub(t)
require.NoError(t, err)
defer cleanupTestHub(hub, testApp)
collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
assert.Nil(t, collection.Fields.GetByName("name"))
oldRecord := core.NewRecord(collection)
oldRecord.Load(map[string]any{
"system": "sys123",
"target": "https://example.com",
"protocol": "http",
"port": 443,
"interval": 60,
"enabled": true,
"res": 1200,
"resAvg1h": 1300,
"resMin1h": 900,
"resMax1h": 1600,
"loss1h": 5,
"updated": "2026-04-29 12:00:00.000Z",
})
newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
assert.Equal(t, "next12345", newRecord.Id)
assert.Equal(t, "https://example.com", newRecord.GetString("target"))
assert.Equal(t, "http", newRecord.GetString("protocol"))
assert.Equal(t, 443, newRecord.GetInt("port"))
assert.True(t, newRecord.GetBool("enabled"))
assert.Zero(t, newRecord.GetFloat("res"))
assert.Zero(t, newRecord.GetFloat("resAvg1h"))
assert.Zero(t, newRecord.GetFloat("resMin1h"))
assert.Zero(t, newRecord.GetFloat("resMax1h"))
assert.Zero(t, newRecord.GetFloat("loss1h"))
assert.Equal(t, "", newRecord.GetString("updated"))
}
@@ -0,0 +1,159 @@
//go:build testing
package systems
import (
"context"
"crypto/ed25519"
"crypto/rand"
"net"
"sync/atomic"
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
func TestSSHNetworkMonitorReconnectSync(t *testing.T) {
sys, app := newTestSystemWithHub(t)
sys.manager.zfsFetchMap = expirymap.New[zfsFetchState](time.Hour)
t.Cleanup(sys.manager.zfsFetchMap.StopCleaner)
sys.ctx = context.Background()
sys.Status = up
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
signer, err := ssh.NewSignerFromKey(key)
require.NoError(t, err)
config := &ssh.ServerConfig{NoClientAuth: true, ServerVersion: "SSH-2.0-beszel_0.20.0"}
config.AddHostKey(signer)
listener, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
t.Cleanup(func() { _ = listener.Close() })
sys.Host, sys.Port, err = net.SplitHostPort(listener.Addr().String())
require.NoError(t, err)
sys.manager.sshConfig = &ssh.ClientConfig{User: "test", HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: time.Second}
t.Cleanup(sys.closeSSHConnection)
requests := make(chan monitor.SyncRequest, 10)
var failSync atomic.Bool
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
server, channels, reqs, err := ssh.NewServerConn(conn, config)
if err != nil {
_ = conn.Close()
return
}
defer server.Close()
go ssh.DiscardRequests(reqs)
for channel := range channels {
ch, reqs, err := channel.Accept()
if err != nil {
return
}
go func() {
defer ch.Close()
for req := range reqs {
if req.Type != "shell" {
_ = req.Reply(false, nil)
continue
}
_ = req.Reply(true, nil)
var request common.HubRequest[cbor.RawMessage]
if cbor.NewDecoder(ch).Decode(&request) != nil {
return
}
response := common.AgentResponse{}
switch request.Action {
case common.GetData:
response.SystemData = &esystem.CombinedData{}
case common.SyncNetworkMonitors:
var syncReq monitor.SyncRequest
if cbor.Unmarshal(request.Data, &syncReq) != nil {
return
}
requests <- syncReq
if failSync.Load() {
response.Error = "test sync failure"
} else {
response.Data, _ = cbor.Marshal(monitor.SyncResponse{})
}
}
_ = cbor.NewEncoder(ch).Encode(response)
_, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
return
}
}()
}
}()
}
}()
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
probe := core.NewRecord(collection)
probe.Load(map[string]any{"system": sys.Id, "target": "localhost", "protocol": "tcp", "port": 80, "interval": 60, "enabled": true})
require.NoError(t, app.SaveNoValidate(probe))
fetch := func() {
t.Helper()
_, err := sys.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err, "monitor sync failure must not fail stats fetching")
}
receive := func() monitor.SyncRequest {
t.Helper()
select {
case req := <-requests:
require.Equal(t, monitor.SyncActionReplace, req.Action)
return req
case <-time.After(time.Second):
t.Fatal("missing full monitor sync")
return monitor.SyncRequest{}
}
}
fetch()
require.Equal(t, probe.Id, receive().Configs[0].ID)
require.False(t, sys.monitorsNeedSync.Load())
fetch()
require.Empty(t, requests, "steady-state fetch must not resync")
// Simulate loss of the agent process/connection and its in-memory monitors.
require.NoError(t, sys.client.Load().Close())
fetch()
require.Equal(t, probe.Id, receive().Configs[0].ID)
require.False(t, sys.monitorsNeedSync.Load())
// Failed replacements are retried on the next successful stats fetch.
require.NoError(t, sys.client.Load().Close())
failSync.Store(true)
fetch()
receive()
require.True(t, sys.monitorsNeedSync.Load())
failSync.Store(false)
fetch()
receive()
require.False(t, sys.monitorsNeedSync.Load())
probe.Set("enabled", false)
require.NoError(t, app.SaveNoValidate(probe))
require.NoError(t, sys.client.Load().Close())
fetch()
require.Empty(t, receive().Configs, "empty replacement must clear stale monitors")
}
func TestPendingNetworkMonitorSyncQueryFailure(t *testing.T) {
sys, app := newTestSystemWithHub(t)
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
require.NoError(t, err)
sys.monitorsNeedSync.Store(true)
sys.syncPendingNetworkMonitors()
require.True(t, sys.monitorsNeedSync.Load())
}
@@ -0,0 +1,224 @@
//go:build testing
package systems
import (
"fmt"
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/subscriptions"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNetworkMonitorProbePruning(t *testing.T) {
for _, tc := range []struct {
name string
monitors map[string]monitor.Result
fail bool
want map[string]int64
}{
{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
} {
t.Run(tc.name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
// Preserve the distinction between nil and empty across the agent transport.
encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
require.NoError(t, err)
var data system.CombinedData
require.NoError(t, cbor.Unmarshal(encoded, &data))
if tc.fail {
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
}
_, err = sys.createRecords(&data)
if tc.fail {
require.Error(t, err)
} else {
require.NoError(t, err)
}
assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
})
}
}
func TestNetworkMonitorStatsFreshness(t *testing.T) {
for _, realtime := range []bool{false, true} {
name := "sql"
if realtime {
name = "realtime"
}
t.Run(name, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
}
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
for _, id := range []string{"monitor1", "monitor2"} {
record := core.NewRecord(col)
record.Id = id
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
}
data := &system.CombinedData{Monitors: map[string]monitor.Result{
"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
}}
count := func(want int64) {
t.Helper()
got, err := app.CountRecords("network_monitor_stats")
require.NoError(t, err)
assert.Equal(t, want, got)
}
save := func() {
t.Helper()
_, err := sys.createRecords(data)
require.NoError(t, err)
}
save()
count(2)
stored, err := app.FindAllRecords("network_monitor_stats")
require.NoError(t, err)
for _, record := range stored {
result := data.Monitors[record.GetString("monitor")]
assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
}
// A resume can overlap the scheduled update with the same probe.
errs := make(chan error, 4)
for range 4 {
go func() {
_, err := sys.createRecords(data)
errs <- err
}()
}
for range 4 {
require.NoError(t, <-errs)
}
count(2)
// A rolling hourly value can change without a new probe.
result := data.Monitors["monitor1"]
result.AvgResponse1h = 42
data.Monitors["monitor1"] = result
save()
count(2)
record, err := app.FindRecordById("network_monitors", "monitor1")
require.NoError(t, err)
assert.Equal(t, 42, record.GetInt("resAvg1h"))
// Identical response values and failed probes still count as new measurements.
for id, result := range data.Monitors {
result.LastProbeAt = 301000
data.Monitors[id] = result
}
save()
count(4)
// A failed individual insert must remain retryable, even if others commit.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
require.NoError(t, err)
for id, result := range data.Monitors {
result.LastProbeAt = 601000
data.Monitors[id] = result
}
save()
count(5)
assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
require.NoError(t, err)
save()
count(6)
// Failure after inserting stats rolls back the whole transaction and its markers.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
result = data.Monitors["monitor1"]
result.LastProbeAt = 901000
data.Monitors["monitor1"] = result
_, err = sys.createRecords(data)
require.Error(t, err)
count(6)
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
require.NoError(t, err)
save()
count(7)
// Clock rollback is a new probe identity, not a reason to stall writes.
result.LastProbeAt = 500
data.Monitors["monitor1"] = result
save()
count(8)
// Recreated systems intentionally accept the first result without restoring state.
sys = &System{Id: sys.Id, manager: sys.manager}
save()
count(10)
})
}
}
// Observes the committed DB through the hub, not the transaction's app.
type monitorAlertHub struct {
stubHub
handle func(*core.Record, map[string]monitor.Result) error
}
func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
return h.handle(record, results)
}
func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
for _, realtime := range []bool{false, true} {
t.Run(fmt.Sprint(realtime), func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
if realtime {
client := subscriptions.NewDefaultClient()
client.Subscribe("network_monitors/*")
app.SubscriptionsBroker().Register(client)
}
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
record := core.NewRecord(collection)
record.Set("system", sys.Id)
require.NoError(t, app.SaveNoValidate(record))
called := 0
result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
called++
assert.Equal(t, sys.Id, systemRecord.Id)
assert.Equal(t, result, results[record.Id])
saved, err := app.FindRecordById("network_monitors", record.Id)
require.NoError(t, err)
assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
return nil
}}
data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
_, err = sys.createRecords(data)
require.NoError(t, err)
assert.Equal(t, 1, called)
// A transaction that fails after writing monitor stats must not notify.
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
require.NoError(t, err)
_, err = sys.createRecords(data)
require.Error(t, err)
assert.Equal(t, 1, called)
})
}
}
@@ -0,0 +1,184 @@
//go:build testing
package systems
import (
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/blang/semver"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/ws"
"github.com/lxzan/gws"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/require"
)
type monitorSyncClient struct {
gws.BuiltinEventHandler
requests chan common.HubRequest[monitor.SyncRequest]
failSync atomic.Bool
}
func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
defer message.Close()
var req common.HubRequest[cbor.RawMessage]
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
return
}
resp := common.AgentResponse{Id: req.Id}
if req.Action == common.GetData {
resp.SystemData = &esystem.CombinedData{}
} else {
var data monitor.SyncRequest
if err := cbor.Unmarshal(req.Data, &data); err != nil {
return
}
c.requests <- common.HubRequest[monitor.SyncRequest]{Id: req.Id, Action: req.Action, Data: data}
if c.failSync.Load() {
resp.Error = "test sync failure"
} else {
resp.Data, _ = cbor.Marshal(monitor.SyncResponse{})
}
}
response, _ := cbor.Marshal(resp)
_ = conn.WriteMessage(gws.OpcodeBinary, response)
}
// Avoid the production delayed disconnect notification; these tests explicitly
// remove each connection from the manager before reconnecting.
type monitorSyncServer struct{ ws.Handler }
func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
t.Run(version, func(t *testing.T) {
// No transport: attempting to send any request would fail.
sys := &System{agentVersion: semver.MustParse(version)}
require.NoError(t, sys.SyncNetworkMonitors(nil))
result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
require.NoError(t, err)
require.Nil(t, result)
require.NoError(t, sys.DeleteNetworkMonitor("test"))
})
}
}
func TestNetworkMonitorReconnectSync(t *testing.T) {
for _, change := range []string{"delete", "disable", "retry"} {
t.Run(change, func(t *testing.T) {
sys, app := newTestSystemWithHub(t)
record, err := app.FindRecordById("systems", sys.Id)
require.NoError(t, err)
// Suppress unrelated system-stat requests while exercising reconnects.
record.Set("status", paused)
require.NoError(t, app.SaveNoValidate(record))
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
require.NoError(t, err)
probe := core.NewRecord(collection)
probe.Load(map[string]any{
"system": sys.Id, "target": "localhost", "protocol": "tcp",
"port": 80, "interval": 60, "enabled": true,
})
require.NoError(t, app.SaveNoValidate(probe))
sm := NewSystemManager(stubHub{app})
t.Cleanup(func() {
sm.cancel()
_ = sm.RemoveSystem(sys.Id)
sm.smartFetchMap.StopCleaner()
sm.zfsFetchMap.StopCleaner()
})
version := semver.MustParse("0.20.0")
connections := make(chan *ws.WsConn, 1)
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r)
if err != nil {
t.Error(err)
return
}
wsConn := ws.NewWsConnection(conn, version)
conn.Session().Store("wsConn", wsConn)
connections <- wsConn
conn.ReadLoop()
}))
t.Cleanup(server.Close)
client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
connect := func() monitor.SyncRequest {
t.Helper()
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
require.NoError(t, err)
t.Cleanup(func() { _ = conn.NetConn().Close() })
go conn.ReadLoop()
select {
case wsConn := <-connections:
require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
case <-time.After(3 * time.Second):
t.Fatal("websocket connection was not established")
}
select {
case req := <-client.requests:
require.Equal(t, common.SyncNetworkMonitors, req.Action)
require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
return req.Data
case <-time.After(3 * time.Second):
t.Fatal("reconnected agent did not receive a monitor replacement")
return monitor.SyncRequest{}
}
}
client.failSync.Store(change == "retry")
initial := connect()
require.Len(t, initial.Configs, 1)
require.Equal(t, probe.Id, initial.Configs[0].ID)
if change == "retry" {
system, err := sm.GetSystem(sys.Id)
require.NoError(t, err)
require.Eventually(t, system.monitorsNeedSync.Load, time.Second, time.Millisecond)
// A second failed sync must not fail the stats fetch or clear pending state.
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.True(t, system.monitorsNeedSync.Load())
require.Len(t, client.requests, 1)
<-client.requests
client.failSync.Store(false)
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.False(t, system.monitorsNeedSync.Load())
require.Len(t, client.requests, 1)
retry := <-client.requests
require.Equal(t, monitor.SyncActionReplace, retry.Data.Action)
require.Equal(t, initial.Configs, retry.Data.Configs)
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
require.NoError(t, err)
require.Empty(t, client.requests, "successful sync must not repeat on every fetch")
return
}
require.NoError(t, sm.RemoveSystem(sys.Id))
if change == "delete" {
require.NoError(t, app.Delete(probe))
} else {
probe.Set("enabled", false)
require.NoError(t, app.SaveNoValidate(probe))
}
require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
})
}
}
func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
sys, app := newTestSystemWithHub(t)
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
require.NoError(t, err)
_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
require.Error(t, err, "a failed query must not be treated as an empty monitor set")
}
+80
View File
@@ -0,0 +1,80 @@
package systems
import (
"context"
"fmt"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
)
// syncPendingNetworkMonitors runs on WebSocket connect and after successful stats
// fetches. Failed syncs retry on the next update without taking the system down.
func (sys *System) syncPendingNetworkMonitors() {
if !sys.monitorsNeedSync.Swap(false) {
return
}
if err := sys.syncAllNetworkMonitors(); err != nil {
sys.monitorsNeedSync.Store(true)
sys.manager.hub.Logger().Warn("failed to sync monitors to agent", "system", sys.Id, "err", err)
}
}
func (sys *System) syncAllNetworkMonitors() error {
configs, err := sys.manager.GetMonitorConfigsForSystem(sys.Id)
if err != nil {
return fmt.Errorf("failed to load monitors: %w", err)
}
// An empty set must also replace probes retained across a disconnect.
return sys.SyncNetworkMonitors(configs)
}
// SyncNetworkMonitors sends monitor configurations to the agent.
func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
return err
}
// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionUpsert,
Config: config,
RunNow: runNow,
})
if err != nil {
return nil, err
}
if resp.Result == (monitor.Result{}) {
return nil, nil
}
result := resp.Result
return &result, nil
}
// DeleteNetworkMonitor removes a single monitor task from the agent.
func (sys *System) DeleteNetworkMonitor(id string) error {
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
Action: monitor.SyncActionDelete,
Config: monitor.Config{ID: id},
})
return err
}
func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
return monitor.SyncResponse{}, nil
}
timeout := 5 * time.Second
if req.Action == monitor.SyncActionUpsert && req.RunNow {
// Allow the probe to finish, including a timeout result, while preserving
// the normal request budget for transport and response handling.
timeout += monitor.MaxProbeTimeout
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
var result monitor.SyncResponse
return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
}
+153 -3
View File
@@ -9,6 +9,7 @@ import (
"math/rand" "math/rand"
"net" "net"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"time" "time"
@@ -18,6 +19,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws" "github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/container" "github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
"github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/systemd" "github.com/henrygd/beszel/internal/entities/systemd"
@@ -30,6 +32,8 @@ import (
"github.com/lxzan/gws" "github.com/lxzan/gws"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/security"
"github.com/pocketbase/pocketbase/tools/types"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
@@ -52,6 +56,13 @@ type System struct {
smartInterval time.Duration // Interval for periodic SMART data updates smartInterval time.Duration // Interval for periodic SMART data updates
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
zfsInterval time.Duration // Interval for periodic ZFS detail data updates zfsInterval time.Duration // Interval for periodic ZFS detail data updates
// A fresh connection needs a full monitor configuration sync.
monitorsNeedSync atomic.Bool
// Serialize persistence from scheduled updates and resumes through commit.
recordsMu sync.Mutex
// Protected by recordsMu; realtime reads don't consume probes.
lastSavedMonitorProbe map[string]int64
} }
func (sm *SystemManager) NewSystem(systemId string) *System { func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -211,11 +222,15 @@ func (sys *System) handlePaused() {
// createRecords updates the system record and adds system_stats and container_stats records // createRecords updates the system record and adds system_stats and container_stats records
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) { func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
sys.recordsMu.Lock()
defer sys.recordsMu.Unlock()
systemRecord, err := sys.getRecord(sys.manager.hub) systemRecord, err := sys.getRecord(sys.manager.hub)
if err != nil { if err != nil {
return nil, err return nil, err
} }
hub := sys.manager.hub hub := sys.manager.hub
savedMonitorProbes := make(map[string]int64)
err = hub.RunInTransaction(func(txApp core.App) error { err = hub.RunInTransaction(func(txApp core.App) error {
// add system_stats record // add system_stats record
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats") systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
@@ -266,6 +281,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
} }
} }
if data.Monitors != nil {
if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
return err
}
}
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil { if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
return err return err
} }
@@ -287,6 +308,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
return nil return nil
}) })
// Publish only successful inserts after the entire transaction commits.
if err == nil && len(savedMonitorProbes) > 0 {
if sys.lastSavedMonitorProbe == nil {
sys.lastSavedMonitorProbe = savedMonitorProbes
} else {
for id, timestamp := range savedMonitorProbes {
sys.lastSavedMonitorProbe[id] = timestamp
}
}
}
// A non-nil report includes cached results for all remaining monitors.
if err == nil && data.Monitors != nil {
for id := range sys.lastSavedMonitorProbe {
if _, exists := data.Monitors[id]; !exists {
delete(sys.lastSavedMonitorProbe, id)
}
}
}
if err == nil {
if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
}
}
return systemRecord, err return systemRecord, err
} }
@@ -337,7 +381,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
} }
suffix := fmt.Sprintf("%d", i) suffix := fmt.Sprintf("%d", i)
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix)) valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
params["id"+suffix] = makeStableHashId(systemId, service.Name) params["id"+suffix] = MakeStableHashId(systemId, service.Name)
params["name"+suffix] = service.Name params["name"+suffix] = service.Name
params["state"+suffix] = service.State params["state"+suffix] = service.State
params["sub"+suffix] = service.Sub params["sub"+suffix] = service.Sub
@@ -363,6 +407,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
return err return err
} }
func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
if len(monitorResults) == 0 {
return nil
}
var err error
systemId := sys.Id
const monitorCollectionName = "network_monitors"
// If realtime updates are active, we save via PocketBase records to trigger realtime events.
// Otherwise we can do a more efficient direct update via SQL
realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
})
now := time.Now().UTC()
nowMilli := now.UnixMilli()
nowString := now.Format(types.DefaultDateLayout)
var db dbx.Builder
var updateQuery *dbx.Query
if !realtimeActive {
db = app.DB()
monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
setClauses := make([]string, len(monitorFields))
for i, f := range monitorFields {
setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
}
queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
updateQuery = db.NewQuery(queryString)
}
// update network_monitors records
for id, result := range monitorResults {
monitorData := map[string]any{
"id": id,
"res": result.AvgResponse,
"resAvg1h": result.AvgResponse1h,
"resMin1h": result.MinResponse1h,
"resMax1h": result.MaxResponse1h,
"loss1h": result.PacketLoss1h,
"updated": nowString,
}
switch realtimeActive {
case true:
var record *core.Record
record, err = app.FindRecordById(monitorCollectionName, id)
if err == nil {
record.Load(monitorData)
err = app.SaveNoValidate(record)
}
default:
_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
}
if err != nil {
app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
}
}
// handle stats collection — one record per monitor
const statsCollectionName = "network_monitor_stats"
var statsCollection *core.Collection
if realtimeActive {
statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
}
for monitorId, result := range monitorResults {
// Compare identity, not ordering, so agent clock changes don't stall writes.
if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
continue
}
statsRecordData := map[string]any{
"system": systemId,
"monitor": monitorId,
"type": "1m",
"created": nowMilli,
"res_min": result.MinResponse,
"res_max": result.MaxResponse,
"total_count": result.TotalCount,
"success_count": result.SuccessCount,
"res_sum": result.ResponseSum,
}
switch realtimeActive {
case true:
record := core.NewRecord(statsCollection)
record.Load(statsRecordData)
err = app.SaveNoValidate(record)
default:
statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
}
if err != nil {
app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
} else {
savedProbes[monitorId] = result.LastProbeAt
}
}
return nil
}
// createContainerRecords creates container records // createContainerRecords creates container records
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error { func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
if len(data) == 0 { if len(data) == 0 {
@@ -489,7 +633,10 @@ func (sys *System) request(ctx context.Context, action common.WebSocketAction, r
err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1) err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1)
// Keep legacy SSH client/version fields in sync for other code paths. // Keep legacy SSH client/version fields in sync for other code paths.
if sys.sshTransport != nil { if sys.sshTransport != nil {
sys.client.Store(sys.sshTransport.GetClient()) client := sys.sshTransport.GetClient()
if previous := sys.client.Swap(client); client != nil && client != previous {
sys.monitorsNeedSync.Store(true)
}
sys.agentVersion = sys.sshTransport.GetAgentVersion() sys.agentVersion = sys.sshTransport.GetAgentVersion()
} }
return err return err
@@ -547,6 +694,7 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
if sys.WsConn != nil && sys.WsConn.IsConnected() { if sys.WsConn != nil && sys.WsConn.IsConnected() {
wsData, err := sys.fetchDataViaWebSocket(options) wsData, err := sys.fetchDataViaWebSocket(options)
if err == nil { if err == nil {
sys.syncPendingNetworkMonitors()
return wsData, nil return wsData, nil
} }
// close the WebSocket connection if error and try SSH // close the WebSocket connection if error and try SSH
@@ -557,6 +705,7 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
if err != nil { if err != nil {
return nil, err return nil, err
} }
sys.syncPendingNetworkMonitors()
return sshData, nil return sshData, nil
} }
@@ -622,7 +771,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
return &result, err return &result, err
} }
func makeStableHashId(strings ...string) string { func MakeStableHashId(strings ...string) string {
hash := fnv.New32a() hash := fnv.New32a()
for _, str := range strings { for _, str := range strings {
hash.Write([]byte(str)) hash.Write([]byte(str))
@@ -791,6 +940,7 @@ func (s *System) createSSHClient() error {
return err return err
} }
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion())) s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
s.monitorsNeedSync.Store(true)
s.manager.resetFailedSmartFetchState(s.Id) s.manager.resetFailedSmartFetchState(s.Id)
s.manager.resetFailedZfsFetchState(s.Id) s.manager.resetFailedZfsFetchState(s.Id)
return nil return nil
+20 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/henrygd/beszel/internal/hub/ws" "github.com/henrygd/beszel/internal/hub/ws"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap" "github.com/henrygd/beszel/internal/hub/expirymap"
@@ -17,6 +18,7 @@ import (
"github.com/henrygd/beszel" "github.com/henrygd/beszel"
"github.com/blang/semver" "github.com/blang/semver"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/store" "github.com/pocketbase/pocketbase/tools/store"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
@@ -62,6 +64,7 @@ type hubLike interface {
core.App core.App
GetSSHKey(dataDir string) (ssh.Signer, error) GetSSHKey(dataDir string) (ssh.Signer, error)
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
HandleStatusAlerts(status string, systemRecord *core.Record) error HandleStatusAlerts(status string, systemRecord *core.Record) error
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
CancelPendingStatusAlerts(systemID string) CancelPendingStatusAlerts(systemID string)
@@ -346,10 +349,15 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
system := sm.NewSystem(systemId) system := sm.NewSystem(systemId)
system.WsConn = wsConn system.WsConn = wsConn
system.agentVersion = agentVersion system.agentVersion = agentVersion
system.monitorsNeedSync.Store(true)
if err := sm.AddRecord(systemRecord, system); err != nil { if err := sm.AddRecord(systemRecord, system); err != nil {
return err return err
} }
// Sync network monitors to the newly connected agent
go system.syncPendingNetworkMonitors()
return nil return nil
} }
@@ -362,6 +370,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
} }
} }
// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
var configs []monitor.Config
err := sm.hub.DB().
NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
Bind(dbx.Params{"system": systemID}).
All(&configs)
return configs, err
}
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh // resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
// agent reconnect retries ZFS discovery immediately after configuration changes. // agent reconnect retries ZFS discovery immediately after configuration changes.
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) { func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
@@ -397,11 +415,12 @@ func (sm *SystemManager) createSSHClientConfig() error {
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive. // deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
// This is called when a system is paused or goes offline to prevent continued alerts. // This is called when a system is paused or goes offline to prevent continued alerts.
// Monitor incidents remain open: a missing observation does not establish recovery.
func deactivateAlerts(app core.App, systemID string) error { func deactivateAlerts(app core.App, systemID string) error {
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API // Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute() // _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0) alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
if err != nil { if err != nil {
return err return err
} }
+19 -1
View File
@@ -6,6 +6,8 @@ import (
"time" "time"
"github.com/henrygd/beszel/internal/common" "github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/utils" "github.com/henrygd/beszel/internal/hub/utils"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/apis" "github.com/pocketbase/pocketbase/apis"
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
if err != nil { if err != nil {
return return
} }
bytes, err := json.Marshal(data) bytes, err := marshalRealtimeData(data)
if err == nil { if err == nil {
notify(sm.hub, system, fetch.subscription, bytes) notify(sm.hub, system, fetch.subscription, bytes)
} }
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
} }
} }
// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
// the per-monitor results into the derived metric fields the frontend charts expect.
func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
if len(data.Monitors) == 0 {
return json.Marshal(data)
}
monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
for id, result := range data.Monitors {
monitorStats[id] = monitor.Stats{}.FromResult(result)
}
return json.Marshal(struct {
*system.CombinedData
Monitors map[string]monitor.Stats `json:"Monitors"`
}{data, monitorStats})
}
// notify broadcasts realtime data to all clients subscribed to a specific subscription. // notify broadcasts realtime data to all clients subscribed to a specific subscription.
// Custom topics bypass collection rules, so check current access for every // Custom topics bypass collection rules, so check current access for every
// recipient, including clients whose authentication or membership was revoked. // recipient, including clients whose authentication or membership was revoked.
+2 -2
View File
@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
currentIDs := make(map[string]struct{}, len(smartData)) currentIDs := make(map[string]struct{}, len(smartData))
for deviceKey := range smartData { for deviceKey := range smartData {
currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{} currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
} }
err = hub.RunInTransaction(func(txApp core.App) error { err = hub.RunInTransaction(func(txApp core.App) error {
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
} }
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error { func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
recordID := makeStableHashId(sys.Id, deviceKey) recordID := MakeStableHashId(sys.Id, deviceKey)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {
+4 -2
View File
@@ -7,6 +7,7 @@ import (
"testing" "testing"
"time" "time"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/smart" "github.com/henrygd/beszel/internal/entities/smart"
esystem "github.com/henrygd/beszel/internal/entities/system" esystem "github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/hub/expirymap" "github.com/henrygd/beszel/internal/hub/expirymap"
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error { func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
return nil return nil
} }
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil } func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error { func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
return nil return nil
} }
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
}, false)) }, false))
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2) assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA")) recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
require.NoError(t, err) require.NoError(t, err)
assert.EqualValues(t, 42, recordA.GetInt("temp")) assert.EqualValues(t, 42, recordA.GetInt("temp"))
} }
+9 -9
View File
@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName := "nginx.service" serviceName := "nginx.service"
// Call multiple times and ensure same result // Call multiple times and ensure same result
id1 := makeStableHashId(systemId, serviceName) id1 := MakeStableHashId(systemId, serviceName)
id2 := makeStableHashId(systemId, serviceName) id2 := MakeStableHashId(systemId, serviceName)
id3 := makeStableHashId(systemId, serviceName) id3 := MakeStableHashId(systemId, serviceName)
assert.Equal(t, id1, id2) assert.Equal(t, id1, id2)
assert.Equal(t, id2, id3) assert.Equal(t, id2, id3)
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
serviceName1 := "nginx.service" serviceName1 := "nginx.service"
serviceName2 := "apache.service" serviceName2 := "apache.service"
id1 := makeStableHashId(systemId1, serviceName1) id1 := MakeStableHashId(systemId1, serviceName1)
id2 := makeStableHashId(systemId2, serviceName1) id2 := MakeStableHashId(systemId2, serviceName1)
id3 := makeStableHashId(systemId1, serviceName2) id3 := MakeStableHashId(systemId1, serviceName2)
id4 := makeStableHashId(systemId2, serviceName2) id4 := MakeStableHashId(systemId2, serviceName2)
// All IDs should be different // All IDs should be different
assert.NotEqual(t, id1, id2) assert.NotEqual(t, id1, id2)
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
} }
for _, tc := range testCases { for _, tc := range testCases {
id := makeStableHashId(tc.systemId, tc.serviceName) id := MakeStableHashId(tc.systemId, tc.serviceName)
// FNV-32 produces 8 hex characters // FNV-32 produces 8 hex characters
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName) assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
} }
}) })
t.Run("hexadecimal output", func(t *testing.T) { t.Run("hexadecimal output", func(t *testing.T) {
id := makeStableHashId("test-system", "test-service") id := MakeStableHashId("test-system", "test-service")
assert.NotEmpty(t, id) assert.NotEmpty(t, id)
// Should only contain hexadecimal characters // Should only contain hexadecimal characters
+2 -2
View File
@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
} }
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error { func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
recordID := makeStableHashId(sys.Id, pool.Name) recordID := MakeStableHashId(sys.Id, pool.Name)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
if pool == nil { if pool == nil {
continue continue
} }
recordID := makeStableHashId(sys.Id, name) recordID := MakeStableHashId(sys.Id, name)
record, err := app.FindRecordById(collection, recordID) record, err := app.FindRecordById(collection, recordID)
if err != nil { if err != nil {
if !errors.Is(err, sql.ErrNoRows) { if !errors.Is(err, sql.ErrNoRows) {
+6 -6
View File
@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10}, {Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
}} }}
require.NoError(t, sys.saveZfsPools(initial)) require.NoError(t, sys.saveZfsPools(initial))
failedID := makeStableHashId(sys.Id, failedKey) failedID := MakeStableHashId(sys.Id, failedKey)
before, err := app.FindRecordById("zfs_pools", failedID) before, err := app.FindRecordById("zfs_pools", failedID)
require.NoError(t, err) require.NoError(t, err)
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{ partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99}, {Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
}} }}
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData) assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey)) fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
require.NoError(t, err) require.NoError(t, err)
assert.EqualValues(t, 20, fresh.GetInt("alloc")) assert.EqualValues(t, 20, fresh.GetInt("alloc"))
cached, err := app.FindRecordById("zfs_pools", failedID) cached, err := app.FindRecordById("zfs_pools", failedID)
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{ require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 25, Health: "ONLINE"}, "tank": {Total: 100, Used: 25, Health: "ONLINE"},
})) }))
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank")) record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err) require.NoError(t, err)
firstUpdated := record.GetDateTime("updated") firstUpdated := record.GetDateTime("updated")
assert.Equal(t, "ONLINE", record.GetString("health")) assert.Equal(t, "ONLINE", record.GetString("health"))
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
func TestZfsRawCapacityPersistence(t *testing.T) { func TestZfsRawCapacityPersistence(t *testing.T) {
sys, app := newTestSystemWithHub(t) sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}})) require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs")) record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
require.NoError(t, err) require.NoError(t, err)
require.True(t, record.GetBool("raw")) require.True(t, record.GetBool("raw"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}})) require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
key: {DisplayName: "tank", Health: "ONLINE"}, key: {DisplayName: "tank", Health: "ONLINE"},
"tank": {Health: "ONLINE"}, "tank": {Health: "ONLINE"},
})) }))
id := makeStableHashId(sys.Id, key) id := MakeStableHashId(sys.Id, key)
record, err := app.FindRecordById("zfs_pools", id) record, err := app.FindRecordById("zfs_pools", id)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "tank", record.GetString("display_name")) assert.Equal(t, "tank", record.GetString("display_name"))
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
record, err = app.FindRecordById("zfs_pools", id) record, err = app.FindRecordById("zfs_pools", id)
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, "detail name", record.GetString("display_name")) assert.Equal(t, "detail name", record.GetString("display_name"))
_, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank")) _, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
require.NoError(t, err) require.NoError(t, err)
} }
+28 -1
View File
@@ -1,7 +1,11 @@
// Package utils provides utility functions for the hub. // Package utils provides utility functions for the hub.
package utils package utils
import "os" import (
"os"
"github.com/pocketbase/pocketbase/core"
)
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key. // GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
func GetEnv(key string) (value string, exists bool) { func GetEnv(key string) (value string, exists bool) {
@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {
} }
return os.LookupEnv(key) return os.LookupEnv(key)
} }
// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
broker := app.SubscriptionsBroker()
if broker.TotalClients() == 0 {
return false
}
for _, client := range broker.Clients() {
subs := client.Subscriptions(collectionName)
if len(subs) > 0 {
if validateFn == nil {
return true
}
for k := range subs {
filter := subs[k].Query["filter"]
if validateFn(filter) {
return true
}
}
}
}
return false
}
@@ -83,7 +83,8 @@ func init() {
"ContainerHealth", "ContainerHealth",
"SystemdFailed", "SystemdFailed",
"CPUIOWait", "CPUIOWait",
"CPUSteal" "CPUSteal",
"NetworkMonitorLoss"
] ]
}, },
{ {
@@ -119,6 +120,16 @@ func init() {
"system": false, "system": false,
"type": "bool" "type": "bool"
}, },
{
"hidden": true,
"id": "json4000656575",
"maxSize": 0,
"name": "state",
"presentable": false,
"required": false,
"system": false,
"type": "json"
},
{ {
"hidden": true, "hidden": true,
"id": "date1302749137", "id": "date1302749137",
@@ -152,6 +163,7 @@ func init() {
} }
], ],
"indexes": [ "indexes": [
"CREATE INDEX idx_alerts_system_name ON alerts (system, name)",
"CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)" "CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"
], ],
"system": false "system": false
@@ -234,6 +246,20 @@ func init() {
"system": false, "system": false,
"type": "text" "type": "text"
}, },
{
"autogeneratePattern": "",
"hidden": false,
"id": "text3888135399",
"max": 0,
"min": 0,
"name": "monitor_name",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{ {
"hidden": false, "hidden": false,
"id": "number494360628", "id": "number494360628",
@@ -1031,6 +1057,15 @@ func init() {
"required": true, "required": true,
"system": false, "system": false,
"type": "number" "type": "number"
},
{
"hidden": false,
"id": "bool2084032502",
"name": "updatable",
"presentable": false,
"required": false,
"system": false,
"type": "bool"
} }
], ],
"indexes": [ "indexes": [
@@ -1721,6 +1756,7 @@ func init() {
"fields": [ "fields": [
{ {
"autogeneratePattern": "[a-z0-9]{15}", "autogeneratePattern": "[a-z0-9]{15}",
"help": "",
"hidden": false, "hidden": false,
"id": "text3208210256", "id": "text3208210256",
"max": 15, "max": 15,
@@ -1736,6 +1772,7 @@ func init() {
{ {
"cascadeDelete": true, "cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx", "collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false, "hidden": false,
"id": "relation1204987316", "id": "relation1204987316",
"maxSelect": 1, "maxSelect": 1,
@@ -1748,6 +1785,7 @@ func init() {
}, },
{ {
"autogeneratePattern": "", "autogeneratePattern": "",
"help": "",
"hidden": false, "hidden": false,
"id": "text7739291048", "id": "text7739291048",
"max": 0, "max": 0,
@@ -1762,6 +1800,7 @@ func init() {
}, },
{ {
"autogeneratePattern": "", "autogeneratePattern": "",
"help": "",
"hidden": false, "hidden": false,
"id": "text5528164482", "id": "text5528164482",
"max": 0, "max": 0,
@@ -1775,6 +1814,7 @@ func init() {
"type": "text" "type": "text"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "number8862034195", "id": "number8862034195",
"max": null, "max": null,
@@ -1787,6 +1827,7 @@ func init() {
"type": "number" "type": "number"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "number4418907321", "id": "number4418907321",
"max": null, "max": null,
@@ -1799,6 +1840,7 @@ func init() {
"type": "number" "type": "number"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "number2904183765", "id": "number2904183765",
"max": null, "max": null,
@@ -1811,6 +1853,7 @@ func init() {
"type": "number" "type": "number"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "json4466109723", "id": "json4466109723",
"maxSize": 0, "maxSize": 0,
@@ -1821,6 +1864,7 @@ func init() {
"type": "json" "type": "json"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "json9012873456", "id": "json9012873456",
"maxSize": 0, "maxSize": 0,
@@ -1831,6 +1875,7 @@ func init() {
"type": "json" "type": "json"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "json7182045639", "id": "json7182045639",
"maxSize": 0, "maxSize": 0,
@@ -1841,6 +1886,7 @@ func init() {
"type": "json" "type": "json"
}, },
{ {
"help": "",
"hidden": false, "hidden": false,
"id": "date9274163058", "id": "date9274163058",
"max": "", "max": "",
@@ -1860,19 +1906,401 @@ func init() {
"presentable": false, "presentable": false,
"system": false, "system": false,
"type": "autodate" "type": "autodate"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text3578368839",
"max": 0,
"min": 0,
"name": "display_name",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{
"help": "",
"hidden": false,
"id": "bool447994709",
"name": "raw",
"presentable": false,
"required": false,
"system": false,
"type": "bool"
} }
], ],
"id": "pbc_8441057391", "id": "pbc_8441057391",
"indexes": [ "indexes": [
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)" "CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
], ],
"listRule": null, "listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id",
"name": "zfs_pools", "name": "zfs_pools",
"system": false, "system": false,
"type": "base", "type": "base",
"updateRule": null, "updateRule": null,
"viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id"
},
{
"createRule": null,
"deleteRule": null,
"fields": [
{
"autogeneratePattern": "[a-z0-9]{10}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 10,
"min": 6,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false,
"id": "nm_system",
"maxSelect": 1,
"minSelect": 0,
"name": "system",
"presentable": false,
"required": true,
"system": false,
"type": "relation"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "nm_target",
"max": 500,
"min": 1,
"name": "target",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": true,
"system": false,
"type": "text"
},
{
"help": "",
"hidden": false,
"id": "nm_protocol",
"maxSelect": 1,
"name": "protocol",
"presentable": false,
"required": true,
"system": false,
"type": "select",
"values": [
"icmp",
"tcp",
"http",
"dns"
]
},
{
"help": "",
"hidden": false,
"id": "nm_port",
"max": 65535,
"min": 0,
"name": "port",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nm_interval",
"max": 3600,
"min": 1,
"name": "interval",
"onlyInt": true,
"presentable": false,
"required": true,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number926446584",
"max": null,
"min": null,
"name": "res",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number1006954605",
"max": null,
"min": null,
"name": "resAvg1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number4267669802",
"max": null,
"min": null,
"name": "resMin1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number591433223",
"max": null,
"min": null,
"name": "resMax1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "number3726709001",
"max": null,
"min": null,
"name": "loss1h",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nm_enabled",
"name": "enabled",
"presentable": false,
"required": false,
"system": false,
"type": "bool"
},
{
"hidden": false,
"id": "autodate2990389176",
"name": "created",
"onCreate": true,
"onUpdate": false,
"presentable": false,
"system": false,
"type": "autodate"
},
{
"help": "",
"hidden": false,
"id": "date3332085495",
"max": "",
"min": "",
"name": "updated",
"presentable": false,
"required": false,
"system": false,
"type": "date"
}
],
"id": "nm_monitors_001",
"indexes": [
"CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)"
],
"listRule": null,
"name": "network_monitors",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": null "viewRule": null
} },
{
"createRule": null,
"deleteRule": null,
"fields": [
{
"autogeneratePattern": "[a-z0-9]{10}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 10,
"min": 10,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"help": "",
"hidden": false,
"id": "nms_system",
"maxSelect": 1,
"minSelect": 0,
"name": "system",
"presentable": false,
"required": true,
"system": false,
"type": "relation"
},
{
"cascadeDelete": true,
"collectionId": "nm_monitors_001",
"help": "",
"hidden": false,
"id": "nms_monitor",
"maxSelect": 1,
"minSelect": 0,
"name": "monitor",
"presentable": false,
"required": false,
"system": false,
"type": "relation"
},
{
"help": "Number of probe attempts",
"hidden": false,
"id": "nms_total_count",
"max": null,
"min": 0,
"name": "total_count",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Number of successful probe attempts",
"hidden": false,
"id": "nms_success_count",
"max": null,
"min": 0,
"name": "success_count",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Sum of successful response times in microseconds",
"hidden": false,
"id": "nms_res_sum",
"max": null,
"min": 0,
"name": "res_sum",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Response time in microseconds",
"hidden": false,
"id": "nms_res_min",
"max": null,
"min": 0,
"name": "res_min",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "Response time in microseconds",
"hidden": false,
"id": "nms_res_max",
"max": null,
"min": 0,
"name": "res_max",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"help": "",
"hidden": false,
"id": "nms_type",
"maxSelect": 1,
"name": "type",
"presentable": false,
"required": true,
"system": false,
"type": "select",
"values": [
"1m",
"10m",
"20m",
"120m",
"480m"
]
},
{
"help": "",
"hidden": false,
"id": "number2990389176",
"max": null,
"min": null,
"name": "created",
"onlyInt": false,
"presentable": false,
"required": false,
"system": false,
"type": "number"
}
],
"id": "nm_stats_001",
"indexes": [
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)"
],
"listRule": null,
"name": "network_monitor_stats",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": null
}
]` ]`
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false) err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
@@ -1,27 +0,0 @@
package migrations
import (
"github.com/pocketbase/pocketbase/core"
m "github.com/pocketbase/pocketbase/migrations"
)
func init() {
m.Register(func(app core.App) error {
c, err := app.FindCollectionByNameOrId("zfs_pools")
if err != nil {
return err
}
c.Fields.Add(&core.TextField{Name: "display_name"})
c.Fields.Add(&core.BoolField{Name: "raw"})
return app.Save(c)
}, func(app core.App) error {
c, err := app.FindCollectionByNameOrId("zfs_pools")
if err != nil {
return err
}
c.Fields.RemoveByName("display_name")
c.Fields.RemoveByName("raw")
return app.Save(c)
})
}
@@ -1,24 +0,0 @@
package migrations
import (
"github.com/pocketbase/pocketbase/core"
m "github.com/pocketbase/pocketbase/migrations"
)
func init() {
m.Register(func(app core.App) error {
collection, err := app.FindCollectionByNameOrId("containers")
if err != nil {
return err
}
collection.Fields.Add(&core.BoolField{Name: "updatable"})
return app.Save(collection)
}, func(app core.App) error {
collection, err := app.FindCollectionByNameOrId("containers")
if err != nil {
return err
}
collection.Fields.RemoveByName("updatable")
return app.Save(collection)
})
}
+226
View File
@@ -0,0 +1,226 @@
//go:build testing
package records_test
import (
"testing"
"time"
monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/records"
"github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestAverageMonitorStats(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")
require.NoError(t, err)
assert.Nil(t, collection.Fields.GetByName("res_avg"))
assert.Nil(t, collection.Fields.GetByName("loss"))
rm := records.NewRecordManager(hub)
user, err := tests.CreateUser(hub, "monitor-avg@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "monitor-avg-system",
"host": "localhost",
"port": "45876",
"status": "up",
"users": []string{user.Id},
})
require.NoError(t, err)
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id,
"name": "cloudflare",
"target": "1.1.1.1",
"protocol": "icmp",
"interval": 30,
"enabled": true,
})
require.NoError(t, err)
created := time.Now().UnixMilli()
// Unequal probe counts must weight both latency and loss.
recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id,
"monitor": monitor.Id,
"type": "1m",
"created": created,
"res_min": 5,
"res_max": 20,
"total_count": 6, "success_count": 6, "res_sum": 60,
})
require.NoError(t, err)
recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id,
"monitor": monitor.Id,
"type": "1m",
"created": created,
"res_min": 10,
"res_max": 60,
"total_count": 1, "success_count": 1, "res_sum": 22,
})
require.NoError(t, err)
result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)
for _, tc := range []struct {
name, monitor, recordType string
after int64
}{
{"other monitor", "missing", "1m", created - 1},
{"other type", monitor.Id, "10m", created - 1},
{"exclusive cutoff", monitor.Id, "1m", created},
} {
t.Run(tc.name, func(t *testing.T) {
stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)
require.NoError(t, err)
assert.Zero(t, count)
assert.Equal(t, monitorEntity.Stats{}, stats)
})
}
// A failure-only bucket counts toward loss but must not lower latency.
recordB.Set("res_min", 0)
recordB.Set("res_max", 0)
recordB.Set("success_count", 0)
recordB.Set("res_sum", 0)
require.NoError(t, hub.Save(recordB))
result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)
// Sparse monitor records must propagate through every rollup level.
rm.CreateLongerRecords()
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
require.NoError(t, err)
require.Len(t, rollups, 1, recordType)
assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
assert.Equal(t, 7, rollups[0].GetInt("total_count"))
assert.Equal(t, 6, rollups[0].GetInt("success_count"))
assert.Equal(t, 60, rollups[0].GetInt("res_sum"))
// A sibling with a different number of probes must retain its actual
// weight when the next tier combines their underlying counts.
_, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,
"res_min": 100, "res_max": 100,
"total_count": 3, "success_count": 1, "res_sum": 100,
})
require.NoError(t, err)
merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)
require.NoError(t, err)
assert.Equal(t, 2, count)
assert.Equal(t, monitorEntity.Stats{
ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,
TotalCount: 10, SuccessCount: 7, ResponseSum: 160,
}, merged)
}
// All failures produce zero latency, while a genuine zero-microsecond
// success remains a valid minimum (it must not be filtered out as a sentinel).
recordA.Set("success_count", 0)
recordA.Set("res_sum", 0)
require.NoError(t, hub.Save(recordA))
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)
recordA.Set("success_count", 1)
recordA.Set("res_min", 0)
recordA.Set("res_max", 0)
require.NoError(t, hub.Save(recordA))
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
require.NoError(t, err)
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)
}
func TestSparseMonitorRollups(t *testing.T) {
for _, tc := range []struct {
name string
interval int
samples int
disabled bool
}{
{"five minute interval", 300, 2, false},
{"ten minute interval", 600, 1, false},
{"fifteen minute interval", 900, 1, false},
{"empty window", 900, 0, false},
{"disabled with pending history", 300, 2, true},
{"disabled empty window", 900, 0, true},
} {
t.Run(tc.name, func(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
user, err := tests.CreateUser(hub, "sparse-monitor@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",
"users": []string{user.Id},
})
require.NoError(t, err)
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
"interval": tc.interval, "enabled": true,
})
require.NoError(t, err)
now := time.Now()
for i := range tc.samples {
_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
"system": sys.Id, "monitor": monitor.Id, "type": "1m",
"created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),
"res_min": 8, "res_max": 20,
"total_count": 4, "success_count": 3, "res_sum": 36,
})
require.NoError(t, err)
}
// Other collections must still reject fewer than nine minute records.
for _, collection := range []string{"system_stats", "container_stats"} {
stats := `{"cpu":10}`
if collection == "container_stats" {
stats = `[{"name":"test","cpu":10}]`
}
for range 8 {
_, err := tests.CreateRecord(hub, collection, map[string]any{
"system": sys.Id, "type": "1m", "stats": stats,
})
require.NoError(t, err)
}
}
if tc.disabled {
monitor.Set("enabled", false)
require.NoError(t, hub.Save(monitor))
}
records.NewRecordManager(hub).CreateLongerRecords()
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
require.NoError(t, err)
if tc.samples == 0 {
assert.Empty(t, rollups, recordType)
} else {
require.Len(t, rollups, 1, recordType)
assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))
assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))
assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))
}
for _, collection := range []string{"system_stats", "container_stats"} {
count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})
require.NoError(t, err)
assert.Zero(t, count, "%s %s", collection, recordType)
}
}
})
}
}
+140 -28
View File
@@ -3,15 +3,16 @@ package records
import ( import (
"encoding/json" "encoding/json"
"log/slog"
"math" "math"
"time" "time"
"github.com/henrygd/beszel/internal/entities/container" "github.com/henrygd/beszel/internal/entities/container"
"github.com/henrygd/beszel/internal/entities/monitor"
"github.com/henrygd/beszel/internal/entities/system" "github.com/henrygd/beszel/internal/entities/system"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core" "github.com/pocketbase/pocketbase/core"
"github.com/pocketbase/pocketbase/tools/types"
) )
type RecordManager struct { type RecordManager struct {
@@ -39,7 +40,7 @@ type StatsRecord struct {
// Create longer records by averaging shorter records // Create longer records by averaging shorter records
func (rm *RecordManager) CreateLongerRecords() { func (rm *RecordManager) CreateLongerRecords() {
// start := time.Now() now := time.Now().UTC()
longerRecordData := []LongerRecordData{ longerRecordData := []LongerRecordData{
{ {
shorterType: "1m", shorterType: "1m",
@@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() {
} }
// wrap the operations in a transaction // wrap the operations in a transaction
// Pocketbase cron does not handle errors, log them here. // Pocketbase cron does not handle errors, log them here.
rm.app.RunInTransaction(func(txApp core.App) error { err := rm.app.RunInTransaction(func(txApp core.App) error {
var err error var err error
collections := [2]*core.Collection{} collections := [2]*core.Collection{}
collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats") collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")
if err != nil { if err != nil {
slog.Error("Error finding cached collection using system stats:", "err", err)
return err return err
} }
collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats") collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")
if err != nil { if err != nil {
slog.Error("Error finding cached collection using container stats:", "err", err) return err
}
monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats")
if err != nil {
return err return err
} }
var systems RecordIds var systems RecordIds
db := txApp.DB() db := txApp.DB()
db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems) if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil {
return err
}
// loop through all active systems, time periods, and collections // loop through all active systems, time periods, and collections
for _, system := range systems { for _, system := range systems {
@@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() {
recordData := longerRecordData[i] recordData := longerRecordData[i]
// log.Println("processing longer record type", recordData.longerType) // log.Println("processing longer record type", recordData.longerType)
// add one minute padding for longer records because they are created slightly later than the job start time // add one minute padding for longer records because they are created slightly later than the job start time
longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute) longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
// shorter records are created independently of longer records, so we shouldn't need to add padding // shorter records are created independently of longer records, so we shouldn't need to add padding
shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration) shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
// loop through both collections
for _, collection := range collections { for _, collection := range collections {
// check creation time of last longer record if not 10m, since 10m is created every run // check creation time of last longer record if not 10m, since 10m is created every run
if recordData.longerType != "10m" { if recordData.longerType != "10m" {
count, err := txApp.CountRecords( count, err := txApp.CountRecords(collection.Id, dbx.NewExp(
collection.Id, "system = {:system} AND type = {:type} AND created > {:created}",
dbx.NewExp( dbx.Params{
"system = {:system} AND type = {:type} AND created > {:created}", "type": recordData.longerType,
dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod}, "system": system.Id,
), "created": longerRecordPeriod.Format(types.DefaultDateLayout),
) },
))
if err != nil {
return err
}
// continue if longer record exists // continue if longer record exists
if err != nil || count > 0 { if count > 0 {
continue continue
} }
} }
// get shorter records from the past x minutes // get shorter records from the past x minutes
var recordIds RecordIds var recordIds RecordIds
err := txApp.DB(). params := dbx.Params{
"type": recordData.shorterType,
"system": system.Id,
"created": shorterRecordPeriod.Format(types.DefaultDateLayout),
}
err := db.
Select("id"). Select("id").
From(collection.Name). From(collection.Name).
AndWhere(dbx.NewExp( Where(dbx.NewExp(
"system={:system} AND type={:type} AND created > {:created}", "system={:system} AND type={:type} AND created > {:created}",
dbx.Params{ params,
"type": recordData.shorterType,
"system": system.Id,
"created": shorterRecordPeriod,
},
)). )).
OrderBy("created"). OrderBy("created").
All(&recordIds) All(&recordIds)
if err != nil {
return err
}
// continue if not enough shorter records // continue if not enough shorter records
if err != nil || len(recordIds) < recordData.minShorterRecords { if len(recordIds) < recordData.minShorterRecords {
continue continue
} }
// average the shorter records and create longer record // average the shorter records and create longer record
@@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() {
case "system_stats": case "system_stats":
longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds)) longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))
case "container_stats": case "container_stats":
longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds)) longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))
} }
if err := txApp.SaveNoValidate(longerRecord); err != nil { if err := txApp.SaveNoValidate(longerRecord); err != nil {
slog.Error("failed to save longer record", "err", err) txApp.Logger().Error("failed to save longer record", "err", err)
} }
} }
} }
} }
// network_monitor_stats is aggregated per monitor (not per system)
var monitors []struct {
Id string `db:"id"`
System string `db:"system"`
}
// Disabled monitors still have history that must advance through retention tiers.
if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil {
return err
}
for _, monitorRec := range monitors {
for i := range longerRecordData {
recordData := longerRecordData[i]
longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
if recordData.longerType != "10m" {
count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp(
"monitor={:monitor} AND type={:type} AND created>{:created}",
dbx.Params{
"monitor": monitorRec.Id,
"type": recordData.longerType,
"created": longerRecordPeriod.UnixMilli(),
},
))
if err != nil {
return err
}
if count > 0 {
continue
}
}
stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli())
if err != nil {
txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err)
continue
}
// Monitor intervals can exceed the aggregation window, so average
// any available records at every level and skip only empty windows.
if count == 0 {
continue
}
longerRecord := core.NewRecord(monitorStatsColl)
longerRecord.Set("system", monitorRec.System)
longerRecord.Set("monitor", monitorRec.Id)
longerRecord.Set("type", recordData.longerType)
longerRecord.Set("created", now.UnixMilli())
longerRecord.Set("res_min", stats.ResMin)
longerRecord.Set("res_max", stats.ResMax)
longerRecord.Set("total_count", stats.TotalCount)
longerRecord.Set("success_count", stats.SuccessCount)
longerRecord.Set("res_sum", stats.ResponseSum)
if err := txApp.SaveNoValidate(longerRecord); err != nil {
txApp.Logger().Error("failed to save monitor longer record", "err", err)
}
}
}
return nil return nil
}) })
if err != nil {
rm.app.Logger().Error("failed to create longer records", "err", err)
}
}
// log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds()) func getCreatedTimeField(collectionName string, period time.Time) any {
// network_monitor_stats stores created as unix timestamp in ms, not as a date string
if collectionName == "network_monitor_stats" {
return period.UnixMilli()
}
return period.Format(types.DefaultDateLayout)
} }
// Calculate the average stats of a list of system_stats records without reflect // Calculate the average stats of a list of system_stats records without reflect
@@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats {
return result return result
} }
// AverageMonitorStats merges probe counts and response sums, preserving their
// weights through every retention tier. Failed probes do not contribute latency.
func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) {
var result struct {
monitor.Stats
Count int `db:"count"`
}
err := db.Select(
"COUNT(*) AS count",
"COALESCE(SUM(total_count), 0) AS total_count",
"COALESCE(SUM(success_count), 0) AS success_count",
"COALESCE(SUM(res_sum), 0) AS res_sum",
"COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min",
"COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max",
).From("network_monitor_stats").Where(dbx.NewExp(
"monitor={:monitor} AND type={:type} AND created>{:created}",
dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter},
)).One(&result)
if err != nil {
return monitor.Stats{}, 0, err
}
if result.SuccessCount > 0 {
result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount))
}
if result.TotalCount > 0 {
result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount))
}
return result.Stats, result.Count, nil
}
/* Round float to two decimals */ /* Round float to two decimals */
func twoDecimals(value float64) float64 { func twoDecimals(value float64) float64 {
return math.Round(value*100) / 100 return math.Round(value*100) / 100
+10 -18
View File
@@ -3,7 +3,6 @@ package records
import ( import (
"fmt" "fmt"
"log/slog" "log/slog"
"strings"
"time" "time"
"github.com/pocketbase/dbx" "github.com/pocketbase/dbx"
@@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int)
// Deletes system_stats records older than what is displayed in the UI // Deletes system_stats records older than what is displayed in the UI
func deleteOldSystemStats(app core.App) error { func deleteOldSystemStats(app core.App) error {
// Collections to process // Collections to process
collections := [2]string{"system_stats", "container_stats"} collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"}
// Record types and their retention periods // Record types and their retention periods
type RecordDeletionData struct { type RecordDeletionData struct {
@@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error {
} }
now := time.Now().UTC() now := time.Now().UTC()
db := app.DB()
for _, collection := range collections { for _, collection := range collections {
// Build the WHERE clause query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}"))
var conditionParts []string for _, rd := range recordData {
var params dbx.Params = make(map[string]any) if _, err := query.Bind(dbx.Params{
for i := range recordData { "type": rd.recordType,
rd := recordData[i] "created": getCreatedTimeField(collection, now.Add(-rd.retention)),
// Create parameterized condition for this record type }).Execute(); err != nil {
dateParam := fmt.Sprintf("date%d", i) return fmt.Errorf("failed to delete from %s: %v", collection, err)
conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam)) }
params[dateParam] = now.Add(-rd.retention)
}
// Combine conditions with OR
conditionStr := strings.Join(conditionParts, " OR ")
// Construct and execute the full raw query
rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr)
if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil {
return fmt.Errorf("failed to delete from %s: %v", collection, err)
} }
} }
return nil return nil
+86
View File
@@ -0,0 +1,86 @@
//go:build testing
package records_test
import (
"testing"
"time"
"github.com/henrygd/beszel/internal/records"
"github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/tools/types"
"github.com/stretchr/testify/require"
)
func TestLongerRecordsPreventDuplicates(t *testing.T) {
for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} {
for _, tier := range []struct {
shorter, longer string
count int
}{
{"10m", "20m", 2},
{"20m", "120m", 6},
{"120m", "480m", 4},
} {
t.Run(collection+"/"+tier.longer, func(t *testing.T) {
hub, err := tests.NewTestHub(t.TempDir())
require.NoError(t, err)
defer hub.Cleanup()
user, err := tests.CreateUser(hub, "rollup@example.com", "testtesttest")
require.NoError(t, err)
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
"name": "rollup-system", "host": "localhost", "port": "45876",
"status": "up", "users": []string{user.Id},
})
require.NoError(t, err)
created := time.Now().UTC().Add(-time.Minute)
data := map[string]any{
"system": sys.Id, "type": tier.shorter,
"created": created.Format(types.DefaultDateLayout),
}
filter := dbx.HashExp{"system": sys.Id, "type": tier.longer}
switch collection {
case "system_stats":
data["stats"] = `{"cpu":10}`
case "container_stats":
data["stats"] = `[{"name":"test","cpu":10}]`
case "network_monitor_stats":
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
"interval": 30, "enabled": true,
})
require.NoError(t, err)
data["monitor"] = monitor.Id
data["created"] = created.UnixMilli()
data["total_count"] = 1
data["success_count"] = 1
data["res_sum"] = 10
data["res_min"] = 10
data["res_max"] = 10
filter["monitor"] = monitor.Id
}
for range tier.count {
_, err := tests.CreateRecord(hub, collection, data)
require.NoError(t, err)
}
rm := records.NewRecordManager(hub)
rm.CreateLongerRecords()
first, err := hub.FindAllRecords(collection, filter)
require.NoError(t, err)
require.Len(t, first, 1)
// The shorter records remain eligible, but the existing longer
// record must prevent another rollup on a subsequent invocation.
rm.CreateLongerRecords()
second, err := hub.FindAllRecords(collection, filter)
require.NoError(t, err)
require.Len(t, second, 1)
require.Equal(t, first[0].Id, second[0].Id)
})
}
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"name": "beszel", "name": "beszel",
"private": true, "private": true,
"version": "0.19.0", "version": "0.20.0",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "vite --host", "dev": "vite --host",
@@ -61,6 +61,8 @@ export const ActiveAlerts = () => {
<AlertDescription> <AlertDescription>
{info.triggeredDesc ? ( {info.triggeredDesc ? (
info.triggeredDesc() info.triggeredDesc()
) : alert.name === "NetworkMonitorLoss" ? (
<Trans>One or more monitors exceed {alert.value}% loss</Trans>
) : alert.name === "Status" ? ( ) : alert.name === "Status" ? (
<Trans>Connection is down</Trans> <Trans>Connection is down</Trans>
) : info.invert ? ( ) : info.invert ? (
@@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
accessorFn: (record) => { accessorFn: (record) => {
const name = record.name const name = record.name
const info = alertInfo[name] const info = alertInfo[name]
return info?.name().replace("cpu", "CPU") || name const label = info?.name().replace("cpu", "CPU") || name
return record.monitor_name ? `${label}: ${record.monitor_name}` : label
}, },
header: ({ column }) => ( header: ({ column }) => (
<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}> <Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
@@ -239,13 +239,13 @@ export function AlertContent({
/** Alerts that fire on first observation have no duration to configure */ /** Alerts that fire on first observation have no duration to configure */
const noDuration = alertData.noDuration === true const noDuration = alertData.noDuration === true
/** Binary alerts have no threshold to configure */ /** Binary alerts have no threshold to configure */
const noThreshold = !!singleDescription || noDuration const noThreshold = !!singleDescription || alertData.noThreshold === true
/** Whether enabling the alert reveals anything to configure */ /** Whether enabling the alert reveals anything to configure */
const hasControls = !(noThreshold && noDuration) const hasControls = !(noThreshold && noDuration)
const [checked, setChecked] = useState(global ? false : !!alert) const [checked, setChecked] = useState(global ? false : !!alert)
const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10)) const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))
const [value, setValue] = useState(alert?.value || (noThreshold ? 0 : (alertData.start ?? 80))) const [value, setValue] = useState(alert?.value ?? (noThreshold ? 0 : (alertData.start ?? 80)))
const Icon = alertData.icon const Icon = alertData.icon
@@ -319,7 +319,7 @@ export function AlertContent({
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground"> <div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
<Suspense fallback={<div className="h-10" />}> <Suspense fallback={<div className="h-10" />}>
{!noThreshold && ( {!noThreshold && (
<div> <div className={cn(noDuration && "col-span-full")}>
<p id={`v${name}`} className="text-sm block h-6"> <p id={`v${name}`} className="text-sm block h-6">
{alertData.invert ? ( {alertData.invert ? (
<Trans> <Trans>
@@ -66,7 +66,7 @@ export default function AreaChartDefault({
}) { }) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth() const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false }) const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
const sourceData = customData ?? chartData.systemStats const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData) const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled) const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -111,6 +111,8 @@ export default function AreaChartDefault({
}) })
}, [areasKey, displayMaxToggled]) }, [areasKey, displayMaxToggled])
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
return useMemo(() => { return useMemo(() => {
if (displayData.length === 0) { if (displayData.length === 0) {
return null return null
@@ -146,7 +148,7 @@ export default function AreaChartDefault({
axisLine={false} axisLine={false}
/> />
)} )}
{xAxis(chartData)} {XAxis}
<ChartTooltip <ChartTooltip
animationEasing="ease-out" animationEasing="ease-out"
animationDuration={150} animationDuration={150}
@@ -167,5 +169,5 @@ export default function AreaChartDefault({
</AreaChart> </AreaChart>
</ChartContainer> </ChartContainer>
) )
}, [displayData, yAxisWidth, filter, Areas]) }, [displayData, yAxisWidth, filter, Areas, XAxis])
} }
@@ -9,14 +9,21 @@ import { memo } from "react"
export default memo(function ChartTimeSelect({ export default memo(function ChartTimeSelect({
className, className,
agentVersion, agentVersion,
chartTimeStore = $chartTime,
allowRealtime = true,
}: { }: {
className?: string className?: string
agentVersion: SemVer agentVersion: SemVer
chartTimeStore?: typeof $chartTime
allowRealtime?: boolean
}) { }) {
const chartTime = useStore($chartTime) const chartTime = useStore(chartTimeStore)
// remove chart times that are not supported by the system agent version // remove chart times that are not supported by the system agent version
const availableChartTimes = Object.entries(chartTimeData).filter(([_, { minVersion }]) => { const availableChartTimes = Object.entries(chartTimeData).filter(([value, { minVersion }]) => {
if (value === "1m" && !allowRealtime) {
return false
}
if (!minVersion) { if (!minVersion) {
return true return true
} }
@@ -24,7 +31,7 @@ export default memo(function ChartTimeSelect({
}) })
return ( return (
<Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => $chartTime.set(value)}> <Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => chartTimeStore.set(value)}>
<SelectTrigger className={cn(className, "relative ps-10 pe-5")}> <SelectTrigger className={cn(className, "relative ps-10 pe-5")}>
<HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" /> <HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
<SelectValue /> <SelectValue />
@@ -22,6 +22,10 @@ export type DataPoint<T = SystemStatsRecord> = {
order?: number order?: number
strokeOpacity?: number strokeOpacity?: number
activeDot?: boolean activeDot?: boolean
dot?: boolean
/** Which Y axis this series plots against. Defaults to "left". */
yAxisId?: "left" | "right"
strokeDasharray?: string
} }
export default function LineChartDefault({ export default function LineChartDefault({
@@ -30,9 +34,12 @@ export default function LineChartDefault({
max, max,
maxToggled, maxToggled,
tickFormatter, tickFormatter,
tickFormatter2,
contentFormatter, contentFormatter,
dataPoints, dataPoints,
domain, domain,
domain2,
max2,
legend, legend,
itemSorter, itemSorter,
showTotal = false, showTotal = false,
@@ -41,18 +48,24 @@ export default function LineChartDefault({
filter, filter,
truncate = false, truncate = false,
chartProps, chartProps,
connectNulls,
}: { }: {
chartData: ChartData chartData: ChartData
// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData) // biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)
customData?: any[] customData?: any[]
max?: number max?: number
max2?: number
maxToggled?: boolean maxToggled?: boolean
tickFormatter: (value: number, index: number) => string tickFormatter: (value: number, index: number) => string
/** Tick formatter for the right ("right"-yAxisId) axis, when any dataPoint uses it. */
tickFormatter2?: (value: number, index: number) => string
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop // biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
contentFormatter: (item: any, key: string) => ReactNode contentFormatter: (item: any, key: string) => ReactNode
// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types // biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types
dataPoints?: DataPoint<any>[] dataPoints?: DataPoint<any>[]
domain?: AxisDomain domain?: AxisDomain
/** Domain for the right axis, when any dataPoint uses it. */
domain2?: AxisDomain
legend?: boolean legend?: boolean
showTotal?: boolean showTotal?: boolean
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop // biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
@@ -62,10 +75,15 @@ export default function LineChartDefault({
filter?: string filter?: string
truncate?: boolean truncate?: boolean
chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin"> chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">
connectNulls?: boolean
}) { }) {
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth() const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
const hasRightAxis = !!dataPoints?.some((dp) => dp.yAxisId === "right")
// fixed width for the secondary axis rather than measured, since its labels (e.g. loss %) are short
// and predictable, and this avoids depending on a second async width-measurement pass to settle
const rightAxisWidth = 38
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false }) const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
const sourceData = customData ?? chartData.systemStats const sourceData = customData ?? chartData.systemStats ?? []
const [displayData, setDisplayData] = useState(sourceData) const [displayData, setDisplayData] = useState(sourceData)
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled) const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
@@ -83,7 +101,9 @@ export default function LineChartDefault({
}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData]) }, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])
// Use a stable key derived from data point identities and visual properties // Use a stable key derived from data point identities and visual properties
const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity ?? ""}`).join("\0") const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity}${d.dot}${d.yAxisId}${d.strokeDasharray}`).join("\0")
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
const Lines = useMemo(() => { const Lines = useMemo(() => {
return dataPoints?.map((dataPoint, i) => { return dataPoints?.map((dataPoint, i) => {
@@ -94,17 +114,20 @@ export default function LineChartDefault({
return ( return (
<Line <Line
key={dataPoint.label} key={dataPoint.label}
yAxisId={dataPoint.yAxisId ?? "left"}
dataKey={dataPoint.dataKey} dataKey={dataPoint.dataKey}
name={dataPoint.label} name={dataPoint.label}
type="monotoneX" type="monotoneX"
dot={false} dot={dataPoint.dot || false}
strokeWidth={1.5} strokeWidth={1.5}
stroke={color} stroke={color}
strokeOpacity={dataPoint.strokeOpacity} strokeOpacity={dataPoint.strokeOpacity}
strokeDasharray={dataPoint.strokeDasharray}
isAnimationActive={false} isAnimationActive={false}
// stackId={dataPoint.stackId} // stackId={dataPoint.stackId}
order={dataPoint.order || i} order={dataPoint.order || i}
activeDot={dataPoint.activeDot ?? true} activeDot={dataPoint.activeDot ?? true}
connectNulls={connectNulls}
/> />
) )
}) })
@@ -135,6 +158,7 @@ export default function LineChartDefault({
<CartesianGrid vertical={false} /> <CartesianGrid vertical={false} />
{!hideYAxis && ( {!hideYAxis && (
<YAxis <YAxis
yAxisId="left"
direction="ltr" direction="ltr"
orientation={chartData.orientation} orientation={chartData.orientation}
className="tracking-tighter" className="tracking-tighter"
@@ -145,7 +169,20 @@ export default function LineChartDefault({
axisLine={false} axisLine={false}
/> />
)} )}
{xAxis(chartData)} {!hideYAxis && hasRightAxis && (
<YAxis
yAxisId="right"
direction="ltr"
orientation={chartData.orientation === "left" ? "right" : "left"}
className="tracking-tighter"
width={rightAxisWidth}
domain={domain2 ?? [0, max2 ?? "auto"]}
tickFormatter={tickFormatter2 ?? tickFormatter}
tickLine={false}
axisLine={false}
/>
)}
{XAxis}
<ChartTooltip <ChartTooltip
animationEasing="ease-out" animationEasing="ease-out"
animationDuration={150} animationDuration={150}
@@ -166,5 +203,5 @@ export default function LineChartDefault({
</LineChart> </LineChart>
</ChartContainer> </ChartContainer>
) )
}, [displayData, yAxisWidth, filter, Lines]) }, [displayData, yAxisWidth, hasRightAxis, filter, Lines, XAxis])
} }
@@ -11,6 +11,7 @@ import {
HardDriveIcon, HardDriveIcon,
LogsIcon, LogsIcon,
MailIcon, MailIcon,
NetworkIcon,
Server, Server,
ServerIcon, ServerIcon,
SettingsIcon, SettingsIcon,
@@ -122,6 +123,20 @@ export default memo(function CommandPalette({ open, setOpen }: { open: boolean;
<Trans>Page</Trans> <Trans>Page</Trans>
</CommandShortcut> </CommandShortcut>
</CommandItem> </CommandItem>
<CommandItem
onSelect={() => {
navigate(getPagePath($router, "monitors"))
setOpen(false)
}}
>
<NetworkIcon className="me-2 size-4" />
<span>
<Trans>Network Monitors</Trans>
</span>
<CommandShortcut>
<Trans>Page</Trans>
</CommandShortcut>
</CommandItem>
<CommandItem <CommandItem
onSelect={() => { onSelect={() => {
navigate(getPagePath($router, "settings", { name: "general" })) navigate(getPagePath($router, "settings", { name: "general" }))
@@ -16,7 +16,7 @@ import {
import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons" import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"
import { Badge } from "../ui/badge" import { Badge } from "../ui/badge"
import { t } from "@lingui/core/macro" import { t } from "@lingui/core/macro"
import { $allSystemsById, $longestSystemNameLen } from "@/lib/stores" import { $allSystemsById, $longestSystemName } from "@/lib/stores"
import { useStore } from "@nanostores/react" import { useStore } from "@nanostores/react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip" import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
@@ -59,15 +59,22 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
const allSystems = $allSystemsById.get() const allSystems = $allSystemsById.get()
const systemNameA = allSystems[a.original.system]?.name ?? "" const systemNameA = allSystems[a.original.system]?.name ?? ""
const systemNameB = allSystems[b.original.system]?.name ?? "" const systemNameB = allSystems[b.original.system]?.name ?? ""
return systemNameA.localeCompare(systemNameB) const primary = systemNameA.localeCompare(systemNameB)
if (primary !== 0) {
return primary
}
return a.original.name.localeCompare(b.original.name)
}, },
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />, header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => { cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById) const allSystems = useStore($allSystemsById)
const longestName = useStore($longestSystemNameLen) const longestName = useStore($longestSystemName)
return ( return (
<div className="ms-1 max-w-40 truncate" style={{ width: `${longestName / 1.05}ch` }}> <div className="ms-1 relative w-fit max-w-40">
{allSystems[getValue() as string]?.name ?? ""} <span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div> </div>
) )
}, },
@@ -189,12 +196,12 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
<Tooltip> <Tooltip>
<TooltipTrigger <TooltipTrigger
className="shrink-0 rounded-sm text-emerald-600 dark:text-emerald-400 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring" className="shrink-0 rounded-sm text-emerald-600 dark:text-emerald-400 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
aria-label={t`Image update available`} aria-label={t({ message: "Image update available", context: "Docker image" })}
onClick={(event) => event.stopPropagation()} onClick={(event) => event.stopPropagation()}
> >
<CircleArrowUpIcon className="size-4" aria-hidden="true" /> <CircleArrowUpIcon className="size-4" aria-hidden="true" />
</TooltipTrigger> </TooltipTrigger>
<TooltipContent>{t`Image update available`}</TooltipContent> <TooltipContent>{t({ message: "Image update available", context: "Docker image" })}</TooltipContent>
</Tooltip> </Tooltip>
)} )}
</div> </div>
+23
View File
@@ -8,6 +8,7 @@ import {
LogOutIcon, LogOutIcon,
LogsIcon, LogsIcon,
MenuIcon, MenuIcon,
NetworkIcon,
PlusIcon, PlusIcon,
SearchIcon, SearchIcon,
ServerIcon, ServerIcon,
@@ -108,6 +109,13 @@ export default function Navbar() {
<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} /> <HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
<span>S.M.A.R.T.</span> <span>S.M.A.R.T.</span>
</DropdownMenuItem> </DropdownMenuItem>
<DropdownMenuItem
onClick={() => navigate(getPagePath($router, "monitors"))}
className="flex items-center"
>
<NetworkIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
<Trans>Network Monitors</Trans>
</DropdownMenuItem>
<DropdownMenuItem <DropdownMenuItem
onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))} onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}
className="flex items-center" className="flex items-center"
@@ -179,6 +187,21 @@ export default function Navbar() {
</TooltipTrigger> </TooltipTrigger>
<TooltipContent>S.M.A.R.T.</TooltipContent> <TooltipContent>S.M.A.R.T.</TooltipContent>
</Tooltip> </Tooltip>
<Tooltip>
<TooltipTrigger asChild>
<Link
href={getPagePath($router, "monitors")}
className={cn("hidden md:grid", buttonVariants({ variant: "ghost", size: "icon" }))}
aria-label="Network Monitors"
onMouseEnter={() => import("@/components/routes/monitors")}
>
<NetworkIcon className="h-[1.2rem] w-[1.2rem]" strokeWidth={1.5} />
</Link>
</TooltipTrigger>
<TooltipContent>
<Trans>Network Monitors</Trans>
</TooltipContent>
</Tooltip>
<ModeToggle /> <ModeToggle />
<Tooltip> <Tooltip>
<TooltipTrigger asChild> <TooltipTrigger asChild>
@@ -0,0 +1,772 @@
import { useCallback, useEffect, useRef, useState } from "react"
import { Trans, useLingui } from "@lingui/react/macro"
import { useStore } from "@nanostores/react"
import { pb } from "@/lib/api"
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import {
DropdownMenu,
DropdownMenuCheckboxItem,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"
import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"
import { Button } from "@/components/ui/button"
import { Input } from "@/components/ui/input"
import { Label } from "@/components/ui/label"
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
import { Textarea } from "@/components/ui/textarea"
import { ChevronDownIcon, ListIcon, SearchIcon, ServerIcon } from "lucide-react"
import { useToast } from "@/components/ui/use-toast"
import { $systems } from "@/lib/stores"
import { cn, supportsNetworkMonitors } from "@/lib/utils"
import type { NetworkMonitorRecord } from "@/types"
import * as v from "valibot"
type MonitorProtocol = "icmp" | "tcp" | "http" | "dns"
type MonitorValues = {
system: string
target: string
protocol: MonitorProtocol
port: number
interval: string
}
type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {
interval: number
}
type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">
const defaultInterval = 30
const MonitorProtocolSchema = v.picklist(["icmp", "tcp", "http", "dns"])
const MonitorIntervalSchema = v.pipe(v.string(), v.toNumber(), v.minValue(1), v.maxValue(3600))
// Both the single-monitor form and the bulk importer flow through this schema so
// defaults and HTTP target normalization stay in one place.
const NormalizedMonitorValuesSchema = v.pipe(
v.object({
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
protocol: MonitorProtocolSchema,
port: v.number(),
interval: MonitorIntervalSchema,
}),
v.transform((input): NormalizedMonitorValues => {
let { protocol, port } = input
let httpTarget = input.target
if (protocol === "icmp" || protocol === "http" || protocol === "dns") {
if (protocol === "http") {
httpTarget = normalizeHttpTarget(input.target, port)
}
port = 0
} else if (protocol === "tcp" && !port) {
port = 443
}
return {
// HTTP monitors may be entered as bare hostnames, so normalize them to a
// scheme-bearing URL before the payload is sent to PocketBase.
target: protocol === "http" ? httpTarget : input.target,
protocol,
port,
interval: input.interval,
}
}),
v.forward(
v.check((input) => {
if (input.protocol === "icmp" || input.protocol === "http" || input.protocol === "dns") {
return input.port === 0
}
return Number.isInteger(input.port) && input.port >= 1 && input.port <= 65535
}, "Port must be between 1 and 65535"),
["port"]
)
)
// Bulk parsing only trims raw CSV fields. Inference, defaults, and protocol-
// specific validation still go through the shared normalization schema above.
const BulkMonitorSchema = v.object({
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
protocol: v.optional(v.pipe(v.string(), v.trim())),
port: v.optional(v.pipe(v.string(), v.trim())),
interval: v.optional(v.pipe(v.string(), v.trim())),
})
function normalizeHttpTarget(target: string, port = 0) {
const useExplicitPort = port > 0 && port !== 80 && port !== 443
const hasOriginOnlyTarget = /^https?:\/\/[^/?#]+$/i.test(target)
if (!/^https?:\/\//i.test(target)) {
const scheme = port === 80 ? "http" : "https"
return `${scheme}://${target}${useExplicitPort ? `:${port}` : ""}`
}
let parsedUrl: URL
try {
parsedUrl = new URL(target)
} catch {
return target
}
if (!parsedUrl.port && useExplicitPort) {
parsedUrl.port = `${port}`
}
// avoid converting "http://localhost:8090" to "http://localhost:8090/" - keep the original formatting if the URL is just an origin
if (hasOriginOnlyTarget && parsedUrl.pathname === "/" && !parsedUrl.search && !parsedUrl.hash) {
return parsedUrl.origin
}
return parsedUrl.toString()
}
function trimTrailingEmptyFields(fields: string[]) {
let lastValueIndex = fields.length - 1
while (lastValueIndex > 0 && !fields[lastValueIndex]) {
lastValueIndex--
}
return fields.slice(0, lastValueIndex + 1)
}
function buildMonitorPayload(values: MonitorValues, enabled = true) {
const normalizedValues = v.safeParse(NormalizedMonitorValuesSchema, values)
if (!normalizedValues.success) {
throw new Error(normalizedValues.issues[0]?.message || "Invalid monitor")
}
const payload = {
system: values.system,
enabled,
...normalizedValues.output,
}
return payload
}
type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">
function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {
return `${system}${target}${protocol}${port}`
}
function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {
const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")
const parsed = v.safeParse(BulkMonitorSchema, {
target: rawTarget,
protocol: rawProtocol,
port: rawPort,
interval: rawInterval,
})
if (!parsed.success) {
throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)
}
const protocol = (parsed.output.protocol?.toLowerCase() ||
(/^https?:\/\//i.test(parsed.output.target) ? "http" : "icmp")) as MonitorProtocol
return buildMonitorPayload({
system,
target: parsed.output.target,
protocol,
port: parsed.output.port ? Number(parsed.output.port) : 0,
interval: parsed.output.interval || `${defaultInterval}`,
})
}
export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {
const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`
const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`
return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")
}
function SystemMultiSelect({
id,
selectedSystemIds,
onChange,
disabled,
className,
}: {
id: string
selectedSystemIds: Set<string>
onChange: (ids: Set<string>) => void
disabled?: boolean
className?: string
}) {
const systems = useStore($systems)
const { t } = useLingui()
const [search, setSearch] = useState("")
const searchRef = useRef<HTMLInputElement>(null)
const focusSearchOnMount = useCallback((node: HTMLInputElement | null) => {
searchRef.current = node
if (!node) return
// Focus after the menu has completed its own initial focus handling.
const frame = requestAnimationFrame(() => node.focus())
return () => cancelAnimationFrame(frame)
}, [])
const contentRef = useRef<HTMLDivElement>(null)
const query = search.trim().toLocaleLowerCase()
const filteredSystems = systems.filter(
(system) => supportsNetworkMonitors(system) && system.name.toLocaleLowerCase().includes(query)
)
const allSelected = filteredSystems.every((system) => selectedSystemIds.has(system.id))
const anySelected = filteredSystems.some((system) => selectedSystemIds.has(system.id))
const selectFiltered = (selected: boolean) => {
const next = new Set(selectedSystemIds)
for (const system of filteredSystems) {
if (selected) next.add(system.id)
else next.delete(system.id)
}
onChange(next)
}
return (
<DropdownMenu onOpenChange={() => setSearch("")}>
<DropdownMenuTrigger asChild>
<Button
id={id}
disabled={disabled}
type="button"
variant="outline"
className={cn("relative w-full min-w-0 ps-10 pe-10 justify-start font-normal text-start", className)}
>
<ServerIcon className="size-3.5 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
<span className="truncate">
{selectedSystemIds.size === 0
? t`Select systems`
: selectedSystemIds.size === 1
? systems.find((s) => selectedSystemIds.has(s.id))?.name
: t`${selectedSystemIds.size} selected`}
</span>
<ChevronDownIcon className="size-4 absolute end-4 top-1/2 -translate-y-1/2 opacity-50" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent
ref={contentRef}
onKeyDown={(event) => {
if (event.key === "Tab") {
event.preventDefault()
searchRef.current?.focus()
}
}}
align="start"
className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden"
>
<div className="shrink-0 border-b mb-1">
<div className="flex items-center gap-2 px-2.5">
<SearchIcon aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />
<Input
ref={focusSearchOnMount}
value={search}
onChange={(event) => setSearch(event.target.value)}
placeholder={t`Search systems`}
aria-label={t`Search systems`}
className="h-10 min-w-0 rounded-none border-0 bg-transparent px-0 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0"
onKeyDown={(event) => {
if (event.key === "Escape") return
// Keep menu typeahead and form submission from consuming search input.
event.stopPropagation()
if (event.key === "Enter") event.preventDefault()
if (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Tab") {
event.preventDefault()
const items = contentRef.current?.querySelectorAll<HTMLElement>(
'[role^="menuitem"]:not([data-disabled])'
)
const index = event.key === "ArrowUp" || event.shiftKey ? (items?.length ?? 1) - 1 : 0
items?.[index]?.focus()
}
}}
/>
</div>
<div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 px-1 pb-1">
<div className="flex items-center">
<DropdownMenuItem
className="px-1.5 py-1 text-xs text-muted-foreground"
disabled={!filteredSystems.length || allSelected}
onSelect={(event) => {
event.preventDefault()
selectFiltered(true)
}}
>
{query ? <Trans>Select matches</Trans> : <Trans>Select all</Trans>}
</DropdownMenuItem>
<span aria-hidden="true" className="text-xs text-muted-foreground/50">
·
</span>
<DropdownMenuItem
className="px-1.5 py-1 text-xs text-muted-foreground"
disabled={!anySelected}
onSelect={(event) => {
event.preventDefault()
selectFiltered(false)
}}
>
{query ? <Trans>Clear matches</Trans> : <Trans>Clear all</Trans>}
</DropdownMenuItem>
</div>
<span className="px-1.5 text-xs tabular-nums text-muted-foreground">
{t`${selectedSystemIds.size} selected`}
</span>
</div>
</div>
<div className="min-h-0 overflow-y-auto">
{filteredSystems.length === 0 && (
<output className="block px-2.5 py-3 text-sm text-muted-foreground">
<Trans>No systems found.</Trans>
</output>
)}
{filteredSystems.map((sys) => (
<DropdownMenuCheckboxItem
key={sys.id}
checked={selectedSystemIds.has(sys.id)}
onSelect={(event) => event.preventDefault()}
onCheckedChange={(checked) => {
const next = new Set(selectedSystemIds)
if (checked) next.add(sys.id)
else next.delete(sys.id)
onChange(next)
}}
className="group min-w-0 gap-2.5 py-2 ps-2.5"
indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3"
>
<span className="truncate">{sys.name}</span>
</DropdownMenuCheckboxItem>
))}
</div>
</DropdownMenuContent>
</DropdownMenu>
)
}
export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) {
const [open, setOpen] = useState(false)
const [bulkOpen, setBulkOpen] = useState(false)
const [bulkInput, setBulkInput] = useState("")
const [bulkLoading, setBulkLoading] = useState(false)
const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState<Set<string>>(new Set())
const bulkFormRef = useRef<HTMLFormElement>(null)
const { toast } = useToast()
const { t } = useLingui()
const resetBulkForm = () => {
setBulkInput("")
}
const openBulkAdd = (selectedSystemIds?: Set<string>) => {
if (!systemId && selectedSystemIds) {
setBulkSelectedSystemIds(new Set(selectedSystemIds))
}
setOpen(false)
setBulkOpen(true)
}
const openAdd = () => {
setBulkOpen(false)
setOpen(true)
}
async function handleBulkSubmit(e: React.FormEvent) {
e.preventDefault()
setBulkLoading(true)
let closedForSubmit = false
try {
const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds)
if (!targetSystems.length) {
throw new Error("Select at least one system.")
}
const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim())
if (!rawLines.length) {
throw new Error("Enter at least one monitor.")
}
let totalCreated = 0
closedForSubmit = true
for (const system of targetSystems) {
const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system))
const existingMonitorKeys = new Set(
monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor))
)
const newPayloads: typeof payloads = []
for (const payload of payloads) {
const monitorKey = getMonitorIdentityKey(payload)
if (existingMonitorKeys.has(monitorKey)) {
continue
}
existingMonitorKeys.add(monitorKey)
newPayloads.push(payload)
}
if (!newPayloads.length) continue
let batch = pb.createBatch()
let inBatch = 0
for (const payload of newPayloads) {
batch.collection("network_monitors").create(payload)
inBatch++
if (inBatch > 20) {
await batch.send()
batch = pb.createBatch()
inBatch = 0
}
}
if (inBatch) {
await batch.send()
}
totalCreated += newPayloads.length
}
if (!totalCreated) {
throw new Error("No new monitors. All entries already exist.")
}
resetBulkForm()
toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` })
} catch (err: unknown) {
if (closedForSubmit) {
setBulkOpen(true)
}
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
} finally {
setBulkLoading(false)
}
}
return (
<>
<div className="flex gap-0 rounded-lg">
<Button variant="outline" onClick={openAdd} className="rounded-e-none grow">
{/* <PlusIcon className="size-4 me-1" /> */}
<Trans>Add {{ foo: t`Monitor` }}</Trans>
</Button>
<div className="w-px h-full bg-muted"></div>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="outline" className="px-2 rounded-s-none border-s-0" aria-label={`More actions`}>
<ChevronDownIcon className="size-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem onClick={() => openBulkAdd()}>
<ListIcon className="size-4 me-2" />
<Trans>Bulk Add</Trans>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
<Dialog
open={open}
onOpenChange={(nextOpen) => {
setOpen(nextOpen)
}}
>
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} onOpenBulkAdd={openBulkAdd} />
</Dialog>
<Sheet
open={bulkOpen}
onOpenChange={(nextOpen) => {
setBulkOpen(nextOpen)
if (!nextOpen) {
resetBulkForm()
}
}}
>
<SheetContent className="w-full sm:max-w-xl gap-0">
<SheetHeader className="border-b">
<SheetTitle>
<Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans>
</SheetTitle>
<SheetDescription>target[,protocol[,port[,interval]]]</SheetDescription>
</SheetHeader>
<form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">
<div className="flex-1 flex flex-col space-y-4 overflow-auto p-4">
{!systemId && (
<div className="grid gap-2">
<Label htmlFor="bulk-monitor-systems" className="sr-only">
<Trans>Systems</Trans>
</Label>
<SystemMultiSelect
id="bulk-monitor-systems"
selectedSystemIds={bulkSelectedSystemIds}
onChange={setBulkSelectedSystemIds}
disabled={bulkLoading}
className="bg-card"
/>
</div>
)}
<div className="grow flex flex-col gap-2">
<Label htmlFor="bulk-monitors" className="sr-only">
Entries
</Label>
<Textarea
id="bulk-monitors"
value={bulkInput}
onChange={(e) => setBulkInput(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
e.preventDefault()
bulkFormRef.current?.requestSubmit()
}
}}
className="font-mono grow text-sm bg-card"
placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}
required
/>
<p className="text-xs text-muted-foreground">target[,protocol[,port[,interval]]]</p>
</div>
</div>
<SheetFooter className="border-t">
<Button type="submit" disabled={bulkLoading || (!systemId && !bulkSelectedSystemIds.size)}>
<Trans>Add {{ foo: t`Network Monitors` }}</Trans>
</Button>
</SheetFooter>
</form>
</SheetContent>
</Sheet>
</>
)
}
export function EditMonitorDialog({
open,
setOpen,
systemId,
monitor,
}: {
open: boolean
setOpen: (open: boolean) => void
systemId?: string
monitor?: NetworkMonitorRecord
}) {
const hasOpened = useRef(false)
if (!monitor && !hasOpened.current) {
return null
}
hasOpened.current = true
return (
<Dialog open={open} onOpenChange={setOpen}>
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} monitor={monitor} />
</Dialog>
)
}
function MonitorDialogContent({
open,
setOpen,
systemId,
monitor,
onOpenBulkAdd,
}: {
open: boolean
setOpen: (open: boolean) => void
systemId?: string
monitor?: NetworkMonitorRecord
onOpenBulkAdd?: (selectedSystemIds: Set<string>) => void
}) {
const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")
const [target, setTarget] = useState(monitor?.target ?? "")
const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))
const [loading, setLoading] = useState(false)
const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")
const [selectedSystemIds, setSelectedSystemIds] = useState<Set<string>>(new Set())
const systems = useStore($systems)
const { toast } = useToast()
const { t } = useLingui()
const isEditing = !!monitor
// When the dialog is opened, initialize form fields with monitor values (if editing) or defaults (if adding).
useEffect(() => {
if (!open) {
return
}
setProtocol(monitor?.protocol ?? "icmp")
setTarget(monitor?.target ?? "")
setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
setMonitorInterval(String(monitor?.interval ?? defaultInterval))
setSelectedSystemId(monitor?.system ?? "")
setSelectedSystemIds(new Set())
setLoading(false)
}, [open, monitor])
async function handleSubmit(e: React.FormEvent) {
e.preventDefault()
setLoading(true)
const targetSystems = systemId ? [systemId] : monitor ? [selectedSystemId] : Array.from(selectedSystemIds)
const remainingSystemIds = new Set(targetSystems)
try {
if (!targetSystems.length || !targetSystems[0]) throw new Error("Select at least one system.")
const payload = buildMonitorPayload(
{
system: targetSystems[0],
target,
protocol,
port: protocol === "tcp" ? Number(port) : 0,
interval: monitorInterval,
},
monitor ? monitor.enabled : true
)
if (monitor) {
await pb.collection("network_monitors").update(monitor.id, payload)
} else {
for (const system of targetSystems) {
await pb.collection("network_monitors").create({ ...payload, system })
remainingSystemIds.delete(system)
}
}
setOpen(false)
} catch (err: unknown) {
if (!monitor && !systemId) {
// Retain only unfinished systems so retrying cannot duplicate successful creates.
setSelectedSystemIds(remainingSystemIds)
}
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
} finally {
setLoading(false)
}
}
return (
<DialogContent className="max-w-md">
<DialogHeader>
<DialogTitle>
{isEditing ? (
<Trans>Edit {{ foo: t`Network Monitor` }}</Trans>
) : (
<Trans>Add {{ foo: t`Network Monitor` }}</Trans>
)}
</DialogTitle>
<DialogDescription>
<Trans>Configure response monitoring from this agent.</Trans>
</DialogDescription>
</DialogHeader>
<form onSubmit={handleSubmit} className="grid gap-4 tabular-nums">
{!systemId && !isEditing && (
<div className="grid gap-2">
<Label htmlFor="monitor-systems">
<Trans>Systems</Trans>
</Label>
<SystemMultiSelect
id="monitor-systems"
selectedSystemIds={selectedSystemIds}
onChange={setSelectedSystemIds}
disabled={loading}
/>
</div>
)}
{!systemId && isEditing && (
<div className="grid gap-2">
<Label>
<Trans>System</Trans>
</Label>
<Select value={selectedSystemId} onValueChange={setSelectedSystemId} required>
<SelectTrigger>
<SelectValue placeholder={t`Select a system`} />
</SelectTrigger>
<SelectContent>
{systems
.filter((sys) => sys.id === monitor?.system || supportsNetworkMonitors(sys))
.map((sys) => (
<SelectItem key={sys.id} value={sys.id}>
{sys.name}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
)}
<div className="grid gap-2">
<Label>
<Trans>Target</Trans>
</Label>
<Input
value={target}
onChange={(e) => setTarget(e.target.value)}
placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}
required
/>
</div>
<div className="grid gap-2">
<Label>
<Trans>Protocol</Trans>
</Label>
<Select value={protocol} onValueChange={(value) => setProtocol(value as MonitorProtocol)}>
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="icmp">ICMP</SelectItem>
<SelectItem value="tcp">TCP</SelectItem>
<SelectItem value="http">HTTP</SelectItem>
<SelectItem value="dns">DNS</SelectItem>
</SelectContent>
</Select>
</div>
{protocol === "tcp" && (
<div className="grid gap-2">
<Label>
<Trans>Port</Trans>
</Label>
<Input
type="number"
value={port}
onChange={(e) => setPort(e.target.value)}
placeholder="443"
min={1}
max={65535}
/>
</div>
)}
<div className="grid gap-2">
<Label>
<Trans>Interval (seconds)</Trans>
</Label>
<Input
type="number"
value={monitorInterval}
onChange={(e) => setMonitorInterval(e.target.value)}
min={1}
max={3600}
required
/>
</div>
<DialogFooter>
{!isEditing && onOpenBulkAdd && (
<Button
type="button"
variant="outline"
onClick={() => onOpenBulkAdd(selectedSystemIds)}
disabled={loading}
className="me-auto"
>
<ListIcon className="size-4 me-2" />
<Trans>Bulk Add</Trans>
</Button>
)}
<Button
type="submit"
disabled={loading || (!systemId && (isEditing ? !selectedSystemId : !selectedSystemIds.size))}
>
{isEditing ? (
<Trans>Save {{ foo: t`Monitor` }}</Trans>
) : (
<Trans>Add {{ foo: t`Monitor` }}</Trans>
)}
</Button>
</DialogFooter>
</form>
</DialogContent>
)
}
@@ -0,0 +1,411 @@
import type { CellContext, Column, ColumnDef } from "@tanstack/react-table"
import { Button } from "@/components/ui/button"
import { cn, copyToClipboard, decimalString, formatMicroseconds, hourWithSeconds } from "@/lib/utils"
import {
GlobeIcon,
TimerIcon,
WifiOffIcon,
Trash2Icon,
ArrowLeftRightIcon,
MoreHorizontalIcon,
ServerIcon,
ClockIcon,
RefreshCwIcon,
PenBoxIcon,
PauseCircleIcon,
PlayCircleIcon,
CopyIcon,
CopyPlusIcon,
} from "lucide-react"
import { t } from "@lingui/core/macro"
import type { NetworkMonitorRecord, SystemRecord } from "@/types"
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuSeparator,
DropdownMenuSub,
DropdownMenuSubContent,
DropdownMenuSubTrigger,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu"
import { Trans } from "@lingui/react/macro"
import { $allSystemsById, $longestSystemName } from "@/lib/stores"
import { useStore } from "@nanostores/react"
import { SystemStatus } from "@/lib/enums"
import { Checkbox } from "@/components/ui/checkbox"
import { useMemo } from "react"
import { formatBulkMonitorLine } from "@/components/network-monitors-table/monitor-dialog"
import { Badge } from "../ui/badge"
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import { pb } from "@/lib/api"
const protocolColors: Record<string, string> = {
icmp: "bg-blue-500/15! text-blue-600 dark:text-blue-400",
tcp: "bg-purple-500/15! text-purple-600 dark:text-purple-400",
http: "bg-green-500/15! text-green-700 dark:text-green-400",
dns: "bg-amber-500/15! text-amber-600 dark:text-amber-400",
}
const SYSTEM_STATUS_COLORS = {
[SystemStatus.Up]: "bg-green-500",
[SystemStatus.Down]: "bg-red-500",
[SystemStatus.Paused]: "bg-primary/40",
[SystemStatus.Pending]: "bg-yellow-500",
} as const
/**
* A monitor is considered muted if it's disabled or if its associated system is not up.
*/
const isMuted = (record: NetworkMonitorRecord, systemRecord: SystemRecord | undefined) =>
!record.enabled || systemRecord?.status !== SystemStatus.Up
export function getMonitorColumns(
longestTarget = "",
{
onEdit,
onDelete,
onSetEnabled,
}: {
onEdit?: (monitor: NetworkMonitorRecord) => void
onDelete?: (monitors: NetworkMonitorRecord[]) => void | Promise<void>
onSetEnabled?: (monitors: NetworkMonitorRecord[], enabled: boolean) => void | Promise<void>
} = {}
): ColumnDef<NetworkMonitorRecord>[] {
return [
{
id: "select",
header: ({ table }) => (
<Checkbox
className="ms-2"
checked={table.getIsAllRowsSelected() || (table.getIsSomeRowsSelected() && "indeterminate")}
onClick={(event) => event.stopPropagation()}
onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}
aria-label={t`Select all`}
/>
),
cell: ({ row }) => (
<Checkbox
checked={row.getIsSelected()}
onClick={(event) => event.stopPropagation()}
onCheckedChange={(value) => row.toggleSelected(!!value)}
aria-label={t`Select row`}
/>
),
enableSorting: false,
enableHiding: false,
size: 44,
},
{
id: "system",
accessorFn: (record) => record.system,
sortingFn: (a, b) => {
const allSystems = $allSystemsById.get()
const systemNameA = allSystems[a.original.system]?.name ?? ""
const systemNameB = allSystems[b.original.system]?.name ?? ""
const primary = systemNameA.localeCompare(systemNameB)
if (primary !== 0) {
return primary
}
return a.original.target.localeCompare(b.original.target)
},
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
cell: ({ getValue }) => {
const system = useStore($allSystemsById)[getValue() as string] as SystemRecord | undefined
const longestSystemName = useStore($longestSystemName)
const name = system?.name
const status = system?.status as SystemStatus // undefined val is fine but makes lsp mad
return useMemo(
() => (
<div className="ms-1.5 max-w-44 flex gap-2 items-center tabular-nums">
<span className={cn("shrink-0 size-2 rounded-full", SYSTEM_STATUS_COLORS[status])} />
<div className="relative w-fit min-w-0 max-w-full">
<span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestSystemName}
</span>
<span className="absolute inset-0 truncate">{name}</span>
</div>
</div>
),
[status, name]
)
},
},
{
id: "target",
sortingFn: (a, b) => a.original.target.localeCompare(b.original.target),
accessorFn: (record) => getMonitorTarget(record),
header: ({ column }) => <HeaderButton column={column} name={t`Target`} Icon={GlobeIcon} />,
cell: ({ row, getValue }) => {
const monitor = row.original
const { status } = useStore($allSystemsById)[monitor.system] || {}
let color = "bg-green-500"
if (!monitor.enabled || status === SystemStatus.Paused) {
color = "bg-primary/40"
} else if (status === SystemStatus.Down || status === SystemStatus.Pending) {
color = "bg-yellow-500"
}
return (
<div className="ms-1.5 max-w-64 flex gap-2 items-center tabular-nums">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
<div className="relative w-fit min-w-0 max-w-full">
<span className="invisible block overflow-hidden whitespace-nowrap" aria-hidden="true">
{longestTarget}
</span>
<span className="absolute inset-0 truncate">{getValue() as string}</span>
</div>
</div>
)
},
},
{
id: "protocol",
accessorFn: (record) => record.protocol,
header: ({ column }) => <HeaderButton column={column} name={t`Protocol`} Icon={ArrowLeftRightIcon} />,
cell: ({ getValue }) => {
const protocol = getValue() as string
return <Badge className={cn("uppercase", protocolColors[protocol])}>{protocol}</Badge>
},
},
{
id: "interval",
accessorFn: (record) => record.interval,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Interval`} Icon={RefreshCwIcon} />,
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{getValue() as number}s</span>,
},
{
id: "res",
accessorFn: (record) => record.res,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Response`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "res1h",
accessorFn: (record) => record.resAvg1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Avg 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "max1h",
accessorFn: (record) => record.resMax1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Max 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "min1h",
accessorFn: (record) => record.resMin1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Min 1h`} Icon={TimerIcon} />,
cell: responseTimeCell,
},
{
id: "loss",
accessorFn: (record) => record.loss1h,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Loss 1h`} Icon={WifiOffIcon} />,
cell: ({ row }) => {
const { loss1h, res, system } = row.original
const systemRecord = useStore($allSystemsById)[system]
if (loss1h === undefined || (!res && !loss1h)) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
const muted = isMuted(row.original, systemRecord)
let color = "bg-green-500"
if (muted) {
color = "bg-muted-foreground/50"
} else if (loss1h) {
color = loss1h > 20 ? "bg-red-500" : "bg-yellow-500"
}
return (
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
{loss1h === 100 ? loss1h : decimalString(loss1h, loss1h >= 10 ? 1 : 2)}%
</span>
)
},
},
{
id: "updated",
invertSorting: true,
accessorFn: (record) => record.updated,
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
cell: ({ getValue }) => {
const timestamp = getValue() as number
if (!timestamp) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
return <span className="ms-1.5 tabular-nums">{hourWithSeconds(timestamp)}</span>
},
},
{
id: "actions",
enableSorting: false,
enableHiding: false,
header: () => null,
size: 40,
cell: ({ row, table }) => {
const selectedRows = table.getSelectedRowModel().rows
const actionRows =
row.getIsSelected() && selectedRows.length > 1
? selectedRows.map((selectedRow) => selectedRow.original)
: [row.original]
const isBulkAction = actionRows.length > 1
const shouldPause = actionRows.some((monitor) => monitor.enabled)
const bulkCopyContent = actionRows.map((monitor) => formatBulkMonitorLine(monitor)).join("\n")
const allSystems = useStore($allSystemsById)
const otherSystems = useMemo(
() => Object.values(allSystems).filter((s) => !isBulkAction && s.id !== row.original.system),
[allSystems, isBulkAction]
)
return (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button variant="ghost" size="icon" className="size-10">
<span className="sr-only">
<Trans>Open menu</Trans>
</span>
<MoreHorizontalIcon className="w-5" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
{!isBulkAction && (
<DropdownMenuItem
onClick={() => {
onEdit?.(row.original)
}}
>
<PenBoxIcon className="me-2.5 size-4" />
<Trans>Edit</Trans>
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={() => {
onSetEnabled?.(actionRows, !shouldPause)
}}
>
{shouldPause ? (
<>
<PauseCircleIcon className="me-2.5 size-4" />
<Trans>Pause</Trans>
</>
) : (
<>
<PlayCircleIcon className="me-2.5 size-4" />
<Trans>Resume</Trans>
</>
)}
</DropdownMenuItem>
<DropdownMenuItem
onClick={() => {
copyToClipboard(bulkCopyContent)
}}
>
<CopyIcon className="me-2.5 size-4" />
<Trans>Bulk copy</Trans>
</DropdownMenuItem>
{!isBulkAction && otherSystems.length > 0 && (
<DropdownMenuSub>
<DropdownMenuSubTrigger>
<CopyPlusIcon className="me-2.5 size-4" />
<Trans>Copy to system</Trans>
</DropdownMenuSubTrigger>
<DropdownMenuSubContent className="max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">
{otherSystems.map((sys) => (
<DropdownMenuItem
key={sys.id}
onClick={() => {
const { id: _id, system: _system, ...rest } = row.original
pb.collection("network_monitors")
.create({ ...rest, system: sys.id })
.catch(() => {})
}}
>
{sys.name}
</DropdownMenuItem>
))}
</DropdownMenuSubContent>
</DropdownMenuSub>
)}
<DropdownMenuSeparator />
<DropdownMenuItem
onClick={() => {
onDelete?.(actionRows)
}}
>
<Trash2Icon className="me-2.5 size-4" />
<Trans>Delete</Trans>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
)
},
},
]
}
const responseTimeThresholds = {
http: { warning: 800_000, critical: 3_000_000 },
tcp: { warning: 500_000, critical: 2_000_000 },
icmp: { warning: 100_000, critical: 500_000 },
dns: { warning: 150_000, critical: 800_000 },
}
function responseTimeCell(cell: CellContext<NetworkMonitorRecord, unknown>) {
const monitor = cell.row.original
const systemRecord = useStore($allSystemsById)[monitor.system]
const responseTime = cell.getValue() as number | undefined
if (!responseTime) {
return <span className="ms-1.5 text-muted-foreground">-</span>
}
const muted = isMuted(monitor, systemRecord)
let color = "bg-green-500"
if (muted) {
color = "bg-muted-foreground/50"
} else if (responseTime > responseTimeThresholds[monitor.protocol].warning) {
color = "bg-yellow-500"
}
if (!muted && responseTime > responseTimeThresholds[monitor.protocol].critical) {
color = "bg-red-500"
}
return (
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
<span className={cn("shrink-0 size-2 rounded-full", color)} />
{formatMicroseconds(responseTime)}
</span>
)
}
function HeaderButton({
column,
name,
Icon,
}: {
column: Column<NetworkMonitorRecord>
name: string
Icon: React.ElementType
}) {
const isSorted = column.getIsSorted()
return (
<Button
className={cn(
"h-9 px-3 flex items-center gap-2 duration-50",
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
)}
variant="ghost"
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
>
{Icon && <Icon className="size-4" />}
{name}
</Button>
)
}
@@ -0,0 +1,574 @@
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import { t } from "@lingui/core/macro"
import { Trans } from "@lingui/react/macro"
import {
type ColumnFiltersState,
flexRender,
getCoreRowModel,
getFilteredRowModel,
getSortedRowModel,
type Row,
type RowSelectionState,
type SortingState,
type Table as TableType,
useReactTable,
type VisibilityState,
} from "@tanstack/react-table"
import { useVirtualizer, type VirtualItem } from "@tanstack/react-virtual"
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog"
import { Button, buttonVariants } from "@/components/ui/button"
import { memo, useCallback, useMemo, useRef, useState } from "react"
import { getMonitorColumns } from "@/components/network-monitors-table/network-monitors-columns"
import { Card, CardHeader, CardTitle } from "@/components/ui/card"
import { Input } from "@/components/ui/input"
import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { useToast } from "@/components/ui/use-toast"
import { isReadOnlyUser } from "@/lib/api"
import { pb } from "@/lib/api"
import { $allSystemsById, $direction, $userSettings } from "@/lib/stores"
import {
cn,
isVisuallyLonger,
matchesFilterGroups,
parseFilterGroups,
parseSemVer,
useBrowserStorage,
} from "@/lib/utils"
import type { ChartData, NetworkMonitorRecord } from "@/types"
import { AddMonitorDialog, EditMonitorDialog } from "./monitor-dialog"
import { ArrowLeftRightIcon, EthernetPortIcon, LoaderCircleIcon, ServerIcon, XIcon } from "lucide-react"
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
import ChartTimeSelect from "@/components/charts/chart-time-select"
import { LossChart, AvgMinMaxResponseChart } from "@/components/routes/system/charts/monitors-charts"
import { useNetworkMonitorStats } from "@/lib/use-network-monitors"
import { useStore } from "@nanostores/react"
import { atom } from "nanostores"
import { Separator } from "../ui/separator"
import { $router, Link } from "../router"
import { getPagePath } from "@nanostores/router"
export default function NetworkMonitorsTableNew({
systemId,
monitors,
isLoading,
}: {
systemId?: string
monitors: NetworkMonitorRecord[]
isLoading: boolean
}) {
const [sorting, setSorting] = useBrowserStorage<SortingState>(
`sort-np-target-${systemId ? 1 : 0}`,
[{ id: systemId ? "target" : "system", desc: false }],
sessionStorage
)
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
const [rowSelection, setRowSelection] = useState<RowSelectionState>({})
const [globalFilter, setGlobalFilter] = useState("")
const [deleteOpen, setDeleteOpen] = useState(false)
const [pendingDeleteIds, setPendingDeleteIds] = useState<string[]>([])
const [editingMonitor, setEditingMonitor] = useState<NetworkMonitorRecord>()
const { toast } = useToast()
const canManageMonitors = !isReadOnlyUser()
const longestTarget = useMemo(() => {
let longestTarget = ""
for (const p of monitors) {
if (isVisuallyLonger(getMonitorTarget(p), longestTarget)) {
longestTarget = getMonitorTarget(p)
}
}
return longestTarget
}, [monitors])
const runMonitorBatch = useCallback(
async (ids: string[], enqueue: (batch: ReturnType<typeof pb.createBatch>, id: string) => void) => {
let batch = pb.createBatch()
let inBatch = 0
for (const id of ids) {
enqueue(batch, id)
if (++inBatch >= 20) {
await batch.send()
batch = pb.createBatch()
inBatch = 0
}
}
if (inBatch) {
await batch.send()
}
},
[]
)
const handleDeleteRequest = useCallback(
async (monitorsToDelete: NetworkMonitorRecord[]) => {
if (!monitorsToDelete.length) {
return
}
const ids = monitorsToDelete.map((monitor) => monitor.id)
if (ids.length === 1) {
try {
await pb.collection("network_monitors").delete(ids[0])
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to delete monitors.`,
})
}
return
}
setPendingDeleteIds(ids)
setDeleteOpen(true)
},
[toast]
)
const handleBulkDelete = async () => {
setDeleteOpen(false)
if (!pendingDeleteIds.length) {
return
}
try {
await runMonitorBatch(pendingDeleteIds, (batch, id) => batch.collection("network_monitors").delete(id))
setPendingDeleteIds([])
setRowSelection({})
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to delete monitors.`,
})
}
}
const handleSetEnabled = useCallback(
async (monitorsToUpdate: NetworkMonitorRecord[], enabled: boolean) => {
if (!monitorsToUpdate.length) {
return
}
const pendingUpdates = monitorsToUpdate.filter((monitor) => monitor.enabled !== enabled)
if (!pendingUpdates.length) {
return
}
try {
if (pendingUpdates.length === 1) {
await pb.collection("network_monitors").update(pendingUpdates[0].id, { enabled })
return
}
await runMonitorBatch(
pendingUpdates.map((monitor) => monitor.id),
(batch, id) => batch.collection("network_monitors").update(id, { enabled })
)
if (monitorsToUpdate.length > 1) {
setRowSelection({})
}
} catch (err: unknown) {
toast({
variant: "destructive",
title: t`Error`,
description: (err as Error)?.message || t`Failed to update monitors.`,
})
}
},
[runMonitorBatch, toast]
)
const columns = useMemo(() => {
let columns = getMonitorColumns(longestTarget, {
onEdit: setEditingMonitor,
onDelete: handleDeleteRequest,
onSetEnabled: handleSetEnabled,
})
columns = systemId ? columns.filter((col) => col.id !== "system") : columns
columns = canManageMonitors ? columns : columns.filter((col) => col.id !== "actions")
return columns
}, [canManageMonitors, handleDeleteRequest, handleSetEnabled, systemId, longestTarget])
const table = useReactTable({
data: monitors,
columns,
getRowId: (row) => row.id,
getCoreRowModel: getCoreRowModel(),
getSortedRowModel: getSortedRowModel(),
getFilteredRowModel: getFilteredRowModel(),
onSortingChange: setSorting,
onColumnFiltersChange: setColumnFilters,
onColumnVisibilityChange: setColumnVisibility,
onRowSelectionChange: setRowSelection,
defaultColumn: {
sortUndefined: "last",
size: 900,
minSize: 0,
},
state: {
sorting,
columnFilters,
columnVisibility,
rowSelection,
globalFilter,
},
onGlobalFilterChange: setGlobalFilter,
globalFilterFn: (row, _columnId, filterValue) => {
const value = (filterValue as string).trim()
if (!value) return true
const monitor = row.original
const systemName = $allSystemsById.get()[monitor.system]?.name ?? ""
const searchString = `${getMonitorTarget(monitor)}${monitor.protocol}${systemName}`.toLocaleLowerCase()
return matchesFilterGroups(searchString, parseFilterGroups(value))
},
})
const rows = table.getRowModel().rows
const visibleColumns = table.getVisibleLeafColumns()
return (
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
<CardHeader className="p-0 mb-3 sm:mb-4">
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
<div className="px-2 sm:px-1">
<CardTitle className="mb-2">
<Trans>Network Monitors</Trans>
</CardTitle>
<div className="text-sm text-muted-foreground flex items-center flex-wrap">
<Trans>Response time monitoring from agents.</Trans>
</div>
</div>
<div className="md:ms-auto flex items-center gap-2">
{monitors.length > 0 && (
<div className="relative">
<Input
placeholder={t`Filter...`}
value={globalFilter}
onChange={(e) => setGlobalFilter(e.target.value)}
className="ms-auto px-4 w-full max-w-full md:w-50"
/>
{globalFilter && (
<Button
type="button"
variant="ghost"
size="icon"
aria-label={t`Clear`}
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
onClick={() => setGlobalFilter("")}
>
<XIcon className="h-4 w-4" />
</Button>
)}
</div>
)}
{canManageMonitors ? <AddMonitorDialog systemId={systemId} monitors={monitors} /> : null}
{canManageMonitors ? (
<EditMonitorDialog
systemId={systemId}
monitor={editingMonitor}
open={!!editingMonitor}
setOpen={(open) => {
if (!open) {
setEditingMonitor(undefined)
}
}}
/>
) : null}
<AlertDialog
open={deleteOpen}
onOpenChange={(open) => {
setDeleteOpen(open)
if (!open) {
setPendingDeleteIds([])
}
}}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
<Trans>Are you sure?</Trans>
</AlertDialogTitle>
<AlertDialogDescription>
<Trans>This will permanently delete all selected records from the database.</Trans>
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>
<Trans>Cancel</Trans>
</AlertDialogCancel>
<AlertDialogAction
className={cn(buttonVariants({ variant: "destructive" }))}
onClick={handleBulkDelete}
>
<Trans>Continue</Trans>
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</div>
</div>
</CardHeader>
<div className="rounded-md">
<NetworkMonitorsTable
table={table}
rows={rows}
colLength={visibleColumns.length}
rowSelection={rowSelection}
isLoading={isLoading}
/>
</div>
</Card>
)
}
const NetworkMonitorsTable = memo(function NetworkMonitorTable({
table,
rows,
colLength,
rowSelection,
isLoading,
}: {
table: TableType<NetworkMonitorRecord>
rows: Row<NetworkMonitorRecord>[]
colLength: number
rowSelection: RowSelectionState
isLoading: boolean
}) {
const scrollRef = useRef<HTMLDivElement>(null)
const [sheetOpen, setSheetOpen] = useState(false)
const [activeMonitorId, setActiveMonitorId] = useState<string | null>(null)
const activeMonitor = activeMonitorId
? table.options.data.find((monitor) => monitor.id === activeMonitorId)
: undefined
const openSheet = useCallback((monitor: NetworkMonitorRecord) => {
setActiveMonitorId(monitor.id)
setSheetOpen(true)
}, [])
const virtualizer = useVirtualizer<HTMLDivElement, HTMLTableRowElement>({
count: rows.length,
estimateSize: () => 54,
getScrollElement: () => scrollRef.current,
overscan: 5,
})
const virtualRows = virtualizer.getVirtualItems()
const paddingTop = Math.max(0, virtualRows[0]?.start ?? 0 - virtualizer.options.scrollMargin)
const paddingBottom = Math.max(0, virtualizer.getTotalSize() - (virtualRows[virtualRows.length - 1]?.end ?? 0))
return (
<div
className={cn(
"h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto border rounded-md",
(!rows.length || rows.length > 2) && "min-h-50"
)}
ref={scrollRef}
>
<div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>
<table className="text-sm w-full h-full text-nowrap">
<NetworkMonitorTableHead table={table} />
<TableBody>
{rows.length ? (
virtualRows.map((virtualRow) => {
const row = rows[virtualRow.index]
return (
<NetworkMonitorTableRow
key={row.id}
row={row}
virtualRow={virtualRow}
isSelected={row.getIsSelected()}
rowSelection={rowSelection}
openSheet={openSheet}
/>
)
})
) : (
<TableRow>
<TableCell colSpan={colLength} className="h-37 text-center pointer-events-none">
{isLoading ? (
<LoaderCircleIcon className="animate-spin size-10 opacity-60 mx-auto" />
) : (
<Trans>No results.</Trans>
)}
</TableCell>
</TableRow>
)}
</TableBody>
</table>
</div>
<NetworkMonitorSheet
open={sheetOpen}
onOpenChange={(nextOpen) => {
setSheetOpen(nextOpen)
}}
monitor={activeMonitor}
/>
</div>
)
})
function NetworkMonitorTableHead({ table }: { table: TableType<NetworkMonitorRecord> }) {
return (
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
{table.getHeaderGroups().map((headerGroup) => (
<tr key={headerGroup.id}>
{headerGroup.headers.map((header) => {
return (
<TableHead className="px-2" key={header.id}>
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
</TableHead>
)
})}
</tr>
))}
</TableHeader>
)
}
const NetworkMonitorTableRow = memo(function NetworkMonitorTableRow({
row,
virtualRow,
isSelected,
rowSelection: _rowSelection,
openSheet,
}: {
row: Row<NetworkMonitorRecord>
virtualRow: VirtualItem
isSelected: boolean
// Menus depend on the entire selection, including changes to other rows.
rowSelection: RowSelectionState
openSheet: (monitor: NetworkMonitorRecord) => void
}) {
return (
<TableRow
data-state={isSelected && "selected"}
className="cursor-pointer transition-opacity"
onClick={() => openSheet(row.original)}
>
{row.getVisibleCells().map((cell) => (
<TableCell
key={cell.id}
className="py-0"
style={{
width: `${cell.column.getSize()}px`,
height: virtualRow.size,
}}
>
{flexRender(cell.column.columnDef.cell, cell.getContext())}
</TableCell>
))}
</TableRow>
)
})
function NetworkMonitorSheet({
open,
onOpenChange,
monitor,
}: {
open: boolean
onOpenChange: (open: boolean) => void
monitor?: NetworkMonitorRecord
}) {
if (!monitor) {
return null
}
return <NetworkMonitorSheetContent key={monitor.system} open={open} onOpenChange={onOpenChange} monitor={monitor} />
}
function NetworkMonitorSheetContent({
open,
onOpenChange,
monitor,
}: {
open: boolean
onOpenChange: (open: boolean) => void
monitor: NetworkMonitorRecord
}) {
// Keep monitor exploration independent of the system charts' time range.
const [chartTimeStore] = useState(() => {
const defaultTime = $userSettings.get().chartTime
return atom(defaultTime === "1m" ? "1h" : defaultTime)
})
const chartTime = useStore(chartTimeStore)
const direction = useStore($direction)
const system = useStore($allSystemsById)[monitor.system]
const monitorStats = useNetworkMonitorStats({
systemId: monitor.system,
monitorId: monitor.id,
chartTime,
enabled: open,
})
const chartData = useMemo<ChartData>(
() => ({
agentVersion: parseSemVer(system?.info?.v),
orientation: direction === "rtl" ? "right" : "left",
chartTime,
}),
[system?.info?.v, direction, chartTime]
)
const hasMonitorStats = monitorStats.some((record) => record.stats?.[monitor.id] != null)
const monitorLabel = getMonitorTarget(monitor)
return (
<Sheet open={open} onOpenChange={onOpenChange}>
<SheetContent className="w-full sm:max-w-220 overflow-auto p-4 sm:p-6">
<SheetHeader className="mb-0 border-b p-0 pb-4">
<SheetTitle>{monitorLabel}</SheetTitle>
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
<ServerIcon className="size-3.5 text-muted-foreground" />
<Link className="hover:underline" href={getPagePath($router, "system", { id: system?.id ?? "" })}>
{system?.name ?? ""}
</Link>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<ArrowLeftRightIcon className="size-3.5 text-muted-foreground" />
{monitor.protocol.toUpperCase()}
{monitor.protocol === "tcp" && monitor.port > 0 && (
<>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<EthernetPortIcon className="size-3.5 text-muted-foreground" />
<span>{monitor.port}</span>
</>
)}
</SheetDescription>
</SheetHeader>
<div className="grid gap-4">
<ChartTimeSelect
className="bg-card"
agentVersion={chartData.agentVersion}
chartTimeStore={chartTimeStore}
allowRealtime={false}
/>
<AvgMinMaxResponseChart
monitorStats={monitorStats}
monitor={monitor}
chartData={chartData}
empty={!hasMonitorStats}
/>
<LossChart
monitorStats={monitorStats}
grid={false}
monitors={[monitor]}
chartData={chartData}
empty={!hasMonitorStats}
showFilter={false}
/>
</div>
</SheetContent>
</Sheet>
)
}
+1
View File
@@ -4,6 +4,7 @@ const routes = {
home: "/", home: "/",
containers: "/containers", containers: "/containers",
smart: "/smart", smart: "/smart",
monitors: "/monitors",
system: `/system/:id`, system: `/system/:id`,
settings: `/settings/:name?`, settings: `/settings/:name?`,
forgot_password: `/forgot-password`, forgot_password: `/forgot-password`,
@@ -0,0 +1,33 @@
import { useLingui } from "@lingui/react/macro"
import { memo, useEffect } from "react"
import NetworkMonitorsTableNew from "@/components/network-monitors-table/network-monitors-table"
import { ActiveAlerts } from "@/components/active-alerts"
import { FooterRepoLink } from "@/components/footer-repo-link"
import { useNetworkMonitors } from "@/lib/use-network-monitors"
import { $allSystemsById } from "@/lib/stores"
import { supportsNetworkMonitors } from "@/lib/utils"
import { useStore } from "@nanostores/react"
export default memo(() => {
const { t } = useLingui()
const { monitors, isLoading } = useNetworkMonitors({})
const systems = useStore($allSystemsById)
const visibleMonitors = monitors.filter((monitor) => {
const system = systems[monitor.system]
return !system || supportsNetworkMonitors(system)
})
useEffect(() => {
document.title = `${t`Network Monitors`} / Beszel`
}, [t])
return (
<>
<div className="grid gap-4">
<ActiveAlerts />
<NetworkMonitorsTableNew monitors={visibleMonitors} isLoading={isLoading} />
</div>
<FooterRepoLink />
</>
)
})
@@ -78,7 +78,7 @@ export default function AlertsHistoryDataTable() {
let unsubscribe: (() => void) | undefined let unsubscribe: (() => void) | undefined
const pbOptions = { const pbOptions = {
expand: "system", expand: "system",
fields: "id,name,value,state,created,resolved,expand.system.name", fields: "id,name,monitor_name,value,state,created,resolved,expand.system.name",
} }
// Initial load // Initial load
pb.collection<AlertsHistoryRecord>("alerts_history") pb.collection<AlertsHistoryRecord>("alerts_history")
@@ -199,7 +199,7 @@ export default function AlertsHistoryDataTable() {
if (!selectedRows.length) return if (!selectedRows.length) return
const cells: Record<string, (record: AlertsHistoryRecord) => string> = { const cells: Record<string, (record: AlertsHistoryRecord) => string> = {
system: (record) => record.expand?.system?.name || record.system, system: (record) => record.expand?.system?.name || record.system,
name: (record) => alertInfo[record.name]?.name() || record.name, name: (record) => [alertInfo[record.name]?.name() || record.name, record.monitor_name].filter(Boolean).join(": "),
value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""), value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),
state: (record) => (record.resolved ? t`Resolved` : t`Active`), state: (record) => (record.resolved ? t`Resolved` : t`Active`),
created: (record) => formatShortDate(record.created), created: (record) => formatShortDate(record.created),
+30 -7
View File
@@ -1,6 +1,6 @@
import { memo, useState } from "react" import { memo, useState } from "react"
import { Trans } from "@lingui/react/macro" import { Trans } from "@lingui/react/macro"
import { compareSemVer, parseSemVer } from "@/lib/utils" import { compareSemVer, parseSemVer, supportsNetworkMonitors } from "@/lib/utils"
import type { GPUData } from "@/types" import type { GPUData } from "@/types"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import InfoBar from "./system/info-bar" import InfoBar from "./system/info-bar"
@@ -12,9 +12,15 @@ import { ZfsCharts } from "./system/charts/storage-pool-charts"
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts" import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts" import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts" import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables" import {
LazyContainersTable,
LazyNetworkMonitorsTable,
LazySmartTable,
LazySystemdTable,
LazyZfsTable,
} from "./system/lazy-tables"
import { LoadAverageChart } from "./system/charts/load-average-chart" import { LoadAverageChart } from "./system/charts/load-average-chart"
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react" import { ContainerIcon, CpuIcon, HardDriveIcon, NetworkIcon, TerminalSquareIcon } from "lucide-react"
import { GpuIcon } from "../ui/icons" import { GpuIcon } from "../ui/icons"
import SystemdTable from "../systemd-table/systemd-table" import SystemdTable from "../systemd-table/systemd-table"
import ContainersTable from "../containers-table/containers-table" import ContainersTable from "../containers-table/containers-table"
@@ -65,9 +71,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
const hasSystemd = system.info.sv const hasSystemd = system.info.sv
const hasGpu = hasGpuData || hasGpuPowerData const hasGpu = hasGpuData || hasGpuPowerData
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0 const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
const hasNetworkMonitors = supportsNetworkMonitors(system)
// keep tabsRef in sync for keyboard navigation // keep tabsRef in sync for keyboard navigation
const tabs = ["core", "disk"] const tabs = ["core", "network", "disk"]
if (hasGpu) tabs.push("gpu") if (hasGpu) tabs.push("gpu")
if (hasContainers) tabs.push("containers") if (hasContainers) tabs.push("containers")
if (hasSystemd) tabs.push("services") if (hasSystemd) tabs.push("services")
@@ -153,6 +160,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
{hasContainersTable && <LazyContainersTable systemId={system.id} />} {hasContainersTable && <LazyContainersTable systemId={system.id} />}
{hasSystemd && <LazySystemdTable systemId={system.id} />} {hasSystemd && <LazySystemdTable systemId={system.id} />}
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
</> </>
) )
} }
@@ -165,6 +174,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<CpuIcon className="size-3.5" /> <CpuIcon className="size-3.5" />
<Trans context="Core system metrics">Core</Trans> <Trans context="Core system metrics">Core</Trans>
</TabsTrigger> </TabsTrigger>
<TabsTrigger value="network" className="w-full flex items-center gap-1.5">
<NetworkIcon className="size-3.5" />
<Trans>Network</Trans>
</TabsTrigger>
<TabsTrigger value="disk" className="w-full flex items-center gap-1.5"> <TabsTrigger value="disk" className="w-full flex items-center gap-1.5">
<HardDriveIcon className="size-3.5" /> <HardDriveIcon className="size-3.5" />
<Trans>Disk</Trans> <Trans>Disk</Trans>
@@ -192,17 +205,27 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}> <TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}>
<div className="grid xl:grid-cols-2 gap-4"> <div className="grid xl:grid-cols-2 gap-4">
<CpuChart {...coreProps} /> <CpuChart {...coreProps} />
<MemoryChart {...coreProps} />
<LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} /> <LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />
<BandwidthChart {...coreProps} systemStats={systemStats} /> <MemoryChart {...coreProps} />
<SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
<TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} /> <TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} />
<FanChart {...coreProps} /> <FanChart {...coreProps} />
<BatteryChart system={system} {...coreProps} /> <BatteryChart system={system} {...coreProps} />
<SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
{pageBottomExtraMargin > 0 && <div style={{ marginBottom: pageBottomExtraMargin }}></div>} {pageBottomExtraMargin > 0 && <div style={{ marginBottom: pageBottomExtraMargin }}></div>}
</div> </div>
</TabsContent> </TabsContent>
<TabsContent value="network" forceMount className={activeTab === "network" ? "contents" : "hidden"}>
{mountedTabs.has("network") && (
<>
<div className="grid xl:grid-cols-2 gap-4">
<BandwidthChart {...coreProps} systemStats={systemStats} />
</div>
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
</>
)}
</TabsContent>
<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}> <TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>
{mountedTabs.has("disk") && ( {mountedTabs.has("disk") && (
<> <>
@@ -1,7 +1,13 @@
import { timeTicks } from "d3-time"
import { getPbTimestamp, pb } from "@/lib/api" import { getPbTimestamp, pb } from "@/lib/api"
import { chartTimeData } from "@/lib/utils" import { chartTimeData } from "@/lib/utils"
import type { ChartData, ChartTimes, ContainerStatsRecord, SystemStatsRecord } from "@/types" import type {
ChartData,
ChartDataContainer,
ChartTimes,
ContainerStatsRecord,
NetworkMonitorStatsRecord,
SystemStatsRecord,
} from "@/types"
type ChartTimeData = { type ChartTimeData = {
time: number time: number
@@ -17,31 +23,10 @@ export const cache = new Map<
ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"] ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]
>() >()
// create ticks and domain for charts
export function getTimeData(chartTime: ChartTimes, lastCreated: number) {
const cached = cache.get("td") as ChartTimeData | undefined
if (cached && cached.chartTime === chartTime) {
if (!lastCreated || cached.time >= lastCreated) {
return cached.data
}
}
// const buffer = chartTime === "1m" ? 400 : 20_000
const now = new Date(Date.now())
const startTime = chartTimeData[chartTime].getOffset(now)
const ticks = timeTicks(startTime, now, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
const data = {
ticks,
domain: [chartTimeData[chartTime].getOffset(now).getTime(), now.getTime()],
}
cache.set("td", { time: now.getTime(), data, chartTime })
return data
}
/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place. /** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.
* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */ * Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */
export function appendData<T extends { created: string | number | null }>( export function appendData<T extends { created: string | number | null }>(
prev: T[], prev: T[] = [],
newRecords: T[], newRecords: T[],
expectedInterval: number, expectedInterval: number,
maxLen?: number maxLen?: number
@@ -66,17 +51,18 @@ export function appendData<T extends { created: string | number | null }>(
return result return result
} }
export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>( export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | NetworkMonitorStatsRecord>(
collection: string, collection: string,
systemId: string, systemId: string,
chartTime: ChartTimes chartTime: ChartTimes,
cachedStats?: { created: string | number | null }[],
createdIsNumber?: boolean
): Promise<T[]> { ): Promise<T[]> {
const cachedStats = cache.get(`${systemId}_${chartTime}_${collection}`) as T[] | undefined
const lastCached = cachedStats?.at(-1)?.created as number const lastCached = cachedStats?.at(-1)?.created as number
return await pb.collection<T>(collection).getFullList({ return await pb.collection<T>(collection).getFullList({
filter: pb.filter("system={:id} && created > {:created} && type={:type}", { filter: pb.filter("system={:id} && created > {:created} && type={:type}", {
id: systemId, id: systemId,
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined), created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, createdIsNumber),
type: chartTimeData[chartTime].type, type: chartTimeData[chartTime].type,
}), }),
fields: "created,stats", fields: "created,stats",
@@ -84,11 +70,11 @@ export async function getStats<T extends SystemStatsRecord | ContainerStatsRecor
}) })
} }
export function makeContainerData(containers: ContainerStatsRecord[]): ChartData["containerData"] { export function makeContainerData(containers: ContainerStatsRecord[]): ChartDataContainer[] {
const result = [] as ChartData["containerData"] const result = [] as ChartDataContainer[]
for (const { created, stats } of containers) { for (const { created, stats } of containers) {
if (!created) { if (!created) {
result.push({ created: null } as ChartData["containerData"][0]) result.push({ created: null } as ChartDataContainer)
continue continue
} }
result.push(makeContainerPoint(new Date(created).getTime(), stats)) result.push(makeContainerPoint(new Date(created).getTime(), stats))
@@ -97,11 +83,8 @@ export function makeContainerData(containers: ContainerStatsRecord[]): ChartData
} }
/** Transform a single realtime container stats message into a ChartDataContainer point. */ /** Transform a single realtime container stats message into a ChartDataContainer point. */
export function makeContainerPoint( export function makeContainerPoint(created: number, stats: ContainerStatsRecord["stats"]): ChartDataContainer {
created: number, const point: ChartDataContainer = { created } as ChartDataContainer
stats: ContainerStatsRecord["stats"]
): ChartData["containerData"][0] {
const point: ChartData["containerData"][0] = { created } as ChartData["containerData"][0]
for (const container of stats) { for (const container of stats) {
;(point as Record<string, unknown>)[container.n] = container ;(point as Record<string, unknown>)[container.n] = container
} }
@@ -0,0 +1,212 @@
import { getMonitorTarget } from "@/lib/network-monitor-utils"
import LineChartDefault from "@/components/charts/line-chart"
import type { DataPoint } from "@/components/charts/line-chart"
import { decimalString, formatMicroseconds, matchesFilterGroups, parseFilterGroups, toFixedFloat } from "@/lib/utils"
import { $monitorFilter } from "@/lib/stores"
import { useLingui } from "@lingui/react/macro"
import { ChartCard, FilterBar } from "../chart-card"
import type { ChartData, MonitorStats, NetworkMonitorRecord, NetworkMonitorStatsRecord } from "@/types"
import { useMemo } from "react"
import { useStore } from "@nanostores/react"
type MonitorChartProps = {
monitorStats: NetworkMonitorStatsRecord[]
grid?: boolean
monitors: NetworkMonitorRecord[]
chartData: ChartData
empty: boolean
showFilter?: boolean
/** Prepended to the chart title, e.g. a target/system name (rendered as "{titlePrefix} — Response"). */
titlePrefix?: string
}
type MonitorChartBaseProps = MonitorChartProps & {
metric: keyof MonitorStats
title: string
description: string
tickFormatter: (value: number) => string
contentFormatter: ({ value }: { value: number | string }) => string | number
domain?: [number | "auto", number | "auto"]
}
function MonitorChart({
monitorStats,
grid,
monitors,
chartData,
empty,
metric,
title,
description,
tickFormatter,
contentFormatter,
domain,
showFilter = monitors.length > 1,
}: MonitorChartBaseProps) {
const storedFilter = useStore($monitorFilter)
const filter = showFilter ? storedFilter : ""
const { dataPoints, visibleKeys } = useMemo(() => {
const sortedMonitors = [...monitors].sort((a, b) => b.resAvg1h - a.resAvg1h)
const count = sortedMonitors.length
const points: DataPoint<NetworkMonitorStatsRecord>[] = []
const visibleIDs: string[] = []
const filterGroups = parseFilterGroups(filter)
const dot = chartData.chartTime === "1m"
for (let i = 0; i < count; i++) {
const p = sortedMonitors[i]
const label = getMonitorTarget(p)
const labelLower = label.toLowerCase()
const filtered = filterGroups.length > 0 && !matchesFilterGroups(labelLower, filterGroups)
if (filtered) {
continue
}
visibleIDs.push(p.id)
points.push({
order: i,
label,
dataKey: (record: NetworkMonitorStatsRecord) => record.stats?.[p.id]?.[metric] ?? null,
dot,
color: count <= 5 ? i + 1 : `hsl(${(i * 360) / count}, var(--chart-saturation), var(--chart-lightness))`,
})
}
return { dataPoints: points, visibleKeys: visibleIDs }
}, [monitors, filter, metric, chartData.chartTime])
const filteredMonitorStats = useMemo(() => {
if (!visibleKeys.length) return monitorStats
return monitorStats.filter((record) => visibleKeys.some((id) => record.stats?.[id] != null))
}, [monitorStats, visibleKeys])
const legend = dataPoints.length < 10 && showFilter
return (
<ChartCard
legend={legend || !showFilter}
cornerEl={showFilter ? <FilterBar store={$monitorFilter} /> : undefined}
empty={empty}
title={title}
description={description}
grid={grid}
>
<LineChartDefault
truncate
chartData={chartData}
customData={filteredMonitorStats}
dataPoints={dataPoints}
domain={domain ?? ["auto", "auto"]}
connectNulls
tickFormatter={tickFormatter}
contentFormatter={contentFormatter}
legend={legend}
filter={filter}
/>
</ChartCard>
)
}
interface AvgMinMaxResponseChartProps {
monitorStats: NetworkMonitorStatsRecord[]
monitor: NetworkMonitorRecord | null
chartData: ChartData
empty: boolean
}
export function AvgMinMaxResponseChart({ monitorStats, monitor, chartData, empty }: AvgMinMaxResponseChartProps) {
const { t } = useLingui()
const { chartTime } = chartData
const hasLongInterval = (monitor?.interval ?? 61) > 60
// only one monitor is relevant for this chart
const dataPoints: DataPoint<NetworkMonitorStatsRecord>[] = useMemo(() => {
const dataFn = (metric: keyof MonitorStats) => (record: NetworkMonitorStatsRecord) =>
record.stats?.[monitor?.id ?? ""]?.[metric] ?? "-"
const avgPoint = {
label: "Avg",
dataKey: dataFn("res_avg"),
color: 1,
order: 0,
}
if (chartTime === "1m" || (hasLongInterval && chartTime === "1h")) {
// avg, min, max are all the same for 1m interval, so just show avg
return [avgPoint]
}
return [
{
label: "Max",
dataKey: dataFn("res_max"),
color: 3,
order: 0,
},
avgPoint,
{
label: "Min",
dataKey: dataFn("res_min"),
color: 2,
order: 2,
},
]
}, [chartTime, hasLongInterval, monitor?.id])
const data = useMemo(() => {
if (!monitor) return []
return monitorStats.filter((record) => record.stats && monitor.id in record.stats)
}, [monitor, monitorStats])
const legend = dataPoints.length > 1
return (
<ChartCard
legend={true}
empty={empty}
title={t`Response`}
description={t`Average, minimum, and maximum response time`}
grid={false}
>
<LineChartDefault
truncate
chartData={chartData}
customData={data}
dataPoints={dataPoints}
domain={["auto", "auto"]}
connectNulls
legend={legend}
tickFormatter={(value) => formatMicroseconds(value, false)}
contentFormatter={({ value }) => {
if (typeof value !== "number") {
return value
}
return formatMicroseconds(value)
}}
/>
</ChartCard>
)
}
export function LossChart({ monitorStats, grid, monitors, chartData, empty, titlePrefix }: MonitorChartProps) {
const { t } = useLingui()
const lossTitle = t({ message: "Loss", context: "Packet loss" })
const title = titlePrefix ? `${titlePrefix} — ${lossTitle}` : lossTitle
return (
<MonitorChart
monitorStats={monitorStats}
grid={grid}
monitors={monitors}
chartData={chartData}
empty={empty}
metric="loss"
title={title}
description={t`Packet loss (%)`}
domain={[0, 100]}
tickFormatter={(value) => `${toFixedFloat(value, value >= 10 ? 0 : 1)}%`}
contentFormatter={({ value }) => {
if (typeof value !== "number") {
return value
}
return `${decimalString(value, 2)}%`
}}
/>
)
}
@@ -1,6 +1,7 @@
import { lazy } from "react" import { lazy } from "react"
import { useIntersectionObserver } from "@/lib/use-intersection-observer" import { useIntersectionObserver } from "@/lib/use-intersection-observer"
import { cn } from "@/lib/utils" import { cn } from "@/lib/utils"
import { useNetworkMonitors } from "@/lib/use-network-monitors"
const ContainersTable = lazy(() => import("../../containers-table/containers-table")) const ContainersTable = lazy(() => import("../../containers-table/containers-table"))
@@ -45,3 +46,19 @@ export function LazySystemdTable({ systemId }: { systemId: string }) {
</div> </div>
) )
} }
const NetworkMonitorsTable = lazy(() => import("../../network-monitors-table/network-monitors-table"))
export function LazyNetworkMonitorsTable({ systemId }: { systemId: string }) {
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
return (
<div ref={ref} className={cn(isIntersecting && "contents")}>
{isIntersecting && <SystemNetworkMonitorsTable systemId={systemId} />}
</div>
)
}
function SystemNetworkMonitorsTable({ systemId }: { systemId: string }) {
const { monitors, isLoading } = useNetworkMonitors({ systemId })
return <NetworkMonitorsTable systemId={systemId} monitors={monitors} isLoading={isLoading} />
}
@@ -43,7 +43,6 @@ import {
toFixedFloat, toFixedFloat,
formatTemperature, formatTemperature,
cn, cn,
getVisualStringWidth,
secondsToString, secondsToString,
hourWithSeconds, hourWithSeconds,
formatShortDate, formatShortDate,
@@ -117,9 +116,9 @@ function formatDataUnits(units: number): string {
const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated" const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"
export const createColumns = ( export const createColumns = (
longestName: number, longestName: string,
longestModel: number, longestModel: string,
longestDevice: number longestDevice: string
): ColumnDef<SmartDeviceRecord>[] => [ ): ColumnDef<SmartDeviceRecord>[] => [
{ {
id: "system", id: "system",
@@ -134,8 +133,11 @@ export const createColumns = (
cell: ({ getValue }) => { cell: ({ getValue }) => {
const allSystems = useStore($allSystemsById) const allSystems = useStore($allSystemsById)
return ( return (
<div className="ms-1.5 max-w-40 block truncate" style={{ width: `${longestName / 1.05}ch` }}> <div className="ms-1.5 relative w-fit max-w-44">
{allSystems[getValue() as string]?.name ?? ""} <span className="invisible block whitespace-nowrap" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
</div> </div>
) )
}, },
@@ -145,12 +147,11 @@ export const createColumns = (
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name), sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />, header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,
cell: ({ getValue }) => ( cell: ({ getValue }) => (
<div <div className="font-medium ms-1 relative w-fit max-w-44" title={getValue() as string}>
className="font-medium max-w-40 truncate ms-1" <span className="invisible block whitespace-nowrap" aria-hidden="true">
title={getValue() as string} {longestDevice}
style={{ width: `${longestDevice / 1.05}ch` }} </span>
> <span className="absolute inset-0 truncate">{getValue() as string}</span>
{getValue() as string}
</div> </div>
), ),
}, },
@@ -161,12 +162,11 @@ export const createColumns = (
<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} /> <HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />
), ),
cell: ({ getValue }) => ( cell: ({ getValue }) => (
<div <div className="ms-1 relative w-fit max-w-44" title={getValue() as string}>
className="max-w-48 truncate ms-1" <span className="invisible block whitespace-nowrap" aria-hidden="true">
title={getValue() as string} {longestModel}
style={{ width: `${longestModel / 1.05}ch` }} </span>
> <span className="absolute inset-0 truncate">{getValue() as string}</span>
{getValue() as string}
</div> </div>
), ),
}, },
@@ -320,7 +320,7 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
// Calculate the right width for the columns based on the longest strings among the displayed devices // Calculate the right width for the columns based on the longest strings among the displayed devices
const { longestName, longestModel, longestDevice } = useMemo(() => { const { longestName, longestModel, longestDevice } = useMemo(() => {
const result = { longestName: 0, longestModel: 0, longestDevice: 0 } const result = { longestName: "", longestModel: "", longestDevice: "" }
if (!smartDevices || Object.keys(allSystems).length === 0) { if (!smartDevices || Object.keys(allSystems).length === 0) {
return result return result
} }
@@ -329,10 +329,16 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
if (!systemId && !seenSystems.has(device.system)) { if (!systemId && !seenSystems.has(device.system)) {
seenSystems.add(device.system) seenSystems.add(device.system)
const name = allSystems[device.system]?.name ?? "" const name = allSystems[device.system]?.name ?? ""
result.longestName = Math.max(result.longestName, getVisualStringWidth(name)) if (name.length > result.longestName.length) {
result.longestName = name
}
}
if ((device.model ?? "").length > result.longestModel.length) {
result.longestModel = device.model ?? ""
}
if ((device.name ?? "").length > result.longestDevice.length) {
result.longestDevice = device.name ?? ""
} }
result.longestModel = Math.max(result.longestModel, getVisualStringWidth(device.model ?? ""))
result.longestDevice = Math.max(result.longestDevice, getVisualStringWidth(device.name ?? ""))
} }
return result return result
}, [smartDevices, systemId, allSystems]) }, [smartDevices, systemId, allSystems])
@@ -26,7 +26,7 @@ import type {
SystemStatsRecord, SystemStatsRecord,
} from "@/types" } from "@/types"
import { $router, navigate } from "../../router" import { $router, navigate } from "../../router"
import { appendData, cache, getStats, getTimeData, makeContainerData, makeContainerPoint } from "./chart-data" import { appendData, cache, getStats, makeContainerData, makeContainerPoint } from "./chart-data"
export type SystemData = ReturnType<typeof useSystemData> export type SystemData = ReturnType<typeof useSystemData>
@@ -185,16 +185,11 @@ export function useSystemData(id: string) {
const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v]) const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])
const chartData: ChartData = useMemo(() => { const chartData: ChartData = useMemo(() => {
const lastCreated = Math.max(
(systemStats.at(-1)?.created as number) ?? 0,
(containerData.at(-1)?.created as number) ?? 0
)
return { return {
systemStats, systemStats,
containerData, containerData,
chartTime, chartTime,
orientation: direction === "rtl" ? "right" : "left", orientation: direction === "rtl" ? "right" : "left",
...getTimeData(chartTime, lastCreated),
agentVersion, agentVersion,
} }
}, [systemStats, containerData, direction]) }, [systemStats, containerData, direction])
@@ -234,8 +229,8 @@ export function useSystemData(id: string) {
} }
Promise.allSettled([ Promise.allSettled([
getStats<SystemStatsRecord>("system_stats", systemId, chartTime), getStats<SystemStatsRecord>("system_stats", systemId, chartTime, cachedSystemStats),
getStats<ContainerStatsRecord>("container_stats", systemId, chartTime), getStats<ContainerStatsRecord>("container_stats", systemId, chartTime, cachedContainerData),
]).then(([systemStats, containerStats]) => { ]).then(([systemStats, containerStats]) => {
// Ignore responses for a previous system or chart time // Ignore responses for a previous system or chart time
if (requestId !== statsRequestId.current) { if (requestId !== statsRequestId.current) {
@@ -327,7 +322,7 @@ export function useSystemData(id: string) {
// derived values // derived values
const isLongerChart = !["1m", "1h"].includes(chartTime) const isLongerChart = !["1m", "1h"].includes(chartTime)
const showMax = maxValues && isLongerChart const showMax = maxValues && isLongerChart
const dataEmpty = !chartLoading && chartData.systemStats.length === 0 const dataEmpty = !chartLoading && chartData.systemStats?.length === 0
const lastGpus = systemStats.at(-1)?.stats?.g const lastGpus = systemStats.at(-1)?.stats?.g
const isPodman = details?.podman ?? system.info?.p ?? false const isPodman = details?.podman ?? system.info?.p ?? false
@@ -26,7 +26,7 @@ import { memo, useMemo, useRef, useState } from "react"
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip" import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
import { isReadOnlyUser, pb } from "@/lib/api" import { isReadOnlyUser, pb } from "@/lib/api"
import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums" import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"
import { $longestSystemNameLen, $userSettings } from "@/lib/stores" import { $longestSystemName, $userSettings } from "@/lib/stores"
import { import {
cn, cn,
copyToClipboard, copyToClipboard,
@@ -135,7 +135,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
Icon: ServerIcon, Icon: ServerIcon,
cell: (info) => { cell: (info) => {
const { name, id } = info.row.original const { name, id } = info.row.original
const longestName = useStore($longestSystemNameLen) const longestName = useStore($longestSystemName)
const linkUrl = getPagePath($router, "system", { id }) const linkUrl = getPagePath($router, "system", { id })
return ( return (
@@ -145,8 +145,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
<Link <Link
href={linkUrl} href={linkUrl}
tabIndex={-1} tabIndex={-1}
className="truncate z-10 relative" className="relative w-fit max-w-48 z-10"
style={{ width: `${longestName / 1.05}ch` }}
onMouseEnter={(e) => { onMouseEnter={(e) => {
// set title on hover if text is truncated to show full name // set title on hover if text is truncated to show full name
const a = e.currentTarget const a = e.currentTarget
@@ -157,7 +156,10 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
} }
}} }}
> >
{name} <span className="invisible block" aria-hidden="true">
{longestName}
</span>
<span className="absolute inset-0 truncate">{name}</span>
</Link> </Link>
</span> </span>
<Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link> <Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>
+66 -24
View File
@@ -1,11 +1,11 @@
import type { JSX } from "react"
import { useLingui } from "@lingui/react/macro" import { useLingui } from "@lingui/react/macro"
import * as React from "react" import * as React from "react"
import * as RechartsPrimitive from "recharts" import * as RechartsPrimitive from "recharts"
import { chartTimeData, cn } from "@/lib/utils" import { chartTimeData, cn } from "@/lib/utils"
import type { ChartData } from "@/types" import type { ChartTimes } from "@/types"
import { Separator } from "./separator" import { Separator } from "./separator"
import { AxisDomain } from "recharts/types/util/types" import type { AxisDomain } from "recharts/types/util/types"
import { timeTicks } from "d3-time"
// Format: { THEME_NAME: CSS_SELECTOR } // Format: { THEME_NAME: CSS_SELECTOR }
const THEMES = { light: "", dark: ".dark" } as const const THEMES = { light: "", dark: ".dark" } as const
@@ -101,7 +101,7 @@ const ChartTooltipContent = React.forwardRef<
labelKey?: string labelKey?: string
unit?: string unit?: string
filter?: string filter?: string
contentFormatter?: (item: any, key: string) => React.ReactNode | string contentFormatter?: (item: unknown, key: string) => React.ReactNode | string
truncate?: boolean truncate?: boolean
showTotal?: boolean showTotal?: boolean
totalLabel?: React.ReactNode totalLabel?: React.ReactNode
@@ -175,7 +175,13 @@ const ChartTooltipContent = React.forwardRef<
} }
const totalKey = "__total__" const totalKey = "__total__"
const totalItem: any = { const totalItem: {
value: number
name: string
dataKey: string
color: string | undefined
payload?: unknown
} = {
value: totalValue, value: totalValue,
name: totalName, name: totalName,
dataKey: totalKey, dataKey: totalKey,
@@ -222,6 +228,11 @@ const ChartTooltipContent = React.forwardRef<
return null return null
} }
payload = payload.filter((item) => item.value != null)
if (!payload.length) {
return null
}
// const nestLabel = payload.length === 1 && indicator !== 'dot' // const nestLabel = payload.length === 1 && indicator !== 'dot'
const nestLabel = false const nestLabel = false
@@ -400,26 +411,57 @@ function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key:
return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config] return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]
} }
let cachedAxis: JSX.Element interface XAxisData {
const xAxis = ({ domain, ticks, chartTime }: ChartData) => { el: React.ReactElement
if (cachedAxis && domain[0] === cachedAxis.props.domain[0]) { domain: [number, number]
return cachedAxis }
const xAxisCache = new Map<ChartTimes, XAxisData>()
function createXAxisData(chartTime: ChartTimes): XAxisData {
// console.log("Creating XAxis for", chartTime, new Date())
const axisEndTime = Date.now() + 500
const axisEndDate = new Date(axisEndTime)
const startTime = chartTimeData[chartTime].getOffset(axisEndDate)
const ticks = timeTicks(startTime, axisEndDate, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
const domain: [number, number] = [startTime.getTime(), axisEndTime]
return {
domain,
el: (
<RechartsPrimitive.XAxis
dataKey="created"
domain={domain}
ticks={ticks}
allowDataOverflow
type="number"
scale="time"
minTickGap={12}
tickMargin={8}
axisLine={false}
tickFormatter={chartTimeData[chartTime].format}
/>
),
} }
cachedAxis = ( }
<RechartsPrimitive.XAxis
dataKey="created" function xAxis(chartTime: ChartTimes, lastCreated: number) {
domain={domain} if (!lastCreated) {
ticks={ticks} return null
allowDataOverflow }
type="number" const cachedAxis = xAxisCache.get(chartTime)
scale="time"
minTickGap={12} const expectedInterval = chartTimeData[chartTime].expectedInterval
tickMargin={8} const conservativeEndTime = Date.now() - expectedInterval / 2
axisLine={false} const axisEndTime = Math.max(lastCreated, conservativeEndTime)
tickFormatter={chartTimeData[chartTime].format}
/> if (cachedAxis && axisEndTime < cachedAxis.domain[1]) {
) return cachedAxis.el
return cachedAxis }
const axisData = createXAxisData(chartTime)
xAxisCache.set(chartTime, axisData)
return axisData.el
} }
export { export {
@@ -90,8 +90,10 @@ DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName
const DropdownMenuCheckboxItem = React.forwardRef< const DropdownMenuCheckboxItem = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>, React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> & {
>(({ className, children, checked, ...props }, ref) => ( indicatorClassName?: string
}
>(({ className, children, checked, indicatorClassName, ...props }, ref) => (
<DropdownMenuPrimitive.CheckboxItem <DropdownMenuPrimitive.CheckboxItem
ref={ref} ref={ref}
className={cn( className={cn(
@@ -101,7 +103,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<
checked={checked} checked={checked}
{...props} {...props}
> >
<span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center"> <span className={cn("absolute left-2 flex h-3.5 w-3.5 items-center justify-center", indicatorClassName)}>
<DropdownMenuPrimitive.ItemIndicator> <DropdownMenuPrimitive.ItemIndicator>
<Check className="h-4 w-4" /> <Check className="h-4 w-4" />
</DropdownMenuPrimitive.ItemIndicator> </DropdownMenuPrimitive.ItemIndicator>
+1 -1
View File
@@ -41,7 +41,7 @@ const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTML
<tr <tr
ref={ref} ref={ref}
className={cn( className={cn(
"border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted!", "border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted/40!",
className className
)} )}
{...props} {...props}
+204 -191
View File
@@ -1,5 +1,5 @@
import { t } from "@lingui/core/macro" import { t } from "@lingui/core/macro"
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, ServerCrashIcon, ServerIcon } from "lucide-react" import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
import type { RecordSubscription } from "pocketbase" import type { RecordSubscription } from "pocketbase"
import { EthernetIcon, GpuIcon } from "@/components/ui/icons" import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
import { $alerts } from "@/lib/stores" import { $alerts } from "@/lib/stores"
@@ -9,210 +9,223 @@ import { ThermometerIcon, BatteryMediumIcon, HourglassIcon } from "@/components/
/** Alert info for each alert type */ /** Alert info for each alert type */
export const alertInfo: Record<string, AlertInfo> = { export const alertInfo: Record<string, AlertInfo> = {
Status: { Status: {
name: () => t`Status`, name: () => t`Status`,
unit: "", unit: "",
icon: ServerIcon, icon: ServerIcon,
desc: () => t`Triggers when status switches between up and down`, desc: () => t`Triggers when status switches between up and down`,
/** "for x minutes" is appended to desc when only one value */ /** "for x minutes" is appended to desc when only one value */
singleDesc: () => `${t`System`} ${t`Down`}`, singleDesc: () => `${t`System`} ${t`Down`}`,
}, },
CPU: { CPU: {
name: () => t`CPU Usage`, name: () => t`CPU Usage`,
unit: "%", unit: "%",
icon: CpuIcon, icon: CpuIcon,
desc: () => t`Triggers when CPU usage exceeds a threshold`, desc: () => t`Triggers when CPU usage exceeds a threshold`,
}, },
CPUIOWait: { CPUIOWait: {
name: () => t`CPU I/O Wait`, name: () => t`CPU I/O Wait`,
unit: "%", unit: "%",
icon: CpuIcon, icon: CpuIcon,
desc: () => t`Triggers when CPU I/O wait exceeds a threshold`, desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
}, },
CPUSteal: { CPUSteal: {
name: () => t`CPU Steal Time`, name: () => t`CPU Steal Time`,
unit: "%", unit: "%",
icon: CpuIcon, icon: CpuIcon,
desc: () => t`Triggers when CPU steal time exceeds a threshold`, desc: () => t`Triggers when CPU steal time exceeds a threshold`,
}, },
Memory: { Memory: {
name: () => t`Memory Usage`, name: () => t`Memory Usage`,
unit: "%", unit: "%",
icon: MemoryStickIcon, icon: MemoryStickIcon,
desc: () => t`Triggers when memory usage exceeds a threshold`, desc: () => t`Triggers when memory usage exceeds a threshold`,
}, },
Disk: { Disk: {
name: () => t`Disk Usage`, name: () => t`Disk Usage`,
unit: "%", unit: "%",
icon: HardDriveIcon, icon: HardDriveIcon,
desc: () => t`Triggers when usage of any disk exceeds a threshold`, desc: () => t`Triggers when usage of any disk exceeds a threshold`,
}, },
Bandwidth: { Bandwidth: {
name: () => t`Bandwidth`, name: () => t`Bandwidth`,
unit: " MB/s", unit: " MB/s",
icon: EthernetIcon, icon: EthernetIcon,
desc: () => t`Triggers when combined up/down exceeds a threshold`, desc: () => t`Triggers when combined up/down exceeds a threshold`,
max: 250, max: 250,
}, },
GPU: { NetworkMonitorLoss: {
name: () => t`GPU Usage`, name: () => t`Network Monitor Loss`,
unit: "%", unit: "%",
icon: GpuIcon, icon: NetworkIcon,
desc: () => t`Triggers when GPU usage exceeds a threshold`, desc: () => t`Triggers when one hour loss exceeds a threshold`,
}, // note: () => t`Uses available history after three probes.`,
Temperature: { noDuration: true,
name: () => t`Temperature`, min: 0,
unit: "°C", max: 99.9,
icon: ThermometerIcon, step: 0.1,
desc: () => t`Triggers when any sensor exceeds a threshold`, start: 5,
}, },
LoadAvg1: { GPU: {
name: () => t`Load Average 1m`, name: () => t`GPU Usage`,
unit: "", unit: "%",
icon: HourglassIcon, icon: GpuIcon,
max: 100, desc: () => t`Triggers when GPU usage exceeds a threshold`,
min: 0.1, },
start: 10, Temperature: {
step: 0.1, name: () => t`Temperature`,
desc: () => t`Triggers when 1 minute load average exceeds a threshold`, unit: "°C",
}, icon: ThermometerIcon,
LoadAvg5: { desc: () => t`Triggers when any sensor exceeds a threshold`,
name: () => t`Load Average 5m`, },
unit: "", LoadAvg1: {
icon: HourglassIcon, name: () => t`Load Average 1m`,
max: 100, unit: "",
min: 0.1, icon: HourglassIcon,
start: 10, max: 100,
step: 0.1, min: 0.1,
desc: () => t`Triggers when 5 minute load average exceeds a threshold`, start: 10,
}, step: 0.1,
LoadAvg15: { desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
name: () => t`Load Average 15m`, },
unit: "", LoadAvg5: {
icon: HourglassIcon, name: () => t`Load Average 5m`,
min: 0.1, unit: "",
max: 100, icon: HourglassIcon,
start: 10, max: 100,
step: 0.1, min: 0.1,
desc: () => t`Triggers when 15 minute load average exceeds a threshold`, start: 10,
}, step: 0.1,
Battery: { desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
name: () => t`Battery`, },
unit: "%", LoadAvg15: {
icon: BatteryMediumIcon, name: () => t`Load Average 15m`,
desc: () => t`Triggers when battery charge drops below a threshold`, unit: "",
start: 20, icon: HourglassIcon,
invert: true, min: 0.1,
}, max: 100,
ContainerHealth: { start: 10,
name: () => t`Container Health`, step: 0.1,
unit: "", desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
icon: ContainerIcon, },
desc: () => t`Triggers when a container's health check reports unhealthy`, Battery: {
note: () => name: () => t`Battery`,
t`Notifications may include recent container log excerpts.`, unit: "%",
triggeredDesc: () => t`One or more containers are unhealthy`, icon: BatteryMediumIcon,
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`, desc: () => t`Triggers when battery charge drops below a threshold`,
start: 20,
invert: true,
},
ContainerHealth: {
name: () => t`Container Health`,
unit: "",
icon: ContainerIcon,
desc: () => t`Triggers when a container's health check reports unhealthy`,
note: () =>
t`Notifications may include recent container log excerpts.`,
triggeredDesc: () => t`One or more containers are unhealthy`,
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
}, },
SystemdFailed: { SystemdFailed: {
name: () => t`Failed Services`, name: () => t`Failed Services`,
unit: "", unit: "",
icon: ServerCrashIcon, icon: ServerCrashIcon,
desc: () => t`Triggers when any systemd service enters the failed state`, desc: () => t`Triggers when any systemd service enters the failed state`,
triggeredDesc: () => t`One or more services are in a failed state`, triggeredDesc: () => t`One or more services are in a failed state`,
/** Fires on first observation - the agent only polls systemd every 10 minutes */ /** Fires on first observation - the agent only polls systemd every 10 minutes */
noDuration: true, noDuration: true,
}, noThreshold: true,
},
} as const } as const
/** Helper to manage user alerts */ /** Helper to manage user alerts */
export const alertManager = (() => { export const alertManager = (() => {
const collection = pb.collection<AlertRecord>("alerts") const collection = pb.collection<AlertRecord>("alerts")
let unsub: () => void let unsub: () => void
/** Fields to fetch from alerts collection */ /** Fields to fetch from alerts collection */
const fields = "id,name,system,value,min,triggered" const fields = "id,name,system,value,min,triggered"
/** Fetch alerts from collection */ /** Fetch alerts from collection */
async function fetchAlerts(): Promise<AlertRecord[]> { async function fetchAlerts(): Promise<AlertRecord[]> {
return await collection.getFullList<AlertRecord>({ fields, sort: "updated" }) return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
} }
/** Format alerts into a map of system id to alert name to alert record */ /** Format alerts into a map of system id to alert name to alert record */
function add(alerts: AlertRecord[]) { function add(alerts: AlertRecord[]) {
for (const alert of alerts) { for (const alert of alerts) {
const systemId = alert.system const systemId = alert.system
const systemAlerts = $alerts.get()[systemId] ?? new Map() const systemAlerts = $alerts.get()[systemId] ?? new Map()
const newAlerts = new Map(systemAlerts) const newAlerts = new Map(systemAlerts)
newAlerts.set(alert.name, alert) newAlerts.set(alert.name, alert)
$alerts.setKey(systemId, newAlerts) $alerts.setKey(systemId, newAlerts)
} }
} }
function remove(alerts: Pick<AlertRecord, "name" | "system">[]) { function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
for (const alert of alerts) { for (const alert of alerts) {
const systemId = alert.system const systemId = alert.system
const systemAlerts = $alerts.get()[systemId] const systemAlerts = $alerts.get()[systemId]
const newAlerts = new Map(systemAlerts) const newAlerts = new Map(systemAlerts)
newAlerts.delete(alert.name) newAlerts.delete(alert.name)
$alerts.setKey(systemId, newAlerts) $alerts.setKey(systemId, newAlerts)
} }
} }
const actionFns = { const actionFns = {
create: add, create: add,
update: add, update: add,
delete: remove, delete: remove,
} }
// batch alert updates to prevent unnecessary re-renders when adding many alerts at once // batch alert updates to prevent unnecessary re-renders when adding many alerts at once
const batchUpdate = (() => { const batchUpdate = (() => {
const batch = new Map<string, RecordSubscription<AlertRecord>>() const batch = new Map<string, RecordSubscription<AlertRecord>>()
let timeout: ReturnType<typeof setTimeout> let timeout: ReturnType<typeof setTimeout>
return (data: RecordSubscription<AlertRecord>) => { return (data: RecordSubscription<AlertRecord>) => {
const { record } = data const { record } = data
batch.set(`${record.system}${record.name}`, data) batch.set(`${record.system}${record.name}`, data)
clearTimeout(timeout) clearTimeout(timeout)
timeout = setTimeout(() => { timeout = setTimeout(() => {
const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]> const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
for (const { action, record } of batch.values()) { for (const { action, record } of batch.values()) {
groups[action]?.push(record) groups[action]?.push(record)
} }
for (const key in groups) { for (const key in groups) {
if (groups[key].length) { if (groups[key].length) {
actionFns[key as keyof typeof actionFns]?.(groups[key]) actionFns[key as keyof typeof actionFns]?.(groups[key])
} }
} }
batch.clear() batch.clear()
}, 50) }, 50)
} }
})() })()
async function subscribe() { async function subscribe() {
unsub = await collection.subscribe("*", batchUpdate, { fields }) unsub = await collection.subscribe("*", batchUpdate, { fields })
} }
function unsubscribe() { function unsubscribe() {
unsub?.() unsub?.()
} }
async function refresh() { async function refresh() {
const records = await fetchAlerts() const records = await fetchAlerts()
add(records) add(records)
} }
return { return {
/** Add alerts to store */ /** Add alerts to store */
add, add,
/** Remove alerts from store */ /** Remove alerts from store */
remove, remove,
/** Subscribe to alerts */ /** Subscribe to alerts */
subscribe, subscribe,
/** Unsubscribe from alerts */ /** Unsubscribe from alerts */
unsubscribe, unsubscribe,
/** Refresh alerts with latest data from hub */ /** Refresh alerts with latest data from hub */
refresh, refresh,
} }
})() })()
+4 -1
View File
@@ -106,8 +106,11 @@ export async function updateUserSettings() {
} }
} }
export function getPbTimestamp(timeString: ChartTimes, d?: Date) { export function getPbTimestamp(timeString: ChartTimes, d?: Date, createdIsNumber?: boolean) {
d ||= chartTimeData[timeString].getOffset(new Date()) d ||= chartTimeData[timeString].getOffset(new Date())
if (createdIsNumber) {
return d.getTime()
}
const year = d.getUTCFullYear() const year = d.getUTCFullYear()
const month = String(d.getUTCMonth() + 1).padStart(2, "0") const month = String(d.getUTCMonth() + 1).padStart(2, "0")
const day = String(d.getUTCDate()).padStart(2, "0") const day = String(d.getUTCDate()).padStart(2, "0")
@@ -0,0 +1,17 @@
import type { MonitorStats, NetworkMonitorRecord, RawMonitorStatsRecord } from "@/types"
/** Derive chart metrics from the counts and response sum stored at every retention tier. */
export function getMonitorStats(record: RawMonitorStatsRecord): MonitorStats {
return {
res_avg: record.success_count > 0 ? record.res_sum / record.success_count : 0,
res_min: record.res_min,
res_max: record.res_max,
loss: record.total_count > 0 ? ((record.total_count - record.success_count) / record.total_count) * 100 : 0,
}
}
export function getMonitorTarget(monitor: Pick<NetworkMonitorRecord, "target" | "protocol" | "port">) {
if (monitor.protocol !== "tcp") return monitor.target
const host = monitor.target.includes(":") && !monitor.target.startsWith("[") ? `[${monitor.target}]` : monitor.target
return `${host}:${monitor.port}`
}
+5 -4
View File
@@ -64,6 +64,9 @@ export const $containerFilter = atom("")
/** Temperature chart filter */ /** Temperature chart filter */
export const $temperatureFilter = atom("") export const $temperatureFilter = atom("")
/** Filter for network monitor charts (compare page and per-system monitor charts) */
export const $monitorFilter = atom("")
/** Fan-speed chart filter */ /** Fan-speed chart filter */
export const $fanFilter = atom("") export const $fanFilter = atom("")
@@ -73,7 +76,5 @@ export const $copyContent = atom("")
/** Direction for localization */ /** Direction for localization */
export const $direction = atom<"ltr" | "rtl">("ltr") export const $direction = atom<"ltr" | "rtl">("ltr")
/** Longest system name length. Used to set table column width. I know this /** Longest system name string. Used to reserve width in virtualized tables. */
* is stupid but the table is virtualized and I know this will work. export const $longestSystemName = atom("")
*/
export const $longestSystemNameLen = atom(8)
+19 -14
View File
@@ -5,20 +5,17 @@ import {
$allSystemsById, $allSystemsById,
$allSystemsByName, $allSystemsByName,
$downSystems, $downSystems,
$longestSystemNameLen, $longestSystemName,
$pausedSystems, $pausedSystems,
$upSystems, $upSystems,
} from "@/lib/stores" } from "@/lib/stores"
import { getVisualStringWidth, updateFavicon } from "@/lib/utils" import { isVisuallyLonger, updateFavicon } from "@/lib/utils"
import type { SystemRecord } from "@/types" import type { SystemRecord } from "@/types"
import { SystemStatus } from "./enums" import { SystemStatus } from "./enums"
const COLLECTION = pb.collection<SystemRecord>("systems") const COLLECTION = pb.collection<SystemRecord>("systems")
const FIELDS_DEFAULT = "id,name,host,port,info,status" const FIELDS_DEFAULT = "id,name,host,port,info,status"
/** Maximum system name length for display purposes */
const MAX_SYSTEM_NAME_LENGTH = 22
let initialized = false let initialized = false
// biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks // biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks
let unsub: (() => void) | undefined | void let unsub: (() => void) | undefined | void
@@ -44,7 +41,7 @@ export function init() {
} }
if (!newSystem) { if (!newSystem) {
onSystemsChanged(newSystems, undefined) onSystemsChanged(newSystems, newSystem, oldSystem)
return return
} }
@@ -68,20 +65,28 @@ export function init() {
} }
// run things that need to be done when systems change // run things that need to be done when systems change
onSystemsChanged(newSystems, newSystem) onSystemsChanged(newSystems, newSystem, oldSystem)
}) })
} }
/** Update the longest system name length and favicon based on system status */ /** Update the longest system name string and favicon based on system status */
function onSystemsChanged(_: Record<string, SystemRecord>, changedSystem: SystemRecord | undefined) { function onSystemsChanged(systems: Record<string, SystemRecord>, newSystem?: SystemRecord, oldSystem?: SystemRecord) {
const downSystemsStore = $downSystems.get() const downSystemsStore = $downSystems.get()
const downSystems = Object.values(downSystemsStore) const downSystems = Object.values(downSystemsStore)
// Update longest system name length // if the old system's old name was the longest, we need to find the new longest name
const longestName = $longestSystemNameLen.get() // otherwise, if the changed system's new name is longer than the current longest, update it
const nameLen = Math.min(MAX_SYSTEM_NAME_LENGTH, getVisualStringWidth(changedSystem?.name || "")) const longestName = $longestSystemName.get()
if (nameLen > longestName) { if (oldSystem?.name === longestName && oldSystem.name !== newSystem?.name) {
$longestSystemNameLen.set(nameLen) let newLongest = ""
for (const id in systems) {
if (isVisuallyLonger(systems[id].name, newLongest)) {
newLongest = systems[id].name
}
}
$longestSystemName.set(newLongest)
} else if (newSystem && newSystem.name !== longestName && isVisuallyLonger(newSystem.name, longestName)) {
$longestSystemName.set(newSystem.name)
} }
updateFavicon(downSystems.length) updateFavicon(downSystems.length)

Some files were not shown because too many files have changed in this diff Show More