mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-20 15:04:29 +00:00
Compare commits
22
Commits
network-monitors
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
97e6f64bdc | ||
|
|
4a5915b141 | ||
|
|
e68372dce4 | ||
|
|
c52f3acb94 | ||
|
|
c09eb8c6df | ||
|
|
a0dc19eacf | ||
|
|
912bc50874 | ||
|
|
c54dbfba7c | ||
|
|
dd3f7d58b5 | ||
|
|
0509053a69 | ||
|
|
187dc886a9 | ||
|
|
2784460621 | ||
|
|
b0bc727411 | ||
|
|
6937453282 | ||
|
|
97de1471d7 | ||
|
|
bd7e359dcd | ||
|
|
d2352e8882 | ||
|
|
f7fd3ef403 | ||
|
|
db3afeabd9 | ||
|
|
b347599928 | ||
|
|
6e5440c21f | ||
|
|
bb1b39928e |
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
# Node.js dependencies
|
# Node.js dependencies
|
||||||
node_modules
|
node_modules/
|
||||||
internalsite/node_modules
|
**/node_modules/
|
||||||
|
|
||||||
# Go build artifacts and binaries
|
# Go build artifacts and binaries
|
||||||
build
|
build
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ jobs:
|
|||||||
# henrygd/beszel-agent:alpine
|
# henrygd/beszel-agent:alpine
|
||||||
- image: henrygd/beszel-agent
|
- image: henrygd/beszel-agent
|
||||||
dockerfile: ./internal/dockerfile_agent_alpine
|
dockerfile: ./internal/dockerfile_agent_alpine
|
||||||
|
flavor: latest=false
|
||||||
registry: docker.io
|
registry: docker.io
|
||||||
username_secret: DOCKERHUB_USERNAME
|
username_secret: DOCKERHUB_USERNAME
|
||||||
password_secret: DOCKERHUB_TOKEN
|
password_secret: DOCKERHUB_TOKEN
|
||||||
@@ -55,6 +56,7 @@ jobs:
|
|||||||
# henrygd/beszel-agent-nvidia:slim
|
# henrygd/beszel-agent-nvidia:slim
|
||||||
- image: henrygd/beszel-agent-nvidia
|
- image: henrygd/beszel-agent-nvidia
|
||||||
dockerfile: ./internal/dockerfile_agent_nvidia_slim
|
dockerfile: ./internal/dockerfile_agent_nvidia_slim
|
||||||
|
flavor: latest=false
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
registry: docker.io
|
registry: docker.io
|
||||||
username_secret: DOCKERHUB_USERNAME
|
username_secret: DOCKERHUB_USERNAME
|
||||||
@@ -123,6 +125,7 @@ jobs:
|
|||||||
# ghcr.io/henrygd/beszel-agent-nvidia:slim
|
# ghcr.io/henrygd/beszel-agent-nvidia:slim
|
||||||
- image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia
|
- image: ghcr.io/${{ github.repository }}/beszel-agent-nvidia
|
||||||
dockerfile: ./internal/dockerfile_agent_nvidia_slim
|
dockerfile: ./internal/dockerfile_agent_nvidia_slim
|
||||||
|
flavor: latest=false
|
||||||
platforms: linux/amd64,linux/arm64
|
platforms: linux/amd64,linux/arm64
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
@@ -150,6 +153,7 @@ jobs:
|
|||||||
# ghcr.io/henrygd/beszel-agent:alpine
|
# ghcr.io/henrygd/beszel-agent:alpine
|
||||||
- image: ghcr.io/${{ github.repository }}/beszel-agent
|
- image: ghcr.io/${{ github.repository }}/beszel-agent
|
||||||
dockerfile: ./internal/dockerfile_agent_alpine
|
dockerfile: ./internal/dockerfile_agent_alpine
|
||||||
|
flavor: latest=false
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
password_secret: GITHUB_TOKEN
|
password_secret: GITHUB_TOKEN
|
||||||
@@ -159,7 +163,7 @@ jobs:
|
|||||||
type=semver,pattern={{major}}.{{minor}}-alpine
|
type=semver,pattern={{major}}.{{minor}}-alpine
|
||||||
type=semver,pattern={{major}}-alpine
|
type=semver,pattern={{major}}-alpine
|
||||||
|
|
||||||
# henrygd/beszel-agent (keep at bottom so it gets built after :alpine and gets the latest tag)
|
# henrygd/beszel-agent
|
||||||
- image: henrygd/beszel-agent
|
- image: henrygd/beszel-agent
|
||||||
dockerfile: ./internal/dockerfile_agent
|
dockerfile: ./internal/dockerfile_agent
|
||||||
registry: docker.io
|
registry: docker.io
|
||||||
@@ -200,6 +204,8 @@ jobs:
|
|||||||
uses: docker/metadata-action@v6
|
uses: docker/metadata-action@v6
|
||||||
with:
|
with:
|
||||||
images: ${{ matrix.image }}
|
images: ${{ matrix.image }}
|
||||||
|
# Variant images must not overwrite the standard image's latest tag.
|
||||||
|
flavor: ${{ matrix.flavor || 'latest=auto' }}
|
||||||
tags: ${{ matrix.tags }}
|
tags: ${{ matrix.tags }}
|
||||||
|
|
||||||
# https://github.com/docker/login-action
|
# https://github.com/docker/login-action
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ type Agent struct {
|
|||||||
keys []gossh.PublicKey // SSH public keys
|
keys []gossh.PublicKey // SSH public keys
|
||||||
smartManager *SmartManager // Manages SMART data
|
smartManager *SmartManager // Manages SMART data
|
||||||
systemdManager *systemdManager // Manages systemd services
|
systemdManager *systemdManager // Manages systemd services
|
||||||
|
monitorManager *MonitorManager // Manages network monitors
|
||||||
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
|
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -122,6 +123,9 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
|
|||||||
// initialize handler registry
|
// initialize handler registry
|
||||||
agent.handlerRegistry = NewHandlerRegistry()
|
agent.handlerRegistry = NewHandlerRegistry()
|
||||||
|
|
||||||
|
// initialize monitor manager
|
||||||
|
agent.monitorManager = newMonitorManager()
|
||||||
|
|
||||||
agent.storagePoolManager = newStoragePoolManager()
|
agent.storagePoolManager = newStoragePoolManager()
|
||||||
|
|
||||||
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
|
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
|
||||||
@@ -192,6 +196,11 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if a.monitorManager != nil {
|
||||||
|
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
|
||||||
|
slog.Debug("Monitors", "data", data.Monitors)
|
||||||
|
}
|
||||||
|
|
||||||
// skip updating systemd services if cache time is not the default 60sec interval
|
// skip updating systemd services if cache time is not the default 60sec interval
|
||||||
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
||||||
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
||||||
|
|||||||
@@ -153,6 +153,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
|
|||||||
// }
|
// }
|
||||||
func (c *ConnectionManager) stop() error {
|
func (c *ConnectionManager) stop() error {
|
||||||
_ = c.agent.StopServer()
|
_ = c.agent.StopServer()
|
||||||
|
c.agent.monitorManager.Stop()
|
||||||
c.closeWebSocket()
|
c.closeWebSocket()
|
||||||
return health.CleanUp()
|
return health.CleanUp()
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-2
@@ -1119,7 +1119,6 @@ func TestCalculateGPUAverage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
||||||
// Save original PATH
|
|
||||||
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1213,7 +1212,7 @@ echo "[]"`
|
|||||||
{
|
{
|
||||||
name: "no gpu tools available",
|
name: "no gpu tools available",
|
||||||
setupCommands: func(_ string) error {
|
setupCommands: func(_ string) error {
|
||||||
t.Setenv("PATH", "")
|
// The subtest already restricts PATH to its empty temporary directory.
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -51,6 +52,7 @@ func NewHandlerRegistry() *HandlerRegistry {
|
|||||||
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
||||||
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
||||||
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
||||||
|
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
|
||||||
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
||||||
|
|
||||||
return registry
|
return registry
|
||||||
@@ -223,3 +225,21 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
|
|||||||
|
|
||||||
return hctx.SendResponse(details, hctx.RequestID)
|
return hctx.SendResponse(details, hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
////////////////////////////////////////////////////////////////////////////
|
||||||
|
////////////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
|
||||||
|
type SyncNetworkMonitorsHandler struct{}
|
||||||
|
|
||||||
|
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
|
||||||
|
var req monitor.SyncRequest
|
||||||
|
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return hctx.SendResponse(resp, hctx.RequestID)
|
||||||
|
}
|
||||||
|
|||||||
@@ -201,12 +201,9 @@ func mdraidSmartStatus(health mdraidHealth) string {
|
|||||||
if health.mismatchCnt > 0 {
|
if health.mismatchCnt > 0 {
|
||||||
return "WARNING"
|
return "WARNING"
|
||||||
}
|
}
|
||||||
// "check" scans for consistency problems without repairing mismatches.
|
// "check" and "repair" are requested consistency scans, not evidence of
|
||||||
// With no mismatches, keep it green while reporting progress attributes.
|
// array failure. With no health issues above, keep scrubbing green while
|
||||||
switch syncAction {
|
// reporting the sync action and progress attributes.
|
||||||
case "repair":
|
|
||||||
return "WARNING"
|
|
||||||
}
|
|
||||||
switch state {
|
switch state {
|
||||||
case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly":
|
case "clean", "active", "active-idle", "write-pending", "read-auto", "readonly":
|
||||||
return "PASSED"
|
return "PASSED"
|
||||||
|
|||||||
@@ -174,8 +174,25 @@ func TestMdraidSmartStatus(t *testing.T) {
|
|||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", mismatchCnt: 1}); got != "WARNING" {
|
||||||
t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got)
|
t.Fatalf("mdraidSmartStatus(clean+mismatch) = %q, want WARNING", got)
|
||||||
}
|
}
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean", syncAction: "repair"}); got != "WARNING" {
|
for _, tc := range []struct {
|
||||||
t.Fatalf("mdraidSmartStatus(repair) = %q, want WARNING", got)
|
name string
|
||||||
|
health mdraidHealth
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"clean", mdraidHealth{arrayState: "clean"}, "PASSED"},
|
||||||
|
{"active", mdraidHealth{arrayState: "active"}, "PASSED"},
|
||||||
|
{"mismatch", mdraidHealth{arrayState: "active", mismatchCnt: 1}, "WARNING"},
|
||||||
|
{"degraded", mdraidHealth{arrayState: "active", degraded: 1}, "FAILED"},
|
||||||
|
{"faulty member", mdraidHealth{arrayState: "active", faultyDisks: 1}, "FAILED"},
|
||||||
|
{"inactive", mdraidHealth{arrayState: "inactive"}, "FAILED"},
|
||||||
|
{"unknown", mdraidHealth{arrayState: "unknown"}, "UNKNOWN"},
|
||||||
|
} {
|
||||||
|
t.Run("repair/"+tc.name, func(t *testing.T) {
|
||||||
|
tc.health.syncAction = "repair"
|
||||||
|
if got := mdraidSmartStatus(tc.health); got != tc.want {
|
||||||
|
t.Fatalf("mdraidSmartStatus(%+v) = %q, want %s", tc.health, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" {
|
if got := mdraidSmartStatus(mdraidHealth{arrayState: "clean"}); got != "PASSED" {
|
||||||
t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got)
|
t.Fatalf("mdraidSmartStatus(clean) = %q, want PASSED", got)
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MonitorManager manages network monitor configurations and task lifetimes.
|
||||||
|
type MonitorManager struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
monitors map[string]*monitorTask // keyed by monitor ID
|
||||||
|
probe monitorProbe
|
||||||
|
resumeGuard monitorResumeGuard
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMonitorManager() *MonitorManager {
|
||||||
|
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
|
||||||
|
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncMonitors replaces all monitor tasks with the given configs.
|
||||||
|
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
|
||||||
|
pm.mu.Lock()
|
||||||
|
defer pm.mu.Unlock()
|
||||||
|
|
||||||
|
// Build set of new keys
|
||||||
|
newKeys := make(map[string]monitor.Config, len(configs))
|
||||||
|
for _, cfg := range configs {
|
||||||
|
if cfg.ID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
newKeys[cfg.ID] = cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop removed monitors
|
||||||
|
for key, task := range pm.monitors {
|
||||||
|
if _, exists := newKeys[key]; !exists {
|
||||||
|
task.cancel()
|
||||||
|
delete(pm.monitors, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start new monitors and restart tasks whose config changed.
|
||||||
|
for key, cfg := range newKeys {
|
||||||
|
task, exists := pm.monitors[key]
|
||||||
|
if exists && task.config == cfg {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
task.cancel()
|
||||||
|
}
|
||||||
|
task = newMonitorTaskFromExisting(cfg, task)
|
||||||
|
task.resumeGuard = &pm.resumeGuard
|
||||||
|
pm.resumeGuard.start()
|
||||||
|
pm.monitors[key] = task
|
||||||
|
pm.startMonitor(task)
|
||||||
|
}
|
||||||
|
if len(pm.monitors) == 0 {
|
||||||
|
pm.resumeGuard.shutdown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleSyncRequest applies a full or incremental monitor sync request.
|
||||||
|
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
|
||||||
|
switch req.Action {
|
||||||
|
case monitor.SyncActionReplace:
|
||||||
|
pm.SyncMonitors(req.Configs)
|
||||||
|
return monitor.SyncResponse{}, nil
|
||||||
|
case monitor.SyncActionUpsert:
|
||||||
|
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
|
||||||
|
if err != nil {
|
||||||
|
return monitor.SyncResponse{}, err
|
||||||
|
}
|
||||||
|
if result == nil {
|
||||||
|
return monitor.SyncResponse{}, nil
|
||||||
|
}
|
||||||
|
return monitor.SyncResponse{Result: *result}, nil
|
||||||
|
case monitor.SyncActionDelete:
|
||||||
|
if req.Config.ID == "" {
|
||||||
|
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
|
||||||
|
}
|
||||||
|
pm.DeleteMonitor(req.Config.ID)
|
||||||
|
return monitor.SyncResponse{}, nil
|
||||||
|
default:
|
||||||
|
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertMonitor creates or replaces a single monitor task.
|
||||||
|
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
|
||||||
|
if config.ID == "" {
|
||||||
|
return nil, errors.New("missing monitor ID")
|
||||||
|
}
|
||||||
|
|
||||||
|
pm.mu.Lock()
|
||||||
|
task, exists := pm.monitors[config.ID]
|
||||||
|
if exists && task.config == config {
|
||||||
|
pm.mu.Unlock()
|
||||||
|
if !runNow {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return task.runProbe(pm.probe), nil
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
task.cancel()
|
||||||
|
}
|
||||||
|
task = newMonitorTaskFromExisting(config, task)
|
||||||
|
task.resumeGuard = &pm.resumeGuard
|
||||||
|
pm.resumeGuard.start()
|
||||||
|
pm.monitors[config.ID] = task
|
||||||
|
pm.mu.Unlock()
|
||||||
|
|
||||||
|
if runNow {
|
||||||
|
result := task.runProbe(pm.probe)
|
||||||
|
pm.startMonitor(task)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
pm.startMonitor(task)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteMonitor stops and removes a single monitor task.
|
||||||
|
func (pm *MonitorManager) DeleteMonitor(id string) {
|
||||||
|
if id == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pm.mu.Lock()
|
||||||
|
defer pm.mu.Unlock()
|
||||||
|
if task, exists := pm.monitors[id]; exists {
|
||||||
|
task.cancel()
|
||||||
|
delete(pm.monitors, id)
|
||||||
|
}
|
||||||
|
if len(pm.monitors) == 0 {
|
||||||
|
pm.resumeGuard.shutdown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
|
||||||
|
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
|
||||||
|
pm.mu.RLock()
|
||||||
|
defer pm.mu.RUnlock()
|
||||||
|
|
||||||
|
results := make(map[string]monitor.Result, len(pm.monitors))
|
||||||
|
now := time.Now()
|
||||||
|
duration := time.Duration(durationMs) * time.Millisecond
|
||||||
|
|
||||||
|
for _, task := range pm.monitors {
|
||||||
|
result, ok := task.history.result(duration, now)
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
results[task.config.ID] = result
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops all monitor tasks.
|
||||||
|
func (pm *MonitorManager) Stop() {
|
||||||
|
pm.mu.Lock()
|
||||||
|
defer pm.mu.Unlock()
|
||||||
|
for key, task := range pm.monitors {
|
||||||
|
task.cancel()
|
||||||
|
delete(pm.monitors, key)
|
||||||
|
}
|
||||||
|
pm.resumeGuard.shutdown()
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Monitors run at user-defined intervals (e.g., every 10s).
|
||||||
|
// To keep memory usage low and constant, data is stored in two layers:
|
||||||
|
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
|
||||||
|
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
|
||||||
|
// wall-clock minute. Samples collected within the same minute are aggregated
|
||||||
|
// (sum, min, max, count) into a single bucket.
|
||||||
|
//
|
||||||
|
// Short-term requests (<= 61s) use raw samples.
|
||||||
|
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
|
||||||
|
// of individual data points.
|
||||||
|
|
||||||
|
const (
|
||||||
|
// monitorRawRetention is the duration to keep individual samples
|
||||||
|
monitorRawRetention = 61 * time.Second
|
||||||
|
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
|
||||||
|
monitorMinuteBucketLen int32 = 61
|
||||||
|
)
|
||||||
|
|
||||||
|
// monitorHistory owns retention and aggregation, independently of probe execution.
|
||||||
|
type monitorHistory struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
sampleCount int64
|
||||||
|
samples []monitorSample
|
||||||
|
buckets [monitorMinuteBucketLen]monitorBucket
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMonitorHistory() *monitorHistory {
|
||||||
|
// Start small for typical intervals; append grows the buffer for faster probes.
|
||||||
|
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *monitorHistory) clone() *monitorHistory {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
cloned := newMonitorHistory()
|
||||||
|
cloned.samples = append(cloned.samples, h.samples...)
|
||||||
|
cloned.buckets = h.buckets
|
||||||
|
cloned.sampleCount = h.sampleCount
|
||||||
|
return cloned
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return h.resultLocked(duration, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
h.addSampleLocked(sample)
|
||||||
|
result, _ := h.resultLocked(time.Minute, sample.timestamp)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorSample stores one monitor attempt and its collection time.
|
||||||
|
type monitorSample struct {
|
||||||
|
responseUs int64 // -1 means loss
|
||||||
|
timestamp time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorBucket stores one minute of aggregated monitor data.
|
||||||
|
type monitorBucket struct {
|
||||||
|
minute int32
|
||||||
|
filled bool
|
||||||
|
stats monitorAggregate
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorAggregate accumulates successful response stats and total sample counts.
|
||||||
|
type monitorAggregate struct {
|
||||||
|
sumUs int64
|
||||||
|
minUs int64
|
||||||
|
maxUs int64
|
||||||
|
totalCount int64
|
||||||
|
successCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// newMonitorAggregate initializes an aggregate with an unset minimum value.
|
||||||
|
func newMonitorAggregate() monitorAggregate {
|
||||||
|
return monitorAggregate{minUs: math.MaxInt64}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addResponse folds a single monitor sample into the aggregate.
|
||||||
|
func (agg *monitorAggregate) addResponse(responseUs int64) {
|
||||||
|
agg.totalCount++
|
||||||
|
if responseUs < 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
agg.successCount++
|
||||||
|
agg.sumUs += responseUs
|
||||||
|
if responseUs < agg.minUs {
|
||||||
|
agg.minUs = responseUs
|
||||||
|
}
|
||||||
|
if responseUs > agg.maxUs {
|
||||||
|
agg.maxUs = responseUs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAggregate merges another aggregate into this one.
|
||||||
|
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
|
||||||
|
if other.totalCount == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
agg.totalCount += other.totalCount
|
||||||
|
agg.successCount += other.successCount
|
||||||
|
agg.sumUs += other.sumUs
|
||||||
|
if other.successCount == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
|
||||||
|
agg.minUs = other.minUs
|
||||||
|
}
|
||||||
|
if other.maxUs > agg.maxUs {
|
||||||
|
agg.maxUs = other.maxUs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasData reports whether the aggregate contains any samples.
|
||||||
|
func (agg monitorAggregate) hasData() bool {
|
||||||
|
return agg.totalCount > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// result converts the aggregate into the monitor result format.
|
||||||
|
func (agg monitorAggregate) result() monitor.Result {
|
||||||
|
avg := agg.avgResponse()
|
||||||
|
result := monitor.Result{
|
||||||
|
AvgResponse: avg,
|
||||||
|
MinResponse: agg.minUs,
|
||||||
|
MaxResponse: agg.maxUs,
|
||||||
|
PacketLoss: agg.lossPercentage(),
|
||||||
|
TotalCount: agg.totalCount,
|
||||||
|
SuccessCount: agg.successCount,
|
||||||
|
ResponseSum: agg.sumUs,
|
||||||
|
}
|
||||||
|
if agg.successCount == 0 {
|
||||||
|
result.MinResponse, result.MaxResponse = 0, 0
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// avgResponse returns the rounded average of successful samples.
|
||||||
|
func (agg monitorAggregate) avgResponse() int64 {
|
||||||
|
if agg.successCount == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return agg.sumUs / agg.successCount
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// lossPercentage returns the rounded failure rate for the aggregate.
|
||||||
|
func (agg monitorAggregate) lossPercentage() float64 {
|
||||||
|
if agg.totalCount == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
|
||||||
|
}
|
||||||
|
|
||||||
|
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
|
||||||
|
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
||||||
|
agg := h.aggregateLocked(duration, now)
|
||||||
|
if !agg.hasData() {
|
||||||
|
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
|
||||||
|
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
|
||||||
|
// the most recent sample so realtime requests still report current status.
|
||||||
|
agg = h.latestSampleAggregateLocked()
|
||||||
|
}
|
||||||
|
hourAgg := h.aggregateLocked(time.Hour, now)
|
||||||
|
if !agg.hasData() {
|
||||||
|
return monitor.Result{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
result := agg.result()
|
||||||
|
if len(h.samples) > 0 {
|
||||||
|
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
|
||||||
|
}
|
||||||
|
|
||||||
|
result.AvgResponse1h = hourAgg.avgResponse()
|
||||||
|
result.MinResponse1h = hourAgg.minUs
|
||||||
|
result.MaxResponse1h = hourAgg.maxUs
|
||||||
|
result.PacketLoss1h = hourAgg.lossPercentage()
|
||||||
|
result.SampleCount = h.sampleCount
|
||||||
|
|
||||||
|
if hourAgg.successCount == 0 {
|
||||||
|
result.MinResponse1h, result.MaxResponse1h = 0, 0
|
||||||
|
}
|
||||||
|
return result, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
|
||||||
|
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
|
||||||
|
agg := newMonitorAggregate()
|
||||||
|
if len(h.samples) == 0 {
|
||||||
|
return agg
|
||||||
|
}
|
||||||
|
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
|
||||||
|
return agg
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregateLocked collects monitor data for the requested time window.
|
||||||
|
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
|
||||||
|
cutoff := now.Add(-duration)
|
||||||
|
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
|
||||||
|
if duration <= monitorRawRetention {
|
||||||
|
return aggregateSamplesSince(h.samples, cutoff)
|
||||||
|
}
|
||||||
|
return aggregateBucketsSince(h.buckets[:], cutoff, now)
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
|
||||||
|
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
|
||||||
|
agg := newMonitorAggregate()
|
||||||
|
for _, sample := range samples {
|
||||||
|
if sample.timestamp.Before(cutoff) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
agg.addResponse(sample.responseUs)
|
||||||
|
}
|
||||||
|
return agg
|
||||||
|
}
|
||||||
|
|
||||||
|
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
|
||||||
|
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
|
||||||
|
agg := newMonitorAggregate()
|
||||||
|
startMinute := int32(cutoff.Unix() / 60)
|
||||||
|
endMinute := int32(now.Unix() / 60)
|
||||||
|
for _, bucket := range buckets {
|
||||||
|
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
agg.addAggregate(bucket.stats)
|
||||||
|
}
|
||||||
|
return agg
|
||||||
|
}
|
||||||
|
|
||||||
|
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
|
||||||
|
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
|
||||||
|
h.sampleCount++
|
||||||
|
cutoff := sample.timestamp.Add(-monitorRawRetention)
|
||||||
|
start := 0
|
||||||
|
for i := range h.samples {
|
||||||
|
if !h.samples[i].timestamp.Before(cutoff) {
|
||||||
|
start = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if i == len(h.samples)-1 {
|
||||||
|
start = len(h.samples)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if start > 0 {
|
||||||
|
size := copy(h.samples, h.samples[start:])
|
||||||
|
h.samples = h.samples[:size]
|
||||||
|
}
|
||||||
|
h.samples = append(h.samples, sample)
|
||||||
|
|
||||||
|
minute := int32(sample.timestamp.Unix() / 60)
|
||||||
|
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
|
||||||
|
bucket := &h.buckets[minute%monitorMinuteBucketLen]
|
||||||
|
if !bucket.filled || bucket.minute != minute {
|
||||||
|
bucket.minute = minute
|
||||||
|
bucket.filled = true
|
||||||
|
bucket.stats = newMonitorAggregate()
|
||||||
|
}
|
||||||
|
bucket.stats.addResponse(sample.responseUs)
|
||||||
|
}
|
||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fxamacker/cbor/v2"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMonitorHistoryWindowCounts(t *testing.T) {
|
||||||
|
history := newMonitorHistory()
|
||||||
|
now := time.Now()
|
||||||
|
// This older success counts toward lifetime warm-up, but not this window.
|
||||||
|
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
|
||||||
|
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
|
||||||
|
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
||||||
|
result, ok := history.result(time.Minute, now)
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.EqualValues(t, 4, result.SampleCount)
|
||||||
|
assert.EqualValues(t, 3, result.TotalCount)
|
||||||
|
assert.EqualValues(t, 2, result.SuccessCount)
|
||||||
|
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
|
||||||
|
assert.EqualValues(t, 15, result.AvgResponse)
|
||||||
|
assert.Equal(t, 33.33, result.PacketLoss)
|
||||||
|
|
||||||
|
encoded, err := cbor.Marshal(result)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var decoded monitor.Result
|
||||||
|
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
|
||||||
|
assert.Equal(t, result, decoded)
|
||||||
|
stats := monitor.Stats{}.FromResult(decoded)
|
||||||
|
assert.Equal(t, result.TotalCount, stats.TotalCount)
|
||||||
|
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
|
||||||
|
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
|
||||||
|
|
||||||
|
// Reads do not consume samples. A short window's latest-sample fallback
|
||||||
|
// carries the count for that single failure, not the minute or lifetime count.
|
||||||
|
repeated, _ := history.result(time.Minute, now)
|
||||||
|
assert.Equal(t, result, repeated)
|
||||||
|
fallback, ok := history.result(time.Second, now)
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.EqualValues(t, 1, fallback.TotalCount)
|
||||||
|
assert.Zero(t, fallback.SuccessCount)
|
||||||
|
assert.Zero(t, fallback.ResponseSum)
|
||||||
|
assert.Equal(t, 100.0, fallback.PacketLoss)
|
||||||
|
assert.EqualValues(t, 4, fallback.SampleCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
|
||||||
|
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
||||||
|
history := newMonitorHistory()
|
||||||
|
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
||||||
|
|
||||||
|
agg := history.aggregateLocked(time.Minute, now)
|
||||||
|
require.True(t, agg.hasData())
|
||||||
|
assert.Equal(t, int64(2), agg.totalCount)
|
||||||
|
assert.Equal(t, int64(1), agg.successCount)
|
||||||
|
result := agg.result()
|
||||||
|
assert.Equal(t, int64(20), result.AvgResponse)
|
||||||
|
assert.Equal(t, int64(20), result.MinResponse)
|
||||||
|
assert.Equal(t, int64(20), result.MaxResponse)
|
||||||
|
assert.Equal(t, 50.0, result.PacketLoss)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
|
||||||
|
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
|
||||||
|
history := newMonitorHistory()
|
||||||
|
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
|
||||||
|
agg := history.aggregateLocked(10*time.Minute, now)
|
||||||
|
require.True(t, agg.hasData())
|
||||||
|
assert.Equal(t, int64(4), agg.totalCount)
|
||||||
|
assert.Equal(t, int64(3), agg.successCount)
|
||||||
|
result := agg.result()
|
||||||
|
assert.Equal(t, int64(30), result.AvgResponse)
|
||||||
|
assert.Equal(t, int64(20), result.MinResponse)
|
||||||
|
assert.Equal(t, int64(40), result.MaxResponse)
|
||||||
|
assert.Equal(t, 25.0, result.PacketLoss)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
|
||||||
|
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
||||||
|
history := newMonitorHistory()
|
||||||
|
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
|
||||||
|
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
|
||||||
|
|
||||||
|
require.Len(t, history.samples, 1)
|
||||||
|
assert.Equal(t, int64(20), history.samples[0].responseUs)
|
||||||
|
|
||||||
|
agg := history.aggregateLocked(10*time.Minute, now)
|
||||||
|
require.True(t, agg.hasData())
|
||||||
|
assert.Equal(t, int64(2), agg.totalCount)
|
||||||
|
assert.Equal(t, int64(2), agg.successCount)
|
||||||
|
result := agg.result()
|
||||||
|
assert.Equal(t, int64(15), result.AvgResponse)
|
||||||
|
assert.Equal(t, int64(10), result.MinResponse)
|
||||||
|
assert.Equal(t, int64(20), result.MaxResponse)
|
||||||
|
assert.Equal(t, 0.0, result.PacketLoss)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
|
||||||
|
history := newMonitorHistory()
|
||||||
|
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
|
||||||
|
_, ok := history.result(time.Minute, start)
|
||||||
|
require.False(t, ok)
|
||||||
|
first := history.record(monitorSample{responseUs: 20, timestamp: start})
|
||||||
|
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
|
||||||
|
for minute := 0; minute < 5; minute++ {
|
||||||
|
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
|
||||||
|
// Realtime reads must not consume freshness for the persistence request.
|
||||||
|
for _, window := range []time.Duration{time.Second, time.Minute} {
|
||||||
|
result, ok := history.result(window, now)
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
|
||||||
|
assert.Equal(t, int64(20), result.AvgResponse)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next := start.Add(5 * time.Minute)
|
||||||
|
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
|
||||||
|
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
|
||||||
|
assert.Equal(t, float64(100), failed.PacketLoss)
|
||||||
|
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
|
||||||
|
assert.Equal(t, float64(100), repeated.PacketLoss)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHistorySampleCount(t *testing.T) {
|
||||||
|
history := newMonitorHistory()
|
||||||
|
now := time.Now()
|
||||||
|
// Both failed and successful probes count, including older samples so
|
||||||
|
// monitors with hourly intervals can finish warming up.
|
||||||
|
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
|
||||||
|
for i, response := range []int64{10, -1, 20} {
|
||||||
|
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
|
||||||
|
assert.EqualValues(t, i+2, result.SampleCount)
|
||||||
|
}
|
||||||
|
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.EqualValues(t, 4, result.SampleCount)
|
||||||
|
}
|
||||||
@@ -0,0 +1,312 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"runtime"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/net/icmp"
|
||||||
|
"golang.org/x/net/ipv4"
|
||||||
|
"golang.org/x/net/ipv6"
|
||||||
|
|
||||||
|
"log/slog"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Match the numeric RTT independently of the localized label used by Windows.
|
||||||
|
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
|
||||||
|
|
||||||
|
var icmpSequence atomic.Uint32
|
||||||
|
|
||||||
|
type icmpPacketConn interface {
|
||||||
|
Close() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
|
||||||
|
// all native methods fail, the choice is cached so subsequent monitors skip
|
||||||
|
// the trial-and-error overhead.
|
||||||
|
type icmpMethod uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
icmpUntried icmpMethod = iota // haven't tried yet
|
||||||
|
icmpRaw // privileged raw socket
|
||||||
|
icmpDatagram // unprivileged datagram socket
|
||||||
|
icmpExecFallback // shell out to system ping command
|
||||||
|
)
|
||||||
|
|
||||||
|
// icmpFamily holds the network parameters and cached detection result for one address family.
|
||||||
|
type icmpFamily struct {
|
||||||
|
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
|
||||||
|
dgramNetwork string // e.g. "udp4" or "udp6"
|
||||||
|
listenAddr string // "0.0.0.0" or "::"
|
||||||
|
echoType icmp.Type // outgoing echo request type
|
||||||
|
replyType icmp.Type // expected echo reply type
|
||||||
|
proto int // IANA protocol number for parsing replies
|
||||||
|
isIPv6 bool
|
||||||
|
mode icmpMethod // cached detection result (guarded by icmpModeMu)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
icmpV4 = icmpFamily{
|
||||||
|
rawNetwork: "ip4:icmp",
|
||||||
|
dgramNetwork: "udp4",
|
||||||
|
listenAddr: "0.0.0.0",
|
||||||
|
echoType: ipv4.ICMPTypeEcho,
|
||||||
|
replyType: ipv4.ICMPTypeEchoReply,
|
||||||
|
proto: 1,
|
||||||
|
}
|
||||||
|
icmpV6 = icmpFamily{
|
||||||
|
rawNetwork: "ip6:ipv6-icmp",
|
||||||
|
dgramNetwork: "udp6",
|
||||||
|
listenAddr: "::",
|
||||||
|
echoType: ipv6.ICMPTypeEchoRequest,
|
||||||
|
replyType: ipv6.ICMPTypeEchoReply,
|
||||||
|
proto: 58,
|
||||||
|
isIPv6: true,
|
||||||
|
}
|
||||||
|
icmpModeMu sync.Mutex
|
||||||
|
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
|
||||||
|
return icmp.ListenPacket(network, listenAddr)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// monitorICMP sends an ICMP echo request and measures round-trip response.
|
||||||
|
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
|
||||||
|
// unprivileged datagram, or exec fallback) is detected once per address
|
||||||
|
// family and cached for subsequent monitors.
|
||||||
|
// Returns response in microseconds, or -1 and an error on failure.
|
||||||
|
func monitorICMP(ctx context.Context, target string) (int64, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
family, ip, err := resolveICMPTarget(ctx, target)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
icmpModeMu.Lock()
|
||||||
|
if family.mode == icmpUntried {
|
||||||
|
family.mode = detectICMPMode(family, icmpListen)
|
||||||
|
}
|
||||||
|
mode := family.mode
|
||||||
|
icmpModeMu.Unlock()
|
||||||
|
|
||||||
|
switch mode {
|
||||||
|
case icmpRaw:
|
||||||
|
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
|
||||||
|
case icmpDatagram:
|
||||||
|
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
|
||||||
|
case icmpExecFallback:
|
||||||
|
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
|
||||||
|
default:
|
||||||
|
return -1, errors.New("unsupported ICMP mode")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveICMPTarget resolves a target hostname or IP to determine the address
|
||||||
|
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
|
||||||
|
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
|
||||||
|
if ip := net.ParseIP(target); ip != nil {
|
||||||
|
if ip.To4() != nil {
|
||||||
|
return &icmpV4, ip.To4(), nil
|
||||||
|
}
|
||||||
|
return &icmpV6, ip, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
|
||||||
|
if err != nil || len(ips) == 0 {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
for _, ip := range ips {
|
||||||
|
if v4 := ip.To4(); v4 != nil {
|
||||||
|
return &icmpV4, v4, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &icmpV6, ips[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
|
||||||
|
label := "IPv4"
|
||||||
|
if family.isIPv6 {
|
||||||
|
label = "IPv6"
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err := listen(family.rawNetwork, family.listenAddr)
|
||||||
|
slog.Debug("ICMP raw socket test", "family", label, "err", err)
|
||||||
|
if err == nil {
|
||||||
|
conn.Close()
|
||||||
|
return icmpRaw
|
||||||
|
}
|
||||||
|
|
||||||
|
conn, err = listen(family.dgramNetwork, family.listenAddr)
|
||||||
|
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
|
||||||
|
if err == nil {
|
||||||
|
conn.Close()
|
||||||
|
return icmpDatagram
|
||||||
|
}
|
||||||
|
|
||||||
|
return icmpExecFallback
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
|
||||||
|
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
|
||||||
|
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
return monitorICMPPacket(ctx, conn, family, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
// Closing the socket interrupts both reads and writes on cancellation.
|
||||||
|
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
|
||||||
|
defer stop()
|
||||||
|
|
||||||
|
// Prepare correlation data before starting the round-trip timer. The token
|
||||||
|
// also distinguishes delayed replies after the 16-bit sequence wraps.
|
||||||
|
token := make([]byte, 16)
|
||||||
|
if _, err := rand.Read(token); err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
echo := &icmp.Echo{
|
||||||
|
ID: os.Getpid() & 0xffff,
|
||||||
|
Seq: int(icmpSequence.Add(1) & 0xffff),
|
||||||
|
Data: token,
|
||||||
|
}
|
||||||
|
// Linux ping sockets replace the Echo ID with their bound port. Darwin
|
||||||
|
// datagram sockets and raw sockets preserve the supplied ID.
|
||||||
|
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
|
||||||
|
echo.ID = local.Port
|
||||||
|
}
|
||||||
|
targetIP := icmpAddrIP(dst)
|
||||||
|
msg := &icmp.Message{
|
||||||
|
Type: family.echoType,
|
||||||
|
Code: 0,
|
||||||
|
Body: echo,
|
||||||
|
}
|
||||||
|
msgBytes, err := msg.Marshal(nil)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set deadline before sending
|
||||||
|
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
buf := make([]byte, 1500)
|
||||||
|
start := time.Now()
|
||||||
|
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read reply
|
||||||
|
for {
|
||||||
|
n, peer, err := conn.ReadFrom(buf)
|
||||||
|
received := time.Now()
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
if !targetIP.Equal(icmpAddrIP(peer)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
reply, err := icmp.ParseMessage(family.proto, buf[:n])
|
||||||
|
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
body, ok := reply.Body.(*icmp.Echo)
|
||||||
|
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
|
||||||
|
return received.Sub(start).Microseconds(), nil
|
||||||
|
}
|
||||||
|
// Keep waiting for our reply without extending the original deadline.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func icmpAddrIP(addr net.Addr) net.IP {
|
||||||
|
switch addr := addr.(type) {
|
||||||
|
case *net.IPAddr:
|
||||||
|
return addr.IP
|
||||||
|
case *net.UDPAddr:
|
||||||
|
return addr.IP
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// pingCommand selects the executable and arguments for the supported agent platforms.
|
||||||
|
// The context deadline enforces the timeout: -W has incompatible meanings across
|
||||||
|
// Linux, BSD IPv4 ping, and macOS ping6.
|
||||||
|
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
|
||||||
|
family := "-4"
|
||||||
|
if isIPv6 {
|
||||||
|
family = "-6"
|
||||||
|
}
|
||||||
|
switch goos {
|
||||||
|
case "windows":
|
||||||
|
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
|
||||||
|
case "linux":
|
||||||
|
return "ping", []string{family, "-n", "-c", "1", target}, nil
|
||||||
|
case "darwin", "freebsd", "openbsd":
|
||||||
|
command := "ping"
|
||||||
|
if isIPv6 {
|
||||||
|
command = "ping6"
|
||||||
|
}
|
||||||
|
return command, []string{"-n", "-c", "1", target}, nil
|
||||||
|
default:
|
||||||
|
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
|
||||||
|
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
cmd := exec.CommandContext(ctx, name, args...)
|
||||||
|
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
|
||||||
|
cmd.Env = append(os.Environ(), "LC_ALL=C")
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return -1, ctx.Err()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return -1, fmt.Errorf("%s failed: %w", name, err)
|
||||||
|
}
|
||||||
|
return parsePingResponse(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsePingResponse returns the reported RTT, never subprocess execution time.
|
||||||
|
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
|
||||||
|
func parsePingResponse(output []byte) (int64, error) {
|
||||||
|
matches := pingTimeRegex.FindSubmatch(output)
|
||||||
|
if len(matches) < 2 {
|
||||||
|
return -1, errors.New("ping output contains no round-trip time")
|
||||||
|
}
|
||||||
|
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
|
||||||
|
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
|
||||||
|
return -1, errors.New("invalid round-trip time in ping output")
|
||||||
|
}
|
||||||
|
return int64(math.Round(ms * 1000)), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,433 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/net/icmp"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testICMPPacketConn struct{}
|
||||||
|
|
||||||
|
func (testICMPPacketConn) Close() error { return nil }
|
||||||
|
|
||||||
|
type blockingICMPConn struct {
|
||||||
|
net.PacketConn
|
||||||
|
reading chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
|
||||||
|
close(c.reading)
|
||||||
|
return c.PacketConn.ReadFrom(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorICMPPacketCancellation(t *testing.T) {
|
||||||
|
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer conn.Close()
|
||||||
|
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-blocking.reading:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("probe did not begin reading")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
require.Error(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("cancellation did not interrupt the socket read")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorICMPExecCancellation(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("test uses a POSIX shell stub for ping")
|
||||||
|
}
|
||||||
|
dir := t.TempDir()
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
|
||||||
|
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
require.ErrorIs(t, err, context.DeadlineExceeded)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("cancellation did not terminate ping")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPingCommand(t *testing.T) {
|
||||||
|
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
|
||||||
|
for _, ipv6 := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
|
||||||
|
target, family := "192.0.2.1", "-4"
|
||||||
|
if ipv6 {
|
||||||
|
target, family = "2001:db8::1", "-6"
|
||||||
|
}
|
||||||
|
name, args, err := pingCommand(goos, target, ipv6)
|
||||||
|
require.NoError(t, err)
|
||||||
|
wantName := "ping"
|
||||||
|
wantArgs := []string{"-n", "-c", "1", target}
|
||||||
|
switch goos {
|
||||||
|
case "windows":
|
||||||
|
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
|
||||||
|
case "linux":
|
||||||
|
wantArgs = append([]string{family}, wantArgs...)
|
||||||
|
default:
|
||||||
|
if ipv6 {
|
||||||
|
wantName = "ping6"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Equal(t, wantName, name)
|
||||||
|
assert.Equal(t, wantArgs, args)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
|
||||||
|
require.Error(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParsePingResponse(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
output string
|
||||||
|
wantUs int64
|
||||||
|
}{
|
||||||
|
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
|
||||||
|
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
|
||||||
|
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
|
||||||
|
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
|
||||||
|
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
|
||||||
|
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
|
||||||
|
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
|
||||||
|
{"rounding", "time=0.1236 ms", 124},
|
||||||
|
{"empty", "", -1},
|
||||||
|
{"timeout", "Request timed out.", -1},
|
||||||
|
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
|
||||||
|
{"malformed", "time=oops ms", -1},
|
||||||
|
{"negative", "time=-1 ms", -1},
|
||||||
|
{"overflow", "time=999999999999999999999 ms", -1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
responseUs, err := parsePingResponse([]byte(tc.output))
|
||||||
|
if tc.wantUs < 0 {
|
||||||
|
require.Error(t, err)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, tc.wantUs, responseUs)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorICMPExecOutput(t *testing.T) {
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
t.Skip("test uses a POSIX shell stub for ping")
|
||||||
|
}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
output string
|
||||||
|
exit int
|
||||||
|
wantUs int64
|
||||||
|
}{
|
||||||
|
{"success", "time=1.234 ms", 0, 1234},
|
||||||
|
{"missing RTT", "unrecognized output", 0, -1},
|
||||||
|
{"failed command with RTT", "time=1.234 ms", 1, -1},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
// Also verify an inherited locale cannot override the C locale.
|
||||||
|
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
|
||||||
|
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
|
||||||
|
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
||||||
|
t.Setenv("LC_ALL", "de_DE.UTF-8")
|
||||||
|
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
|
||||||
|
if tc.wantUs < 0 {
|
||||||
|
require.Error(t, err)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, tc.wantUs, responseUs)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type icmpTestReply struct {
|
||||||
|
data []byte
|
||||||
|
peer net.Addr
|
||||||
|
}
|
||||||
|
|
||||||
|
type scriptedICMPConn struct {
|
||||||
|
net.PacketConn
|
||||||
|
local net.Addr
|
||||||
|
onWrite func([]byte, net.Addr)
|
||||||
|
replies []icmpTestReply
|
||||||
|
reads int
|
||||||
|
deadlineSets int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
|
||||||
|
|
||||||
|
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
|
||||||
|
c.deadlineSets++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
|
||||||
|
c.onWrite(data, dst)
|
||||||
|
return len(data), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
|
||||||
|
c.reads++
|
||||||
|
if len(c.replies) == 0 {
|
||||||
|
return 0, nil, os.ErrDeadlineExceeded
|
||||||
|
}
|
||||||
|
reply := c.replies[0]
|
||||||
|
c.replies = c.replies[1:]
|
||||||
|
return copy(buf, reply.data), reply.peer, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorICMPReplyCorrelation(t *testing.T) {
|
||||||
|
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
||||||
|
for _, datagram := range []bool{false, true} {
|
||||||
|
network := family.rawNetwork
|
||||||
|
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
|
||||||
|
if family.isIPv6 {
|
||||||
|
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
|
||||||
|
}
|
||||||
|
var dst net.Addr = &net.IPAddr{IP: ip}
|
||||||
|
var wrongPeer net.Addr = &net.IPAddr{IP: other}
|
||||||
|
if datagram {
|
||||||
|
network = family.dgramNetwork
|
||||||
|
dst = &net.UDPAddr{IP: ip}
|
||||||
|
wrongPeer = &net.UDPAddr{IP: other}
|
||||||
|
}
|
||||||
|
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
|
||||||
|
for _, eventuallyMatches := range []bool{false, true} {
|
||||||
|
ending := "timeout"
|
||||||
|
if eventuallyMatches {
|
||||||
|
ending = "success"
|
||||||
|
}
|
||||||
|
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
|
||||||
|
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
|
||||||
|
if datagram {
|
||||||
|
conn.local = &net.UDPAddr{Port: 12345}
|
||||||
|
if runtime.GOOS == "linux" {
|
||||||
|
// Deliberately differ from the process ID.
|
||||||
|
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
conn.onWrite = func(data []byte, target net.Addr) {
|
||||||
|
require.Equal(t, dst, target)
|
||||||
|
request, err := icmp.ParseMessage(family.proto, data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
echo := request.Body.(*icmp.Echo)
|
||||||
|
expectedID := os.Getpid() & 0xffff
|
||||||
|
if datagram && runtime.GOOS == "linux" {
|
||||||
|
expectedID = conn.local.(*net.UDPAddr).Port
|
||||||
|
}
|
||||||
|
require.Equal(t, expectedID, echo.ID)
|
||||||
|
reply := &icmp.Message{Type: family.replyType, Body: echo}
|
||||||
|
valid, err := reply.Marshal(nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
peer := dst
|
||||||
|
switch mismatch {
|
||||||
|
case "source":
|
||||||
|
peer = wrongPeer
|
||||||
|
case "id":
|
||||||
|
echo.ID ^= 1
|
||||||
|
case "sequence":
|
||||||
|
echo.Seq ^= 1
|
||||||
|
case "payload":
|
||||||
|
echo.Data[0] ^= 1
|
||||||
|
case "type":
|
||||||
|
reply.Type = family.echoType
|
||||||
|
case "code":
|
||||||
|
reply.Code = 1
|
||||||
|
}
|
||||||
|
invalid, err := reply.Marshal(nil)
|
||||||
|
require.NoError(t, err)
|
||||||
|
if mismatch == "malformed" {
|
||||||
|
invalid = invalid[:2]
|
||||||
|
}
|
||||||
|
conn.replies = []icmpTestReply{{invalid, peer}}
|
||||||
|
if eventuallyMatches {
|
||||||
|
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
||||||
|
if eventuallyMatches {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, elapsed, int64(0))
|
||||||
|
} else {
|
||||||
|
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
|
||||||
|
assert.Equal(t, int64(-1), elapsed)
|
||||||
|
}
|
||||||
|
assert.Equal(t, 2, conn.reads)
|
||||||
|
assert.Equal(t, 1, conn.deadlineSets)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorICMPLoopback(t *testing.T) {
|
||||||
|
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
||||||
|
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
|
||||||
|
t.Run(network, func(t *testing.T) {
|
||||||
|
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("ICMP socket unavailable: %v", err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
ip := net.ParseIP("127.0.0.1")
|
||||||
|
if family.isIPv6 {
|
||||||
|
ip = net.ParseIP("::1")
|
||||||
|
}
|
||||||
|
var dst net.Addr = &net.IPAddr{IP: ip}
|
||||||
|
if network == family.dgramNetwork {
|
||||||
|
dst = &net.UDPAddr{IP: ip}
|
||||||
|
}
|
||||||
|
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, elapsed, int64(0))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectICMPMode(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
family *icmpFamily
|
||||||
|
rawErr error
|
||||||
|
udpErr error
|
||||||
|
want icmpMethod
|
||||||
|
wantNetworks []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "IPv4 prefers raw socket when available",
|
||||||
|
family: &icmpV4,
|
||||||
|
want: icmpRaw,
|
||||||
|
wantNetworks: []string{"ip4:icmp"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IPv4 uses datagram when raw unavailable",
|
||||||
|
family: &icmpV4,
|
||||||
|
rawErr: errors.New("operation not permitted"),
|
||||||
|
want: icmpDatagram,
|
||||||
|
wantNetworks: []string{"ip4:icmp", "udp4"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IPv4 falls back to exec when both unavailable",
|
||||||
|
family: &icmpV4,
|
||||||
|
rawErr: errors.New("operation not permitted"),
|
||||||
|
udpErr: errors.New("protocol not supported"),
|
||||||
|
want: icmpExecFallback,
|
||||||
|
wantNetworks: []string{"ip4:icmp", "udp4"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IPv6 prefers raw socket when available",
|
||||||
|
family: &icmpV6,
|
||||||
|
want: icmpRaw,
|
||||||
|
wantNetworks: []string{"ip6:ipv6-icmp"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IPv6 uses datagram when raw unavailable",
|
||||||
|
family: &icmpV6,
|
||||||
|
rawErr: errors.New("operation not permitted"),
|
||||||
|
want: icmpDatagram,
|
||||||
|
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "IPv6 falls back to exec when both unavailable",
|
||||||
|
family: &icmpV6,
|
||||||
|
rawErr: errors.New("operation not permitted"),
|
||||||
|
udpErr: errors.New("protocol not supported"),
|
||||||
|
want: icmpExecFallback,
|
||||||
|
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
calls := make([]string, 0, 2)
|
||||||
|
listen := func(network, listenAddr string) (icmpPacketConn, error) {
|
||||||
|
require.Equal(t, tt.family.listenAddr, listenAddr)
|
||||||
|
calls = append(calls, network)
|
||||||
|
switch network {
|
||||||
|
case tt.family.rawNetwork:
|
||||||
|
if tt.rawErr != nil {
|
||||||
|
return nil, tt.rawErr
|
||||||
|
}
|
||||||
|
case tt.family.dgramNetwork:
|
||||||
|
if tt.udpErr != nil {
|
||||||
|
return nil, tt.udpErr
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatalf("unexpected network %q", network)
|
||||||
|
}
|
||||||
|
return testICMPPacketConn{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
|
||||||
|
assert.Equal(t, tt.wantNetworks, calls)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveICMPTarget(t *testing.T) {
|
||||||
|
t.Run("IPv4 literal", func(t *testing.T) {
|
||||||
|
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, family)
|
||||||
|
assert.False(t, family.isIPv6)
|
||||||
|
assert.Equal(t, "127.0.0.1", ip.String())
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("IPv6 literal", func(t *testing.T) {
|
||||||
|
family, ip, err := resolveICMPTarget(context.Background(), "::1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, family)
|
||||||
|
assert.True(t, family.isIPv6)
|
||||||
|
assert.Equal(t, "::1", ip.String())
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
|
||||||
|
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, family)
|
||||||
|
assert.False(t, family.isIPv6)
|
||||||
|
assert.Equal(t, "127.0.0.1", ip.String())
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
|
||||||
|
// Implementations must honor cancellation and bound their execution time.
|
||||||
|
type monitorProbe func(context.Context, monitor.Config) (int64, error)
|
||||||
|
|
||||||
|
func networkMonitorProbe(client *http.Client) monitorProbe {
|
||||||
|
return func(ctx context.Context, config monitor.Config) (int64, error) {
|
||||||
|
switch config.Protocol {
|
||||||
|
case "icmp":
|
||||||
|
return monitorICMP(ctx, config.Target)
|
||||||
|
case "tcp":
|
||||||
|
return monitorTCP(ctx, config.Target, config.Port)
|
||||||
|
case "http":
|
||||||
|
return monitorHTTP(ctx, client, config.Target)
|
||||||
|
case "dns":
|
||||||
|
return monitorDNS(ctx, config.Target)
|
||||||
|
default:
|
||||||
|
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorTCP measures connection establishment time, including address fallback
|
||||||
|
// but excluding DNS resolution.
|
||||||
|
// Returns -1 and an error on failure.
|
||||||
|
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Resolve DNS first, outside the timing window but within the probe deadline.
|
||||||
|
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
if len(ips) == 0 {
|
||||||
|
return -1, errors.New("no addresses resolved for TCP monitor")
|
||||||
|
}
|
||||||
|
portString := fmt.Sprintf("%d", port)
|
||||||
|
deadline, _ := ctx.Deadline()
|
||||||
|
|
||||||
|
// Share the remaining probe budget across addresses so an unresponsive
|
||||||
|
// first address cannot consume all the time available for alternatives.
|
||||||
|
start := time.Now()
|
||||||
|
for i, ip := range ips {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
|
||||||
|
var conn net.Conn
|
||||||
|
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
responseUs := time.Since(start).Microseconds()
|
||||||
|
conn.Close()
|
||||||
|
return responseUs, nil
|
||||||
|
}
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
|
||||||
|
func monitorDNS(ctx context.Context, target string) (int64, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
||||||
|
if err != nil || len(ips) == 0 {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
return time.Since(start).Microseconds(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
|
||||||
|
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
|
||||||
|
if client == nil {
|
||||||
|
client = http.DefaultClient
|
||||||
|
}
|
||||||
|
start := time.Now()
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
resp.Body.Close()
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
|
||||||
|
}
|
||||||
|
return time.Since(start).Microseconds(), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
monitorResumeHeartbeat = 10 * time.Second
|
||||||
|
// Allow scheduling jitter without mistaking an ordinary tick for resume.
|
||||||
|
monitorResumeGap = 2 * monitorResumeHeartbeat
|
||||||
|
monitorResumePause = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// monitorResumeGuard detects likely suspend/resume using wall time. A long
|
||||||
|
// process stall or forward clock adjustment can also trigger the bounded pause.
|
||||||
|
// One heartbeat is shared by all configured monitors.
|
||||||
|
type monitorResumeGuard struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
stop chan struct{}
|
||||||
|
lastTick time.Time
|
||||||
|
pauseUntil time.Time
|
||||||
|
generation uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *monitorResumeGuard) start() {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.stop != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
stop := make(chan struct{})
|
||||||
|
g.stop = stop
|
||||||
|
g.lastTick = time.Now().Round(0)
|
||||||
|
g.pauseUntil = time.Time{}
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(monitorResumeHeartbeat)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
g.mu.Lock()
|
||||||
|
if g.stop == stop {
|
||||||
|
g.observe(time.Now())
|
||||||
|
}
|
||||||
|
g.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *monitorResumeGuard) shutdown() {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.stop != nil {
|
||||||
|
close(g.stop)
|
||||||
|
g.stop = nil
|
||||||
|
g.generation++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// observe requires mu. Strip the monotonic component because it can stop during
|
||||||
|
// suspend. Read the current time rather than the ticker's queued timestamp.
|
||||||
|
func (g *monitorResumeGuard) observe(now time.Time) {
|
||||||
|
now = now.Round(0)
|
||||||
|
if now.Sub(g.lastTick) > monitorResumeGap {
|
||||||
|
g.pauseUntil = now.Add(monitorResumePause)
|
||||||
|
g.generation++
|
||||||
|
}
|
||||||
|
g.lastTick = now
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshot also observes time so a probe waking before the heartbeat detects
|
||||||
|
// resume itself. A changed generation invalidates probes spanning suspend.
|
||||||
|
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
|
||||||
|
if g == nil {
|
||||||
|
return 0, true
|
||||||
|
}
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.stop == nil {
|
||||||
|
return g.generation, true
|
||||||
|
}
|
||||||
|
g.observe(time.Now())
|
||||||
|
return g.generation, !g.lastTick.Before(g.pauseUntil)
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func simulateMonitorSleep(g *monitorResumeGuard) {
|
||||||
|
g.mu.Lock()
|
||||||
|
g.lastTick = time.Now().Add(-time.Hour).Round(0)
|
||||||
|
g.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorResumePause(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
var g monitorResumeGuard
|
||||||
|
g.start()
|
||||||
|
defer g.shutdown()
|
||||||
|
generation, allowed := g.snapshot()
|
||||||
|
require.True(t, allowed)
|
||||||
|
// Heartbeats alone must keep the guard current between infrequent probes.
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
synctest.Wait()
|
||||||
|
steadyGeneration, allowed := g.snapshot()
|
||||||
|
require.True(t, allowed)
|
||||||
|
require.Equal(t, generation, steadyGeneration)
|
||||||
|
// The probe, rather than the heartbeat, must detect this gap.
|
||||||
|
simulateMonitorSleep(&g)
|
||||||
|
next, allowed := g.snapshot()
|
||||||
|
assert.False(t, allowed)
|
||||||
|
assert.NotEqual(t, generation, next)
|
||||||
|
time.Sleep(9 * time.Second)
|
||||||
|
_, allowed = g.snapshot()
|
||||||
|
assert.False(t, allowed)
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
end, allowed := g.snapshot()
|
||||||
|
assert.True(t, allowed)
|
||||||
|
assert.Equal(t, next, end)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorResumeGuardLifecycle(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
|
||||||
|
defer pm.Stop()
|
||||||
|
assert.Nil(t, pm.resumeGuard.stop)
|
||||||
|
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
|
||||||
|
stop := pm.resumeGuard.stop
|
||||||
|
require.NotNil(t, stop)
|
||||||
|
pm.DeleteMonitor("a")
|
||||||
|
assert.Equal(t, stop, pm.resumeGuard.stop)
|
||||||
|
pm.DeleteMonitor("b")
|
||||||
|
assert.Nil(t, pm.resumeGuard.stop)
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
default:
|
||||||
|
t.Fatal("heartbeat was not stopped")
|
||||||
|
}
|
||||||
|
time.Sleep(time.Hour)
|
||||||
|
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, allowed := pm.resumeGuard.snapshot()
|
||||||
|
assert.True(t, allowed, "idle time must not trigger a resume pause")
|
||||||
|
pm.SyncMonitors(nil)
|
||||||
|
assert.Nil(t, pm.resumeGuard.stop)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
var g monitorResumeGuard
|
||||||
|
g.start()
|
||||||
|
defer g.shutdown()
|
||||||
|
task := newMonitorTask(monitor.Config{ID: "test"})
|
||||||
|
defer task.cancel()
|
||||||
|
task.resumeGuard = &g
|
||||||
|
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
||||||
|
simulateMonitorSleep(&g)
|
||||||
|
return 0, errors.New("network not ready")
|
||||||
|
})
|
||||||
|
assert.Nil(t, result)
|
||||||
|
assert.Empty(t, task.history.samples)
|
||||||
|
// Explicit requests may still run during the pause and record real failures.
|
||||||
|
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
||||||
|
return 0, errors.New("unreachable")
|
||||||
|
})
|
||||||
|
require.NotNil(t, result)
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
|
||||||
|
calls.Add(1)
|
||||||
|
return 1, nil
|
||||||
|
})
|
||||||
|
defer pm.Stop()
|
||||||
|
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
|
||||||
|
simulateMonitorSleep(&pm.resumeGuard)
|
||||||
|
pm.resumeGuard.snapshot()
|
||||||
|
time.Sleep(9 * time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Zero(t, calls.Load())
|
||||||
|
assert.Empty(t, pm.GetResults(1000))
|
||||||
|
time.Sleep(2 * time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Positive(t, calls.Load())
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"math/rand"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (pm *MonitorManager) startMonitor(task *monitorTask) {
|
||||||
|
interval := time.Duration(task.config.Interval) * time.Second
|
||||||
|
if interval < time.Second {
|
||||||
|
interval = 30 * time.Second
|
||||||
|
}
|
||||||
|
delay := getStagger(interval.Milliseconds())
|
||||||
|
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
|
||||||
|
go runMonitorSchedule(task.ctx, interval, delay, func() {
|
||||||
|
if _, allowed := task.resumeGuard.snapshot(); allowed {
|
||||||
|
task.runProbe(pm.probe)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
|
||||||
|
// naturally drop missed ticks rather than building an execution backlog.
|
||||||
|
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
|
||||||
|
timer := time.NewTimer(delay)
|
||||||
|
defer timer.Stop()
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-timer.C:
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run()
|
||||||
|
ticker := time.NewTicker(interval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
run()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// getStagger returns an initial delay between half an interval and one interval.
|
||||||
|
func getStagger(intervalMilli int64) time.Duration {
|
||||||
|
delay := rand.Intn(int(intervalMilli))
|
||||||
|
if delay < int(intervalMilli)/2 {
|
||||||
|
delay += int(intervalMilli) / 2
|
||||||
|
}
|
||||||
|
return time.Duration(delay) * time.Millisecond
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMonitorScheduleTiming(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
var calls atomic.Int32
|
||||||
|
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
|
||||||
|
synctest.Wait()
|
||||||
|
time.Sleep(4 * time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 0, int(calls.Load()))
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 1, int(calls.Load()))
|
||||||
|
time.Sleep(10 * time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 2, int(calls.Load()))
|
||||||
|
cancel()
|
||||||
|
synctest.Wait()
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 2, int(calls.Load()))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorScheduleSlowProbe(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
defer cancel()
|
||||||
|
var calls atomic.Int32
|
||||||
|
release := make(chan struct{})
|
||||||
|
go runMonitorSchedule(ctx, time.Second, 0, func() {
|
||||||
|
calls.Add(1)
|
||||||
|
select {
|
||||||
|
case <-release:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
})
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 1, int(calls.Load()))
|
||||||
|
time.Sleep(time.Minute)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
|
||||||
|
time.Sleep(time.Second)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 2, int(calls.Load()))
|
||||||
|
cancel()
|
||||||
|
synctest.Wait()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
var calls atomic.Int32
|
||||||
|
release := make(chan struct{})
|
||||||
|
cfg := monitor.Config{ID: "test", Interval: 10}
|
||||||
|
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
||||||
|
assert.Equal(t, cfg, config)
|
||||||
|
calls.Add(1)
|
||||||
|
<-release
|
||||||
|
return 42, nil
|
||||||
|
})
|
||||||
|
defer pm.Stop()
|
||||||
|
task := newMonitorTask(cfg)
|
||||||
|
pm.monitors[cfg.ID] = task
|
||||||
|
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
|
||||||
|
synctest.Wait()
|
||||||
|
results := make(chan *monitor.Result, 2)
|
||||||
|
for range 2 {
|
||||||
|
go func() {
|
||||||
|
result, _ := pm.UpsertMonitor(cfg, true)
|
||||||
|
results <- result
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Equal(t, 1, int(calls.Load()))
|
||||||
|
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
|
||||||
|
close(release)
|
||||||
|
synctest.Wait()
|
||||||
|
first, second := <-results, <-results
|
||||||
|
require.NotNil(t, first)
|
||||||
|
require.NotNil(t, second)
|
||||||
|
assert.Equal(t, int64(42), first.AvgResponse)
|
||||||
|
assert.Equal(t, first, second)
|
||||||
|
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
|
||||||
|
assert.Len(t, task.history.samples, 1)
|
||||||
|
// A later explicit request must still perform a fresh probe.
|
||||||
|
_, err := pm.UpsertMonitor(cfg, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 2, int(calls.Load()))
|
||||||
|
assert.Len(t, task.history.samples, 2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
cfg := monitor.Config{ID: "test", Interval: 10}
|
||||||
|
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
||||||
|
if config.Interval == 10 {
|
||||||
|
<-ctx.Done()
|
||||||
|
return 0, ctx.Err()
|
||||||
|
}
|
||||||
|
return 30, nil
|
||||||
|
})
|
||||||
|
defer pm.Stop()
|
||||||
|
task := newMonitorTask(cfg)
|
||||||
|
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
|
||||||
|
pm.monitors[cfg.ID] = task
|
||||||
|
results := make(chan *monitor.Result, 2)
|
||||||
|
for range 2 {
|
||||||
|
go func() {
|
||||||
|
result, _ := pm.UpsertMonitor(cfg, true)
|
||||||
|
results <- result
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
synctest.Wait()
|
||||||
|
updated := cfg
|
||||||
|
updated.Interval = 20
|
||||||
|
result, err := pm.UpsertMonitor(updated, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
assert.Equal(t, int64(20), result.AvgResponse)
|
||||||
|
assert.Zero(t, result.PacketLoss)
|
||||||
|
synctest.Wait()
|
||||||
|
assert.Nil(t, <-results)
|
||||||
|
assert.Nil(t, <-results)
|
||||||
|
assert.Len(t, task.history.samples, 1)
|
||||||
|
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
<-ctx.Done()
|
||||||
|
return 0, ctx.Err()
|
||||||
|
})
|
||||||
|
defer pm.Stop()
|
||||||
|
start := time.Now()
|
||||||
|
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
assert.Equal(t, 3*time.Second, time.Since(start))
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss)
|
||||||
|
assert.NoError(t, pm.monitors["test"].ctx.Err())
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log/slog"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
const monitorFailureLogInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// monitorTask coordinates a probe and its history for one immutable configuration.
|
||||||
|
type monitorTask struct {
|
||||||
|
config monitor.Config
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
history *monitorHistory
|
||||||
|
resumeGuard *monitorResumeGuard
|
||||||
|
runMu sync.Mutex
|
||||||
|
inflight *monitorRun
|
||||||
|
lastFailureLog int64 // Unix nanoseconds
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitorRun struct {
|
||||||
|
done chan struct{}
|
||||||
|
result *monitor.Result // published by closing done; never mutated afterwards
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMonitorTask(config monitor.Config) *monitorTask {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
|
||||||
|
// Serialize cancellation with publication, so canceled probes cannot enter
|
||||||
|
// history copied into a replacement task.
|
||||||
|
task.cancel = func() {
|
||||||
|
task.runMu.Lock()
|
||||||
|
cancel()
|
||||||
|
task.runMu.Unlock()
|
||||||
|
}
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
|
||||||
|
task := newMonitorTask(config)
|
||||||
|
if existing != nil {
|
||||||
|
task.history = existing.history.clone()
|
||||||
|
}
|
||||||
|
return task
|
||||||
|
}
|
||||||
|
|
||||||
|
// runProbe shares an in-flight check between scheduled and immediate requests.
|
||||||
|
// Every completed check contributes exactly one sample, regardless of how many
|
||||||
|
// callers were waiting for it. No task or history lock is held during network I/O.
|
||||||
|
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
|
||||||
|
task.runMu.Lock()
|
||||||
|
if task.ctx.Err() != nil {
|
||||||
|
task.runMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if run := task.inflight; run != nil {
|
||||||
|
task.runMu.Unlock()
|
||||||
|
select {
|
||||||
|
case <-task.ctx.Done():
|
||||||
|
return nil
|
||||||
|
case <-run.done:
|
||||||
|
if task.ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return copyMonitorResult(run.result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
run := &monitorRun{done: make(chan struct{})}
|
||||||
|
task.inflight = run
|
||||||
|
task.runMu.Unlock()
|
||||||
|
|
||||||
|
generation, _ := task.resumeGuard.snapshot()
|
||||||
|
responseUs, err := probe(task.ctx, task.config)
|
||||||
|
var logFailure bool
|
||||||
|
task.runMu.Lock()
|
||||||
|
currentGeneration, _ := task.resumeGuard.snapshot()
|
||||||
|
if task.ctx.Err() == nil && generation == currentGeneration {
|
||||||
|
now := time.Now()
|
||||||
|
if err != nil {
|
||||||
|
responseUs = -1
|
||||||
|
logAt := now.UnixNano()
|
||||||
|
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
|
||||||
|
logFailure = true
|
||||||
|
task.lastFailureLog = logAt
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
task.lastFailureLog = 0
|
||||||
|
}
|
||||||
|
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
|
||||||
|
run.result = &result
|
||||||
|
}
|
||||||
|
|
||||||
|
task.inflight = nil
|
||||||
|
close(run.done)
|
||||||
|
task.runMu.Unlock()
|
||||||
|
if logFailure {
|
||||||
|
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
|
||||||
|
}
|
||||||
|
if task.ctx.Err() != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return copyMonitorResult(run.result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyMonitorResult(result *monitor.Result) *monitor.Result {
|
||||||
|
if result == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
copy := *result
|
||||||
|
return ©
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log/slog"
|
||||||
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMonitorFailureLogCooldown(t *testing.T) {
|
||||||
|
var logs bytes.Buffer
|
||||||
|
previous := slog.Default()
|
||||||
|
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
|
||||||
|
t.Cleanup(func() { slog.SetDefault(previous) })
|
||||||
|
|
||||||
|
synctest.Test(t, func(t *testing.T) {
|
||||||
|
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
|
||||||
|
defer task.cancel()
|
||||||
|
failure := errors.New("connection refused")
|
||||||
|
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
|
||||||
|
var samples int64
|
||||||
|
check := func(wantLog bool) {
|
||||||
|
t.Helper()
|
||||||
|
logs.Reset()
|
||||||
|
result := task.runProbe(probe)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
samples++
|
||||||
|
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
|
||||||
|
if !wantLog {
|
||||||
|
assert.Empty(t, logs.String())
|
||||||
|
} else {
|
||||||
|
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
||||||
|
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
check(true)
|
||||||
|
check(false)
|
||||||
|
time.Sleep(5*time.Minute - time.Nanosecond)
|
||||||
|
check(false)
|
||||||
|
time.Sleep(time.Nanosecond)
|
||||||
|
check(true)
|
||||||
|
check(false)
|
||||||
|
time.Sleep(5 * time.Minute)
|
||||||
|
check(true)
|
||||||
|
check(false)
|
||||||
|
|
||||||
|
// Recovery clears the cooldown.
|
||||||
|
failure = nil
|
||||||
|
check(false)
|
||||||
|
failure = errors.New("connection refused again")
|
||||||
|
check(true)
|
||||||
|
|
||||||
|
// Another monitor has its own cooldown.
|
||||||
|
other := newMonitorTask(task.config)
|
||||||
|
defer other.cancel()
|
||||||
|
logs.Reset()
|
||||||
|
require.NotNil(t, other.runProbe(probe))
|
||||||
|
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
||||||
|
|
||||||
|
// A canceled probe must not publish a failure or emit a warning.
|
||||||
|
logs.Reset()
|
||||||
|
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
||||||
|
other.cancel()
|
||||||
|
return -1, context.Canceled
|
||||||
|
})
|
||||||
|
assert.Nil(t, result)
|
||||||
|
assert.Empty(t, logs.String())
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,524 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/binary"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
||||||
|
|
||||||
|
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
||||||
|
result, ok := results["monitor-1"]
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, int64(30), result.AvgResponse)
|
||||||
|
assert.Equal(t, int64(25), result.AvgResponse1h)
|
||||||
|
assert.Equal(t, int64(30), result.MinResponse)
|
||||||
|
assert.Equal(t, int64(10), result.MinResponse1h)
|
||||||
|
assert.Equal(t, int64(30), result.MaxResponse)
|
||||||
|
assert.Equal(t, int64(40), result.MaxResponse1h)
|
||||||
|
assert.Equal(t, 50.0, result.PacketLoss)
|
||||||
|
assert.Equal(t, 20.0, result.PacketLoss1h)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
||||||
|
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
||||||
|
|
||||||
|
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
||||||
|
result, ok := results["monitor-1"]
|
||||||
|
require.True(t, ok)
|
||||||
|
assert.Equal(t, int64(0), result.AvgResponse)
|
||||||
|
assert.Equal(t, int64(0), result.AvgResponse1h)
|
||||||
|
assert.Equal(t, int64(0), result.MinResponse)
|
||||||
|
assert.Equal(t, int64(0), result.MinResponse1h)
|
||||||
|
assert.Equal(t, int64(0), result.MaxResponse)
|
||||||
|
assert.Equal(t, int64(0), result.MaxResponse1h)
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss)
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss1h)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
|
||||||
|
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
||||||
|
assert.Equal(t, "monitor-1", cfg.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
|
||||||
|
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
||||||
|
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
|
||||||
|
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
|
||||||
|
defer pm.Stop()
|
||||||
|
|
||||||
|
_, validExists := pm.monitors[validCfg.ID]
|
||||||
|
_, invalidExists := pm.monitors[invalidCfg.ID]
|
||||||
|
assert.True(t, validExists)
|
||||||
|
assert.False(t, invalidExists)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
|
||||||
|
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
||||||
|
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
|
||||||
|
|
||||||
|
keptTask := newMonitorTask(keepCfg)
|
||||||
|
removedTask := newMonitorTask(removeCfg)
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{
|
||||||
|
keepCfg.ID: keptTask,
|
||||||
|
removeCfg.ID: removedTask,
|
||||||
|
}
|
||||||
|
|
||||||
|
pm.SyncMonitors([]monitor.Config{keepCfg})
|
||||||
|
|
||||||
|
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
|
||||||
|
_, exists := pm.monitors[removeCfg.ID]
|
||||||
|
assert.False(t, exists)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-removedTask.ctx.Done():
|
||||||
|
default:
|
||||||
|
t.Fatal("expected removed monitor task to be cancelled")
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-keptTask.ctx.Done():
|
||||||
|
t.Fatal("expected existing monitor task to remain active")
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
|
||||||
|
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
|
||||||
|
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
|
||||||
|
originalTask := newMonitorTask(originalCfg)
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{
|
||||||
|
originalCfg.ID: originalTask,
|
||||||
|
}
|
||||||
|
|
||||||
|
pm.SyncMonitors([]monitor.Config{updatedCfg})
|
||||||
|
defer pm.Stop()
|
||||||
|
|
||||||
|
restartedTask := pm.monitors[updatedCfg.ID]
|
||||||
|
assert.NotSame(t, originalTask, restartedTask)
|
||||||
|
assert.Equal(t, updatedCfg, restartedTask.config)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-originalTask.ctx.Done():
|
||||||
|
default:
|
||||||
|
t.Fatal("expected changed monitor task to be cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
pm := &MonitorManager{
|
||||||
|
monitors: make(map[string]*monitorTask),
|
||||||
|
probe: networkMonitorProbe(server.Client()),
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
||||||
|
Action: monitor.SyncActionUpsert,
|
||||||
|
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
|
||||||
|
RunNow: true,
|
||||||
|
})
|
||||||
|
defer pm.Stop()
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
|
||||||
|
assert.Equal(t, 0.0, resp.Result.PacketLoss)
|
||||||
|
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
|
||||||
|
|
||||||
|
task := pm.monitors["monitor-1"]
|
||||||
|
require.NotNil(t, task)
|
||||||
|
task.history.mu.Lock()
|
||||||
|
defer task.history.mu.Unlock()
|
||||||
|
require.Len(t, task.history.samples, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
|
||||||
|
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
||||||
|
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
|
||||||
|
existingTask := newMonitorTask(originalCfg)
|
||||||
|
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
|
||||||
|
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
|
||||||
|
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
|
||||||
|
|
||||||
|
result, err := pm.UpsertMonitor(updatedCfg, false)
|
||||||
|
defer pm.Stop()
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, result)
|
||||||
|
|
||||||
|
updatedTask := pm.monitors[updatedCfg.ID]
|
||||||
|
require.NotNil(t, updatedTask)
|
||||||
|
assert.NotSame(t, existingTask, updatedTask)
|
||||||
|
assert.Equal(t, updatedCfg, updatedTask.config)
|
||||||
|
|
||||||
|
updatedTask.history.mu.Lock()
|
||||||
|
defer updatedTask.history.mu.Unlock()
|
||||||
|
require.Len(t, updatedTask.history.samples, 1)
|
||||||
|
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
|
||||||
|
|
||||||
|
agg := updatedTask.history.aggregateLocked(time.Hour, now)
|
||||||
|
require.True(t, agg.hasData())
|
||||||
|
assert.Equal(t, int64(2), agg.totalCount)
|
||||||
|
assert.Equal(t, int64(2), agg.successCount)
|
||||||
|
assert.Equal(t, int64(18), agg.avgResponse())
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-existingTask.ctx.Done():
|
||||||
|
default:
|
||||||
|
t.Fatal("expected original monitor task to be cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
|
||||||
|
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
||||||
|
task := newMonitorTask(config)
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.monitors = map[string]*monitorTask{config.ID: task}
|
||||||
|
|
||||||
|
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
||||||
|
Action: monitor.SyncActionDelete,
|
||||||
|
Config: monitor.Config{ID: config.ID},
|
||||||
|
})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, exists := pm.monitors[config.ID]
|
||||||
|
assert.False(t, exists)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-task.ctx.Done():
|
||||||
|
default:
|
||||||
|
t.Fatal("expected deleted monitor task to be cancelled")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerGetRandomDelay(t *testing.T) {
|
||||||
|
for i := 1000; i < 360_000; i += 1000 {
|
||||||
|
delay := getStagger(int64(i))
|
||||||
|
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
|
||||||
|
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorHTTP(t *testing.T) {
|
||||||
|
t.Run("success", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, responseUs, int64(0))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("server error", func(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
http.Error(w, "boom", http.StatusInternalServerError)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
||||||
|
assert.Equal(t, int64(-1), responseUs)
|
||||||
|
require.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorTCP(t *testing.T) {
|
||||||
|
t.Run("success", func(t *testing.T) {
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer listener.Close()
|
||||||
|
|
||||||
|
accepted := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(accepted)
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err == nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
||||||
|
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, responseUs, int64(0))
|
||||||
|
<-accepted
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("connection failure", func(t *testing.T) {
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
||||||
|
require.NoError(t, listener.Close())
|
||||||
|
|
||||||
|
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
||||||
|
assert.Equal(t, int64(-1), responseUs)
|
||||||
|
require.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorTCPAddressFallback(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
ips []string
|
||||||
|
loss bool
|
||||||
|
}{
|
||||||
|
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
|
||||||
|
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
|
||||||
|
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer listener.Close()
|
||||||
|
original := net.DefaultResolver
|
||||||
|
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
|
||||||
|
defer func() { net.DefaultResolver = original }()
|
||||||
|
|
||||||
|
// Verify the resolver preserves the intended order, so success cannot
|
||||||
|
// accidentally bypass the failed first address in the regression case.
|
||||||
|
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, tc.ips, ips)
|
||||||
|
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
|
||||||
|
if tc.loss {
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, int64(-1), responseUs)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, responseUs, int64(0))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// tcpMonitorTestResolver supplies multiple A records without external DNS.
|
||||||
|
func tcpMonitorTestResolver(ips []string) *net.Resolver {
|
||||||
|
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||||
|
client, server := net.Pipe()
|
||||||
|
go func() {
|
||||||
|
defer server.Close()
|
||||||
|
// net.Resolver uses TCP framing when its connection is not a PacketConn.
|
||||||
|
var size uint16
|
||||||
|
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
packet := make([]byte, size)
|
||||||
|
if _, err := io.ReadFull(server, packet); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var msg dnsmessage.Message
|
||||||
|
if err := msg.Unpack(packet); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
msg.Header.Response = true
|
||||||
|
msg.Header.RecursionAvailable = true
|
||||||
|
for _, question := range msg.Questions {
|
||||||
|
if question.Type != dnsmessage.TypeA {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, ip := range ips {
|
||||||
|
msg.Answers = append(msg.Answers, dnsmessage.Resource{
|
||||||
|
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
|
||||||
|
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
packet, err := msg.Pack()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
|
||||||
|
_, _ = server.Write(append(response, packet...))
|
||||||
|
}()
|
||||||
|
return client, nil
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorDNS(t *testing.T) {
|
||||||
|
t.Run("success", func(t *testing.T) {
|
||||||
|
responseUs, err := monitorDNS(context.Background(), "localhost")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.GreaterOrEqual(t, responseUs, int64(0))
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("lookup failure", func(t *testing.T) {
|
||||||
|
responseUs, err := monitorDNS(context.Background(), "")
|
||||||
|
assert.Equal(t, int64(-1), responseUs)
|
||||||
|
require.Error(t, err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
|
||||||
|
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
|
||||||
|
t.Run(action, func(t *testing.T) {
|
||||||
|
started := make(chan struct{})
|
||||||
|
canceled := make(chan struct{})
|
||||||
|
release := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
close(started)
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
close(canceled)
|
||||||
|
case <-release:
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
defer close(release)
|
||||||
|
pm := newMonitorManager()
|
||||||
|
defer pm.Stop()
|
||||||
|
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
|
||||||
|
task := newMonitorTask(cfg)
|
||||||
|
// Seed history to ensure a canceled RunNow does not return an old result.
|
||||||
|
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
|
||||||
|
pm.monitors[cfg.ID] = task
|
||||||
|
done := make(chan *monitor.Result, 1)
|
||||||
|
go func() {
|
||||||
|
result, _ := pm.UpsertMonitor(cfg, true)
|
||||||
|
done <- result
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("probe did not start")
|
||||||
|
}
|
||||||
|
updated := cfg
|
||||||
|
updated.Interval--
|
||||||
|
switch action {
|
||||||
|
case "stop":
|
||||||
|
pm.Stop()
|
||||||
|
case "delete":
|
||||||
|
pm.DeleteMonitor(cfg.ID)
|
||||||
|
case "upsert":
|
||||||
|
_, err := pm.UpsertMonitor(updated, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
case "sync replace":
|
||||||
|
pm.SyncMonitors([]monitor.Config{updated})
|
||||||
|
case "sync remove":
|
||||||
|
pm.SyncMonitors(nil)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-canceled:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("active HTTP request was not canceled")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case result := <-done:
|
||||||
|
assert.Nil(t, result)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("RunNow did not return after cancellation")
|
||||||
|
}
|
||||||
|
task.history.mu.Lock()
|
||||||
|
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
|
||||||
|
task.history.mu.Unlock()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorResolutionCancellation(t *testing.T) {
|
||||||
|
for _, protocol := range []string{"tcp", "dns", "icmp"} {
|
||||||
|
t.Run(protocol, func(t *testing.T) {
|
||||||
|
started := make(chan struct{}, 1)
|
||||||
|
original := net.DefaultResolver
|
||||||
|
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||||
|
select {
|
||||||
|
case started <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
<-ctx.Done()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}}
|
||||||
|
defer func() { net.DefaultResolver = original }()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
switch protocol {
|
||||||
|
case "tcp":
|
||||||
|
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
|
||||||
|
case "dns":
|
||||||
|
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
|
||||||
|
case "icmp":
|
||||||
|
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
|
||||||
|
}
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("lookup did not start")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
require.Error(t, err)
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("lookup did not cancel")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
select {
|
||||||
|
case <-r.Context().Done():
|
||||||
|
case <-release:
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
defer close(release)
|
||||||
|
pm := newMonitorManager()
|
||||||
|
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
|
||||||
|
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
|
||||||
|
defer task.cancel()
|
||||||
|
|
||||||
|
result := task.runProbe(pm.probe)
|
||||||
|
require.NotNil(t, result)
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss)
|
||||||
|
assert.Equal(t, 100.0, result.PacketLoss1h)
|
||||||
|
require.Len(t, task.history.samples, 1)
|
||||||
|
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
|
||||||
|
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// Version is the current version of the application.
|
// Version is the current version of the application.
|
||||||
Version = "0.19.0"
|
Version = "0.20.0"
|
||||||
// AppName is the name of the application.
|
// AppName is the name of the application.
|
||||||
AppName = "beszel"
|
AppName = "beszel"
|
||||||
)
|
)
|
||||||
@@ -19,3 +19,6 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
|||||||
|
|
||||||
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
||||||
var MinVersionZfsData = semver.MustParse("0.18.9")
|
var MinVersionZfsData = semver.MustParse("0.18.9")
|
||||||
|
|
||||||
|
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
|
||||||
|
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
|
||||||
|
|||||||
@@ -7,22 +7,23 @@ require (
|
|||||||
github.com/coreos/go-systemd/v22 v22.7.0
|
github.com/coreos/go-systemd/v22 v22.7.0
|
||||||
github.com/distribution/reference v0.6.0
|
github.com/distribution/reference v0.6.0
|
||||||
github.com/ebitengine/purego v0.11.0
|
github.com/ebitengine/purego v0.11.0
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3
|
github.com/fxamacker/cbor/v2 v2.9.4
|
||||||
github.com/gliderlabs/ssh v0.3.8
|
github.com/gliderlabs/ssh v0.3.8
|
||||||
github.com/lxzan/gws v1.10.1
|
github.com/lxzan/gws v1.10.2
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0
|
github.com/nicholas-fedor/shoutrrr v0.21.0
|
||||||
github.com/opencontainers/go-digest v1.0.0
|
github.com/opencontainers/go-digest v1.0.0
|
||||||
github.com/pocketbase/dbx v1.12.0
|
github.com/pocketbase/dbx v1.12.0
|
||||||
github.com/pocketbase/pocketbase v0.40.2
|
github.com/pocketbase/pocketbase v0.40.4
|
||||||
github.com/shirou/gopsutil/v4 v4.26.8
|
github.com/shirou/gopsutil/v4 v4.26.8
|
||||||
github.com/spf13/cast v1.10.0
|
github.com/spf13/cast v1.10.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/spf13/pflag v1.0.10
|
github.com/spf13/pflag v1.0.10
|
||||||
github.com/stretchr/testify v1.12.1
|
github.com/stretchr/testify v1.12.1
|
||||||
golang.org/x/crypto v0.56.0
|
golang.org/x/crypto v0.57.0
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa
|
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
|
||||||
golang.org/x/net v0.58.0
|
golang.org/x/net v0.59.0
|
||||||
golang.org/x/sys v0.47.0
|
golang.org/x/oauth2 v0.37.0
|
||||||
|
golang.org/x/sys v0.48.0
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
howett.net/plist v1.0.1
|
howett.net/plist v1.0.1
|
||||||
)
|
)
|
||||||
@@ -32,7 +33,7 @@ require (
|
|||||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||||
github.com/disintegration/imaging v1.6.2 // indirect
|
github.com/disintegration/imaging v1.6.2 // indirect
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||||
github.com/eclipse/paho.golang v0.23.0 // indirect
|
github.com/eclipse/paho.golang v0.23.0 // indirect
|
||||||
github.com/fatih/color v1.19.0 // indirect
|
github.com/fatih/color v1.19.0 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||||
@@ -51,18 +52,23 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||||
github.com/x448/float16 v0.8.4 // indirect
|
github.com/x448/float16 v0.8.4 // indirect
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/image v0.45.0 // indirect
|
golang.org/x/image v0.46.0 // indirect
|
||||||
golang.org/x/oauth2 v0.36.0 // indirect
|
golang.org/x/mod v0.41.0 // indirect
|
||||||
golang.org/x/sync v0.22.0 // indirect
|
golang.org/x/sync v0.23.0 // indirect
|
||||||
golang.org/x/term v0.45.0 // indirect
|
golang.org/x/term v0.46.0 // indirect
|
||||||
golang.org/x/text v0.41.0 // indirect
|
golang.org/x/text v0.42.0 // indirect
|
||||||
|
golang.org/x/tools v0.50.0 // indirect
|
||||||
|
mellium.im/reader v0.1.0 // indirect
|
||||||
|
mellium.im/sasl v0.3.2 // indirect
|
||||||
|
mellium.im/xmlstream v0.15.4 // indirect
|
||||||
|
mellium.im/xmpp v0.23.0 // indirect
|
||||||
modernc.org/libc v1.74.4 // indirect
|
modernc.org/libc v1.74.4 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.12.1 // indirect
|
modernc.org/memory v1.12.1 // indirect
|
||||||
|
|||||||
@@ -19,8 +19,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
|||||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
||||||
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
||||||
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
||||||
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
||||||
@@ -31,8 +31,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q=
|
github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
||||||
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
||||||
@@ -77,18 +77,18 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
|
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
|
||||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||||
github.com/lxzan/gws v1.10.1 h1:1xG+tDOV0lgDeVPf0wNT74u3cn0K3LpcavRrTPTrMwQ=
|
github.com/lxzan/gws v1.10.2 h1:htReTvcY89iMk1ScVtUbk6J96kIZWaafj6r/lasK/NA=
|
||||||
github.com/lxzan/gws v1.10.1/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
|
github.com/lxzan/gws v1.10.2/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
|
||||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0 h1:hMAxIYlfAeZ1FcTDgU0kUOvVXUsOirWo8IWlnzGLkac=
|
github.com/nicholas-fedor/shoutrrr v0.21.0 h1:as/mEwdaZMijCVu0FkTUEXashhvC3Y7C5g9dsXMcmQc=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.20.0/go.mod h1:hgde37yNWCXh8+N6WemyDRMNYLOFTf326GsBx8Z7CFA=
|
github.com/nicholas-fedor/shoutrrr v0.21.0/go.mod h1:dgg4kJv9K0tLXBH/1TXiSibNbM2hcd4SK6xb0sglyU4=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
|
github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||||
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
||||||
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
||||||
@@ -98,10 +98,10 @@ github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA
|
|||||||
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
||||||
github.com/pocketbase/pocketbase v0.40.2 h1:7gTqvt3bmilkphyZZ1QNhX19g3BXHqT7ynDyU81RVT4=
|
github.com/pocketbase/pocketbase v0.40.4 h1:0SvSUreR3NhUMCs9LchE59oEG53efZ3cKiMGyAGBN9U=
|
||||||
github.com/pocketbase/pocketbase v0.40.2/go.mod h1:jc3YuyToy+ZXM4CeO7uSCN/htgR8yv+tjSE3eJZ8eh8=
|
github.com/pocketbase/pocketbase v0.40.4/go.mod h1:2mU+80FLiY1fb13WZRg8Xx/lKg4nTjgKVJMigyRH6k0=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||||
@@ -136,37 +136,37 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
|||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
|
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
|
||||||
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
|
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
|
||||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
||||||
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
||||||
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||||
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||||
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
||||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
|
||||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
||||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
||||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||||
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
||||||
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
||||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
@@ -176,6 +176,14 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
||||||
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||||
|
mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
|
||||||
|
mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
|
||||||
|
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
|
||||||
|
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
|
||||||
|
mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI=
|
||||||
|
mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4=
|
||||||
|
mellium.im/xmpp v0.23.0 h1:rvKvOvMdIURCLaAWEJN8J0QpO3AJYCDKjxLsqtTPSjY=
|
||||||
|
mellium.im/xmpp v0.23.0/go.mod h1:GHDKlKKQe0LNmD9YqExyxnFEEBiz84KGqnfiA2VNzb8=
|
||||||
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
||||||
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
||||||
|
|||||||
@@ -20,10 +20,11 @@ type hubLike interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type AlertManager struct {
|
type AlertManager struct {
|
||||||
hub hubLike
|
hub hubLike
|
||||||
stopOnce sync.Once
|
stopOnce sync.Once
|
||||||
pendingAlerts sync.Map
|
pendingAlerts sync.Map
|
||||||
alertsCache *AlertsCache
|
alertsCache *AlertsCache
|
||||||
|
networkMonitors *networkMonitorCache
|
||||||
}
|
}
|
||||||
|
|
||||||
type AlertMessageData struct {
|
type AlertMessageData struct {
|
||||||
@@ -107,8 +108,9 @@ var supportsTitle = map[string]struct{}{
|
|||||||
// NewAlertManager creates a new AlertManager instance.
|
// NewAlertManager creates a new AlertManager instance.
|
||||||
func NewAlertManager(app hubLike) *AlertManager {
|
func NewAlertManager(app hubLike) *AlertManager {
|
||||||
am := &AlertManager{
|
am := &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
|
networkMonitors: newNetworkMonitorCache(app),
|
||||||
}
|
}
|
||||||
am.bindEvents()
|
am.bindEvents()
|
||||||
return am
|
return am
|
||||||
@@ -116,6 +118,7 @@ func NewAlertManager(app hubLike) *AlertManager {
|
|||||||
|
|
||||||
// Bind events to the alerts collection lifecycle
|
// Bind events to the alerts collection lifecycle
|
||||||
func (am *AlertManager) bindEvents() {
|
func (am *AlertManager) bindEvents() {
|
||||||
|
am.bindNetworkMonitorAlertEvents()
|
||||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||||
|
|||||||
@@ -29,6 +29,13 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
|
|||||||
return e.BadRequestError("Bad data", err)
|
return e.BadRequestError("Bad data", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if reqData.Name == alertNameNetworkMonitorLoss {
|
||||||
|
if reqData.Value < 0 || reqData.Value >= 100 {
|
||||||
|
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
||||||
|
}
|
||||||
|
reqData.Min = 0
|
||||||
|
}
|
||||||
|
|
||||||
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||||
@@ -533,6 +534,20 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
|
|
||||||
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
|
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
|
||||||
scenarios = append(scenarios, beszelTests.ApiScenario{
|
scenarios = append(scenarios, beszelTests.ApiScenario{
|
||||||
|
BeforeTestFunc: func(tb testing.TB, _ *pbTests.TestApp, e *core.ServeEvent) {
|
||||||
|
if !strings.HasPrefix(url, "mqtt://") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Keep the real MQTT rejection path, but advance its library's
|
||||||
|
// fixed timeout using virtual time instead of waiting 10 seconds.
|
||||||
|
e.Router.BindFunc(func(re *core.RequestEvent) error {
|
||||||
|
var err error
|
||||||
|
synctest.Test(tb.(*testing.T), func(t *testing.T) {
|
||||||
|
err = re.Next()
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
},
|
||||||
Name: "readonly cannot send to " + url,
|
Name: "readonly cannot send to " + url,
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
URL: "/api/beszel/test-notification",
|
URL: "/api/beszel/test-notification",
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package alerts
|
package alerts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
@@ -18,6 +19,9 @@ type CachedAlertData struct {
|
|||||||
Triggered bool
|
Triggered bool
|
||||||
Min uint8
|
Min uint8
|
||||||
PendingSince time.Time
|
PendingSince time.Time
|
||||||
|
// Immutable after publication; decoded only when the alert record changes.
|
||||||
|
MonitorStates map[string]string
|
||||||
|
MonitorStatesValid bool
|
||||||
// Created types.DateTime
|
// Created types.DateTime
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -30,11 +34,18 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
|
|||||||
a.Triggered = record.GetBool("triggered")
|
a.Triggered = record.GetBool("triggered")
|
||||||
a.Min = uint8(record.GetInt("min"))
|
a.Min = uint8(record.GetInt("min"))
|
||||||
a.PendingSince = record.GetDateTime("pending_since").Time()
|
a.PendingSince = record.GetDateTime("pending_since").Time()
|
||||||
|
if a.Name == alertNameNetworkMonitorLoss {
|
||||||
|
var state networkMonitorAlertState
|
||||||
|
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
|
||||||
|
a.MonitorStates = state.Monitors
|
||||||
|
}
|
||||||
// a.Created = record.GetDateTime("created")
|
// a.Created = record.GetDateTime("created")
|
||||||
}
|
}
|
||||||
|
|
||||||
// AlertsCache provides an in-memory cache for system alerts.
|
// AlertsCache provides an in-memory cache for system alerts.
|
||||||
type AlertsCache struct {
|
type AlertsCache struct {
|
||||||
|
// Serialize lazy loads with updates so a late load cannot replace newer state.
|
||||||
|
loadMu sync.Mutex
|
||||||
app core.App
|
app core.App
|
||||||
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
||||||
populated bool
|
populated bool
|
||||||
@@ -69,6 +80,8 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
|
|||||||
|
|
||||||
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
||||||
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
||||||
|
c.loadMu.Lock()
|
||||||
|
defer c.loadMu.Unlock()
|
||||||
if !force && c.populated {
|
if !force && c.populated {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -78,7 +91,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
}
|
}
|
||||||
c.store.RemoveAll()
|
c.store.RemoveAll()
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
c.Update(record)
|
c.update(record)
|
||||||
}
|
}
|
||||||
c.populated = true
|
c.populated = true
|
||||||
return nil
|
return nil
|
||||||
@@ -86,6 +99,12 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
|
|
||||||
// Update adds or updates an alert record in the cache.
|
// Update adds or updates an alert record in the cache.
|
||||||
func (c *AlertsCache) Update(record *core.Record) {
|
func (c *AlertsCache) Update(record *core.Record) {
|
||||||
|
c.loadMu.Lock()
|
||||||
|
defer c.loadMu.Unlock()
|
||||||
|
c.update(record)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *AlertsCache) update(record *core.Record) {
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -102,6 +121,8 @@ func (c *AlertsCache) Update(record *core.Record) {
|
|||||||
|
|
||||||
// Delete removes an alert record from the cache.
|
// Delete removes an alert record from the cache.
|
||||||
func (c *AlertsCache) Delete(record *core.Record) {
|
func (c *AlertsCache) Delete(record *core.Record) {
|
||||||
|
c.loadMu.Lock()
|
||||||
|
defer c.loadMu.Unlock()
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -115,18 +136,23 @@ func (c *AlertsCache) Delete(record *core.Record) {
|
|||||||
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
||||||
systemStore, ok := c.store.GetOk(systemID)
|
systemStore, ok := c.store.GetOk(systemID)
|
||||||
if !ok {
|
if !ok {
|
||||||
// Populate cache for this system
|
c.loadMu.Lock()
|
||||||
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
defer c.loadMu.Unlock()
|
||||||
if err != nil {
|
systemStore, ok = c.store.GetOk(systemID)
|
||||||
return nil
|
if !ok {
|
||||||
|
// Populate cache for this system
|
||||||
|
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
systemStore = store.New(map[string]CachedAlertData{})
|
||||||
|
for _, record := range records {
|
||||||
|
var ca CachedAlertData
|
||||||
|
ca.PopulateFromRecord(record)
|
||||||
|
systemStore.Set(record.Id, ca)
|
||||||
|
}
|
||||||
|
c.store.Set(systemID, systemStore)
|
||||||
}
|
}
|
||||||
systemStore = store.New(map[string]CachedAlertData{})
|
|
||||||
for _, record := range records {
|
|
||||||
var ca CachedAlertData
|
|
||||||
ca.PopulateFromRecord(record)
|
|
||||||
systemStore.Set(record.Id, ca)
|
|
||||||
}
|
|
||||||
c.store.Set(systemID, systemStore)
|
|
||||||
}
|
}
|
||||||
all := systemStore.GetAll()
|
all := systemStore.GetAll()
|
||||||
alerts := make([]CachedAlertData, 0, len(all))
|
alerts := make([]CachedAlertData, 0, len(all))
|
||||||
|
|||||||
@@ -9,6 +9,12 @@ import (
|
|||||||
|
|
||||||
// On triggered alert record delete, set matching alert history record to resolved
|
// On triggered alert record delete, set matching alert history record to resolved
|
||||||
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
||||||
|
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
||||||
|
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
if !e.Record.GetBool("triggered") {
|
if !e.Record.GetBool("triggered") {
|
||||||
return e.Next()
|
return e.Next()
|
||||||
}
|
}
|
||||||
@@ -18,6 +24,10 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
|||||||
|
|
||||||
// On alert record update, update alert history record
|
// On alert record update, update alert history record
|
||||||
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
||||||
|
// Network monitor incidents have separate history entries per monitor.
|
||||||
|
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
original := e.Record.Original()
|
original := e.Record.Original()
|
||||||
new := e.Record
|
new := e.Record
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,269 @@
|
|||||||
|
package alerts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"math"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
|
||||||
|
|
||||||
|
// networkMonitorAlertState is this alert type's persisted runtime state.
|
||||||
|
// Monitor IDs map to their open history entries independently of history retention.
|
||||||
|
type networkMonitorAlertState struct {
|
||||||
|
Monitors map[string]string `json:"monitors"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
|
||||||
|
// Hidden fields are still writable through the record API unless protected.
|
||||||
|
protectState := func(e *core.RecordRequestEvent) error {
|
||||||
|
e.Record.Set("state", e.Record.Original().Get("state"))
|
||||||
|
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
|
||||||
|
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
|
||||||
|
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
|
||||||
|
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
|
||||||
|
}
|
||||||
|
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
||||||
|
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
|
||||||
|
return e.ForbiddenError("You do not have access to this system", nil)
|
||||||
|
}
|
||||||
|
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
|
||||||
|
value := e.Record.GetFloat("value")
|
||||||
|
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
|
||||||
|
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
||||||
|
}
|
||||||
|
e.Record.Set("min", 0)
|
||||||
|
}
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
|
||||||
|
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
|
||||||
|
cleanup := func(e *core.RecordEvent) error {
|
||||||
|
if err := e.Next(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
|
||||||
|
}
|
||||||
|
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
|
||||||
|
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
return cleanup(e)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
|
||||||
|
// using its exact payload (dashboard requests can replace the cached payload).
|
||||||
|
// Omitted results and disconnected systems never imply recovery.
|
||||||
|
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
|
||||||
|
if systemRecord.GetString("status") != "up" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
|
||||||
|
if len(alerts) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
monitors, err := am.networkMonitors.get(systemRecord.Id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// The cache only predicts a transition. Reload and recheck under the DB
|
||||||
|
// transaction before persisting, including current system/monitor status.
|
||||||
|
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
|
||||||
|
}
|
||||||
|
|
||||||
|
// networkMonitorTransitionPending does no IO and never mutates cached maps.
|
||||||
|
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
|
||||||
|
for _, alert := range alerts {
|
||||||
|
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for id := range alert.MonitorStates {
|
||||||
|
if _, enabled := monitors[id]; !enabled {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for id, result := range results {
|
||||||
|
interval, enabled := monitors[id]
|
||||||
|
if !enabled || !monitorResultReady(result, interval, now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, active := alert.MonitorStates[id]
|
||||||
|
if (result.PacketLoss1h > alert.Value) != active {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
|
||||||
|
var messages []AlertMessageData
|
||||||
|
err := app.RunInTransaction(func(tx core.App) error {
|
||||||
|
// Read configuration inside the transaction so concurrent threshold changes,
|
||||||
|
// disabling, and evaluations cannot overwrite each other's incident state.
|
||||||
|
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
|
||||||
|
if err != nil || len(alerts) == 0 {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
system, err := tx.FindRecordById("systems", systemID)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
// System deletion cascades to its alerts.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
enabled := make(map[string]*core.Record, len(monitors))
|
||||||
|
for _, m := range monitors {
|
||||||
|
enabled[m.Id] = m
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
for _, alert := range alerts {
|
||||||
|
var state networkMonitorAlertState
|
||||||
|
if err := alert.UnmarshalJSONField("state", &state); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
states := state.Monitors
|
||||||
|
if states == nil {
|
||||||
|
states = map[string]string{}
|
||||||
|
}
|
||||||
|
changed := false
|
||||||
|
// Removing or disabling a monitor closes its incident silently.
|
||||||
|
for id, historyID := range states {
|
||||||
|
if _, ok := enabled[id]; !ok {
|
||||||
|
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
delete(states, id)
|
||||||
|
changed = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if system.GetString("status") == "up" {
|
||||||
|
for _, m := range monitors {
|
||||||
|
result, ok := results[m.Id]
|
||||||
|
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
historyID, active := states[m.Id]
|
||||||
|
triggered := result.PacketLoss1h > alert.GetFloat("value")
|
||||||
|
if triggered == active {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
label := m.GetString("target")
|
||||||
|
if m.GetString("protocol") == "tcp" {
|
||||||
|
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
|
||||||
|
}
|
||||||
|
if triggered {
|
||||||
|
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
history := core.NewRecord(collection)
|
||||||
|
history.Load(map[string]any{
|
||||||
|
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
|
||||||
|
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
|
||||||
|
})
|
||||||
|
if err := tx.Save(history); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
states[m.Id] = history.Id
|
||||||
|
} else {
|
||||||
|
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
delete(states, m.Id)
|
||||||
|
}
|
||||||
|
changed = true
|
||||||
|
state, comparison := "loss", "exceeds"
|
||||||
|
if !triggered {
|
||||||
|
state, comparison = "recovered", "is at or below"
|
||||||
|
}
|
||||||
|
messages = append(messages, AlertMessageData{
|
||||||
|
UserID: alert.GetString("user"), SystemID: systemID,
|
||||||
|
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
|
||||||
|
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
|
||||||
|
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if changed || alert.GetBool("triggered") != (len(states) > 0) {
|
||||||
|
alert.Set("state", networkMonitorAlertState{Monitors: states})
|
||||||
|
alert.Set("triggered", len(states) > 0)
|
||||||
|
if err := tx.Save(alert); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Match other alert types: persist transitions before delivery, and respect
|
||||||
|
// the user's existing notification destinations and quiet hours.
|
||||||
|
for _, message := range messages {
|
||||||
|
if err := am.SendAlert(message); err != nil {
|
||||||
|
app.Logger().Error("Failed to send network monitor alert", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
|
||||||
|
// Three completed attempts provide a short warm-up, including after an agent
|
||||||
|
// restart.
|
||||||
|
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Never interpret an empty one-hour window as zero loss.
|
||||||
|
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
|
||||||
|
age := now.Sub(time.UnixMilli(result.LastProbeAt))
|
||||||
|
return age >= -time.Minute && age <= maxAge
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
|
||||||
|
record, err := app.FindRecordById("alerts_history", id)
|
||||||
|
if errors.Is(err, sql.ErrNoRows) {
|
||||||
|
// History can be purged independently.
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !record.GetDateTime("resolved").IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
record.Set("resolved", now.UTC())
|
||||||
|
return app.Save(record)
|
||||||
|
}
|
||||||
|
|
||||||
|
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
|
||||||
|
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, record := range records {
|
||||||
|
record.Set("resolved", time.Now().UTC())
|
||||||
|
if err := app.Save(record); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,513 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package alerts_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/alerts"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
|
||||||
|
t.Helper()
|
||||||
|
hub, system, alert := systemdTestSetup(t, false)
|
||||||
|
t.Cleanup(hub.Cleanup)
|
||||||
|
alert.Set("name", "NetworkMonitorLoss")
|
||||||
|
alert.Set("value", 5)
|
||||||
|
require.NoError(t, hub.Save(alert))
|
||||||
|
var monitors []*core.Record
|
||||||
|
for _, name := range []string{"gateway", "website"} {
|
||||||
|
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
||||||
|
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
monitors = append(monitors, record)
|
||||||
|
}
|
||||||
|
// Avoid starting a system update worker in tests.
|
||||||
|
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
system.Set("status", "up")
|
||||||
|
return hub, system, alert, monitors
|
||||||
|
}
|
||||||
|
|
||||||
|
func monitorResult(loss float64) monitor.Result {
|
||||||
|
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
|
||||||
|
check := func(active bool, open, sent int) {
|
||||||
|
t.Helper()
|
||||||
|
record, err := hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, active, record.GetBool("triggered"))
|
||||||
|
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.EqualValues(t, open, total)
|
||||||
|
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
|
||||||
|
}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
check(true, 1, 1)
|
||||||
|
message := hub.TestMailer.Messages()[count]
|
||||||
|
assert.Contains(t, message.Text, "gateway.example.com")
|
||||||
|
assert.Contains(t, message.Text, "10.00%")
|
||||||
|
assert.Contains(t, message.Text, "5.00%")
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
check(true, 1, 1)
|
||||||
|
// Persisted monitor state prevents duplicate notifications after a hub restart.
|
||||||
|
am = alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results[monitors[1].Id] = monitorResult(20)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
check(true, 2, 2)
|
||||||
|
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
check(true, 1, 3)
|
||||||
|
results[monitors[1].Id] = monitorResult(0)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
check(false, 0, 4)
|
||||||
|
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, histories, 2)
|
||||||
|
for _, history := range histories {
|
||||||
|
assert.NotEmpty(t, history.GetString("monitor_name"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
protocol, target, label string
|
||||||
|
port int
|
||||||
|
}{
|
||||||
|
{"icmp", "gateway.example.com", "gateway.example.com", 0},
|
||||||
|
{"http", "https://example.com/health", "https://example.com/health", 0},
|
||||||
|
{"tcp", "example.com", "example.com:8443", 8443},
|
||||||
|
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
|
||||||
|
} {
|
||||||
|
t.Run(tc.label, func(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
m := monitors[0]
|
||||||
|
m.Set("protocol", tc.protocol)
|
||||||
|
m.Set("target", tc.target)
|
||||||
|
m.Set("port", tc.port)
|
||||||
|
require.NoError(t, hub.Save(m))
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
|
||||||
|
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, histories, 1)
|
||||||
|
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
|
||||||
|
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
apply := func(loss float64) {
|
||||||
|
result := monitorResult(loss)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: result}
|
||||||
|
switch scenario {
|
||||||
|
case "missing":
|
||||||
|
results = nil
|
||||||
|
case "stale":
|
||||||
|
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
case "expired hourly window":
|
||||||
|
monitors[0].Set("interval", 3600)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
case "future":
|
||||||
|
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
case "warmup":
|
||||||
|
result.SampleCount = 2
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
case "no probes":
|
||||||
|
result.SampleCount = 0
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
case "down":
|
||||||
|
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
case "paused":
|
||||||
|
record, err := hub.FindRecordById("systems", system.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
record.Set("status", "paused")
|
||||||
|
require.NoError(t, hub.Save(record))
|
||||||
|
}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
}
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
apply(100)
|
||||||
|
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
||||||
|
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
|
||||||
|
apply(0)
|
||||||
|
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
|
||||||
|
record, err := hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, record.GetBool("triggered"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertCleanup(t *testing.T) {
|
||||||
|
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
|
||||||
|
t.Run(scenario, func(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
switch scenario {
|
||||||
|
case "disable monitor":
|
||||||
|
monitors[0].Set("enabled", false)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
case "delete monitor":
|
||||||
|
require.NoError(t, hub.Delete(monitors[0]))
|
||||||
|
case "disable alert":
|
||||||
|
require.NoError(t, hub.Delete(alert))
|
||||||
|
case "delete system":
|
||||||
|
require.NoError(t, hub.Delete(system))
|
||||||
|
case "purge history":
|
||||||
|
history, err := hub.FindAllRecords("alerts_history")
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, record := range history {
|
||||||
|
require.NoError(t, hub.Delete(record))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
||||||
|
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
||||||
|
require.NoError(t, err)
|
||||||
|
if scenario == "disable monitor" || scenario == "delete monitor" {
|
||||||
|
assert.EqualValues(t, 1, open)
|
||||||
|
record, err := hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.True(t, record.GetBool("triggered"))
|
||||||
|
require.NoError(t, hub.Delete(monitors[1]))
|
||||||
|
record, err = hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, record.GetBool("triggered"))
|
||||||
|
} else {
|
||||||
|
assert.Zero(t, open)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
|
||||||
|
require.NoError(t, err)
|
||||||
|
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
|
||||||
|
require.NoError(t, err)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
other, err = hub.FindRecordById("alerts", other.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, other.GetBool("triggered"))
|
||||||
|
// Editing the threshold re-evaluates on the next batch, without losing state.
|
||||||
|
alert, err = hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
alert.Set("value", 15)
|
||||||
|
require.NoError(t, hub.Save(alert))
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
alert, err = hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.False(t, alert.GetBool("triggered"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertAPI(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
value float64
|
||||||
|
direct, denied, patch bool
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{name: "zero threshold", value: 0, status: 200},
|
||||||
|
{name: "fractional threshold", value: 5.5, status: 200},
|
||||||
|
{name: "negative threshold", value: -1, status: 400},
|
||||||
|
{name: "unreachable threshold", value: 100, status: 400},
|
||||||
|
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
|
||||||
|
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
|
||||||
|
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
|
||||||
|
{name: "direct private state", value: 5, direct: true, status: 200},
|
||||||
|
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
owner := user.Id
|
||||||
|
if tc.denied {
|
||||||
|
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
|
||||||
|
require.NoError(t, err)
|
||||||
|
owner = other.Id
|
||||||
|
}
|
||||||
|
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
|
||||||
|
require.NoError(t, err)
|
||||||
|
token, err := user.NewAuthToken()
|
||||||
|
require.NoError(t, err)
|
||||||
|
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
|
||||||
|
url, method := "/api/beszel/user-alerts", "POST"
|
||||||
|
if tc.direct {
|
||||||
|
url = "/api/collections/alerts/records"
|
||||||
|
body["system"], body["user"] = systems[0].Id, user.Id
|
||||||
|
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
|
||||||
|
}
|
||||||
|
if tc.patch {
|
||||||
|
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
|
||||||
|
require.NoError(t, err)
|
||||||
|
url += "/" + alert.Id
|
||||||
|
method = "PATCH"
|
||||||
|
body["triggered"] = false
|
||||||
|
}
|
||||||
|
content := `"success":true`
|
||||||
|
if tc.direct {
|
||||||
|
content = `"name":"NetworkMonitorLoss"`
|
||||||
|
}
|
||||||
|
if tc.status == 400 {
|
||||||
|
content = `"status":400`
|
||||||
|
}
|
||||||
|
if tc.status == 403 {
|
||||||
|
content = `"status":403`
|
||||||
|
}
|
||||||
|
scenario := beszelTests.ApiScenario{
|
||||||
|
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
|
||||||
|
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
|
||||||
|
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
|
||||||
|
}
|
||||||
|
scenario.Test(t)
|
||||||
|
records, err := hub.FindAllRecords("alerts")
|
||||||
|
require.NoError(t, err)
|
||||||
|
if tc.status != 200 || tc.denied {
|
||||||
|
assert.Empty(t, records)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
require.Len(t, records, 1)
|
||||||
|
assert.Equal(t, tc.value, records[0].GetFloat("value"))
|
||||||
|
assert.Zero(t, records[0].GetInt("min"))
|
||||||
|
state := struct {
|
||||||
|
Monitors map[string]string `json:"monitors"`
|
||||||
|
}{}
|
||||||
|
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
|
||||||
|
states := state.Monitors
|
||||||
|
if tc.patch {
|
||||||
|
assert.Equal(t, map[string]string{"real": "history"}, states)
|
||||||
|
assert.True(t, records[0].GetBool("triggered"))
|
||||||
|
} else {
|
||||||
|
assert.Empty(t, states)
|
||||||
|
assert.False(t, records[0].GetBool("triggered"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type monitorCountingHub struct {
|
||||||
|
*beszelTests.TestHub
|
||||||
|
transactions atomic.Int64
|
||||||
|
beforeTransaction func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
|
||||||
|
h.transactions.Add(1)
|
||||||
|
if h.beforeTransaction != nil {
|
||||||
|
h.beforeTransaction()
|
||||||
|
}
|
||||||
|
return h.App.RunInTransaction(fn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count actual SQL on both DB connections, including queries through record APIs.
|
||||||
|
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
|
||||||
|
t.Helper()
|
||||||
|
count := &atomic.Int64{}
|
||||||
|
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
|
||||||
|
db := builder.(*dbx.DB)
|
||||||
|
old := db.LogFunc
|
||||||
|
db.LogFunc = func(string, ...any) { count.Add(1) }
|
||||||
|
t.Cleanup(func() { db.LogFunc = old })
|
||||||
|
}
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
counted := &monitorCountingHub{TestHub: hub}
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
||||||
|
sql := monitorSQLCounter(t, hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
||||||
|
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
|
||||||
|
noWork := func() {
|
||||||
|
t.Helper()
|
||||||
|
sql.Store(0)
|
||||||
|
counted.transactions.Store(0)
|
||||||
|
for range 100 {
|
||||||
|
evaluate()
|
||||||
|
}
|
||||||
|
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
|
||||||
|
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
|
||||||
|
}
|
||||||
|
// One-time lazy loads are permitted, including on hub restart.
|
||||||
|
evaluate()
|
||||||
|
assert.Positive(t, sql.Load())
|
||||||
|
noWork()
|
||||||
|
// Realtime metric saves invoke record hooks but must not invalidate config.
|
||||||
|
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
monitors[0] = fresh
|
||||||
|
monitors[0].Set("loss1h", 0)
|
||||||
|
monitors[0].Set("res", 100)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
noWork()
|
||||||
|
results[monitors[0].Id] = monitorResult(10)
|
||||||
|
evaluate()
|
||||||
|
assert.Positive(t, sql.Load(), "transitions must still be persisted")
|
||||||
|
assert.EqualValues(t, 1, counted.transactions.Load())
|
||||||
|
noWork()
|
||||||
|
// Missing and stale observations must not enter the transaction either.
|
||||||
|
results = nil
|
||||||
|
noWork()
|
||||||
|
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
|
||||||
|
noWork()
|
||||||
|
results[monitors[0].Id] = monitorResult(0)
|
||||||
|
evaluate()
|
||||||
|
noWork()
|
||||||
|
require.NoError(t, hub.Delete(alert))
|
||||||
|
noWork()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
// Widening the interval makes this observation fresh. A stale interval cache
|
||||||
|
// would miss the failure indefinitely, even though results keep arriving.
|
||||||
|
result := monitorResult(10)
|
||||||
|
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
|
||||||
|
results[monitors[0].Id] = result
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
||||||
|
monitors[0].Set("interval", 120)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
||||||
|
// Disable, then re-enable the same ID: its new failure must be detected.
|
||||||
|
monitors[0].Set("enabled", false)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
monitors[0].Set("enabled", true)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
|
||||||
|
// A new monitor must also become eligible without restarting the hub.
|
||||||
|
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
||||||
|
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
results[created.Id] = monitorResult(10)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
|
||||||
|
// Threshold changes refresh cached config and preserve the active incidents.
|
||||||
|
alert, err = hub.FindRecordById("alerts", alert.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
alert.Set("value", 15)
|
||||||
|
require.NoError(t, hub.Save(alert))
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
|
||||||
|
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
|
||||||
|
t.Run(change, func(t *testing.T) {
|
||||||
|
hub, system, alert, monitors := networkAlertSetup(t)
|
||||||
|
counted := &monitorCountingHub{TestHub: hub}
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
// Change the DB after the cache predicts a transition, before its transaction.
|
||||||
|
counted.beforeTransaction = func() {
|
||||||
|
counted.beforeTransaction = nil
|
||||||
|
switch change {
|
||||||
|
case "threshold":
|
||||||
|
alert.Set("value", 20)
|
||||||
|
require.NoError(t, hub.Save(alert))
|
||||||
|
case "disable alert":
|
||||||
|
require.NoError(t, hub.Delete(alert))
|
||||||
|
case "disable monitor":
|
||||||
|
monitors[0].Set("enabled", false)
|
||||||
|
require.NoError(t, hub.Save(monitors[0]))
|
||||||
|
case "down":
|
||||||
|
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
results[monitors[0].Id] = monitorResult(10)
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.EqualValues(t, 1, counted.transactions.Load())
|
||||||
|
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
||||||
|
histories, err := hub.CountRecords("alerts_history")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Zero(t, histories)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
|
||||||
|
hub, system, _, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, 8)
|
||||||
|
for range 8 {
|
||||||
|
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
|
||||||
|
hub, system, _, monitors := networkAlertSetup(t)
|
||||||
|
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
||||||
|
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
count := hub.TestMailer.TotalSend()
|
||||||
|
results[monitors[0].Id] = monitorResult(10)
|
||||||
|
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
||||||
|
histories, err := hub.CountRecords("alerts_history")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Zero(t, histories)
|
||||||
|
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
// A failed transition must not be published to the cache and mask the retry.
|
||||||
|
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
||||||
|
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
||||||
|
}
|
||||||
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
|
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
|
||||||
if len(alerts) == 0 {
|
if len(alerts) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,8 +11,9 @@ import (
|
|||||||
|
|
||||||
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
||||||
return &AlertManager{
|
return &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
|
networkMonitors: newNetworkMonitorCache(app),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package alerts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
)
|
||||||
|
|
||||||
|
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
|
||||||
|
// needed for the alert fast path. Names and targets are read only on transitions.
|
||||||
|
// Returned maps are immutable; configuration changes invalidate the whole entry.
|
||||||
|
type networkMonitorCache struct {
|
||||||
|
app core.App
|
||||||
|
mu sync.RWMutex
|
||||||
|
systems map[string]map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
|
||||||
|
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
|
||||||
|
invalidate := func(e *core.RecordEvent) error {
|
||||||
|
c.invalidate(e.Record.GetString("system"))
|
||||||
|
return e.Next()
|
||||||
|
}
|
||||||
|
app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
|
||||||
|
app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
|
||||||
|
app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
old := e.Record.Original()
|
||||||
|
// Realtime metric saves also invoke this hook. They must not evict config.
|
||||||
|
if old.GetString("system") != e.Record.GetString("system") ||
|
||||||
|
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
|
||||||
|
old.GetInt("interval") != e.Record.GetInt("interval") {
|
||||||
|
c.invalidate(old.GetString("system"))
|
||||||
|
c.invalidate(e.Record.GetString("system"))
|
||||||
|
}
|
||||||
|
return e.Next()
|
||||||
|
})
|
||||||
|
app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
c.invalidate(e.Record.Id)
|
||||||
|
return e.Next()
|
||||||
|
})
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *networkMonitorCache) invalidate(systemID string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
delete(c.systems, systemID)
|
||||||
|
c.mu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
monitors, ok := c.systems[systemID]
|
||||||
|
c.mu.RUnlock()
|
||||||
|
if ok {
|
||||||
|
return monitors, nil
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if monitors, ok := c.systems[systemID]; ok {
|
||||||
|
return monitors, nil
|
||||||
|
}
|
||||||
|
// Keep the lock through the load so a concurrent config change cannot be
|
||||||
|
// invalidated first and then overwritten by the older query result.
|
||||||
|
var rows []struct {
|
||||||
|
ID string `db:"id"`
|
||||||
|
Interval int `db:"interval"`
|
||||||
|
}
|
||||||
|
if err := c.app.DB().Select("id", "interval").From("network_monitors").
|
||||||
|
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
monitors = make(map[string]int, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
monitors[row.ID] = row.Interval
|
||||||
|
}
|
||||||
|
c.systems[systemID] = monitors
|
||||||
|
return monitors, nil
|
||||||
|
}
|
||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
"testing/synctest"
|
||||||
|
|
||||||
"github.com/nicholas-fedor/shoutrrr/pkg/types"
|
"github.com/nicholas-fedor/shoutrrr/pkg/types"
|
||||||
"golang.org/x/net/dns/dnsmessage"
|
"golang.org/x/net/dns/dnsmessage"
|
||||||
@@ -178,39 +179,45 @@ func TestPublicNotificationTCP(t *testing.T) {
|
|||||||
} {
|
} {
|
||||||
t.Run(rawURL, func(t *testing.T) {
|
t.Run(rawURL, func(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
|
// MQTT waits for a fixed library timeout even after a dial failure.
|
||||||
|
// Virtual time preserves the full send/cleanup path without that delay.
|
||||||
t.Run("internal destination", func(t *testing.T) {
|
t.Run("internal destination", func(t *testing.T) {
|
||||||
err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test")
|
synctest.Test(t, func(t *testing.T) {
|
||||||
if !errors.Is(err, errInternalDestination) {
|
err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test")
|
||||||
t.Fatalf("expected blocked destination, got %v", err)
|
if !errors.Is(err, errInternalDestination) {
|
||||||
}
|
t.Fatalf("expected blocked destination, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
})
|
})
|
||||||
t.Run("public destination uses injected dialer", func(t *testing.T) {
|
t.Run("public destination uses injected dialer", func(t *testing.T) {
|
||||||
var calls atomic.Int32
|
synctest.Test(t, func(t *testing.T) {
|
||||||
stopped := errors.New("test dial stopped")
|
var calls atomic.Int32
|
||||||
service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{
|
stopped := errors.New("test dial stopped")
|
||||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{
|
||||||
calls.Add(1)
|
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
||||||
if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") {
|
calls.Add(1)
|
||||||
t.Errorf("unexpected dial: %s %s", network, address)
|
if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") {
|
||||||
}
|
t.Errorf("unexpected dial: %s %s", network, address)
|
||||||
if err := checkNotificationAddress(address); err != nil {
|
}
|
||||||
t.Error(err)
|
if err := checkNotificationAddress(address); err != nil {
|
||||||
}
|
t.Error(err)
|
||||||
return nil, stopped
|
}
|
||||||
},
|
return nil, stopped
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if closer, ok := service.(io.Closer); ok {
|
||||||
|
defer closer.Close()
|
||||||
|
}
|
||||||
|
if err := service.Send("test", &types.Params{}); err == nil {
|
||||||
|
t.Fatal("expected dial failure")
|
||||||
|
}
|
||||||
|
if calls.Load() == 0 {
|
||||||
|
t.Fatal("custom dialer was not used")
|
||||||
|
}
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if closer, ok := service.(io.Closer); ok {
|
|
||||||
defer closer.Close()
|
|
||||||
}
|
|
||||||
if err := service.Send("test", &types.Params{}); err == nil {
|
|
||||||
t.Fatal("expected dial failure")
|
|
||||||
}
|
|
||||||
if calls.Load() == 0 {
|
|
||||||
t.Fatal("custom dialer was not used")
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,8 @@ const (
|
|||||||
GetSystemdInfo
|
GetSystemdInfo
|
||||||
// Request ZFS detail data from agent
|
// Request ZFS detail data from agent
|
||||||
GetZfsData
|
GetZfsData
|
||||||
|
// Sync network monitor configuration to agent
|
||||||
|
SyncNetworkMonitors
|
||||||
// Add new actions here...
|
// Add new actions here...
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ RUN apk add --no-cache ca-certificates && update-ca-certificates
|
|||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
|
|
||||||
RUN rm -rf /tmp/*
|
RUN rm -rf /tmp/*
|
||||||
|
|
||||||
|
|||||||
@@ -10,14 +10,14 @@ COPY . ./
|
|||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
|
|
||||||
RUN rm -rf /tmp/*
|
RUN rm -rf /tmp/*
|
||||||
|
|
||||||
# --------------------------
|
# --------------------------
|
||||||
# Final image: default scratch-based agent
|
# Final image: default scratch-based agent
|
||||||
# --------------------------
|
# --------------------------
|
||||||
FROM alpine:3.23
|
FROM alpine:3.24
|
||||||
COPY --from=builder /agent /agent
|
COPY --from=builder /agent /agent
|
||||||
|
|
||||||
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
|
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
|
||||||
|
|||||||
@@ -10,13 +10,13 @@ COPY . ./
|
|||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
|
|
||||||
# --------------------------
|
# --------------------------
|
||||||
# Final image
|
# Final image
|
||||||
# Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume
|
# Note: must cap_add: [CAP_PERFMON] and mount /dev/dri/ as volume
|
||||||
# --------------------------
|
# --------------------------
|
||||||
FROM alpine:3.23
|
FROM alpine:3.24
|
||||||
|
|
||||||
COPY --from=builder /agent /agent
|
COPY --from=builder /agent /agent
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ COPY . ./
|
|||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
|
|
||||||
# --------------------------
|
# --------------------------
|
||||||
# Smartmontools builder stage
|
# Smartmontools builder stage
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ RUN set -eux; \
|
|||||||
if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \
|
if [ "$TARGETARCH" = "arm" ] && [ -n "$TARGETVARIANT" ]; then \
|
||||||
export GOARM="${TARGETVARIANT#v}"; \
|
export GOARM="${TARGETVARIANT#v}"; \
|
||||||
fi; \
|
fi; \
|
||||||
CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH \
|
CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH \
|
||||||
go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
go build -tags glibc -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
|
|
||||||
# --------------------------
|
# --------------------------
|
||||||
@@ -70,7 +70,9 @@ RUN set -eux; \
|
|||||||
# --------------------------
|
# --------------------------
|
||||||
FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder
|
FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
# zfsutils-linux is distributed in Debian's contrib component.
|
||||||
|
RUN sed -i 's/Components: main/Components: main contrib/' /etc/apt/sources.list.d/debian.sources \
|
||||||
|
&& apt-get update && apt-get install -y --no-install-recommends \
|
||||||
zfsutils-linux \
|
zfsutils-linux \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ RUN update-ca-certificates
|
|||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub
|
RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /beszel ./internal/cmd/hub
|
||||||
|
|
||||||
# ? -------------------------
|
# ? -------------------------
|
||||||
FROM scratch
|
FROM scratch
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package monitor
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
|
||||||
|
// Hub requests that run a probe must allow this time in addition to transport overhead.
|
||||||
|
const MaxProbeTimeout = 10 * time.Second
|
||||||
|
|
||||||
|
type SyncAction uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
// SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
|
||||||
|
SyncActionReplace SyncAction = iota
|
||||||
|
// SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
|
||||||
|
SyncActionUpsert
|
||||||
|
// SyncActionDelete indicates an incremental sync where the provided config should be removed.
|
||||||
|
SyncActionDelete
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config defines a network monitor task sent from hub to agent.
|
||||||
|
type Config struct {
|
||||||
|
// ID is the stable network_monitors record ID generated by the hub.
|
||||||
|
ID string `cbor:"0,keyasint"`
|
||||||
|
Target string `cbor:"1,keyasint"`
|
||||||
|
Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
|
||||||
|
Port uint16 `cbor:"3,keyasint,omitempty"`
|
||||||
|
Interval uint16 `cbor:"4,keyasint"` // seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncRequest defines an incremental or full monitor sync request sent to the agent.
|
||||||
|
type SyncRequest struct {
|
||||||
|
Action SyncAction `cbor:"0,keyasint"`
|
||||||
|
Config Config `cbor:"1,keyasint,omitempty"`
|
||||||
|
Configs []Config `cbor:"2,keyasint,omitempty"`
|
||||||
|
RunNow bool `cbor:"3,keyasint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncResponse returns the immediate result for an upsert when requested.
|
||||||
|
type SyncResponse struct {
|
||||||
|
Result Result `cbor:"0,keyasint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Result holds aggregated monitor results for a single target.
|
||||||
|
//
|
||||||
|
// 0: avg response in microseconds
|
||||||
|
//
|
||||||
|
// 1: 1h average response in microseconds
|
||||||
|
//
|
||||||
|
// 2: min response in microseconds
|
||||||
|
//
|
||||||
|
// 3: 1h min response in microseconds
|
||||||
|
//
|
||||||
|
// 4: max response in microseconds
|
||||||
|
//
|
||||||
|
// 5: 1h max response in microseconds
|
||||||
|
//
|
||||||
|
// 6: packet loss percentage (0-100)
|
||||||
|
//
|
||||||
|
// 7: 1h packet loss percentage (0-100)
|
||||||
|
type Result struct {
|
||||||
|
AvgResponse int64 `cbor:"0,keyasint,omitempty"`
|
||||||
|
AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
|
||||||
|
MinResponse int64 `cbor:"2,keyasint,omitempty"`
|
||||||
|
MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
|
||||||
|
MaxResponse int64 `cbor:"4,keyasint,omitempty"`
|
||||||
|
MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
|
||||||
|
PacketLoss float64 `cbor:"6,keyasint,omitempty"`
|
||||||
|
PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
|
||||||
|
// LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
|
||||||
|
LastProbeAt int64 `cbor:"8,keyasint"`
|
||||||
|
// SampleCount includes all completed probes since this monitor started.
|
||||||
|
// Used for alert warm-up even when the interval is longer than 20 minutes.
|
||||||
|
SampleCount int64 `cbor:"9,keyasint,omitempty"`
|
||||||
|
// Counts and sum cover the current response window (or latest-sample
|
||||||
|
// fallback), not the hourly window or lifetime SampleCount.
|
||||||
|
TotalCount int64 `cbor:"10,keyasint"`
|
||||||
|
SuccessCount int64 `cbor:"11,keyasint"`
|
||||||
|
ResponseSum int64 `cbor:"12,keyasint"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stats holds response times in microseconds and packet loss percentage (0-100).
|
||||||
|
type Stats struct {
|
||||||
|
ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
|
||||||
|
ResMin float64 `json:"res_min" db:"res_min"`
|
||||||
|
ResMax float64 `json:"res_max" db:"res_max"`
|
||||||
|
Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
|
||||||
|
TotalCount int64 `json:"-" db:"total_count"`
|
||||||
|
SuccessCount int64 `json:"-" db:"success_count"`
|
||||||
|
ResponseSum int64 `json:"-" db:"res_sum"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Stats) FromResult(result Result) Stats {
|
||||||
|
return Stats{
|
||||||
|
ResAvg: float64(result.AvgResponse),
|
||||||
|
ResMin: float64(result.MinResponse),
|
||||||
|
ResMax: float64(result.MaxResponse),
|
||||||
|
Loss: result.PacketLoss,
|
||||||
|
TotalCount: result.TotalCount,
|
||||||
|
SuccessCount: result.SuccessCount,
|
||||||
|
ResponseSum: result.ResponseSum,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -210,5 +211,6 @@ type CombinedData struct {
|
|||||||
Details *Details `cbor:"4,keyasint,omitempty"`
|
Details *Details `cbor:"4,keyasint,omitempty"`
|
||||||
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
|
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
|
||||||
// that omitted systemd data (for example, a short-cache dashboard request).
|
// that omitted systemd data (for example, a short-cache dashboard request).
|
||||||
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
|
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
|
||||||
|
Monitors map[string]monitor.Result `cbor:"6,keyasint"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -978,11 +978,18 @@ func TestAgentWebSocketIntegration(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify system status
|
// A connected WebSocket does not mean the hub has finished verifying
|
||||||
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id)
|
// the agent and updating the system. Wait for the database state rather
|
||||||
require.NoError(t, err)
|
// than assuming that work completes within a fixed sleep under load.
|
||||||
status := updatedSystemRecord.GetString("status")
|
var status string
|
||||||
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value")
|
require.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||||
|
updatedSystemRecord, err := testApp.FindRecordById("systems", systemRecord.Id)
|
||||||
|
if !assert.NoError(c, err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
status = updatedSystemRecord.GetString("status")
|
||||||
|
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
|
||||||
|
}, 5*time.Second, 20*time.Millisecond)
|
||||||
|
|
||||||
t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint)
|
t.Logf("%s - System status: %s, Fingerprint: %s", tc.description, status, finalFingerprint)
|
||||||
})
|
})
|
||||||
@@ -1142,42 +1149,43 @@ func TestMultipleSystemsWithSameUniversalToken(t *testing.T) {
|
|||||||
|
|
||||||
// Verify system creation/reuse behavior
|
// Verify system creation/reuse behavior
|
||||||
if tc.expectConnection {
|
if tc.expectConnection {
|
||||||
// Count systems after connection
|
expectedSystemsAfter := systemsBeforeCount
|
||||||
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
systemsAfterCount := len(systemsAfter)
|
|
||||||
|
|
||||||
if tc.expectNewSystem {
|
if tc.expectNewSystem {
|
||||||
// Should have created a new system
|
expectedSystemsAfter++
|
||||||
systemCount++
|
systemCount++
|
||||||
assert.Equal(t, systemsBeforeCount+1, systemsAfterCount, "Should have created a new system")
|
|
||||||
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should match expected")
|
|
||||||
} else {
|
|
||||||
// Should have reused existing system
|
|
||||||
assert.Equal(t, systemsBeforeCount, systemsAfterCount, "Should not have created a new system")
|
|
||||||
assert.Equal(t, systemCount, systemsAfterCount, "Total system count should remain the same")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
// WebSocket connection precedes the hub's asynchronous system
|
||||||
|
// setup. Re-read all database state until setup is complete.
|
||||||
|
var systemId, status string
|
||||||
|
require.EventuallyWithT(t, func(c *assert.CollectT) {
|
||||||
|
systemsAfter, err := testApp.FindRecordsByFilter("systems", "users ~ {:userId}", "", -1, 0, map[string]any{"userId": userRecord.Id})
|
||||||
|
if !assert.NoError(c, err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.Len(c, systemsAfter, expectedSystemsAfter, "System creation/reuse should match expected behavior")
|
||||||
|
assert.Len(c, systemsAfter, systemCount, "Total system count should match expected")
|
||||||
|
|
||||||
// Verify that a fingerprint record exists for this fingerprint
|
fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{
|
||||||
fingerprints, err := testApp.FindRecordsByFilter("fingerprints", "token = {:token} && fingerprint = {:fingerprint}", "", -1, 0, map[string]any{
|
"token": universalToken,
|
||||||
"token": universalToken,
|
"fingerprint": tc.agentFingerprint,
|
||||||
"fingerprint": tc.agentFingerprint,
|
})
|
||||||
})
|
if !assert.NoError(c, err) || !assert.Len(c, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination") {
|
||||||
require.NoError(t, err)
|
return
|
||||||
require.Len(t, fingerprints, 1, "Should have exactly one fingerprint record for this token+fingerprint combination")
|
}
|
||||||
|
|
||||||
fingerprint := fingerprints[0]
|
fingerprint := fingerprints[0]
|
||||||
assert.Equal(t, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token")
|
assert.Equal(c, universalToken, fingerprint.GetString("token"), "Fingerprint should have the universal token")
|
||||||
assert.Equal(t, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint")
|
assert.Equal(c, tc.agentFingerprint, fingerprint.GetString("fingerprint"), "Fingerprint should match agent's fingerprint")
|
||||||
|
|
||||||
// Verify system status
|
systemId = fingerprint.GetString("system")
|
||||||
systemId := fingerprint.GetString("system")
|
system, err := testApp.FindRecordById("systems", systemId)
|
||||||
system, err := testApp.FindRecordById("systems", systemId)
|
if !assert.NoError(c, err) {
|
||||||
require.NoError(t, err)
|
return
|
||||||
status := system.GetString("status")
|
}
|
||||||
assert.Equal(t, tc.expectSystemStatus, status, "System status should match expected value")
|
status = system.GetString("status")
|
||||||
|
assert.Equal(c, tc.expectSystemStatus, status, "System status should match expected value")
|
||||||
|
}, 5*time.Second, 20*time.Millisecond)
|
||||||
|
|
||||||
t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem)
|
t.Logf("%s - System ID: %s, Status: %s, New System: %v", tc.description, systemId, status, tc.expectNewSystem)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{
|
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
|
||||||
list: &systemScopedReadRule,
|
list: &systemScopedReadRule,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -108,6 +108,16 @@ func setCollectionAuthSettings(app core.App) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
|
||||||
|
list: &systemScopedReadRule,
|
||||||
|
view: &systemScopedReadRule,
|
||||||
|
create: &systemScopedWriteRule,
|
||||||
|
update: &systemScopedWriteRule,
|
||||||
|
delete: &systemScopedWriteRule,
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
|
if err := applyCollectionRules(app, []string{"system_details"}, collectionRules{
|
||||||
list: &systemScopedReadRule,
|
list: &systemScopedReadRule,
|
||||||
view: &systemScopedReadRule,
|
view: &systemScopedReadRule,
|
||||||
|
|||||||
@@ -106,9 +106,12 @@ func (h *Hub) StartHub() error {
|
|||||||
|
|
||||||
// TODO: move to users package
|
// TODO: move to users package
|
||||||
// handle default values for user / user_settings creation
|
// handle default values for user / user_settings creation
|
||||||
|
h.App.OnRecordAuthWithOAuth2Request("users").BindFunc(h.um.InitializeOAuthUserRole)
|
||||||
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
|
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
|
||||||
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
|
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
|
||||||
|
|
||||||
|
bindNetworkMonitorsEvents(h)
|
||||||
|
|
||||||
pb, ok := h.App.(*pocketbase.PocketBase)
|
pb, ok := h.App.(*pocketbase.PocketBase)
|
||||||
if !ok {
|
if !ok {
|
||||||
return errors.New("not a pocketbase app")
|
return errors.New("not a pocketbase app")
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
package hub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/henrygd/beszel/internal/hub/systems"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
|
||||||
|
func generateMonitorID(systemId string, config monitor.Config) string {
|
||||||
|
args := []string{systemId, config.Target, config.Protocol}
|
||||||
|
// only use port for TCP monitors, since for other protocols it's not relevant as standalone value
|
||||||
|
if config.Protocol == "tcp" {
|
||||||
|
args = append(args, strconv.FormatUint(uint64(config.Port), 10))
|
||||||
|
}
|
||||||
|
return systems.MakeStableHashId(args...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
|
||||||
|
func bindNetworkMonitorsEvents(hub *Hub) {
|
||||||
|
// on create, make sure the id is set to a stable hash
|
||||||
|
hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
systemID := e.Record.GetString("system")
|
||||||
|
config := monitorConfigFromRecord(e.Record)
|
||||||
|
id := generateMonitorID(systemID, *config)
|
||||||
|
e.Record.Set("id", id)
|
||||||
|
return e.Next()
|
||||||
|
})
|
||||||
|
|
||||||
|
// sync monitor to agent on creation and persist the first result immediately when available
|
||||||
|
hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
err := e.Next()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !e.Record.GetBool("enabled") {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// If connected, run the monitor immediately. Paused systems may be absent
|
||||||
|
// from the manager; their monitors will sync when they reconnect.
|
||||||
|
system, err := hub.sm.GetSystem(e.Record.GetString("system"))
|
||||||
|
if err == nil && system.Status == "up" {
|
||||||
|
go hub.upsertNetworkMonitor(e.Record, true)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
// On API update requests, if the monitor config changed in a way that requires a new ID, create a new
|
||||||
|
// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
|
||||||
|
hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
|
||||||
|
systemID := e.Record.GetString("system")
|
||||||
|
// only tcp uses port - set other protocols port to zero
|
||||||
|
if e.Record.GetString("protocol") != "tcp" {
|
||||||
|
e.Record.Set("port", 0)
|
||||||
|
}
|
||||||
|
ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
|
||||||
|
if ID != e.Record.Id {
|
||||||
|
newRecord := copyMonitorToNewRecord(e.Record, ID)
|
||||||
|
if err := e.App.Save(newRecord); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := e.App.Delete(e.Record); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
err := e.Next()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if e.Record.GetBool("enabled") {
|
||||||
|
// if the monitor is enabled, sync the updated config to the agent now
|
||||||
|
runNow := !e.Record.Original().GetBool("enabled")
|
||||||
|
err = hub.upsertNetworkMonitor(e.Record, runNow)
|
||||||
|
} else {
|
||||||
|
// if the monitor is paused, remove it from the agent
|
||||||
|
err = hub.deleteNetworkMonitor(e.Record)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
// sync monitor to agent on delete
|
||||||
|
hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
||||||
|
if err := hub.deleteNetworkMonitor(e.Record); err != nil {
|
||||||
|
hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
|
||||||
|
}
|
||||||
|
return e.Next()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// monitorConfigFromRecord builds a monitor config from a network_monitors record.
|
||||||
|
func monitorConfigFromRecord(record *core.Record) *monitor.Config {
|
||||||
|
return &monitor.Config{
|
||||||
|
ID: record.Id,
|
||||||
|
Target: record.GetString("target"),
|
||||||
|
Protocol: record.GetString("protocol"),
|
||||||
|
Port: uint16(record.GetInt("port")),
|
||||||
|
Interval: uint16(record.GetInt("interval")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setMonitorResultFields stores the latest monitor result values on the record.
|
||||||
|
func setMonitorResultFields(record *core.Record, result monitor.Result) {
|
||||||
|
nowString := time.Now().UTC().Format(types.DefaultDateLayout)
|
||||||
|
record.Set("res", result.AvgResponse)
|
||||||
|
record.Set("resAvg1h", result.AvgResponse1h)
|
||||||
|
record.Set("resMin1h", result.MinResponse1h)
|
||||||
|
record.Set("resMax1h", result.MaxResponse1h)
|
||||||
|
record.Set("loss1h", result.PacketLoss1h)
|
||||||
|
record.Set("updated", nowString)
|
||||||
|
}
|
||||||
|
|
||||||
|
// copyMonitorToNewRecord creates a new record with the same field values as the old one.
|
||||||
|
// This is used when the monitor config changes in a way that requires a new ID, so we need
|
||||||
|
// to create a new record with the new ID and delete the old one.
|
||||||
|
func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
|
||||||
|
collection := oldRecord.Collection()
|
||||||
|
newRecord := core.NewRecord(collection)
|
||||||
|
newRecord.Id = newID
|
||||||
|
fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}
|
||||||
|
for _, field := range fields {
|
||||||
|
newRecord.Set(field, oldRecord.Get(field))
|
||||||
|
}
|
||||||
|
return newRecord
|
||||||
|
}
|
||||||
|
|
||||||
|
// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
|
||||||
|
// is true, it will also trigger an immediate monitor run and update the record with the result.
|
||||||
|
func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
|
||||||
|
systemID := record.GetString("system")
|
||||||
|
system, err := h.sm.GetSystem(systemID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
|
||||||
|
if err != nil || result == nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
setMonitorResultFields(record, *result)
|
||||||
|
return h.App.SaveNoValidate(record)
|
||||||
|
}
|
||||||
|
|
||||||
|
// deleteNetworkMonitor removes the record's monitor from the target system.
|
||||||
|
func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
|
||||||
|
systemID := record.GetString("system")
|
||||||
|
system, err := h.sm.GetSystem(systemID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return system.DeleteNetworkMonitor(record.Id)
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
package hub
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/pocketbase/pocketbase/apis"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
|
||||||
|
for _, batch := range []bool{false, true} {
|
||||||
|
name := "single"
|
||||||
|
if batch {
|
||||||
|
name = "batch"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
hub, testApp, err := createTestHub(t)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer cleanupTestHub(hub, testApp)
|
||||||
|
bindNetworkMonitorsEvents(hub)
|
||||||
|
|
||||||
|
user, err := createTestUser(hub)
|
||||||
|
require.NoError(t, err)
|
||||||
|
system, err := createTestRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "Paused", "host": "localhost", "port": "45876",
|
||||||
|
"status": "paused", "users": []string{user.Id},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
// Paused systems are not loaded into the manager at startup.
|
||||||
|
_, err = hub.sm.GetSystem(system.Id)
|
||||||
|
require.Error(t, err)
|
||||||
|
|
||||||
|
payload := func(target string) map[string]any {
|
||||||
|
return map[string]any{
|
||||||
|
"system": system.Id, "target": target, "protocol": "icmp",
|
||||||
|
"interval": 60, "enabled": true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
url := "/api/collections/network_monitors/records"
|
||||||
|
var body any = payload("1.1.1.1")
|
||||||
|
count := 1
|
||||||
|
if batch {
|
||||||
|
body = map[string]any{"requests": []map[string]any{
|
||||||
|
{"method": "POST", "url": url, "body": payload("1.1.1.1")},
|
||||||
|
{"method": "POST", "url": url, "body": payload("8.8.8.8")},
|
||||||
|
}}
|
||||||
|
url = "/api/batch"
|
||||||
|
count = 2
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(body)
|
||||||
|
require.NoError(t, err)
|
||||||
|
token, err := user.NewAuthToken()
|
||||||
|
require.NoError(t, err)
|
||||||
|
router, err := apis.NewRouter(hub)
|
||||||
|
require.NoError(t, err)
|
||||||
|
handler, err := router.BuildMux()
|
||||||
|
require.NoError(t, err)
|
||||||
|
request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
|
||||||
|
request.Header.Set("Content-Type", "application/json")
|
||||||
|
request.Header.Set("Authorization", token)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(response, request)
|
||||||
|
assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
|
||||||
|
|
||||||
|
records, err := hub.FindAllRecords("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, records, count)
|
||||||
|
for _, record := range records {
|
||||||
|
assert.Equal(t, system.Id, record.GetString("system"))
|
||||||
|
assert.True(t, record.GetBool("enabled"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateMonitorID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
systemID string
|
||||||
|
config monitor.Config
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "HTTP monitor on example.com",
|
||||||
|
systemID: "sys123",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "http",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 0,
|
||||||
|
Interval: 60,
|
||||||
|
},
|
||||||
|
expected: "a20a5827",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "HTTP monitor on example.com with different port",
|
||||||
|
systemID: "sys123",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "http",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 8080,
|
||||||
|
Interval: 60,
|
||||||
|
},
|
||||||
|
expected: "a20a5827",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "HTTP monitor on example.com with different system ID",
|
||||||
|
systemID: "sys1234",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "http",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 80,
|
||||||
|
Interval: 60,
|
||||||
|
},
|
||||||
|
expected: "ab602ae7",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Same monitor, different interval",
|
||||||
|
systemID: "sys1234",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "http",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 80,
|
||||||
|
Interval: 120,
|
||||||
|
},
|
||||||
|
expected: "ab602ae7",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ICMP monitor on 1.1.1.1",
|
||||||
|
systemID: "sys456",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "icmp",
|
||||||
|
Target: "1.1.1.1",
|
||||||
|
Port: 0,
|
||||||
|
Interval: 10,
|
||||||
|
},
|
||||||
|
expected: "6d13a4a4",
|
||||||
|
}, {
|
||||||
|
name: "ICMP monitor on 1.1.1.1 with different system ID",
|
||||||
|
systemID: "sys4567",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "icmp",
|
||||||
|
Target: "1.1.1.1",
|
||||||
|
Port: 0,
|
||||||
|
Interval: 10,
|
||||||
|
},
|
||||||
|
expected: "ddd6c81",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "TCP monitor on example.com with port 443",
|
||||||
|
systemID: "sys789",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "tcp",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 443,
|
||||||
|
Interval: 30,
|
||||||
|
},
|
||||||
|
expected: "677b991",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "TCP monitor on example.com with port 8443",
|
||||||
|
systemID: "sys789",
|
||||||
|
config: monitor.Config{
|
||||||
|
Protocol: "tcp",
|
||||||
|
Target: "example.com",
|
||||||
|
Port: 8443,
|
||||||
|
Interval: 30,
|
||||||
|
},
|
||||||
|
expected: "84167969",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got := generateMonitorID(tt.systemID, tt.config)
|
||||||
|
assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
|
||||||
|
hub, testApp, err := createTestHub(t)
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer cleanupTestHub(hub, testApp)
|
||||||
|
|
||||||
|
collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, collection.Fields.GetByName("name"))
|
||||||
|
|
||||||
|
oldRecord := core.NewRecord(collection)
|
||||||
|
oldRecord.Load(map[string]any{
|
||||||
|
"system": "sys123",
|
||||||
|
"target": "https://example.com",
|
||||||
|
"protocol": "http",
|
||||||
|
"port": 443,
|
||||||
|
"interval": 60,
|
||||||
|
"enabled": true,
|
||||||
|
"res": 1200,
|
||||||
|
"resAvg1h": 1300,
|
||||||
|
"resMin1h": 900,
|
||||||
|
"resMax1h": 1600,
|
||||||
|
"loss1h": 5,
|
||||||
|
"updated": "2026-04-29 12:00:00.000Z",
|
||||||
|
})
|
||||||
|
|
||||||
|
newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
|
||||||
|
|
||||||
|
assert.Equal(t, "next12345", newRecord.Id)
|
||||||
|
assert.Equal(t, "https://example.com", newRecord.GetString("target"))
|
||||||
|
assert.Equal(t, "http", newRecord.GetString("protocol"))
|
||||||
|
assert.Equal(t, 443, newRecord.GetInt("port"))
|
||||||
|
assert.True(t, newRecord.GetBool("enabled"))
|
||||||
|
assert.Zero(t, newRecord.GetFloat("res"))
|
||||||
|
assert.Zero(t, newRecord.GetFloat("resAvg1h"))
|
||||||
|
assert.Zero(t, newRecord.GetFloat("resMin1h"))
|
||||||
|
assert.Zero(t, newRecord.GetFloat("resMax1h"))
|
||||||
|
assert.Zero(t, newRecord.GetFloat("loss1h"))
|
||||||
|
assert.Equal(t, "", newRecord.GetString("updated"))
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"net"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fxamacker/cbor/v2"
|
||||||
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
esystem "github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSSHNetworkMonitorReconnectSync(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
sys.manager.zfsFetchMap = expirymap.New[zfsFetchState](time.Hour)
|
||||||
|
t.Cleanup(sys.manager.zfsFetchMap.StopCleaner)
|
||||||
|
sys.ctx = context.Background()
|
||||||
|
sys.Status = up
|
||||||
|
_, key, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
signer, err := ssh.NewSignerFromKey(key)
|
||||||
|
require.NoError(t, err)
|
||||||
|
config := &ssh.ServerConfig{NoClientAuth: true, ServerVersion: "SSH-2.0-beszel_0.20.0"}
|
||||||
|
config.AddHostKey(signer)
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = listener.Close() })
|
||||||
|
sys.Host, sys.Port, err = net.SplitHostPort(listener.Addr().String())
|
||||||
|
require.NoError(t, err)
|
||||||
|
sys.manager.sshConfig = &ssh.ClientConfig{User: "test", HostKeyCallback: ssh.InsecureIgnoreHostKey(), Timeout: time.Second}
|
||||||
|
t.Cleanup(sys.closeSSHConnection)
|
||||||
|
requests := make(chan monitor.SyncRequest, 10)
|
||||||
|
var failSync atomic.Bool
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
conn, err := listener.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
server, channels, reqs, err := ssh.NewServerConn(conn, config)
|
||||||
|
if err != nil {
|
||||||
|
_ = conn.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer server.Close()
|
||||||
|
go ssh.DiscardRequests(reqs)
|
||||||
|
for channel := range channels {
|
||||||
|
ch, reqs, err := channel.Accept()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer ch.Close()
|
||||||
|
for req := range reqs {
|
||||||
|
if req.Type != "shell" {
|
||||||
|
_ = req.Reply(false, nil)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_ = req.Reply(true, nil)
|
||||||
|
var request common.HubRequest[cbor.RawMessage]
|
||||||
|
if cbor.NewDecoder(ch).Decode(&request) != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
response := common.AgentResponse{}
|
||||||
|
switch request.Action {
|
||||||
|
case common.GetData:
|
||||||
|
response.SystemData = &esystem.CombinedData{}
|
||||||
|
case common.SyncNetworkMonitors:
|
||||||
|
var syncReq monitor.SyncRequest
|
||||||
|
if cbor.Unmarshal(request.Data, &syncReq) != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
requests <- syncReq
|
||||||
|
if failSync.Load() {
|
||||||
|
response.Error = "test sync failure"
|
||||||
|
} else {
|
||||||
|
response.Data, _ = cbor.Marshal(monitor.SyncResponse{})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = cbor.NewEncoder(ch).Encode(response)
|
||||||
|
_, _ = ch.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{0}))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
probe := core.NewRecord(collection)
|
||||||
|
probe.Load(map[string]any{"system": sys.Id, "target": "localhost", "protocol": "tcp", "port": 80, "interval": 60, "enabled": true})
|
||||||
|
require.NoError(t, app.SaveNoValidate(probe))
|
||||||
|
fetch := func() {
|
||||||
|
t.Helper()
|
||||||
|
_, err := sys.fetchDataFromAgent(common.DataRequestOptions{})
|
||||||
|
require.NoError(t, err, "monitor sync failure must not fail stats fetching")
|
||||||
|
}
|
||||||
|
receive := func() monitor.SyncRequest {
|
||||||
|
t.Helper()
|
||||||
|
select {
|
||||||
|
case req := <-requests:
|
||||||
|
require.Equal(t, monitor.SyncActionReplace, req.Action)
|
||||||
|
return req
|
||||||
|
case <-time.After(time.Second):
|
||||||
|
t.Fatal("missing full monitor sync")
|
||||||
|
return monitor.SyncRequest{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fetch()
|
||||||
|
require.Equal(t, probe.Id, receive().Configs[0].ID)
|
||||||
|
require.False(t, sys.monitorsNeedSync.Load())
|
||||||
|
fetch()
|
||||||
|
require.Empty(t, requests, "steady-state fetch must not resync")
|
||||||
|
|
||||||
|
// Simulate loss of the agent process/connection and its in-memory monitors.
|
||||||
|
require.NoError(t, sys.client.Load().Close())
|
||||||
|
fetch()
|
||||||
|
require.Equal(t, probe.Id, receive().Configs[0].ID)
|
||||||
|
require.False(t, sys.monitorsNeedSync.Load())
|
||||||
|
|
||||||
|
// Failed replacements are retried on the next successful stats fetch.
|
||||||
|
require.NoError(t, sys.client.Load().Close())
|
||||||
|
failSync.Store(true)
|
||||||
|
fetch()
|
||||||
|
receive()
|
||||||
|
require.True(t, sys.monitorsNeedSync.Load())
|
||||||
|
failSync.Store(false)
|
||||||
|
fetch()
|
||||||
|
receive()
|
||||||
|
require.False(t, sys.monitorsNeedSync.Load())
|
||||||
|
|
||||||
|
probe.Set("enabled", false)
|
||||||
|
require.NoError(t, app.SaveNoValidate(probe))
|
||||||
|
require.NoError(t, sys.client.Load().Close())
|
||||||
|
fetch()
|
||||||
|
require.Empty(t, receive().Configs, "empty replacement must clear stale monitors")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPendingNetworkMonitorSyncQueryFailure(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
sys.monitorsNeedSync.Store(true)
|
||||||
|
sys.syncPendingNetworkMonitors()
|
||||||
|
require.True(t, sys.monitorsNeedSync.Load())
|
||||||
|
}
|
||||||
@@ -0,0 +1,224 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/fxamacker/cbor/v2"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/subscriptions"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNetworkMonitorProbePruning(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
monitors map[string]monitor.Result
|
||||||
|
fail bool
|
||||||
|
want map[string]int64
|
||||||
|
}{
|
||||||
|
{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
|
||||||
|
{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
|
||||||
|
{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
|
||||||
|
{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
|
||||||
|
// Preserve the distinction between nil and empty across the agent transport.
|
||||||
|
encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
|
||||||
|
require.NoError(t, err)
|
||||||
|
var data system.CombinedData
|
||||||
|
require.NoError(t, cbor.Unmarshal(encoded, &data))
|
||||||
|
if tc.fail {
|
||||||
|
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
_, err = sys.createRecords(&data)
|
||||||
|
if tc.fail {
|
||||||
|
require.Error(t, err)
|
||||||
|
} else {
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorStatsFreshness(t *testing.T) {
|
||||||
|
for _, realtime := range []bool{false, true} {
|
||||||
|
name := "sql"
|
||||||
|
if realtime {
|
||||||
|
name = "realtime"
|
||||||
|
}
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
if realtime {
|
||||||
|
client := subscriptions.NewDefaultClient()
|
||||||
|
client.Subscribe("network_monitors/*")
|
||||||
|
app.SubscriptionsBroker().Register(client)
|
||||||
|
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
|
||||||
|
}
|
||||||
|
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, id := range []string{"monitor1", "monitor2"} {
|
||||||
|
record := core.NewRecord(col)
|
||||||
|
record.Id = id
|
||||||
|
record.Set("system", sys.Id)
|
||||||
|
require.NoError(t, app.SaveNoValidate(record))
|
||||||
|
}
|
||||||
|
data := &system.CombinedData{Monitors: map[string]monitor.Result{
|
||||||
|
"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
|
||||||
|
"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
|
||||||
|
}}
|
||||||
|
count := func(want int64) {
|
||||||
|
t.Helper()
|
||||||
|
got, err := app.CountRecords("network_monitor_stats")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, want, got)
|
||||||
|
}
|
||||||
|
save := func() {
|
||||||
|
t.Helper()
|
||||||
|
_, err := sys.createRecords(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
save()
|
||||||
|
count(2)
|
||||||
|
stored, err := app.FindAllRecords("network_monitor_stats")
|
||||||
|
require.NoError(t, err)
|
||||||
|
for _, record := range stored {
|
||||||
|
result := data.Monitors[record.GetString("monitor")]
|
||||||
|
assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
|
||||||
|
assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
|
||||||
|
assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
|
||||||
|
}
|
||||||
|
// A resume can overlap the scheduled update with the same probe.
|
||||||
|
errs := make(chan error, 4)
|
||||||
|
for range 4 {
|
||||||
|
go func() {
|
||||||
|
_, err := sys.createRecords(data)
|
||||||
|
errs <- err
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
for range 4 {
|
||||||
|
require.NoError(t, <-errs)
|
||||||
|
}
|
||||||
|
count(2)
|
||||||
|
|
||||||
|
// A rolling hourly value can change without a new probe.
|
||||||
|
result := data.Monitors["monitor1"]
|
||||||
|
result.AvgResponse1h = 42
|
||||||
|
data.Monitors["monitor1"] = result
|
||||||
|
save()
|
||||||
|
count(2)
|
||||||
|
record, err := app.FindRecordById("network_monitors", "monitor1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 42, record.GetInt("resAvg1h"))
|
||||||
|
|
||||||
|
// Identical response values and failed probes still count as new measurements.
|
||||||
|
for id, result := range data.Monitors {
|
||||||
|
result.LastProbeAt = 301000
|
||||||
|
data.Monitors[id] = result
|
||||||
|
}
|
||||||
|
save()
|
||||||
|
count(4)
|
||||||
|
|
||||||
|
// A failed individual insert must remain retryable, even if others commit.
|
||||||
|
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
for id, result := range data.Monitors {
|
||||||
|
result.LastProbeAt = 601000
|
||||||
|
data.Monitors[id] = result
|
||||||
|
}
|
||||||
|
save()
|
||||||
|
count(5)
|
||||||
|
assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
|
||||||
|
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
|
||||||
|
_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
save()
|
||||||
|
count(6)
|
||||||
|
|
||||||
|
// Failure after inserting stats rolls back the whole transaction and its markers.
|
||||||
|
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
result = data.Monitors["monitor1"]
|
||||||
|
result.LastProbeAt = 901000
|
||||||
|
data.Monitors["monitor1"] = result
|
||||||
|
_, err = sys.createRecords(data)
|
||||||
|
require.Error(t, err)
|
||||||
|
count(6)
|
||||||
|
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
|
||||||
|
_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
save()
|
||||||
|
count(7)
|
||||||
|
|
||||||
|
// Clock rollback is a new probe identity, not a reason to stall writes.
|
||||||
|
result.LastProbeAt = 500
|
||||||
|
data.Monitors["monitor1"] = result
|
||||||
|
save()
|
||||||
|
count(8)
|
||||||
|
|
||||||
|
// Recreated systems intentionally accept the first result without restoring state.
|
||||||
|
sys = &System{Id: sys.Id, manager: sys.manager}
|
||||||
|
save()
|
||||||
|
count(10)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Observes the committed DB through the hub, not the transaction's app.
|
||||||
|
type monitorAlertHub struct {
|
||||||
|
stubHub
|
||||||
|
handle func(*core.Record, map[string]monitor.Result) error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
|
||||||
|
return h.handle(record, results)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
|
||||||
|
for _, realtime := range []bool{false, true} {
|
||||||
|
t.Run(fmt.Sprint(realtime), func(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
if realtime {
|
||||||
|
client := subscriptions.NewDefaultClient()
|
||||||
|
client.Subscribe("network_monitors/*")
|
||||||
|
app.SubscriptionsBroker().Register(client)
|
||||||
|
}
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
record := core.NewRecord(collection)
|
||||||
|
record.Set("system", sys.Id)
|
||||||
|
require.NoError(t, app.SaveNoValidate(record))
|
||||||
|
called := 0
|
||||||
|
result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
|
||||||
|
sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
|
||||||
|
called++
|
||||||
|
assert.Equal(t, sys.Id, systemRecord.Id)
|
||||||
|
assert.Equal(t, result, results[record.Id])
|
||||||
|
saved, err := app.FindRecordById("network_monitors", record.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
|
||||||
|
return nil
|
||||||
|
}}
|
||||||
|
data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
|
||||||
|
_, err = sys.createRecords(data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 1, called)
|
||||||
|
// A transaction that fails after writing monitor stats must not notify.
|
||||||
|
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = sys.createRecords(data)
|
||||||
|
require.Error(t, err)
|
||||||
|
assert.Equal(t, 1, called)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/blang/semver"
|
||||||
|
"github.com/fxamacker/cbor/v2"
|
||||||
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
esystem "github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/henrygd/beszel/internal/hub/ws"
|
||||||
|
"github.com/lxzan/gws"
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
type monitorSyncClient struct {
|
||||||
|
gws.BuiltinEventHandler
|
||||||
|
requests chan common.HubRequest[monitor.SyncRequest]
|
||||||
|
failSync atomic.Bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
|
||||||
|
defer message.Close()
|
||||||
|
var req common.HubRequest[cbor.RawMessage]
|
||||||
|
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resp := common.AgentResponse{Id: req.Id}
|
||||||
|
if req.Action == common.GetData {
|
||||||
|
resp.SystemData = &esystem.CombinedData{}
|
||||||
|
} else {
|
||||||
|
var data monitor.SyncRequest
|
||||||
|
if err := cbor.Unmarshal(req.Data, &data); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.requests <- common.HubRequest[monitor.SyncRequest]{Id: req.Id, Action: req.Action, Data: data}
|
||||||
|
if c.failSync.Load() {
|
||||||
|
resp.Error = "test sync failure"
|
||||||
|
} else {
|
||||||
|
resp.Data, _ = cbor.Marshal(monitor.SyncResponse{})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
response, _ := cbor.Marshal(resp)
|
||||||
|
_ = conn.WriteMessage(gws.OpcodeBinary, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Avoid the production delayed disconnect notification; these tests explicitly
|
||||||
|
// remove each connection from the manager before reconnecting.
|
||||||
|
type monitorSyncServer struct{ ws.Handler }
|
||||||
|
|
||||||
|
func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
|
||||||
|
|
||||||
|
func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
|
||||||
|
for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
|
||||||
|
t.Run(version, func(t *testing.T) {
|
||||||
|
// No transport: attempting to send any request would fail.
|
||||||
|
sys := &System{agentVersion: semver.MustParse(version)}
|
||||||
|
require.NoError(t, sys.SyncNetworkMonitors(nil))
|
||||||
|
result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Nil(t, result)
|
||||||
|
require.NoError(t, sys.DeleteNetworkMonitor("test"))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNetworkMonitorReconnectSync(t *testing.T) {
|
||||||
|
for _, change := range []string{"delete", "disable", "retry"} {
|
||||||
|
t.Run(change, func(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
record, err := app.FindRecordById("systems", sys.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
// Suppress unrelated system-stat requests while exercising reconnects.
|
||||||
|
record.Set("status", paused)
|
||||||
|
require.NoError(t, app.SaveNoValidate(record))
|
||||||
|
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
||||||
|
require.NoError(t, err)
|
||||||
|
probe := core.NewRecord(collection)
|
||||||
|
probe.Load(map[string]any{
|
||||||
|
"system": sys.Id, "target": "localhost", "protocol": "tcp",
|
||||||
|
"port": 80, "interval": 60, "enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, app.SaveNoValidate(probe))
|
||||||
|
|
||||||
|
sm := NewSystemManager(stubHub{app})
|
||||||
|
t.Cleanup(func() {
|
||||||
|
sm.cancel()
|
||||||
|
_ = sm.RemoveSystem(sys.Id)
|
||||||
|
sm.smartFetchMap.StopCleaner()
|
||||||
|
sm.zfsFetchMap.StopCleaner()
|
||||||
|
})
|
||||||
|
version := semver.MustParse("0.20.0")
|
||||||
|
connections := make(chan *ws.WsConn, 1)
|
||||||
|
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, r)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
wsConn := ws.NewWsConnection(conn, version)
|
||||||
|
conn.Session().Store("wsConn", wsConn)
|
||||||
|
connections <- wsConn
|
||||||
|
conn.ReadLoop()
|
||||||
|
}))
|
||||||
|
t.Cleanup(server.Close)
|
||||||
|
client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
|
||||||
|
connect := func() monitor.SyncRequest {
|
||||||
|
t.Helper()
|
||||||
|
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = conn.NetConn().Close() })
|
||||||
|
go conn.ReadLoop()
|
||||||
|
select {
|
||||||
|
case wsConn := <-connections:
|
||||||
|
require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("websocket connection was not established")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case req := <-client.requests:
|
||||||
|
require.Equal(t, common.SyncNetworkMonitors, req.Action)
|
||||||
|
require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
|
||||||
|
return req.Data
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("reconnected agent did not receive a monitor replacement")
|
||||||
|
return monitor.SyncRequest{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
client.failSync.Store(change == "retry")
|
||||||
|
initial := connect()
|
||||||
|
require.Len(t, initial.Configs, 1)
|
||||||
|
require.Equal(t, probe.Id, initial.Configs[0].ID)
|
||||||
|
if change == "retry" {
|
||||||
|
system, err := sm.GetSystem(sys.Id)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Eventually(t, system.monitorsNeedSync.Load, time.Second, time.Millisecond)
|
||||||
|
// A second failed sync must not fail the stats fetch or clear pending state.
|
||||||
|
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.True(t, system.monitorsNeedSync.Load())
|
||||||
|
require.Len(t, client.requests, 1)
|
||||||
|
<-client.requests
|
||||||
|
client.failSync.Store(false)
|
||||||
|
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.False(t, system.monitorsNeedSync.Load())
|
||||||
|
require.Len(t, client.requests, 1)
|
||||||
|
retry := <-client.requests
|
||||||
|
require.Equal(t, monitor.SyncActionReplace, retry.Data.Action)
|
||||||
|
require.Equal(t, initial.Configs, retry.Data.Configs)
|
||||||
|
_, err = system.fetchDataFromAgent(common.DataRequestOptions{})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, client.requests, "successful sync must not repeat on every fetch")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
require.NoError(t, sm.RemoveSystem(sys.Id))
|
||||||
|
if change == "delete" {
|
||||||
|
require.NoError(t, app.Delete(probe))
|
||||||
|
} else {
|
||||||
|
probe.Set("enabled", false)
|
||||||
|
require.NoError(t, app.SaveNoValidate(probe))
|
||||||
|
}
|
||||||
|
require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
|
||||||
|
sys, app := newTestSystemWithHub(t)
|
||||||
|
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
|
||||||
|
require.NoError(t, err)
|
||||||
|
_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
|
||||||
|
require.Error(t, err, "a failed query must not be treated as an empty monitor set")
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package systems
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel"
|
||||||
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
)
|
||||||
|
|
||||||
|
// syncPendingNetworkMonitors runs on WebSocket connect and after successful stats
|
||||||
|
// fetches. Failed syncs retry on the next update without taking the system down.
|
||||||
|
func (sys *System) syncPendingNetworkMonitors() {
|
||||||
|
if !sys.monitorsNeedSync.Swap(false) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := sys.syncAllNetworkMonitors(); err != nil {
|
||||||
|
sys.monitorsNeedSync.Store(true)
|
||||||
|
sys.manager.hub.Logger().Warn("failed to sync monitors to agent", "system", sys.Id, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *System) syncAllNetworkMonitors() error {
|
||||||
|
configs, err := sys.manager.GetMonitorConfigsForSystem(sys.Id)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to load monitors: %w", err)
|
||||||
|
}
|
||||||
|
// An empty set must also replace probes retained across a disconnect.
|
||||||
|
return sys.SyncNetworkMonitors(configs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SyncNetworkMonitors sends monitor configurations to the agent.
|
||||||
|
func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
|
||||||
|
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
|
||||||
|
func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
|
||||||
|
resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
|
||||||
|
Action: monitor.SyncActionUpsert,
|
||||||
|
Config: config,
|
||||||
|
RunNow: runNow,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.Result == (monitor.Result{}) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
result := resp.Result
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteNetworkMonitor removes a single monitor task from the agent.
|
||||||
|
func (sys *System) DeleteNetworkMonitor(id string) error {
|
||||||
|
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
|
||||||
|
Action: monitor.SyncActionDelete,
|
||||||
|
Config: monitor.Config{ID: id},
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
|
||||||
|
if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
|
||||||
|
return monitor.SyncResponse{}, nil
|
||||||
|
}
|
||||||
|
timeout := 5 * time.Second
|
||||||
|
if req.Action == monitor.SyncActionUpsert && req.RunNow {
|
||||||
|
// Allow the probe to finish, including a timeout result, while preserving
|
||||||
|
// the normal request budget for transport and response handling.
|
||||||
|
timeout += monitor.MaxProbeTimeout
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||||
|
defer cancel()
|
||||||
|
var result monitor.SyncResponse
|
||||||
|
return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"math/rand"
|
"math/rand"
|
||||||
"net"
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -18,6 +19,7 @@ import (
|
|||||||
"github.com/henrygd/beszel/internal/hub/ws"
|
"github.com/henrygd/beszel/internal/hub/ws"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
@@ -30,6 +32,8 @@ import (
|
|||||||
"github.com/lxzan/gws"
|
"github.com/lxzan/gws"
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/security"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -52,6 +56,13 @@ type System struct {
|
|||||||
smartInterval time.Duration // Interval for periodic SMART data updates
|
smartInterval time.Duration // Interval for periodic SMART data updates
|
||||||
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
||||||
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
||||||
|
|
||||||
|
// A fresh connection needs a full monitor configuration sync.
|
||||||
|
monitorsNeedSync atomic.Bool
|
||||||
|
// Serialize persistence from scheduled updates and resumes through commit.
|
||||||
|
recordsMu sync.Mutex
|
||||||
|
// Protected by recordsMu; realtime reads don't consume probes.
|
||||||
|
lastSavedMonitorProbe map[string]int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sm *SystemManager) NewSystem(systemId string) *System {
|
func (sm *SystemManager) NewSystem(systemId string) *System {
|
||||||
@@ -211,11 +222,15 @@ func (sys *System) handlePaused() {
|
|||||||
|
|
||||||
// createRecords updates the system record and adds system_stats and container_stats records
|
// createRecords updates the system record and adds system_stats and container_stats records
|
||||||
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
|
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
|
||||||
|
sys.recordsMu.Lock()
|
||||||
|
defer sys.recordsMu.Unlock()
|
||||||
|
|
||||||
systemRecord, err := sys.getRecord(sys.manager.hub)
|
systemRecord, err := sys.getRecord(sys.manager.hub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
hub := sys.manager.hub
|
hub := sys.manager.hub
|
||||||
|
savedMonitorProbes := make(map[string]int64)
|
||||||
err = hub.RunInTransaction(func(txApp core.App) error {
|
err = hub.RunInTransaction(func(txApp core.App) error {
|
||||||
// add system_stats record
|
// add system_stats record
|
||||||
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
|
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
|
||||||
@@ -266,6 +281,12 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if data.Monitors != nil {
|
||||||
|
if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -287,6 +308,29 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Publish only successful inserts after the entire transaction commits.
|
||||||
|
if err == nil && len(savedMonitorProbes) > 0 {
|
||||||
|
if sys.lastSavedMonitorProbe == nil {
|
||||||
|
sys.lastSavedMonitorProbe = savedMonitorProbes
|
||||||
|
} else {
|
||||||
|
for id, timestamp := range savedMonitorProbes {
|
||||||
|
sys.lastSavedMonitorProbe[id] = timestamp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A non-nil report includes cached results for all remaining monitors.
|
||||||
|
if err == nil && data.Monitors != nil {
|
||||||
|
for id := range sys.lastSavedMonitorProbe {
|
||||||
|
if _, exists := data.Monitors[id]; !exists {
|
||||||
|
delete(sys.lastSavedMonitorProbe, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
|
||||||
|
hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
return systemRecord, err
|
return systemRecord, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,7 +381,7 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
|
|||||||
}
|
}
|
||||||
suffix := fmt.Sprintf("%d", i)
|
suffix := fmt.Sprintf("%d", i)
|
||||||
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
|
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
|
||||||
params["id"+suffix] = makeStableHashId(systemId, service.Name)
|
params["id"+suffix] = MakeStableHashId(systemId, service.Name)
|
||||||
params["name"+suffix] = service.Name
|
params["name"+suffix] = service.Name
|
||||||
params["state"+suffix] = service.State
|
params["state"+suffix] = service.State
|
||||||
params["sub"+suffix] = service.Sub
|
params["sub"+suffix] = service.Sub
|
||||||
@@ -363,6 +407,106 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
|
||||||
|
if len(monitorResults) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
systemId := sys.Id
|
||||||
|
const monitorCollectionName = "network_monitors"
|
||||||
|
|
||||||
|
// If realtime updates are active, we save via PocketBase records to trigger realtime events.
|
||||||
|
// Otherwise we can do a more efficient direct update via SQL
|
||||||
|
realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
|
||||||
|
return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
|
||||||
|
})
|
||||||
|
|
||||||
|
now := time.Now().UTC()
|
||||||
|
nowMilli := now.UnixMilli()
|
||||||
|
nowString := now.Format(types.DefaultDateLayout)
|
||||||
|
var db dbx.Builder
|
||||||
|
var updateQuery *dbx.Query
|
||||||
|
if !realtimeActive {
|
||||||
|
db = app.DB()
|
||||||
|
monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
|
||||||
|
setClauses := make([]string, len(monitorFields))
|
||||||
|
for i, f := range monitorFields {
|
||||||
|
setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
|
||||||
|
}
|
||||||
|
queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
|
||||||
|
updateQuery = db.NewQuery(queryString)
|
||||||
|
}
|
||||||
|
|
||||||
|
// update network_monitors records
|
||||||
|
for id, result := range monitorResults {
|
||||||
|
monitorData := map[string]any{
|
||||||
|
"id": id,
|
||||||
|
"res": result.AvgResponse,
|
||||||
|
"resAvg1h": result.AvgResponse1h,
|
||||||
|
"resMin1h": result.MinResponse1h,
|
||||||
|
"resMax1h": result.MaxResponse1h,
|
||||||
|
"loss1h": result.PacketLoss1h,
|
||||||
|
"updated": nowString,
|
||||||
|
}
|
||||||
|
switch realtimeActive {
|
||||||
|
case true:
|
||||||
|
var record *core.Record
|
||||||
|
record, err = app.FindRecordById(monitorCollectionName, id)
|
||||||
|
if err == nil {
|
||||||
|
record.Load(monitorData)
|
||||||
|
err = app.SaveNoValidate(record)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handle stats collection — one record per monitor
|
||||||
|
const statsCollectionName = "network_monitor_stats"
|
||||||
|
|
||||||
|
var statsCollection *core.Collection
|
||||||
|
if realtimeActive {
|
||||||
|
statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
|
||||||
|
}
|
||||||
|
|
||||||
|
for monitorId, result := range monitorResults {
|
||||||
|
// Compare identity, not ordering, so agent clock changes don't stall writes.
|
||||||
|
if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
statsRecordData := map[string]any{
|
||||||
|
"system": systemId,
|
||||||
|
"monitor": monitorId,
|
||||||
|
"type": "1m",
|
||||||
|
"created": nowMilli,
|
||||||
|
"res_min": result.MinResponse,
|
||||||
|
"res_max": result.MaxResponse,
|
||||||
|
"total_count": result.TotalCount,
|
||||||
|
"success_count": result.SuccessCount,
|
||||||
|
"res_sum": result.ResponseSum,
|
||||||
|
}
|
||||||
|
switch realtimeActive {
|
||||||
|
case true:
|
||||||
|
record := core.NewRecord(statsCollection)
|
||||||
|
record.Load(statsRecordData)
|
||||||
|
err = app.SaveNoValidate(record)
|
||||||
|
default:
|
||||||
|
statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
|
||||||
|
_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
|
||||||
|
} else {
|
||||||
|
savedProbes[monitorId] = result.LastProbeAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// createContainerRecords creates container records
|
// createContainerRecords creates container records
|
||||||
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
|
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
|
||||||
if len(data) == 0 {
|
if len(data) == 0 {
|
||||||
@@ -489,7 +633,10 @@ func (sys *System) request(ctx context.Context, action common.WebSocketAction, r
|
|||||||
err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1)
|
err := sys.sshTransport.RequestWithRetry(ctx, action, req, dest, 1)
|
||||||
// Keep legacy SSH client/version fields in sync for other code paths.
|
// Keep legacy SSH client/version fields in sync for other code paths.
|
||||||
if sys.sshTransport != nil {
|
if sys.sshTransport != nil {
|
||||||
sys.client.Store(sys.sshTransport.GetClient())
|
client := sys.sshTransport.GetClient()
|
||||||
|
if previous := sys.client.Swap(client); client != nil && client != previous {
|
||||||
|
sys.monitorsNeedSync.Store(true)
|
||||||
|
}
|
||||||
sys.agentVersion = sys.sshTransport.GetAgentVersion()
|
sys.agentVersion = sys.sshTransport.GetAgentVersion()
|
||||||
}
|
}
|
||||||
return err
|
return err
|
||||||
@@ -547,6 +694,7 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
|
|||||||
if sys.WsConn != nil && sys.WsConn.IsConnected() {
|
if sys.WsConn != nil && sys.WsConn.IsConnected() {
|
||||||
wsData, err := sys.fetchDataViaWebSocket(options)
|
wsData, err := sys.fetchDataViaWebSocket(options)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
|
sys.syncPendingNetworkMonitors()
|
||||||
return wsData, nil
|
return wsData, nil
|
||||||
}
|
}
|
||||||
// close the WebSocket connection if error and try SSH
|
// close the WebSocket connection if error and try SSH
|
||||||
@@ -557,6 +705,7 @@ func (sys *System) fetchDataFromAgent(options common.DataRequestOptions) (*syste
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
sys.syncPendingNetworkMonitors()
|
||||||
return sshData, nil
|
return sshData, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -622,7 +771,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
|
|||||||
return &result, err
|
return &result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func makeStableHashId(strings ...string) string {
|
func MakeStableHashId(strings ...string) string {
|
||||||
hash := fnv.New32a()
|
hash := fnv.New32a()
|
||||||
for _, str := range strings {
|
for _, str := range strings {
|
||||||
hash.Write([]byte(str))
|
hash.Write([]byte(str))
|
||||||
@@ -791,6 +940,7 @@ func (s *System) createSSHClient() error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
|
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
|
||||||
|
s.monitorsNeedSync.Store(true)
|
||||||
s.manager.resetFailedSmartFetchState(s.Id)
|
s.manager.resetFailedSmartFetchState(s.Id)
|
||||||
s.manager.resetFailedZfsFetchState(s.Id)
|
s.manager.resetFailedZfsFetchState(s.Id)
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/henrygd/beszel/internal/hub/ws"
|
"github.com/henrygd/beszel/internal/hub/ws"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/hub/expirymap"
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
|
|
||||||
@@ -17,6 +18,7 @@ import (
|
|||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
|
|
||||||
"github.com/blang/semver"
|
"github.com/blang/semver"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
"github.com/pocketbase/pocketbase/tools/store"
|
"github.com/pocketbase/pocketbase/tools/store"
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
@@ -62,6 +64,7 @@ type hubLike interface {
|
|||||||
core.App
|
core.App
|
||||||
GetSSHKey(dataDir string) (ssh.Signer, error)
|
GetSSHKey(dataDir string) (ssh.Signer, error)
|
||||||
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
|
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
|
||||||
|
HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
|
||||||
HandleStatusAlerts(status string, systemRecord *core.Record) error
|
HandleStatusAlerts(status string, systemRecord *core.Record) error
|
||||||
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
|
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
|
||||||
CancelPendingStatusAlerts(systemID string)
|
CancelPendingStatusAlerts(systemID string)
|
||||||
@@ -346,10 +349,15 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
|
|||||||
system := sm.NewSystem(systemId)
|
system := sm.NewSystem(systemId)
|
||||||
system.WsConn = wsConn
|
system.WsConn = wsConn
|
||||||
system.agentVersion = agentVersion
|
system.agentVersion = agentVersion
|
||||||
|
system.monitorsNeedSync.Store(true)
|
||||||
|
|
||||||
if err := sm.AddRecord(systemRecord, system); err != nil {
|
if err := sm.AddRecord(systemRecord, system); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Sync network monitors to the newly connected agent
|
||||||
|
go system.syncPendingNetworkMonitors()
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -362,6 +370,16 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
|
||||||
|
func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
|
||||||
|
var configs []monitor.Config
|
||||||
|
err := sm.hub.DB().
|
||||||
|
NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
|
||||||
|
Bind(dbx.Params{"system": systemID}).
|
||||||
|
All(&configs)
|
||||||
|
return configs, err
|
||||||
|
}
|
||||||
|
|
||||||
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
||||||
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
||||||
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
||||||
@@ -397,11 +415,12 @@ func (sm *SystemManager) createSSHClientConfig() error {
|
|||||||
|
|
||||||
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
|
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
|
||||||
// This is called when a system is paused or goes offline to prevent continued alerts.
|
// This is called when a system is paused or goes offline to prevent continued alerts.
|
||||||
|
// Monitor incidents remain open: a missing observation does not establish recovery.
|
||||||
func deactivateAlerts(app core.App, systemID string) error {
|
func deactivateAlerts(app core.App, systemID string) error {
|
||||||
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
|
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
|
||||||
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
|
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
|
||||||
|
|
||||||
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)
|
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/hub/utils"
|
"github.com/henrygd/beszel/internal/hub/utils"
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/apis"
|
"github.com/pocketbase/pocketbase/apis"
|
||||||
@@ -165,7 +167,7 @@ func (sm *SystemManager) fetchRealtimeDataAndNotify() {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
bytes, err := json.Marshal(data)
|
bytes, err := marshalRealtimeData(data)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
notify(sm.hub, system, fetch.subscription, bytes)
|
notify(sm.hub, system, fetch.subscription, bytes)
|
||||||
}
|
}
|
||||||
@@ -204,6 +206,22 @@ func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
|
||||||
|
// the per-monitor results into the derived metric fields the frontend charts expect.
|
||||||
|
func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
|
||||||
|
if len(data.Monitors) == 0 {
|
||||||
|
return json.Marshal(data)
|
||||||
|
}
|
||||||
|
monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
|
||||||
|
for id, result := range data.Monitors {
|
||||||
|
monitorStats[id] = monitor.Stats{}.FromResult(result)
|
||||||
|
}
|
||||||
|
return json.Marshal(struct {
|
||||||
|
*system.CombinedData
|
||||||
|
Monitors map[string]monitor.Stats `json:"Monitors"`
|
||||||
|
}{data, monitorStats})
|
||||||
|
}
|
||||||
|
|
||||||
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
|
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
|
||||||
// Custom topics bypass collection rules, so check current access for every
|
// Custom topics bypass collection rules, so check current access for every
|
||||||
// recipient, including clients whose authentication or membership was revoked.
|
// recipient, including clients whose authentication or membership was revoked.
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
|
|||||||
|
|
||||||
currentIDs := make(map[string]struct{}, len(smartData))
|
currentIDs := make(map[string]struct{}, len(smartData))
|
||||||
for deviceKey := range smartData {
|
for deviceKey := range smartData {
|
||||||
currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}
|
currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = hub.RunInTransaction(func(txApp core.App) error {
|
err = hub.RunInTransaction(func(txApp core.App) error {
|
||||||
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
|
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
|
||||||
recordID := makeStableHashId(sys.Id, deviceKey)
|
recordID := MakeStableHashId(sys.Id, deviceKey)
|
||||||
|
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
esystem "github.com/henrygd/beszel/internal/entities/system"
|
esystem "github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/hub/expirymap"
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
@@ -28,7 +29,8 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
|
|||||||
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
|
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
|
func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
|
||||||
|
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
|
||||||
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
|
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -212,7 +214,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
|
|||||||
}, false))
|
}, false))
|
||||||
|
|
||||||
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
|
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
|
||||||
recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))
|
recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.EqualValues(t, 42, recordA.GetInt("temp"))
|
assert.EqualValues(t, 42, recordA.GetInt("temp"))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
serviceName := "nginx.service"
|
serviceName := "nginx.service"
|
||||||
|
|
||||||
// Call multiple times and ensure same result
|
// Call multiple times and ensure same result
|
||||||
id1 := makeStableHashId(systemId, serviceName)
|
id1 := MakeStableHashId(systemId, serviceName)
|
||||||
id2 := makeStableHashId(systemId, serviceName)
|
id2 := MakeStableHashId(systemId, serviceName)
|
||||||
id3 := makeStableHashId(systemId, serviceName)
|
id3 := MakeStableHashId(systemId, serviceName)
|
||||||
|
|
||||||
assert.Equal(t, id1, id2)
|
assert.Equal(t, id1, id2)
|
||||||
assert.Equal(t, id2, id3)
|
assert.Equal(t, id2, id3)
|
||||||
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
serviceName1 := "nginx.service"
|
serviceName1 := "nginx.service"
|
||||||
serviceName2 := "apache.service"
|
serviceName2 := "apache.service"
|
||||||
|
|
||||||
id1 := makeStableHashId(systemId1, serviceName1)
|
id1 := MakeStableHashId(systemId1, serviceName1)
|
||||||
id2 := makeStableHashId(systemId2, serviceName1)
|
id2 := MakeStableHashId(systemId2, serviceName1)
|
||||||
id3 := makeStableHashId(systemId1, serviceName2)
|
id3 := MakeStableHashId(systemId1, serviceName2)
|
||||||
id4 := makeStableHashId(systemId2, serviceName2)
|
id4 := MakeStableHashId(systemId2, serviceName2)
|
||||||
|
|
||||||
// All IDs should be different
|
// All IDs should be different
|
||||||
assert.NotEqual(t, id1, id2)
|
assert.NotEqual(t, id1, id2)
|
||||||
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
id := makeStableHashId(tc.systemId, tc.serviceName)
|
id := MakeStableHashId(tc.systemId, tc.serviceName)
|
||||||
// FNV-32 produces 8 hex characters
|
// FNV-32 produces 8 hex characters
|
||||||
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
|
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("hexadecimal output", func(t *testing.T) {
|
t.Run("hexadecimal output", func(t *testing.T) {
|
||||||
id := makeStableHashId("test-system", "test-service")
|
id := MakeStableHashId("test-system", "test-service")
|
||||||
assert.NotEmpty(t, id)
|
assert.NotEmpty(t, id)
|
||||||
|
|
||||||
// Should only contain hexadecimal characters
|
// Should only contain hexadecimal characters
|
||||||
|
|||||||
@@ -129,7 +129,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
||||||
recordID := makeStableHashId(sys.Id, pool.Name)
|
recordID := MakeStableHashId(sys.Id, pool.Name)
|
||||||
|
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -171,7 +171,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
|
|||||||
if pool == nil {
|
if pool == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
recordID := makeStableHashId(sys.Id, name)
|
recordID := MakeStableHashId(sys.Id, name)
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, sql.ErrNoRows) {
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
|
|||||||
@@ -135,14 +135,14 @@ func TestSavePartialBackendInventory(t *testing.T) {
|
|||||||
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
|
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
|
||||||
}}
|
}}
|
||||||
require.NoError(t, sys.saveZfsPools(initial))
|
require.NoError(t, sys.saveZfsPools(initial))
|
||||||
failedID := makeStableHashId(sys.Id, failedKey)
|
failedID := MakeStableHashId(sys.Id, failedKey)
|
||||||
before, err := app.FindRecordById("zfs_pools", failedID)
|
before, err := app.FindRecordById("zfs_pools", failedID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
|
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
|
||||||
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
|
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
|
||||||
}}
|
}}
|
||||||
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
|
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
|
||||||
fresh, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, healthyKey))
|
fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.EqualValues(t, 20, fresh.GetInt("alloc"))
|
assert.EqualValues(t, 20, fresh.GetInt("alloc"))
|
||||||
cached, err := app.FindRecordById("zfs_pools", failedID)
|
cached, err := app.FindRecordById("zfs_pools", failedID)
|
||||||
@@ -169,7 +169,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
|||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||||
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
||||||
}))
|
}))
|
||||||
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
|
record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
firstUpdated := record.GetDateTime("updated")
|
firstUpdated := record.GetDateTime("updated")
|
||||||
assert.Equal(t, "ONLINE", record.GetString("health"))
|
assert.Equal(t, "ONLINE", record.GetString("health"))
|
||||||
@@ -193,7 +193,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
|||||||
func TestZfsRawCapacityPersistence(t *testing.T) {
|
func TestZfsRawCapacityPersistence(t *testing.T) {
|
||||||
sys, app := newTestSystemWithHub(t)
|
sys, app := newTestSystemWithHub(t)
|
||||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
|
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
|
||||||
record, err := app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "btrfs"))
|
record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.True(t, record.GetBool("raw"))
|
require.True(t, record.GetBool("raw"))
|
||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
|
||||||
@@ -210,7 +210,7 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
|
|||||||
key: {DisplayName: "tank", Health: "ONLINE"},
|
key: {DisplayName: "tank", Health: "ONLINE"},
|
||||||
"tank": {Health: "ONLINE"},
|
"tank": {Health: "ONLINE"},
|
||||||
}))
|
}))
|
||||||
id := makeStableHashId(sys.Id, key)
|
id := MakeStableHashId(sys.Id, key)
|
||||||
record, err := app.FindRecordById("zfs_pools", id)
|
record, err := app.FindRecordById("zfs_pools", id)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "tank", record.GetString("display_name"))
|
assert.Equal(t, "tank", record.GetString("display_name"))
|
||||||
@@ -225,6 +225,6 @@ func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
|
|||||||
record, err = app.FindRecordById("zfs_pools", id)
|
record, err = app.FindRecordById("zfs_pools", id)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "detail name", record.GetString("display_name"))
|
assert.Equal(t, "detail name", record.GetString("display_name"))
|
||||||
_, err = app.FindRecordById("zfs_pools", makeStableHashId(sys.Id, "tank"))
|
_, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,11 @@
|
|||||||
// Package utils provides utility functions for the hub.
|
// Package utils provides utility functions for the hub.
|
||||||
package utils
|
package utils
|
||||||
|
|
||||||
import "os"
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
)
|
||||||
|
|
||||||
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
|
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
|
||||||
func GetEnv(key string) (value string, exists bool) {
|
func GetEnv(key string) (value string, exists bool) {
|
||||||
@@ -10,3 +14,26 @@ func GetEnv(key string) (value string, exists bool) {
|
|||||||
}
|
}
|
||||||
return os.LookupEnv(key)
|
return os.LookupEnv(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
|
||||||
|
func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
|
||||||
|
broker := app.SubscriptionsBroker()
|
||||||
|
if broker.TotalClients() == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, client := range broker.Clients() {
|
||||||
|
subs := client.Subscriptions(collectionName)
|
||||||
|
if len(subs) > 0 {
|
||||||
|
if validateFn == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for k := range subs {
|
||||||
|
filter := subs[k].Query["filter"]
|
||||||
|
if validateFn(filter) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|||||||
+431
-3
@@ -83,7 +83,8 @@ func init() {
|
|||||||
"ContainerHealth",
|
"ContainerHealth",
|
||||||
"SystemdFailed",
|
"SystemdFailed",
|
||||||
"CPUIOWait",
|
"CPUIOWait",
|
||||||
"CPUSteal"
|
"CPUSteal",
|
||||||
|
"NetworkMonitorLoss"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -119,6 +120,16 @@ func init() {
|
|||||||
"system": false,
|
"system": false,
|
||||||
"type": "bool"
|
"type": "bool"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"hidden": true,
|
||||||
|
"id": "json4000656575",
|
||||||
|
"maxSize": 0,
|
||||||
|
"name": "state",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "json"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"hidden": true,
|
"hidden": true,
|
||||||
"id": "date1302749137",
|
"id": "date1302749137",
|
||||||
@@ -152,6 +163,7 @@ func init() {
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"indexes": [
|
"indexes": [
|
||||||
|
"CREATE INDEX idx_alerts_system_name ON alerts (system, name)",
|
||||||
"CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"
|
"CREATE UNIQUE INDEX ` + "`" + `idx_MnhEt21L5r` + "`" + ` ON ` + "`" + `alerts` + "`" + ` (\n ` + "`" + `user` + "`" + `,\n ` + "`" + `system` + "`" + `,\n ` + "`" + `name` + "`" + `\n)"
|
||||||
],
|
],
|
||||||
"system": false
|
"system": false
|
||||||
@@ -234,6 +246,20 @@ func init() {
|
|||||||
"system": false,
|
"system": false,
|
||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text3888135399",
|
||||||
|
"max": 0,
|
||||||
|
"min": 0,
|
||||||
|
"name": "monitor_name",
|
||||||
|
"pattern": "",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "number494360628",
|
"id": "number494360628",
|
||||||
@@ -1031,6 +1057,15 @@ func init() {
|
|||||||
"required": true,
|
"required": true,
|
||||||
"system": false,
|
"system": false,
|
||||||
"type": "number"
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "bool2084032502",
|
||||||
|
"name": "updatable",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "bool"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"indexes": [
|
"indexes": [
|
||||||
@@ -1721,6 +1756,7 @@ func init() {
|
|||||||
"fields": [
|
"fields": [
|
||||||
{
|
{
|
||||||
"autogeneratePattern": "[a-z0-9]{15}",
|
"autogeneratePattern": "[a-z0-9]{15}",
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "text3208210256",
|
"id": "text3208210256",
|
||||||
"max": 15,
|
"max": 15,
|
||||||
@@ -1736,6 +1772,7 @@ func init() {
|
|||||||
{
|
{
|
||||||
"cascadeDelete": true,
|
"cascadeDelete": true,
|
||||||
"collectionId": "2hz5ncl8tizk5nx",
|
"collectionId": "2hz5ncl8tizk5nx",
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "relation1204987316",
|
"id": "relation1204987316",
|
||||||
"maxSelect": 1,
|
"maxSelect": 1,
|
||||||
@@ -1748,6 +1785,7 @@ func init() {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"autogeneratePattern": "",
|
"autogeneratePattern": "",
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "text7739291048",
|
"id": "text7739291048",
|
||||||
"max": 0,
|
"max": 0,
|
||||||
@@ -1762,6 +1800,7 @@ func init() {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"autogeneratePattern": "",
|
"autogeneratePattern": "",
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "text5528164482",
|
"id": "text5528164482",
|
||||||
"max": 0,
|
"max": 0,
|
||||||
@@ -1775,6 +1814,7 @@ func init() {
|
|||||||
"type": "text"
|
"type": "text"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "number8862034195",
|
"id": "number8862034195",
|
||||||
"max": null,
|
"max": null,
|
||||||
@@ -1787,6 +1827,7 @@ func init() {
|
|||||||
"type": "number"
|
"type": "number"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "number4418907321",
|
"id": "number4418907321",
|
||||||
"max": null,
|
"max": null,
|
||||||
@@ -1799,6 +1840,7 @@ func init() {
|
|||||||
"type": "number"
|
"type": "number"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "number2904183765",
|
"id": "number2904183765",
|
||||||
"max": null,
|
"max": null,
|
||||||
@@ -1811,6 +1853,7 @@ func init() {
|
|||||||
"type": "number"
|
"type": "number"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "json4466109723",
|
"id": "json4466109723",
|
||||||
"maxSize": 0,
|
"maxSize": 0,
|
||||||
@@ -1821,6 +1864,7 @@ func init() {
|
|||||||
"type": "json"
|
"type": "json"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "json9012873456",
|
"id": "json9012873456",
|
||||||
"maxSize": 0,
|
"maxSize": 0,
|
||||||
@@ -1831,6 +1875,7 @@ func init() {
|
|||||||
"type": "json"
|
"type": "json"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "json7182045639",
|
"id": "json7182045639",
|
||||||
"maxSize": 0,
|
"maxSize": 0,
|
||||||
@@ -1841,6 +1886,7 @@ func init() {
|
|||||||
"type": "json"
|
"type": "json"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
"help": "",
|
||||||
"hidden": false,
|
"hidden": false,
|
||||||
"id": "date9274163058",
|
"id": "date9274163058",
|
||||||
"max": "",
|
"max": "",
|
||||||
@@ -1860,19 +1906,401 @@ func init() {
|
|||||||
"presentable": false,
|
"presentable": false,
|
||||||
"system": false,
|
"system": false,
|
||||||
"type": "autodate"
|
"type": "autodate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text3578368839",
|
||||||
|
"max": 0,
|
||||||
|
"min": 0,
|
||||||
|
"name": "display_name",
|
||||||
|
"pattern": "",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "bool447994709",
|
||||||
|
"name": "raw",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "bool"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"id": "pbc_8441057391",
|
"id": "pbc_8441057391",
|
||||||
"indexes": [
|
"indexes": [
|
||||||
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
|
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
|
||||||
],
|
],
|
||||||
"listRule": null,
|
"listRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id",
|
||||||
"name": "zfs_pools",
|
"name": "zfs_pools",
|
||||||
"system": false,
|
"system": false,
|
||||||
"type": "base",
|
"type": "base",
|
||||||
"updateRule": null,
|
"updateRule": null,
|
||||||
|
"viewRule": "@request.auth.id != \"\" && system.users.id ?= @request.auth.id"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"createRule": null,
|
||||||
|
"deleteRule": null,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "[a-z0-9]{10}",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text3208210256",
|
||||||
|
"max": 10,
|
||||||
|
"min": 6,
|
||||||
|
"name": "id",
|
||||||
|
"pattern": "^[a-z0-9]+$",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": true,
|
||||||
|
"required": true,
|
||||||
|
"system": true,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cascadeDelete": true,
|
||||||
|
"collectionId": "2hz5ncl8tizk5nx",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_system",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"minSelect": 0,
|
||||||
|
"name": "system",
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "relation"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_target",
|
||||||
|
"max": 500,
|
||||||
|
"min": 1,
|
||||||
|
"name": "target",
|
||||||
|
"pattern": "",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_protocol",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"name": "protocol",
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "select",
|
||||||
|
"values": [
|
||||||
|
"icmp",
|
||||||
|
"tcp",
|
||||||
|
"http",
|
||||||
|
"dns"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_port",
|
||||||
|
"max": 65535,
|
||||||
|
"min": 0,
|
||||||
|
"name": "port",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_interval",
|
||||||
|
"max": 3600,
|
||||||
|
"min": 1,
|
||||||
|
"name": "interval",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number926446584",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "res",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number1006954605",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "resAvg1h",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number4267669802",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "resMin1h",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number591433223",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "resMax1h",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number3726709001",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "loss1h",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nm_enabled",
|
||||||
|
"name": "enabled",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "bool"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hidden": false,
|
||||||
|
"id": "autodate2990389176",
|
||||||
|
"name": "created",
|
||||||
|
"onCreate": true,
|
||||||
|
"onUpdate": false,
|
||||||
|
"presentable": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "autodate"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "date3332085495",
|
||||||
|
"max": "",
|
||||||
|
"min": "",
|
||||||
|
"name": "updated",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "date"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": "nm_monitors_001",
|
||||||
|
"indexes": [
|
||||||
|
"CREATE INDEX ` + "`" + `idx_nm_system_enabled` + "`" + ` ON ` + "`" + `network_monitors` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `enabled` + "`" + `)"
|
||||||
|
],
|
||||||
|
"listRule": null,
|
||||||
|
"name": "network_monitors",
|
||||||
|
"system": false,
|
||||||
|
"type": "base",
|
||||||
|
"updateRule": null,
|
||||||
"viewRule": null
|
"viewRule": null
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"createRule": null,
|
||||||
|
"deleteRule": null,
|
||||||
|
"fields": [
|
||||||
|
{
|
||||||
|
"autogeneratePattern": "[a-z0-9]{10}",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "text3208210256",
|
||||||
|
"max": 10,
|
||||||
|
"min": 10,
|
||||||
|
"name": "id",
|
||||||
|
"pattern": "^[a-z0-9]+$",
|
||||||
|
"presentable": false,
|
||||||
|
"primaryKey": true,
|
||||||
|
"required": true,
|
||||||
|
"system": true,
|
||||||
|
"type": "text"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cascadeDelete": true,
|
||||||
|
"collectionId": "2hz5ncl8tizk5nx",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_system",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"minSelect": 0,
|
||||||
|
"name": "system",
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "relation"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cascadeDelete": true,
|
||||||
|
"collectionId": "nm_monitors_001",
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_monitor",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"minSelect": 0,
|
||||||
|
"name": "monitor",
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "relation"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "Number of probe attempts",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_total_count",
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"name": "total_count",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "Number of successful probe attempts",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_success_count",
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"name": "success_count",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "Sum of successful response times in microseconds",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_res_sum",
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"name": "res_sum",
|
||||||
|
"onlyInt": true,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "Response time in microseconds",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_res_min",
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"name": "res_min",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "Response time in microseconds",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_res_max",
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"name": "res_max",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "nms_type",
|
||||||
|
"maxSelect": 1,
|
||||||
|
"name": "type",
|
||||||
|
"presentable": false,
|
||||||
|
"required": true,
|
||||||
|
"system": false,
|
||||||
|
"type": "select",
|
||||||
|
"values": [
|
||||||
|
"1m",
|
||||||
|
"10m",
|
||||||
|
"20m",
|
||||||
|
"120m",
|
||||||
|
"480m"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"help": "",
|
||||||
|
"hidden": false,
|
||||||
|
"id": "number2990389176",
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"name": "created",
|
||||||
|
"onlyInt": false,
|
||||||
|
"presentable": false,
|
||||||
|
"required": false,
|
||||||
|
"system": false,
|
||||||
|
"type": "number"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"id": "nm_stats_001",
|
||||||
|
"indexes": [
|
||||||
|
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_system_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `system` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_monitor_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `monitor` + "`" + `, ` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)",
|
||||||
|
"CREATE INDEX IF NOT EXISTS ` + "`" + `idx_nms_type_created` + "`" + ` ON ` + "`" + `network_monitor_stats` + "`" + ` (` + "`" + `type` + "`" + `, ` + "`" + `created` + "`" + `)"
|
||||||
|
],
|
||||||
|
"listRule": null,
|
||||||
|
"name": "network_monitor_stats",
|
||||||
|
"system": false,
|
||||||
|
"type": "base",
|
||||||
|
"updateRule": null,
|
||||||
|
"viewRule": null
|
||||||
|
}
|
||||||
]`
|
]`
|
||||||
|
|
||||||
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
|
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
package migrations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
m "github.com/pocketbase/pocketbase/migrations"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
m.Register(func(app core.App) error {
|
|
||||||
c, err := app.FindCollectionByNameOrId("zfs_pools")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c.Fields.Add(&core.TextField{Name: "display_name"})
|
|
||||||
c.Fields.Add(&core.BoolField{Name: "raw"})
|
|
||||||
return app.Save(c)
|
|
||||||
}, func(app core.App) error {
|
|
||||||
c, err := app.FindCollectionByNameOrId("zfs_pools")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
c.Fields.RemoveByName("display_name")
|
|
||||||
c.Fields.RemoveByName("raw")
|
|
||||||
|
|
||||||
return app.Save(c)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
package migrations
|
|
||||||
|
|
||||||
import (
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
m "github.com/pocketbase/pocketbase/migrations"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
m.Register(func(app core.App) error {
|
|
||||||
collection, err := app.FindCollectionByNameOrId("containers")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
collection.Fields.Add(&core.BoolField{Name: "updatable"})
|
|
||||||
return app.Save(collection)
|
|
||||||
}, func(app core.App) error {
|
|
||||||
collection, err := app.FindCollectionByNameOrId("containers")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
collection.Fields.RemoveByName("updatable")
|
|
||||||
return app.Save(collection)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,226 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package records_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
monitorEntity "github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
|
"github.com/henrygd/beszel/internal/records"
|
||||||
|
"github.com/henrygd/beszel/internal/tests"
|
||||||
|
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAverageMonitorStats(t *testing.T) {
|
||||||
|
hub, err := tests.NewTestHub(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
collection, err := hub.FindCachedCollectionByNameOrId("network_monitor_stats")
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Nil(t, collection.Fields.GetByName("res_avg"))
|
||||||
|
assert.Nil(t, collection.Fields.GetByName("loss"))
|
||||||
|
|
||||||
|
rm := records.NewRecordManager(hub)
|
||||||
|
user, err := tests.CreateUser(hub, "monitor-avg@example.com", "testtesttest")
|
||||||
|
require.NoError(t, err)
|
||||||
|
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "monitor-avg-system",
|
||||||
|
"host": "localhost",
|
||||||
|
"port": "45876",
|
||||||
|
"status": "up",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
|
||||||
|
"system": sys.Id,
|
||||||
|
"name": "cloudflare",
|
||||||
|
"target": "1.1.1.1",
|
||||||
|
"protocol": "icmp",
|
||||||
|
"interval": 30,
|
||||||
|
"enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
created := time.Now().UnixMilli()
|
||||||
|
// Unequal probe counts must weight both latency and loss.
|
||||||
|
recordA, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
|
||||||
|
"system": sys.Id,
|
||||||
|
"monitor": monitor.Id,
|
||||||
|
"type": "1m",
|
||||||
|
"created": created,
|
||||||
|
"res_min": 5,
|
||||||
|
"res_max": 20,
|
||||||
|
"total_count": 6, "success_count": 6, "res_sum": 60,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
recordB, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
|
||||||
|
"system": sys.Id,
|
||||||
|
"monitor": monitor.Id,
|
||||||
|
"type": "1m",
|
||||||
|
"created": created,
|
||||||
|
"res_min": 10,
|
||||||
|
"res_max": 60,
|
||||||
|
"total_count": 1, "success_count": 1, "res_sum": 22,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
result, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 2, count)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{ResAvg: 11.71, ResMin: 5, ResMax: 60, TotalCount: 7, SuccessCount: 7, ResponseSum: 82}, result)
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, monitor, recordType string
|
||||||
|
after int64
|
||||||
|
}{
|
||||||
|
{"other monitor", "missing", "1m", created - 1},
|
||||||
|
{"other type", monitor.Id, "10m", created - 1},
|
||||||
|
{"exclusive cutoff", monitor.Id, "1m", created},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
stats, count, err := rm.AverageMonitorStats(hub.DB(), tc.monitor, tc.recordType, tc.after)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Zero(t, count)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{}, stats)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// A failure-only bucket counts toward loss but must not lower latency.
|
||||||
|
recordB.Set("res_min", 0)
|
||||||
|
recordB.Set("res_max", 0)
|
||||||
|
recordB.Set("success_count", 0)
|
||||||
|
recordB.Set("res_sum", 0)
|
||||||
|
require.NoError(t, hub.Save(recordB))
|
||||||
|
result, count, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 2, count)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{ResAvg: 10, ResMin: 5, ResMax: 20, Loss: 14.29, TotalCount: 7, SuccessCount: 6, ResponseSum: 60}, result)
|
||||||
|
// Sparse monitor records must propagate through every rollup level.
|
||||||
|
rm.CreateLongerRecords()
|
||||||
|
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
|
||||||
|
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, rollups, 1, recordType)
|
||||||
|
assert.Equal(t, 5.0, rollups[0].GetFloat("res_min"))
|
||||||
|
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
|
||||||
|
assert.Equal(t, 7, rollups[0].GetInt("total_count"))
|
||||||
|
assert.Equal(t, 6, rollups[0].GetInt("success_count"))
|
||||||
|
assert.Equal(t, 60, rollups[0].GetInt("res_sum"))
|
||||||
|
// A sibling with a different number of probes must retain its actual
|
||||||
|
// weight when the next tier combines their underlying counts.
|
||||||
|
_, err = tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
|
||||||
|
"system": sys.Id, "monitor": monitor.Id, "type": recordType, "created": created,
|
||||||
|
"res_min": 100, "res_max": 100,
|
||||||
|
"total_count": 3, "success_count": 1, "res_sum": 100,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
merged, count, err := rm.AverageMonitorStats(hub.DB(), monitor.Id, recordType, created-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, 2, count)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{
|
||||||
|
ResAvg: 22.86, ResMin: 5, ResMax: 100, Loss: 30,
|
||||||
|
TotalCount: 10, SuccessCount: 7, ResponseSum: 160,
|
||||||
|
}, merged)
|
||||||
|
}
|
||||||
|
// All failures produce zero latency, while a genuine zero-microsecond
|
||||||
|
// success remains a valid minimum (it must not be filtered out as a sentinel).
|
||||||
|
recordA.Set("success_count", 0)
|
||||||
|
recordA.Set("res_sum", 0)
|
||||||
|
require.NoError(t, hub.Save(recordA))
|
||||||
|
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, Loss: 100}, result)
|
||||||
|
recordA.Set("success_count", 1)
|
||||||
|
recordA.Set("res_min", 0)
|
||||||
|
recordA.Set("res_max", 0)
|
||||||
|
require.NoError(t, hub.Save(recordA))
|
||||||
|
result, _, err = rm.AverageMonitorStats(hub.DB(), monitor.Id, "1m", created-1)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, monitorEntity.Stats{TotalCount: 7, SuccessCount: 1, Loss: 85.71}, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSparseMonitorRollups(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
interval int
|
||||||
|
samples int
|
||||||
|
disabled bool
|
||||||
|
}{
|
||||||
|
{"five minute interval", 300, 2, false},
|
||||||
|
{"ten minute interval", 600, 1, false},
|
||||||
|
{"fifteen minute interval", 900, 1, false},
|
||||||
|
{"empty window", 900, 0, false},
|
||||||
|
{"disabled with pending history", 300, 2, true},
|
||||||
|
{"disabled empty window", 900, 0, true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
hub, err := tests.NewTestHub(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
user, err := tests.CreateUser(hub, "sparse-monitor@example.com", "testtesttest")
|
||||||
|
require.NoError(t, err)
|
||||||
|
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "sparse-monitor-system", "host": "localhost", "port": "45876", "status": "up",
|
||||||
|
"users": []string{user.Id},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
|
||||||
|
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
|
||||||
|
"interval": tc.interval, "enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
now := time.Now()
|
||||||
|
for i := range tc.samples {
|
||||||
|
_, err := tests.CreateRecord(hub, "network_monitor_stats", map[string]any{
|
||||||
|
"system": sys.Id, "monitor": monitor.Id, "type": "1m",
|
||||||
|
"created": now.Add(-time.Minute - time.Duration(i*tc.interval)*time.Second).UnixMilli(),
|
||||||
|
"res_min": 8, "res_max": 20,
|
||||||
|
"total_count": 4, "success_count": 3, "res_sum": 36,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
// Other collections must still reject fewer than nine minute records.
|
||||||
|
for _, collection := range []string{"system_stats", "container_stats"} {
|
||||||
|
stats := `{"cpu":10}`
|
||||||
|
if collection == "container_stats" {
|
||||||
|
stats = `[{"name":"test","cpu":10}]`
|
||||||
|
}
|
||||||
|
for range 8 {
|
||||||
|
_, err := tests.CreateRecord(hub, collection, map[string]any{
|
||||||
|
"system": sys.Id, "type": "1m", "stats": stats,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.disabled {
|
||||||
|
monitor.Set("enabled", false)
|
||||||
|
require.NoError(t, hub.Save(monitor))
|
||||||
|
}
|
||||||
|
records.NewRecordManager(hub).CreateLongerRecords()
|
||||||
|
for _, recordType := range []string{"10m", "20m", "120m", "480m"} {
|
||||||
|
rollups, err := hub.FindAllRecords("network_monitor_stats", dbx.HashExp{"monitor": monitor.Id, "type": recordType})
|
||||||
|
require.NoError(t, err)
|
||||||
|
if tc.samples == 0 {
|
||||||
|
assert.Empty(t, rollups, recordType)
|
||||||
|
} else {
|
||||||
|
require.Len(t, rollups, 1, recordType)
|
||||||
|
assert.Equal(t, 8.0, rollups[0].GetFloat("res_min"))
|
||||||
|
assert.Equal(t, 20.0, rollups[0].GetFloat("res_max"))
|
||||||
|
assert.Equal(t, 4*tc.samples, rollups[0].GetInt("total_count"))
|
||||||
|
assert.Equal(t, 3*tc.samples, rollups[0].GetInt("success_count"))
|
||||||
|
assert.Equal(t, 36*tc.samples, rollups[0].GetInt("res_sum"))
|
||||||
|
}
|
||||||
|
for _, collection := range []string{"system_stats", "container_stats"} {
|
||||||
|
count, err := hub.CountRecords(collection, dbx.HashExp{"system": sys.Id, "type": recordType})
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Zero(t, count, "%s %s", collection, recordType)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+140
-28
@@ -3,15 +3,16 @@ package records
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"log/slog"
|
|
||||||
"math"
|
"math"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/monitor"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
type RecordManager struct {
|
type RecordManager struct {
|
||||||
@@ -39,7 +40,7 @@ type StatsRecord struct {
|
|||||||
|
|
||||||
// Create longer records by averaging shorter records
|
// Create longer records by averaging shorter records
|
||||||
func (rm *RecordManager) CreateLongerRecords() {
|
func (rm *RecordManager) CreateLongerRecords() {
|
||||||
// start := time.Now()
|
now := time.Now().UTC()
|
||||||
longerRecordData := []LongerRecordData{
|
longerRecordData := []LongerRecordData{
|
||||||
{
|
{
|
||||||
shorterType: "1m",
|
shorterType: "1m",
|
||||||
@@ -69,23 +70,28 @@ func (rm *RecordManager) CreateLongerRecords() {
|
|||||||
}
|
}
|
||||||
// wrap the operations in a transaction
|
// wrap the operations in a transaction
|
||||||
// Pocketbase cron does not handle errors, log them here.
|
// Pocketbase cron does not handle errors, log them here.
|
||||||
rm.app.RunInTransaction(func(txApp core.App) error {
|
err := rm.app.RunInTransaction(func(txApp core.App) error {
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
collections := [2]*core.Collection{}
|
collections := [2]*core.Collection{}
|
||||||
collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")
|
collections[0], err = txApp.FindCachedCollectionByNameOrId("system_stats")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Error finding cached collection using system stats:", "err", err)
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")
|
collections[1], err = txApp.FindCachedCollectionByNameOrId("container_stats")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("Error finding cached collection using container stats:", "err", err)
|
return err
|
||||||
|
}
|
||||||
|
monitorStatsColl, err := txApp.FindCachedCollectionByNameOrId("network_monitor_stats")
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
var systems RecordIds
|
var systems RecordIds
|
||||||
db := txApp.DB()
|
db := txApp.DB()
|
||||||
|
|
||||||
db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems)
|
if err := db.NewQuery("SELECT id FROM systems WHERE status='up'").All(&systems); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// loop through all active systems, time periods, and collections
|
// loop through all active systems, time periods, and collections
|
||||||
for _, system := range systems {
|
for _, system := range systems {
|
||||||
@@ -94,44 +100,52 @@ func (rm *RecordManager) CreateLongerRecords() {
|
|||||||
recordData := longerRecordData[i]
|
recordData := longerRecordData[i]
|
||||||
// log.Println("processing longer record type", recordData.longerType)
|
// log.Println("processing longer record type", recordData.longerType)
|
||||||
// add one minute padding for longer records because they are created slightly later than the job start time
|
// add one minute padding for longer records because they are created slightly later than the job start time
|
||||||
longerRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration + time.Minute)
|
longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
|
||||||
// shorter records are created independently of longer records, so we shouldn't need to add padding
|
// shorter records are created independently of longer records, so we shouldn't need to add padding
|
||||||
shorterRecordPeriod := time.Now().UTC().Add(recordData.longerTimeDuration)
|
shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
|
||||||
// loop through both collections
|
|
||||||
for _, collection := range collections {
|
for _, collection := range collections {
|
||||||
// check creation time of last longer record if not 10m, since 10m is created every run
|
// check creation time of last longer record if not 10m, since 10m is created every run
|
||||||
if recordData.longerType != "10m" {
|
if recordData.longerType != "10m" {
|
||||||
count, err := txApp.CountRecords(
|
count, err := txApp.CountRecords(collection.Id, dbx.NewExp(
|
||||||
collection.Id,
|
"system = {:system} AND type = {:type} AND created > {:created}",
|
||||||
dbx.NewExp(
|
dbx.Params{
|
||||||
"system = {:system} AND type = {:type} AND created > {:created}",
|
"type": recordData.longerType,
|
||||||
dbx.Params{"type": recordData.longerType, "system": system.Id, "created": longerRecordPeriod},
|
"system": system.Id,
|
||||||
),
|
"created": longerRecordPeriod.Format(types.DefaultDateLayout),
|
||||||
)
|
},
|
||||||
|
))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
// continue if longer record exists
|
// continue if longer record exists
|
||||||
if err != nil || count > 0 {
|
if count > 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// get shorter records from the past x minutes
|
// get shorter records from the past x minutes
|
||||||
var recordIds RecordIds
|
var recordIds RecordIds
|
||||||
|
|
||||||
err := txApp.DB().
|
params := dbx.Params{
|
||||||
|
"type": recordData.shorterType,
|
||||||
|
"system": system.Id,
|
||||||
|
"created": shorterRecordPeriod.Format(types.DefaultDateLayout),
|
||||||
|
}
|
||||||
|
|
||||||
|
err := db.
|
||||||
Select("id").
|
Select("id").
|
||||||
From(collection.Name).
|
From(collection.Name).
|
||||||
AndWhere(dbx.NewExp(
|
Where(dbx.NewExp(
|
||||||
"system={:system} AND type={:type} AND created > {:created}",
|
"system={:system} AND type={:type} AND created > {:created}",
|
||||||
dbx.Params{
|
params,
|
||||||
"type": recordData.shorterType,
|
|
||||||
"system": system.Id,
|
|
||||||
"created": shorterRecordPeriod,
|
|
||||||
},
|
|
||||||
)).
|
)).
|
||||||
OrderBy("created").
|
OrderBy("created").
|
||||||
All(&recordIds)
|
All(&recordIds)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// continue if not enough shorter records
|
// continue if not enough shorter records
|
||||||
if err != nil || len(recordIds) < recordData.minShorterRecords {
|
if len(recordIds) < recordData.minShorterRecords {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// average the shorter records and create longer record
|
// average the shorter records and create longer record
|
||||||
@@ -142,20 +156,88 @@ func (rm *RecordManager) CreateLongerRecords() {
|
|||||||
case "system_stats":
|
case "system_stats":
|
||||||
longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))
|
longerRecord.Set("stats", rm.AverageSystemStats(db, recordIds))
|
||||||
case "container_stats":
|
case "container_stats":
|
||||||
|
|
||||||
longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))
|
longerRecord.Set("stats", rm.AverageContainerStats(db, recordIds))
|
||||||
}
|
}
|
||||||
if err := txApp.SaveNoValidate(longerRecord); err != nil {
|
if err := txApp.SaveNoValidate(longerRecord); err != nil {
|
||||||
slog.Error("failed to save longer record", "err", err)
|
txApp.Logger().Error("failed to save longer record", "err", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// network_monitor_stats is aggregated per monitor (not per system)
|
||||||
|
var monitors []struct {
|
||||||
|
Id string `db:"id"`
|
||||||
|
System string `db:"system"`
|
||||||
|
}
|
||||||
|
// Disabled monitors still have history that must advance through retention tiers.
|
||||||
|
if err := db.NewQuery("SELECT id, system FROM network_monitors").All(&monitors); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, monitorRec := range monitors {
|
||||||
|
for i := range longerRecordData {
|
||||||
|
recordData := longerRecordData[i]
|
||||||
|
longerRecordPeriod := now.Add(recordData.longerTimeDuration + time.Minute)
|
||||||
|
shorterRecordPeriod := now.Add(recordData.longerTimeDuration)
|
||||||
|
|
||||||
|
if recordData.longerType != "10m" {
|
||||||
|
count, err := txApp.CountRecords(monitorStatsColl.Id, dbx.NewExp(
|
||||||
|
"monitor={:monitor} AND type={:type} AND created>{:created}",
|
||||||
|
dbx.Params{
|
||||||
|
"monitor": monitorRec.Id,
|
||||||
|
"type": recordData.longerType,
|
||||||
|
"created": longerRecordPeriod.UnixMilli(),
|
||||||
|
},
|
||||||
|
))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if count > 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, count, err := rm.AverageMonitorStats(db, monitorRec.Id, recordData.shorterType, shorterRecordPeriod.UnixMilli())
|
||||||
|
if err != nil {
|
||||||
|
txApp.Logger().Error("failed to average monitor stats", "monitor", monitorRec.Id, "err", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Monitor intervals can exceed the aggregation window, so average
|
||||||
|
// any available records at every level and skip only empty windows.
|
||||||
|
if count == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
longerRecord := core.NewRecord(monitorStatsColl)
|
||||||
|
longerRecord.Set("system", monitorRec.System)
|
||||||
|
longerRecord.Set("monitor", monitorRec.Id)
|
||||||
|
longerRecord.Set("type", recordData.longerType)
|
||||||
|
longerRecord.Set("created", now.UnixMilli())
|
||||||
|
longerRecord.Set("res_min", stats.ResMin)
|
||||||
|
longerRecord.Set("res_max", stats.ResMax)
|
||||||
|
longerRecord.Set("total_count", stats.TotalCount)
|
||||||
|
longerRecord.Set("success_count", stats.SuccessCount)
|
||||||
|
longerRecord.Set("res_sum", stats.ResponseSum)
|
||||||
|
if err := txApp.SaveNoValidate(longerRecord); err != nil {
|
||||||
|
txApp.Logger().Error("failed to save monitor longer record", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
rm.app.Logger().Error("failed to create longer records", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// log.Println("finished creating longer records", "time (ms)", time.Since(start).Milliseconds())
|
func getCreatedTimeField(collectionName string, period time.Time) any {
|
||||||
|
// network_monitor_stats stores created as unix timestamp in ms, not as a date string
|
||||||
|
if collectionName == "network_monitor_stats" {
|
||||||
|
return period.UnixMilli()
|
||||||
|
}
|
||||||
|
return period.Format(types.DefaultDateLayout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Calculate the average stats of a list of system_stats records without reflect
|
// Calculate the average stats of a list of system_stats records without reflect
|
||||||
@@ -596,6 +678,36 @@ func AverageContainerStatsSlice(records [][]container.Stats) []container.Stats {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AverageMonitorStats merges probe counts and response sums, preserving their
|
||||||
|
// weights through every retention tier. Failed probes do not contribute latency.
|
||||||
|
func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordType string, createdAfter int64) (monitor.Stats, int, error) {
|
||||||
|
var result struct {
|
||||||
|
monitor.Stats
|
||||||
|
Count int `db:"count"`
|
||||||
|
}
|
||||||
|
err := db.Select(
|
||||||
|
"COUNT(*) AS count",
|
||||||
|
"COALESCE(SUM(total_count), 0) AS total_count",
|
||||||
|
"COALESCE(SUM(success_count), 0) AS success_count",
|
||||||
|
"COALESCE(SUM(res_sum), 0) AS res_sum",
|
||||||
|
"COALESCE(MIN(CASE WHEN success_count > 0 THEN res_min END), 0) AS res_min",
|
||||||
|
"COALESCE(MAX(CASE WHEN success_count > 0 THEN res_max END), 0) AS res_max",
|
||||||
|
).From("network_monitor_stats").Where(dbx.NewExp(
|
||||||
|
"monitor={:monitor} AND type={:type} AND created>{:created}",
|
||||||
|
dbx.Params{"monitor": monitorID, "type": recordType, "created": createdAfter},
|
||||||
|
)).One(&result)
|
||||||
|
if err != nil {
|
||||||
|
return monitor.Stats{}, 0, err
|
||||||
|
}
|
||||||
|
if result.SuccessCount > 0 {
|
||||||
|
result.ResAvg = twoDecimals(float64(result.ResponseSum) / float64(result.SuccessCount))
|
||||||
|
}
|
||||||
|
if result.TotalCount > 0 {
|
||||||
|
result.Loss = twoDecimals(float64(result.TotalCount-result.SuccessCount) * 100 / float64(result.TotalCount))
|
||||||
|
}
|
||||||
|
return result.Stats, result.Count, nil
|
||||||
|
}
|
||||||
|
|
||||||
/* Round float to two decimals */
|
/* Round float to two decimals */
|
||||||
func twoDecimals(value float64) float64 {
|
func twoDecimals(value float64) float64 {
|
||||||
return math.Round(value*100) / 100
|
return math.Round(value*100) / 100
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package records
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
@@ -60,7 +59,7 @@ func deleteOldAlertsHistory(app core.App, countToKeep, countBeforeDeletion int)
|
|||||||
// Deletes system_stats records older than what is displayed in the UI
|
// Deletes system_stats records older than what is displayed in the UI
|
||||||
func deleteOldSystemStats(app core.App) error {
|
func deleteOldSystemStats(app core.App) error {
|
||||||
// Collections to process
|
// Collections to process
|
||||||
collections := [2]string{"system_stats", "container_stats"}
|
collections := [3]string{"system_stats", "container_stats", "network_monitor_stats"}
|
||||||
|
|
||||||
// Record types and their retention periods
|
// Record types and their retention periods
|
||||||
type RecordDeletionData struct {
|
type RecordDeletionData struct {
|
||||||
@@ -76,24 +75,17 @@ func deleteOldSystemStats(app core.App) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
db := app.DB()
|
||||||
|
|
||||||
for _, collection := range collections {
|
for _, collection := range collections {
|
||||||
// Build the WHERE clause
|
query := db.Delete(collection, dbx.NewExp("type={:type} AND created<{:created}"))
|
||||||
var conditionParts []string
|
for _, rd := range recordData {
|
||||||
var params dbx.Params = make(map[string]any)
|
if _, err := query.Bind(dbx.Params{
|
||||||
for i := range recordData {
|
"type": rd.recordType,
|
||||||
rd := recordData[i]
|
"created": getCreatedTimeField(collection, now.Add(-rd.retention)),
|
||||||
// Create parameterized condition for this record type
|
}).Execute(); err != nil {
|
||||||
dateParam := fmt.Sprintf("date%d", i)
|
return fmt.Errorf("failed to delete from %s: %v", collection, err)
|
||||||
conditionParts = append(conditionParts, fmt.Sprintf("(type = '%s' AND created < {:%s})", rd.recordType, dateParam))
|
}
|
||||||
params[dateParam] = now.Add(-rd.retention)
|
|
||||||
}
|
|
||||||
// Combine conditions with OR
|
|
||||||
conditionStr := strings.Join(conditionParts, " OR ")
|
|
||||||
// Construct and execute the full raw query
|
|
||||||
rawQuery := fmt.Sprintf("DELETE FROM %s WHERE %s", collection, conditionStr)
|
|
||||||
if _, err := app.DB().NewQuery(rawQuery).Bind(params).Execute(); err != nil {
|
|
||||||
return fmt.Errorf("failed to delete from %s: %v", collection, err)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package records_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/records"
|
||||||
|
"github.com/henrygd/beszel/internal/tests"
|
||||||
|
"github.com/pocketbase/dbx"
|
||||||
|
"github.com/pocketbase/pocketbase/tools/types"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLongerRecordsPreventDuplicates(t *testing.T) {
|
||||||
|
for _, collection := range []string{"system_stats", "container_stats", "network_monitor_stats"} {
|
||||||
|
for _, tier := range []struct {
|
||||||
|
shorter, longer string
|
||||||
|
count int
|
||||||
|
}{
|
||||||
|
{"10m", "20m", 2},
|
||||||
|
{"20m", "120m", 6},
|
||||||
|
{"120m", "480m", 4},
|
||||||
|
} {
|
||||||
|
t.Run(collection+"/"+tier.longer, func(t *testing.T) {
|
||||||
|
hub, err := tests.NewTestHub(t.TempDir())
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer hub.Cleanup()
|
||||||
|
|
||||||
|
user, err := tests.CreateUser(hub, "rollup@example.com", "testtesttest")
|
||||||
|
require.NoError(t, err)
|
||||||
|
sys, err := tests.CreateRecord(hub, "systems", map[string]any{
|
||||||
|
"name": "rollup-system", "host": "localhost", "port": "45876",
|
||||||
|
"status": "up", "users": []string{user.Id},
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
created := time.Now().UTC().Add(-time.Minute)
|
||||||
|
data := map[string]any{
|
||||||
|
"system": sys.Id, "type": tier.shorter,
|
||||||
|
"created": created.Format(types.DefaultDateLayout),
|
||||||
|
}
|
||||||
|
filter := dbx.HashExp{"system": sys.Id, "type": tier.longer}
|
||||||
|
switch collection {
|
||||||
|
case "system_stats":
|
||||||
|
data["stats"] = `{"cpu":10}`
|
||||||
|
case "container_stats":
|
||||||
|
data["stats"] = `[{"name":"test","cpu":10}]`
|
||||||
|
case "network_monitor_stats":
|
||||||
|
monitor, err := tests.CreateRecord(hub, "network_monitors", map[string]any{
|
||||||
|
"system": sys.Id, "target": "1.1.1.1", "protocol": "icmp",
|
||||||
|
"interval": 30, "enabled": true,
|
||||||
|
})
|
||||||
|
require.NoError(t, err)
|
||||||
|
data["monitor"] = monitor.Id
|
||||||
|
data["created"] = created.UnixMilli()
|
||||||
|
data["total_count"] = 1
|
||||||
|
data["success_count"] = 1
|
||||||
|
data["res_sum"] = 10
|
||||||
|
data["res_min"] = 10
|
||||||
|
data["res_max"] = 10
|
||||||
|
filter["monitor"] = monitor.Id
|
||||||
|
}
|
||||||
|
for range tier.count {
|
||||||
|
_, err := tests.CreateRecord(hub, collection, data)
|
||||||
|
require.NoError(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
rm := records.NewRecordManager(hub)
|
||||||
|
rm.CreateLongerRecords()
|
||||||
|
first, err := hub.FindAllRecords(collection, filter)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, first, 1)
|
||||||
|
|
||||||
|
// The shorter records remain eligible, but the existing longer
|
||||||
|
// record must prevent another rollup on a subsequent invocation.
|
||||||
|
rm.CreateLongerRecords()
|
||||||
|
second, err := hub.FindAllRecords(collection, filter)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, second, 1)
|
||||||
|
require.Equal(t, first[0].Id, second[0].Id)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "beszel",
|
"name": "beszel",
|
||||||
"private": true,
|
"private": true,
|
||||||
"version": "0.19.0",
|
"version": "0.20.0",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite --host",
|
"dev": "vite --host",
|
||||||
|
|||||||
@@ -61,6 +61,8 @@ export const ActiveAlerts = () => {
|
|||||||
<AlertDescription>
|
<AlertDescription>
|
||||||
{info.triggeredDesc ? (
|
{info.triggeredDesc ? (
|
||||||
info.triggeredDesc()
|
info.triggeredDesc()
|
||||||
|
) : alert.name === "NetworkMonitorLoss" ? (
|
||||||
|
<Trans>One or more monitors exceed {alert.value}% loss</Trans>
|
||||||
) : alert.name === "Status" ? (
|
) : alert.name === "Status" ? (
|
||||||
<Trans>Connection is down</Trans>
|
<Trans>Connection is down</Trans>
|
||||||
) : info.invert ? (
|
) : info.invert ? (
|
||||||
|
|||||||
@@ -30,7 +30,8 @@ export const alertsHistoryColumns: ColumnDef<AlertsHistoryRecord>[] = [
|
|||||||
accessorFn: (record) => {
|
accessorFn: (record) => {
|
||||||
const name = record.name
|
const name = record.name
|
||||||
const info = alertInfo[name]
|
const info = alertInfo[name]
|
||||||
return info?.name().replace("cpu", "CPU") || name
|
const label = info?.name().replace("cpu", "CPU") || name
|
||||||
|
return record.monitor_name ? `${label}: ${record.monitor_name}` : label
|
||||||
},
|
},
|
||||||
header: ({ column }) => (
|
header: ({ column }) => (
|
||||||
<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
|
<Button variant="ghost" onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}>
|
||||||
|
|||||||
@@ -239,13 +239,13 @@ export function AlertContent({
|
|||||||
/** Alerts that fire on first observation have no duration to configure */
|
/** Alerts that fire on first observation have no duration to configure */
|
||||||
const noDuration = alertData.noDuration === true
|
const noDuration = alertData.noDuration === true
|
||||||
/** Binary alerts have no threshold to configure */
|
/** Binary alerts have no threshold to configure */
|
||||||
const noThreshold = !!singleDescription || noDuration
|
const noThreshold = !!singleDescription || alertData.noThreshold === true
|
||||||
/** Whether enabling the alert reveals anything to configure */
|
/** Whether enabling the alert reveals anything to configure */
|
||||||
const hasControls = !(noThreshold && noDuration)
|
const hasControls = !(noThreshold && noDuration)
|
||||||
|
|
||||||
const [checked, setChecked] = useState(global ? false : !!alert)
|
const [checked, setChecked] = useState(global ? false : !!alert)
|
||||||
const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))
|
const [min, setMin] = useState(alert?.min || (noDuration ? 0 : 10))
|
||||||
const [value, setValue] = useState(alert?.value || (noThreshold ? 0 : (alertData.start ?? 80)))
|
const [value, setValue] = useState(alert?.value ?? (noThreshold ? 0 : (alertData.start ?? 80)))
|
||||||
|
|
||||||
const Icon = alertData.icon
|
const Icon = alertData.icon
|
||||||
|
|
||||||
@@ -319,7 +319,7 @@ export function AlertContent({
|
|||||||
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
|
<div className="grid sm:grid-cols-2 mt-1.5 gap-5 px-4 pb-5 tabular-nums text-muted-foreground">
|
||||||
<Suspense fallback={<div className="h-10" />}>
|
<Suspense fallback={<div className="h-10" />}>
|
||||||
{!noThreshold && (
|
{!noThreshold && (
|
||||||
<div>
|
<div className={cn(noDuration && "col-span-full")}>
|
||||||
<p id={`v${name}`} className="text-sm block h-6">
|
<p id={`v${name}`} className="text-sm block h-6">
|
||||||
{alertData.invert ? (
|
{alertData.invert ? (
|
||||||
<Trans>
|
<Trans>
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ export default function AreaChartDefault({
|
|||||||
}) {
|
}) {
|
||||||
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
|
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
|
||||||
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
|
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
|
||||||
const sourceData = customData ?? chartData.systemStats
|
const sourceData = customData ?? chartData.systemStats ?? []
|
||||||
const [displayData, setDisplayData] = useState(sourceData)
|
const [displayData, setDisplayData] = useState(sourceData)
|
||||||
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
|
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
|
||||||
|
|
||||||
@@ -111,6 +111,8 @@ export default function AreaChartDefault({
|
|||||||
})
|
})
|
||||||
}, [areasKey, displayMaxToggled])
|
}, [areasKey, displayMaxToggled])
|
||||||
|
|
||||||
|
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
|
||||||
|
|
||||||
return useMemo(() => {
|
return useMemo(() => {
|
||||||
if (displayData.length === 0) {
|
if (displayData.length === 0) {
|
||||||
return null
|
return null
|
||||||
@@ -146,7 +148,7 @@ export default function AreaChartDefault({
|
|||||||
axisLine={false}
|
axisLine={false}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
{xAxis(chartData)}
|
{XAxis}
|
||||||
<ChartTooltip
|
<ChartTooltip
|
||||||
animationEasing="ease-out"
|
animationEasing="ease-out"
|
||||||
animationDuration={150}
|
animationDuration={150}
|
||||||
@@ -167,5 +169,5 @@ export default function AreaChartDefault({
|
|||||||
</AreaChart>
|
</AreaChart>
|
||||||
</ChartContainer>
|
</ChartContainer>
|
||||||
)
|
)
|
||||||
}, [displayData, yAxisWidth, filter, Areas])
|
}, [displayData, yAxisWidth, filter, Areas, XAxis])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,14 +9,21 @@ import { memo } from "react"
|
|||||||
export default memo(function ChartTimeSelect({
|
export default memo(function ChartTimeSelect({
|
||||||
className,
|
className,
|
||||||
agentVersion,
|
agentVersion,
|
||||||
|
chartTimeStore = $chartTime,
|
||||||
|
allowRealtime = true,
|
||||||
}: {
|
}: {
|
||||||
className?: string
|
className?: string
|
||||||
agentVersion: SemVer
|
agentVersion: SemVer
|
||||||
|
chartTimeStore?: typeof $chartTime
|
||||||
|
allowRealtime?: boolean
|
||||||
}) {
|
}) {
|
||||||
const chartTime = useStore($chartTime)
|
const chartTime = useStore(chartTimeStore)
|
||||||
|
|
||||||
// remove chart times that are not supported by the system agent version
|
// remove chart times that are not supported by the system agent version
|
||||||
const availableChartTimes = Object.entries(chartTimeData).filter(([_, { minVersion }]) => {
|
const availableChartTimes = Object.entries(chartTimeData).filter(([value, { minVersion }]) => {
|
||||||
|
if (value === "1m" && !allowRealtime) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
if (!minVersion) {
|
if (!minVersion) {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -24,7 +31,7 @@ export default memo(function ChartTimeSelect({
|
|||||||
})
|
})
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => $chartTime.set(value)}>
|
<Select defaultValue="1h" value={chartTime} onValueChange={(value: ChartTimes) => chartTimeStore.set(value)}>
|
||||||
<SelectTrigger className={cn(className, "relative ps-10 pe-5")}>
|
<SelectTrigger className={cn(className, "relative ps-10 pe-5")}>
|
||||||
<HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
|
<HistoryIcon className="h-4 w-4 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
|
||||||
<SelectValue />
|
<SelectValue />
|
||||||
|
|||||||
@@ -22,6 +22,10 @@ export type DataPoint<T = SystemStatsRecord> = {
|
|||||||
order?: number
|
order?: number
|
||||||
strokeOpacity?: number
|
strokeOpacity?: number
|
||||||
activeDot?: boolean
|
activeDot?: boolean
|
||||||
|
dot?: boolean
|
||||||
|
/** Which Y axis this series plots against. Defaults to "left". */
|
||||||
|
yAxisId?: "left" | "right"
|
||||||
|
strokeDasharray?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export default function LineChartDefault({
|
export default function LineChartDefault({
|
||||||
@@ -30,9 +34,12 @@ export default function LineChartDefault({
|
|||||||
max,
|
max,
|
||||||
maxToggled,
|
maxToggled,
|
||||||
tickFormatter,
|
tickFormatter,
|
||||||
|
tickFormatter2,
|
||||||
contentFormatter,
|
contentFormatter,
|
||||||
dataPoints,
|
dataPoints,
|
||||||
domain,
|
domain,
|
||||||
|
domain2,
|
||||||
|
max2,
|
||||||
legend,
|
legend,
|
||||||
itemSorter,
|
itemSorter,
|
||||||
showTotal = false,
|
showTotal = false,
|
||||||
@@ -41,18 +48,24 @@ export default function LineChartDefault({
|
|||||||
filter,
|
filter,
|
||||||
truncate = false,
|
truncate = false,
|
||||||
chartProps,
|
chartProps,
|
||||||
|
connectNulls,
|
||||||
}: {
|
}: {
|
||||||
chartData: ChartData
|
chartData: ChartData
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)
|
// biome-ignore lint/suspicious/noExplicitAny: accepts different data source types (systemStats or containerData)
|
||||||
customData?: any[]
|
customData?: any[]
|
||||||
max?: number
|
max?: number
|
||||||
|
max2?: number
|
||||||
maxToggled?: boolean
|
maxToggled?: boolean
|
||||||
tickFormatter: (value: number, index: number) => string
|
tickFormatter: (value: number, index: number) => string
|
||||||
|
/** Tick formatter for the right ("right"-yAxisId) axis, when any dataPoint uses it. */
|
||||||
|
tickFormatter2?: (value: number, index: number) => string
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
|
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
|
||||||
contentFormatter: (item: any, key: string) => ReactNode
|
contentFormatter: (item: any, key: string) => ReactNode
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types
|
// biome-ignore lint/suspicious/noExplicitAny: accepts DataPoint with different generic types
|
||||||
dataPoints?: DataPoint<any>[]
|
dataPoints?: DataPoint<any>[]
|
||||||
domain?: AxisDomain
|
domain?: AxisDomain
|
||||||
|
/** Domain for the right axis, when any dataPoint uses it. */
|
||||||
|
domain2?: AxisDomain
|
||||||
legend?: boolean
|
legend?: boolean
|
||||||
showTotal?: boolean
|
showTotal?: boolean
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
|
// biome-ignore lint/suspicious/noExplicitAny: recharts tooltip item interop
|
||||||
@@ -62,10 +75,15 @@ export default function LineChartDefault({
|
|||||||
filter?: string
|
filter?: string
|
||||||
truncate?: boolean
|
truncate?: boolean
|
||||||
chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">
|
chartProps?: Omit<React.ComponentProps<typeof LineChart>, "data" | "margin">
|
||||||
|
connectNulls?: boolean
|
||||||
}) {
|
}) {
|
||||||
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
|
const { yAxisWidth, updateYAxisWidth } = useYAxisWidth()
|
||||||
|
const hasRightAxis = !!dataPoints?.some((dp) => dp.yAxisId === "right")
|
||||||
|
// fixed width for the secondary axis rather than measured, since its labels (e.g. loss %) are short
|
||||||
|
// and predictable, and this avoids depending on a second async width-measurement pass to settle
|
||||||
|
const rightAxisWidth = 38
|
||||||
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
|
const { isIntersecting, ref } = useIntersectionObserver({ freeze: false })
|
||||||
const sourceData = customData ?? chartData.systemStats
|
const sourceData = customData ?? chartData.systemStats ?? []
|
||||||
const [displayData, setDisplayData] = useState(sourceData)
|
const [displayData, setDisplayData] = useState(sourceData)
|
||||||
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
|
const [displayMaxToggled, setDisplayMaxToggled] = useState(maxToggled)
|
||||||
|
|
||||||
@@ -83,7 +101,9 @@ export default function LineChartDefault({
|
|||||||
}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])
|
}, [displayData, displayMaxToggled, isIntersecting, maxToggled, sourceData])
|
||||||
|
|
||||||
// Use a stable key derived from data point identities and visual properties
|
// Use a stable key derived from data point identities and visual properties
|
||||||
const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity ?? ""}`).join("\0")
|
const linesKey = dataPoints?.map((d) => `${d.label}:${d.strokeOpacity}${d.dot}${d.yAxisId}${d.strokeDasharray}`).join("\0")
|
||||||
|
|
||||||
|
const XAxis = xAxis(chartData.chartTime, displayData.at(-1)?.created)
|
||||||
|
|
||||||
const Lines = useMemo(() => {
|
const Lines = useMemo(() => {
|
||||||
return dataPoints?.map((dataPoint, i) => {
|
return dataPoints?.map((dataPoint, i) => {
|
||||||
@@ -94,17 +114,20 @@ export default function LineChartDefault({
|
|||||||
return (
|
return (
|
||||||
<Line
|
<Line
|
||||||
key={dataPoint.label}
|
key={dataPoint.label}
|
||||||
|
yAxisId={dataPoint.yAxisId ?? "left"}
|
||||||
dataKey={dataPoint.dataKey}
|
dataKey={dataPoint.dataKey}
|
||||||
name={dataPoint.label}
|
name={dataPoint.label}
|
||||||
type="monotoneX"
|
type="monotoneX"
|
||||||
dot={false}
|
dot={dataPoint.dot || false}
|
||||||
strokeWidth={1.5}
|
strokeWidth={1.5}
|
||||||
stroke={color}
|
stroke={color}
|
||||||
strokeOpacity={dataPoint.strokeOpacity}
|
strokeOpacity={dataPoint.strokeOpacity}
|
||||||
|
strokeDasharray={dataPoint.strokeDasharray}
|
||||||
isAnimationActive={false}
|
isAnimationActive={false}
|
||||||
// stackId={dataPoint.stackId}
|
// stackId={dataPoint.stackId}
|
||||||
order={dataPoint.order || i}
|
order={dataPoint.order || i}
|
||||||
activeDot={dataPoint.activeDot ?? true}
|
activeDot={dataPoint.activeDot ?? true}
|
||||||
|
connectNulls={connectNulls}
|
||||||
/>
|
/>
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
@@ -135,6 +158,7 @@ export default function LineChartDefault({
|
|||||||
<CartesianGrid vertical={false} />
|
<CartesianGrid vertical={false} />
|
||||||
{!hideYAxis && (
|
{!hideYAxis && (
|
||||||
<YAxis
|
<YAxis
|
||||||
|
yAxisId="left"
|
||||||
direction="ltr"
|
direction="ltr"
|
||||||
orientation={chartData.orientation}
|
orientation={chartData.orientation}
|
||||||
className="tracking-tighter"
|
className="tracking-tighter"
|
||||||
@@ -145,7 +169,20 @@ export default function LineChartDefault({
|
|||||||
axisLine={false}
|
axisLine={false}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
{xAxis(chartData)}
|
{!hideYAxis && hasRightAxis && (
|
||||||
|
<YAxis
|
||||||
|
yAxisId="right"
|
||||||
|
direction="ltr"
|
||||||
|
orientation={chartData.orientation === "left" ? "right" : "left"}
|
||||||
|
className="tracking-tighter"
|
||||||
|
width={rightAxisWidth}
|
||||||
|
domain={domain2 ?? [0, max2 ?? "auto"]}
|
||||||
|
tickFormatter={tickFormatter2 ?? tickFormatter}
|
||||||
|
tickLine={false}
|
||||||
|
axisLine={false}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
{XAxis}
|
||||||
<ChartTooltip
|
<ChartTooltip
|
||||||
animationEasing="ease-out"
|
animationEasing="ease-out"
|
||||||
animationDuration={150}
|
animationDuration={150}
|
||||||
@@ -166,5 +203,5 @@ export default function LineChartDefault({
|
|||||||
</LineChart>
|
</LineChart>
|
||||||
</ChartContainer>
|
</ChartContainer>
|
||||||
)
|
)
|
||||||
}, [displayData, yAxisWidth, filter, Lines])
|
}, [displayData, yAxisWidth, hasRightAxis, filter, Lines, XAxis])
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import {
|
|||||||
HardDriveIcon,
|
HardDriveIcon,
|
||||||
LogsIcon,
|
LogsIcon,
|
||||||
MailIcon,
|
MailIcon,
|
||||||
|
NetworkIcon,
|
||||||
Server,
|
Server,
|
||||||
ServerIcon,
|
ServerIcon,
|
||||||
SettingsIcon,
|
SettingsIcon,
|
||||||
@@ -122,6 +123,20 @@ export default memo(function CommandPalette({ open, setOpen }: { open: boolean;
|
|||||||
<Trans>Page</Trans>
|
<Trans>Page</Trans>
|
||||||
</CommandShortcut>
|
</CommandShortcut>
|
||||||
</CommandItem>
|
</CommandItem>
|
||||||
|
<CommandItem
|
||||||
|
onSelect={() => {
|
||||||
|
navigate(getPagePath($router, "monitors"))
|
||||||
|
setOpen(false)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<NetworkIcon className="me-2 size-4" />
|
||||||
|
<span>
|
||||||
|
<Trans>Network Monitors</Trans>
|
||||||
|
</span>
|
||||||
|
<CommandShortcut>
|
||||||
|
<Trans>Page</Trans>
|
||||||
|
</CommandShortcut>
|
||||||
|
</CommandItem>
|
||||||
<CommandItem
|
<CommandItem
|
||||||
onSelect={() => {
|
onSelect={() => {
|
||||||
navigate(getPagePath($router, "settings", { name: "general" }))
|
navigate(getPagePath($router, "settings", { name: "general" }))
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import {
|
|||||||
import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"
|
import { EthernetIcon, HourglassIcon, SquareArrowRightEnterIcon } from "../ui/icons"
|
||||||
import { Badge } from "../ui/badge"
|
import { Badge } from "../ui/badge"
|
||||||
import { t } from "@lingui/core/macro"
|
import { t } from "@lingui/core/macro"
|
||||||
import { $allSystemsById, $longestSystemNameLen } from "@/lib/stores"
|
import { $allSystemsById, $longestSystemName } from "@/lib/stores"
|
||||||
import { useStore } from "@nanostores/react"
|
import { useStore } from "@nanostores/react"
|
||||||
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
|
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
|
||||||
|
|
||||||
@@ -59,15 +59,22 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
|
|||||||
const allSystems = $allSystemsById.get()
|
const allSystems = $allSystemsById.get()
|
||||||
const systemNameA = allSystems[a.original.system]?.name ?? ""
|
const systemNameA = allSystems[a.original.system]?.name ?? ""
|
||||||
const systemNameB = allSystems[b.original.system]?.name ?? ""
|
const systemNameB = allSystems[b.original.system]?.name ?? ""
|
||||||
return systemNameA.localeCompare(systemNameB)
|
const primary = systemNameA.localeCompare(systemNameB)
|
||||||
|
if (primary !== 0) {
|
||||||
|
return primary
|
||||||
|
}
|
||||||
|
return a.original.name.localeCompare(b.original.name)
|
||||||
},
|
},
|
||||||
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
|
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
|
||||||
cell: ({ getValue }) => {
|
cell: ({ getValue }) => {
|
||||||
const allSystems = useStore($allSystemsById)
|
const allSystems = useStore($allSystemsById)
|
||||||
const longestName = useStore($longestSystemNameLen)
|
const longestName = useStore($longestSystemName)
|
||||||
return (
|
return (
|
||||||
<div className="ms-1 max-w-40 truncate" style={{ width: `${longestName / 1.05}ch` }}>
|
<div className="ms-1 relative w-fit max-w-40">
|
||||||
{allSystems[getValue() as string]?.name ?? ""}
|
<span className="invisible block whitespace-nowrap" aria-hidden="true">
|
||||||
|
{longestName}
|
||||||
|
</span>
|
||||||
|
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
@@ -189,12 +196,12 @@ export const containerChartCols: ColumnDef<ContainerRecord>[] = [
|
|||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger
|
<TooltipTrigger
|
||||||
className="shrink-0 rounded-sm text-emerald-600 dark:text-emerald-400 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
|
className="shrink-0 rounded-sm text-emerald-600 dark:text-emerald-400 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring"
|
||||||
aria-label={t`Image update available`}
|
aria-label={t({ message: "Image update available", context: "Docker image" })}
|
||||||
onClick={(event) => event.stopPropagation()}
|
onClick={(event) => event.stopPropagation()}
|
||||||
>
|
>
|
||||||
<CircleArrowUpIcon className="size-4" aria-hidden="true" />
|
<CircleArrowUpIcon className="size-4" aria-hidden="true" />
|
||||||
</TooltipTrigger>
|
</TooltipTrigger>
|
||||||
<TooltipContent>{t`Image update available`}</TooltipContent>
|
<TooltipContent>{t({ message: "Image update available", context: "Docker image" })}</TooltipContent>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import {
|
|||||||
LogOutIcon,
|
LogOutIcon,
|
||||||
LogsIcon,
|
LogsIcon,
|
||||||
MenuIcon,
|
MenuIcon,
|
||||||
|
NetworkIcon,
|
||||||
PlusIcon,
|
PlusIcon,
|
||||||
SearchIcon,
|
SearchIcon,
|
||||||
ServerIcon,
|
ServerIcon,
|
||||||
@@ -108,6 +109,13 @@ export default function Navbar() {
|
|||||||
<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
|
<HardDriveIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
|
||||||
<span>S.M.A.R.T.</span>
|
<span>S.M.A.R.T.</span>
|
||||||
</DropdownMenuItem>
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => navigate(getPagePath($router, "monitors"))}
|
||||||
|
className="flex items-center"
|
||||||
|
>
|
||||||
|
<NetworkIcon className="h-4 w-4 me-2.5" strokeWidth={1.5} />
|
||||||
|
<Trans>Network Monitors</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
<DropdownMenuItem
|
<DropdownMenuItem
|
||||||
onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}
|
onClick={() => navigate(getPagePath($router, "settings", { name: "general" }))}
|
||||||
className="flex items-center"
|
className="flex items-center"
|
||||||
@@ -179,6 +187,21 @@ export default function Navbar() {
|
|||||||
</TooltipTrigger>
|
</TooltipTrigger>
|
||||||
<TooltipContent>S.M.A.R.T.</TooltipContent>
|
<TooltipContent>S.M.A.R.T.</TooltipContent>
|
||||||
</Tooltip>
|
</Tooltip>
|
||||||
|
<Tooltip>
|
||||||
|
<TooltipTrigger asChild>
|
||||||
|
<Link
|
||||||
|
href={getPagePath($router, "monitors")}
|
||||||
|
className={cn("hidden md:grid", buttonVariants({ variant: "ghost", size: "icon" }))}
|
||||||
|
aria-label="Network Monitors"
|
||||||
|
onMouseEnter={() => import("@/components/routes/monitors")}
|
||||||
|
>
|
||||||
|
<NetworkIcon className="h-[1.2rem] w-[1.2rem]" strokeWidth={1.5} />
|
||||||
|
</Link>
|
||||||
|
</TooltipTrigger>
|
||||||
|
<TooltipContent>
|
||||||
|
<Trans>Network Monitors</Trans>
|
||||||
|
</TooltipContent>
|
||||||
|
</Tooltip>
|
||||||
<ModeToggle />
|
<ModeToggle />
|
||||||
<Tooltip>
|
<Tooltip>
|
||||||
<TooltipTrigger asChild>
|
<TooltipTrigger asChild>
|
||||||
|
|||||||
@@ -0,0 +1,772 @@
|
|||||||
|
import { useCallback, useEffect, useRef, useState } from "react"
|
||||||
|
import { Trans, useLingui } from "@lingui/react/macro"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
import { pb } from "@/lib/api"
|
||||||
|
import {
|
||||||
|
Dialog,
|
||||||
|
DialogContent,
|
||||||
|
DialogDescription,
|
||||||
|
DialogFooter,
|
||||||
|
DialogHeader,
|
||||||
|
DialogTitle,
|
||||||
|
} from "@/components/ui/dialog"
|
||||||
|
import {
|
||||||
|
DropdownMenu,
|
||||||
|
DropdownMenuCheckboxItem,
|
||||||
|
DropdownMenuContent,
|
||||||
|
DropdownMenuItem,
|
||||||
|
DropdownMenuTrigger,
|
||||||
|
} from "@/components/ui/dropdown-menu"
|
||||||
|
import { Sheet, SheetContent, SheetDescription, SheetFooter, SheetHeader, SheetTitle } from "@/components/ui/sheet"
|
||||||
|
import { Button } from "@/components/ui/button"
|
||||||
|
import { Input } from "@/components/ui/input"
|
||||||
|
import { Label } from "@/components/ui/label"
|
||||||
|
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"
|
||||||
|
import { Textarea } from "@/components/ui/textarea"
|
||||||
|
import { ChevronDownIcon, ListIcon, SearchIcon, ServerIcon } from "lucide-react"
|
||||||
|
import { useToast } from "@/components/ui/use-toast"
|
||||||
|
import { $systems } from "@/lib/stores"
|
||||||
|
import { cn, supportsNetworkMonitors } from "@/lib/utils"
|
||||||
|
import type { NetworkMonitorRecord } from "@/types"
|
||||||
|
import * as v from "valibot"
|
||||||
|
|
||||||
|
type MonitorProtocol = "icmp" | "tcp" | "http" | "dns"
|
||||||
|
|
||||||
|
type MonitorValues = {
|
||||||
|
system: string
|
||||||
|
target: string
|
||||||
|
protocol: MonitorProtocol
|
||||||
|
port: number
|
||||||
|
interval: string
|
||||||
|
}
|
||||||
|
|
||||||
|
type NormalizedMonitorValues = Omit<MonitorValues, "system" | "interval"> & {
|
||||||
|
interval: number
|
||||||
|
}
|
||||||
|
|
||||||
|
type BulkMonitorLineSource = Pick<NetworkMonitorRecord, "target" | "protocol" | "port" | "interval">
|
||||||
|
|
||||||
|
const defaultInterval = 30
|
||||||
|
|
||||||
|
const MonitorProtocolSchema = v.picklist(["icmp", "tcp", "http", "dns"])
|
||||||
|
|
||||||
|
const MonitorIntervalSchema = v.pipe(v.string(), v.toNumber(), v.minValue(1), v.maxValue(3600))
|
||||||
|
|
||||||
|
// Both the single-monitor form and the bulk importer flow through this schema so
|
||||||
|
// defaults and HTTP target normalization stay in one place.
|
||||||
|
const NormalizedMonitorValuesSchema = v.pipe(
|
||||||
|
v.object({
|
||||||
|
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
|
||||||
|
protocol: MonitorProtocolSchema,
|
||||||
|
port: v.number(),
|
||||||
|
interval: MonitorIntervalSchema,
|
||||||
|
}),
|
||||||
|
v.transform((input): NormalizedMonitorValues => {
|
||||||
|
let { protocol, port } = input
|
||||||
|
let httpTarget = input.target
|
||||||
|
if (protocol === "icmp" || protocol === "http" || protocol === "dns") {
|
||||||
|
if (protocol === "http") {
|
||||||
|
httpTarget = normalizeHttpTarget(input.target, port)
|
||||||
|
}
|
||||||
|
port = 0
|
||||||
|
} else if (protocol === "tcp" && !port) {
|
||||||
|
port = 443
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
// HTTP monitors may be entered as bare hostnames, so normalize them to a
|
||||||
|
// scheme-bearing URL before the payload is sent to PocketBase.
|
||||||
|
target: protocol === "http" ? httpTarget : input.target,
|
||||||
|
protocol,
|
||||||
|
port,
|
||||||
|
interval: input.interval,
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
v.forward(
|
||||||
|
v.check((input) => {
|
||||||
|
if (input.protocol === "icmp" || input.protocol === "http" || input.protocol === "dns") {
|
||||||
|
return input.port === 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return Number.isInteger(input.port) && input.port >= 1 && input.port <= 65535
|
||||||
|
}, "Port must be between 1 and 65535"),
|
||||||
|
["port"]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bulk parsing only trims raw CSV fields. Inference, defaults, and protocol-
|
||||||
|
// specific validation still go through the shared normalization schema above.
|
||||||
|
const BulkMonitorSchema = v.object({
|
||||||
|
target: v.pipe(v.string(), v.trim(), v.nonEmpty("target is required")),
|
||||||
|
protocol: v.optional(v.pipe(v.string(), v.trim())),
|
||||||
|
port: v.optional(v.pipe(v.string(), v.trim())),
|
||||||
|
interval: v.optional(v.pipe(v.string(), v.trim())),
|
||||||
|
})
|
||||||
|
|
||||||
|
function normalizeHttpTarget(target: string, port = 0) {
|
||||||
|
const useExplicitPort = port > 0 && port !== 80 && port !== 443
|
||||||
|
const hasOriginOnlyTarget = /^https?:\/\/[^/?#]+$/i.test(target)
|
||||||
|
if (!/^https?:\/\//i.test(target)) {
|
||||||
|
const scheme = port === 80 ? "http" : "https"
|
||||||
|
return `${scheme}://${target}${useExplicitPort ? `:${port}` : ""}`
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsedUrl: URL
|
||||||
|
try {
|
||||||
|
parsedUrl = new URL(target)
|
||||||
|
} catch {
|
||||||
|
return target
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!parsedUrl.port && useExplicitPort) {
|
||||||
|
parsedUrl.port = `${port}`
|
||||||
|
}
|
||||||
|
|
||||||
|
// avoid converting "http://localhost:8090" to "http://localhost:8090/" - keep the original formatting if the URL is just an origin
|
||||||
|
if (hasOriginOnlyTarget && parsedUrl.pathname === "/" && !parsedUrl.search && !parsedUrl.hash) {
|
||||||
|
return parsedUrl.origin
|
||||||
|
}
|
||||||
|
|
||||||
|
return parsedUrl.toString()
|
||||||
|
}
|
||||||
|
|
||||||
|
function trimTrailingEmptyFields(fields: string[]) {
|
||||||
|
let lastValueIndex = fields.length - 1
|
||||||
|
while (lastValueIndex > 0 && !fields[lastValueIndex]) {
|
||||||
|
lastValueIndex--
|
||||||
|
}
|
||||||
|
return fields.slice(0, lastValueIndex + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMonitorPayload(values: MonitorValues, enabled = true) {
|
||||||
|
const normalizedValues = v.safeParse(NormalizedMonitorValuesSchema, values)
|
||||||
|
if (!normalizedValues.success) {
|
||||||
|
throw new Error(normalizedValues.issues[0]?.message || "Invalid monitor")
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
system: values.system,
|
||||||
|
enabled,
|
||||||
|
...normalizedValues.output,
|
||||||
|
}
|
||||||
|
|
||||||
|
return payload
|
||||||
|
}
|
||||||
|
|
||||||
|
type MonitorIdentity = Pick<MonitorValues, "system" | "target" | "protocol" | "port">
|
||||||
|
function getMonitorIdentityKey({ system, target, protocol, port }: MonitorIdentity) {
|
||||||
|
return `${system}${target}${protocol}${port}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseBulkMonitorLine(line: string, lineNumber: number, system: string) {
|
||||||
|
const [rawTarget = "", rawProtocol = "", rawPort = "", rawInterval = ""] = line.split(",")
|
||||||
|
const parsed = v.safeParse(BulkMonitorSchema, {
|
||||||
|
target: rawTarget,
|
||||||
|
protocol: rawProtocol,
|
||||||
|
port: rawPort,
|
||||||
|
interval: rawInterval,
|
||||||
|
})
|
||||||
|
if (!parsed.success) {
|
||||||
|
throw new Error(`Line ${lineNumber}: ${parsed.issues[0]?.message || "invalid monitor entry"}`)
|
||||||
|
}
|
||||||
|
const protocol = (parsed.output.protocol?.toLowerCase() ||
|
||||||
|
(/^https?:\/\//i.test(parsed.output.target) ? "http" : "icmp")) as MonitorProtocol
|
||||||
|
|
||||||
|
return buildMonitorPayload({
|
||||||
|
system,
|
||||||
|
target: parsed.output.target,
|
||||||
|
protocol,
|
||||||
|
port: parsed.output.port ? Number(parsed.output.port) : 0,
|
||||||
|
interval: parsed.output.interval || `${defaultInterval}`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function formatBulkMonitorLine(monitor: BulkMonitorLineSource) {
|
||||||
|
const port = monitor.protocol !== "tcp" || monitor.port === 443 ? "" : `${monitor.port}`
|
||||||
|
const interval = monitor.interval === defaultInterval ? "" : `${monitor.interval}`
|
||||||
|
return trimTrailingEmptyFields([monitor.target, monitor.protocol, port, interval]).join(",")
|
||||||
|
}
|
||||||
|
|
||||||
|
function SystemMultiSelect({
|
||||||
|
id,
|
||||||
|
selectedSystemIds,
|
||||||
|
onChange,
|
||||||
|
disabled,
|
||||||
|
className,
|
||||||
|
}: {
|
||||||
|
id: string
|
||||||
|
selectedSystemIds: Set<string>
|
||||||
|
onChange: (ids: Set<string>) => void
|
||||||
|
disabled?: boolean
|
||||||
|
className?: string
|
||||||
|
}) {
|
||||||
|
const systems = useStore($systems)
|
||||||
|
const { t } = useLingui()
|
||||||
|
const [search, setSearch] = useState("")
|
||||||
|
const searchRef = useRef<HTMLInputElement>(null)
|
||||||
|
const focusSearchOnMount = useCallback((node: HTMLInputElement | null) => {
|
||||||
|
searchRef.current = node
|
||||||
|
if (!node) return
|
||||||
|
// Focus after the menu has completed its own initial focus handling.
|
||||||
|
const frame = requestAnimationFrame(() => node.focus())
|
||||||
|
return () => cancelAnimationFrame(frame)
|
||||||
|
}, [])
|
||||||
|
const contentRef = useRef<HTMLDivElement>(null)
|
||||||
|
const query = search.trim().toLocaleLowerCase()
|
||||||
|
const filteredSystems = systems.filter(
|
||||||
|
(system) => supportsNetworkMonitors(system) && system.name.toLocaleLowerCase().includes(query)
|
||||||
|
)
|
||||||
|
const allSelected = filteredSystems.every((system) => selectedSystemIds.has(system.id))
|
||||||
|
const anySelected = filteredSystems.some((system) => selectedSystemIds.has(system.id))
|
||||||
|
|
||||||
|
const selectFiltered = (selected: boolean) => {
|
||||||
|
const next = new Set(selectedSystemIds)
|
||||||
|
for (const system of filteredSystems) {
|
||||||
|
if (selected) next.add(system.id)
|
||||||
|
else next.delete(system.id)
|
||||||
|
}
|
||||||
|
onChange(next)
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<DropdownMenu onOpenChange={() => setSearch("")}>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button
|
||||||
|
id={id}
|
||||||
|
disabled={disabled}
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
className={cn("relative w-full min-w-0 ps-10 pe-10 justify-start font-normal text-start", className)}
|
||||||
|
>
|
||||||
|
<ServerIcon className="size-3.5 absolute start-4 top-1/2 -translate-y-1/2 opacity-85" />
|
||||||
|
<span className="truncate">
|
||||||
|
{selectedSystemIds.size === 0
|
||||||
|
? t`Select systems`
|
||||||
|
: selectedSystemIds.size === 1
|
||||||
|
? systems.find((s) => selectedSystemIds.has(s.id))?.name
|
||||||
|
: t`${selectedSystemIds.size} selected`}
|
||||||
|
</span>
|
||||||
|
<ChevronDownIcon className="size-4 absolute end-4 top-1/2 -translate-y-1/2 opacity-50" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent
|
||||||
|
ref={contentRef}
|
||||||
|
onKeyDown={(event) => {
|
||||||
|
if (event.key === "Tab") {
|
||||||
|
event.preventDefault()
|
||||||
|
searchRef.current?.focus()
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
align="start"
|
||||||
|
className="w-[var(--radix-dropdown-menu-trigger-width)] max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] flex flex-col overflow-hidden"
|
||||||
|
>
|
||||||
|
<div className="shrink-0 border-b mb-1">
|
||||||
|
<div className="flex items-center gap-2 px-2.5">
|
||||||
|
<SearchIcon aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />
|
||||||
|
<Input
|
||||||
|
ref={focusSearchOnMount}
|
||||||
|
value={search}
|
||||||
|
onChange={(event) => setSearch(event.target.value)}
|
||||||
|
placeholder={t`Search systems`}
|
||||||
|
aria-label={t`Search systems`}
|
||||||
|
className="h-10 min-w-0 rounded-none border-0 bg-transparent px-0 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0"
|
||||||
|
onKeyDown={(event) => {
|
||||||
|
if (event.key === "Escape") return
|
||||||
|
// Keep menu typeahead and form submission from consuming search input.
|
||||||
|
event.stopPropagation()
|
||||||
|
if (event.key === "Enter") event.preventDefault()
|
||||||
|
if (event.key === "ArrowDown" || event.key === "ArrowUp" || event.key === "Tab") {
|
||||||
|
event.preventDefault()
|
||||||
|
const items = contentRef.current?.querySelectorAll<HTMLElement>(
|
||||||
|
'[role^="menuitem"]:not([data-disabled])'
|
||||||
|
)
|
||||||
|
const index = event.key === "ArrowUp" || event.shiftKey ? (items?.length ?? 1) - 1 : 0
|
||||||
|
items?.[index]?.focus()
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 px-1 pb-1">
|
||||||
|
<div className="flex items-center">
|
||||||
|
<DropdownMenuItem
|
||||||
|
className="px-1.5 py-1 text-xs text-muted-foreground"
|
||||||
|
disabled={!filteredSystems.length || allSelected}
|
||||||
|
onSelect={(event) => {
|
||||||
|
event.preventDefault()
|
||||||
|
selectFiltered(true)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{query ? <Trans>Select matches</Trans> : <Trans>Select all</Trans>}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<span aria-hidden="true" className="text-xs text-muted-foreground/50">
|
||||||
|
·
|
||||||
|
</span>
|
||||||
|
<DropdownMenuItem
|
||||||
|
className="px-1.5 py-1 text-xs text-muted-foreground"
|
||||||
|
disabled={!anySelected}
|
||||||
|
onSelect={(event) => {
|
||||||
|
event.preventDefault()
|
||||||
|
selectFiltered(false)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{query ? <Trans>Clear matches</Trans> : <Trans>Clear all</Trans>}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</div>
|
||||||
|
<span className="px-1.5 text-xs tabular-nums text-muted-foreground">
|
||||||
|
{t`${selectedSystemIds.size} selected`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="min-h-0 overflow-y-auto">
|
||||||
|
{filteredSystems.length === 0 && (
|
||||||
|
<output className="block px-2.5 py-3 text-sm text-muted-foreground">
|
||||||
|
<Trans>No systems found.</Trans>
|
||||||
|
</output>
|
||||||
|
)}
|
||||||
|
{filteredSystems.map((sys) => (
|
||||||
|
<DropdownMenuCheckboxItem
|
||||||
|
key={sys.id}
|
||||||
|
checked={selectedSystemIds.has(sys.id)}
|
||||||
|
onSelect={(event) => event.preventDefault()}
|
||||||
|
onCheckedChange={(checked) => {
|
||||||
|
const next = new Set(selectedSystemIds)
|
||||||
|
if (checked) next.add(sys.id)
|
||||||
|
else next.delete(sys.id)
|
||||||
|
onChange(next)
|
||||||
|
}}
|
||||||
|
className="group min-w-0 gap-2.5 py-2 ps-2.5"
|
||||||
|
indicatorClassName="static size-4 shrink-0 rounded border border-input group-data-[state=checked]:border-primary group-data-[state=checked]:bg-primary group-data-[state=checked]:text-primary-foreground [&_svg]:size-3"
|
||||||
|
>
|
||||||
|
<span className="truncate">{sys.name}</span>
|
||||||
|
</DropdownMenuCheckboxItem>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AddMonitorDialog({ systemId, monitors }: { systemId?: string; monitors: NetworkMonitorRecord[] }) {
|
||||||
|
const [open, setOpen] = useState(false)
|
||||||
|
const [bulkOpen, setBulkOpen] = useState(false)
|
||||||
|
const [bulkInput, setBulkInput] = useState("")
|
||||||
|
const [bulkLoading, setBulkLoading] = useState(false)
|
||||||
|
const [bulkSelectedSystemIds, setBulkSelectedSystemIds] = useState<Set<string>>(new Set())
|
||||||
|
const bulkFormRef = useRef<HTMLFormElement>(null)
|
||||||
|
const { toast } = useToast()
|
||||||
|
const { t } = useLingui()
|
||||||
|
|
||||||
|
const resetBulkForm = () => {
|
||||||
|
setBulkInput("")
|
||||||
|
}
|
||||||
|
|
||||||
|
const openBulkAdd = (selectedSystemIds?: Set<string>) => {
|
||||||
|
if (!systemId && selectedSystemIds) {
|
||||||
|
setBulkSelectedSystemIds(new Set(selectedSystemIds))
|
||||||
|
}
|
||||||
|
setOpen(false)
|
||||||
|
setBulkOpen(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
const openAdd = () => {
|
||||||
|
setBulkOpen(false)
|
||||||
|
setOpen(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleBulkSubmit(e: React.FormEvent) {
|
||||||
|
e.preventDefault()
|
||||||
|
setBulkLoading(true)
|
||||||
|
let closedForSubmit = false
|
||||||
|
|
||||||
|
try {
|
||||||
|
const targetSystems = systemId ? [systemId] : Array.from(bulkSelectedSystemIds)
|
||||||
|
if (!targetSystems.length) {
|
||||||
|
throw new Error("Select at least one system.")
|
||||||
|
}
|
||||||
|
const rawLines = bulkInput.split(/\r?\n/).filter((line) => line.trim())
|
||||||
|
if (!rawLines.length) {
|
||||||
|
throw new Error("Enter at least one monitor.")
|
||||||
|
}
|
||||||
|
|
||||||
|
let totalCreated = 0
|
||||||
|
closedForSubmit = true
|
||||||
|
|
||||||
|
for (const system of targetSystems) {
|
||||||
|
const payloads = rawLines.map((line, index) => parseBulkMonitorLine(line, index + 1, system))
|
||||||
|
const existingMonitorKeys = new Set(
|
||||||
|
monitors.filter((monitor) => monitor.system === system).map((monitor) => getMonitorIdentityKey(monitor))
|
||||||
|
)
|
||||||
|
const newPayloads: typeof payloads = []
|
||||||
|
|
||||||
|
for (const payload of payloads) {
|
||||||
|
const monitorKey = getMonitorIdentityKey(payload)
|
||||||
|
if (existingMonitorKeys.has(monitorKey)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
existingMonitorKeys.add(monitorKey)
|
||||||
|
newPayloads.push(payload)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!newPayloads.length) continue
|
||||||
|
|
||||||
|
let batch = pb.createBatch()
|
||||||
|
let inBatch = 0
|
||||||
|
for (const payload of newPayloads) {
|
||||||
|
batch.collection("network_monitors").create(payload)
|
||||||
|
inBatch++
|
||||||
|
if (inBatch > 20) {
|
||||||
|
await batch.send()
|
||||||
|
batch = pb.createBatch()
|
||||||
|
inBatch = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (inBatch) {
|
||||||
|
await batch.send()
|
||||||
|
}
|
||||||
|
totalCreated += newPayloads.length
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!totalCreated) {
|
||||||
|
throw new Error("No new monitors. All entries already exist.")
|
||||||
|
}
|
||||||
|
|
||||||
|
resetBulkForm()
|
||||||
|
toast({ title: t`Monitors created`, description: `${totalCreated} monitor(s) added.` })
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (closedForSubmit) {
|
||||||
|
setBulkOpen(true)
|
||||||
|
}
|
||||||
|
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
|
||||||
|
} finally {
|
||||||
|
setBulkLoading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div className="flex gap-0 rounded-lg">
|
||||||
|
<Button variant="outline" onClick={openAdd} className="rounded-e-none grow">
|
||||||
|
{/* <PlusIcon className="size-4 me-1" /> */}
|
||||||
|
<Trans>Add {{ foo: t`Monitor` }}</Trans>
|
||||||
|
</Button>
|
||||||
|
<div className="w-px h-full bg-muted"></div>
|
||||||
|
<DropdownMenu>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button variant="outline" className="px-2 rounded-s-none border-s-0" aria-label={`More actions`}>
|
||||||
|
<ChevronDownIcon className="size-4" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent align="end">
|
||||||
|
<DropdownMenuItem onClick={() => openBulkAdd()}>
|
||||||
|
<ListIcon className="size-4 me-2" />
|
||||||
|
<Trans>Bulk Add</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
</div>
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
onOpenChange={(nextOpen) => {
|
||||||
|
setOpen(nextOpen)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} onOpenBulkAdd={openBulkAdd} />
|
||||||
|
</Dialog>
|
||||||
|
|
||||||
|
<Sheet
|
||||||
|
open={bulkOpen}
|
||||||
|
onOpenChange={(nextOpen) => {
|
||||||
|
setBulkOpen(nextOpen)
|
||||||
|
if (!nextOpen) {
|
||||||
|
resetBulkForm()
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<SheetContent className="w-full sm:max-w-xl gap-0">
|
||||||
|
<SheetHeader className="border-b">
|
||||||
|
<SheetTitle>
|
||||||
|
<Trans>Bulk Add {{ foo: t`Network Monitors` }}</Trans>
|
||||||
|
</SheetTitle>
|
||||||
|
<SheetDescription>target[,protocol[,port[,interval]]]</SheetDescription>
|
||||||
|
</SheetHeader>
|
||||||
|
<form ref={bulkFormRef} onSubmit={handleBulkSubmit} className="flex h-full flex-col overflow-hidden">
|
||||||
|
<div className="flex-1 flex flex-col space-y-4 overflow-auto p-4">
|
||||||
|
{!systemId && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label htmlFor="bulk-monitor-systems" className="sr-only">
|
||||||
|
<Trans>Systems</Trans>
|
||||||
|
</Label>
|
||||||
|
<SystemMultiSelect
|
||||||
|
id="bulk-monitor-systems"
|
||||||
|
selectedSystemIds={bulkSelectedSystemIds}
|
||||||
|
onChange={setBulkSelectedSystemIds}
|
||||||
|
disabled={bulkLoading}
|
||||||
|
className="bg-card"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="grow flex flex-col gap-2">
|
||||||
|
<Label htmlFor="bulk-monitors" className="sr-only">
|
||||||
|
Entries
|
||||||
|
</Label>
|
||||||
|
<Textarea
|
||||||
|
id="bulk-monitors"
|
||||||
|
value={bulkInput}
|
||||||
|
onChange={(e) => setBulkInput(e.target.value)}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) {
|
||||||
|
e.preventDefault()
|
||||||
|
bulkFormRef.current?.requestSubmit()
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
className="font-mono grow text-sm bg-card"
|
||||||
|
placeholder={["1.1.1.1", "example.com,tcp", "https://example.com,http,,60"].join("\n")}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
<p className="text-xs text-muted-foreground">target[,protocol[,port[,interval]]]</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<SheetFooter className="border-t">
|
||||||
|
<Button type="submit" disabled={bulkLoading || (!systemId && !bulkSelectedSystemIds.size)}>
|
||||||
|
<Trans>Add {{ foo: t`Network Monitors` }}</Trans>
|
||||||
|
</Button>
|
||||||
|
</SheetFooter>
|
||||||
|
</form>
|
||||||
|
</SheetContent>
|
||||||
|
</Sheet>
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function EditMonitorDialog({
|
||||||
|
open,
|
||||||
|
setOpen,
|
||||||
|
systemId,
|
||||||
|
monitor,
|
||||||
|
}: {
|
||||||
|
open: boolean
|
||||||
|
setOpen: (open: boolean) => void
|
||||||
|
systemId?: string
|
||||||
|
monitor?: NetworkMonitorRecord
|
||||||
|
}) {
|
||||||
|
const hasOpened = useRef(false)
|
||||||
|
if (!monitor && !hasOpened.current) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
hasOpened.current = true
|
||||||
|
return (
|
||||||
|
<Dialog open={open} onOpenChange={setOpen}>
|
||||||
|
<MonitorDialogContent open={open} setOpen={setOpen} systemId={systemId} monitor={monitor} />
|
||||||
|
</Dialog>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function MonitorDialogContent({
|
||||||
|
open,
|
||||||
|
setOpen,
|
||||||
|
systemId,
|
||||||
|
monitor,
|
||||||
|
onOpenBulkAdd,
|
||||||
|
}: {
|
||||||
|
open: boolean
|
||||||
|
setOpen: (open: boolean) => void
|
||||||
|
systemId?: string
|
||||||
|
monitor?: NetworkMonitorRecord
|
||||||
|
onOpenBulkAdd?: (selectedSystemIds: Set<string>) => void
|
||||||
|
}) {
|
||||||
|
const [protocol, setProtocol] = useState<MonitorProtocol>(monitor?.protocol ?? "icmp")
|
||||||
|
const [target, setTarget] = useState(monitor?.target ?? "")
|
||||||
|
const [port, setPort] = useState(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
|
||||||
|
const [monitorInterval, setMonitorInterval] = useState(String(monitor?.interval ?? defaultInterval))
|
||||||
|
const [loading, setLoading] = useState(false)
|
||||||
|
const [selectedSystemId, setSelectedSystemId] = useState(monitor?.system ?? "")
|
||||||
|
const [selectedSystemIds, setSelectedSystemIds] = useState<Set<string>>(new Set())
|
||||||
|
const systems = useStore($systems)
|
||||||
|
const { toast } = useToast()
|
||||||
|
const { t } = useLingui()
|
||||||
|
const isEditing = !!monitor
|
||||||
|
|
||||||
|
// When the dialog is opened, initialize form fields with monitor values (if editing) or defaults (if adding).
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
setProtocol(monitor?.protocol ?? "icmp")
|
||||||
|
setTarget(monitor?.target ?? "")
|
||||||
|
setPort(monitor?.protocol === "tcp" && monitor.port ? String(monitor.port) : "")
|
||||||
|
setMonitorInterval(String(monitor?.interval ?? defaultInterval))
|
||||||
|
setSelectedSystemId(monitor?.system ?? "")
|
||||||
|
setSelectedSystemIds(new Set())
|
||||||
|
setLoading(false)
|
||||||
|
}, [open, monitor])
|
||||||
|
|
||||||
|
async function handleSubmit(e: React.FormEvent) {
|
||||||
|
e.preventDefault()
|
||||||
|
setLoading(true)
|
||||||
|
|
||||||
|
const targetSystems = systemId ? [systemId] : monitor ? [selectedSystemId] : Array.from(selectedSystemIds)
|
||||||
|
const remainingSystemIds = new Set(targetSystems)
|
||||||
|
try {
|
||||||
|
if (!targetSystems.length || !targetSystems[0]) throw new Error("Select at least one system.")
|
||||||
|
const payload = buildMonitorPayload(
|
||||||
|
{
|
||||||
|
system: targetSystems[0],
|
||||||
|
target,
|
||||||
|
protocol,
|
||||||
|
port: protocol === "tcp" ? Number(port) : 0,
|
||||||
|
interval: monitorInterval,
|
||||||
|
},
|
||||||
|
monitor ? monitor.enabled : true
|
||||||
|
)
|
||||||
|
if (monitor) {
|
||||||
|
await pb.collection("network_monitors").update(monitor.id, payload)
|
||||||
|
} else {
|
||||||
|
for (const system of targetSystems) {
|
||||||
|
await pb.collection("network_monitors").create({ ...payload, system })
|
||||||
|
remainingSystemIds.delete(system)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setOpen(false)
|
||||||
|
} catch (err: unknown) {
|
||||||
|
if (!monitor && !systemId) {
|
||||||
|
// Retain only unfinished systems so retrying cannot duplicate successful creates.
|
||||||
|
setSelectedSystemIds(remainingSystemIds)
|
||||||
|
}
|
||||||
|
toast({ variant: "destructive", title: t`Error`, description: (err as Error)?.message })
|
||||||
|
} finally {
|
||||||
|
setLoading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<DialogContent className="max-w-md">
|
||||||
|
<DialogHeader>
|
||||||
|
<DialogTitle>
|
||||||
|
{isEditing ? (
|
||||||
|
<Trans>Edit {{ foo: t`Network Monitor` }}</Trans>
|
||||||
|
) : (
|
||||||
|
<Trans>Add {{ foo: t`Network Monitor` }}</Trans>
|
||||||
|
)}
|
||||||
|
</DialogTitle>
|
||||||
|
<DialogDescription>
|
||||||
|
<Trans>Configure response monitoring from this agent.</Trans>
|
||||||
|
</DialogDescription>
|
||||||
|
</DialogHeader>
|
||||||
|
<form onSubmit={handleSubmit} className="grid gap-4 tabular-nums">
|
||||||
|
{!systemId && !isEditing && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label htmlFor="monitor-systems">
|
||||||
|
<Trans>Systems</Trans>
|
||||||
|
</Label>
|
||||||
|
<SystemMultiSelect
|
||||||
|
id="monitor-systems"
|
||||||
|
selectedSystemIds={selectedSystemIds}
|
||||||
|
onChange={setSelectedSystemIds}
|
||||||
|
disabled={loading}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{!systemId && isEditing && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label>
|
||||||
|
<Trans>System</Trans>
|
||||||
|
</Label>
|
||||||
|
<Select value={selectedSystemId} onValueChange={setSelectedSystemId} required>
|
||||||
|
<SelectTrigger>
|
||||||
|
<SelectValue placeholder={t`Select a system`} />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
{systems
|
||||||
|
.filter((sys) => sys.id === monitor?.system || supportsNetworkMonitors(sys))
|
||||||
|
.map((sys) => (
|
||||||
|
<SelectItem key={sys.id} value={sys.id}>
|
||||||
|
{sys.name}
|
||||||
|
</SelectItem>
|
||||||
|
))}
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label>
|
||||||
|
<Trans>Target</Trans>
|
||||||
|
</Label>
|
||||||
|
<Input
|
||||||
|
value={target}
|
||||||
|
onChange={(e) => setTarget(e.target.value)}
|
||||||
|
placeholder={protocol === "http" ? "http://localhost:8090" : "1.1.1.1"}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label>
|
||||||
|
<Trans>Protocol</Trans>
|
||||||
|
</Label>
|
||||||
|
|
||||||
|
<Select value={protocol} onValueChange={(value) => setProtocol(value as MonitorProtocol)}>
|
||||||
|
<SelectTrigger>
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem value="icmp">ICMP</SelectItem>
|
||||||
|
<SelectItem value="tcp">TCP</SelectItem>
|
||||||
|
<SelectItem value="http">HTTP</SelectItem>
|
||||||
|
<SelectItem value="dns">DNS</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
{protocol === "tcp" && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label>
|
||||||
|
<Trans>Port</Trans>
|
||||||
|
</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
value={port}
|
||||||
|
onChange={(e) => setPort(e.target.value)}
|
||||||
|
placeholder="443"
|
||||||
|
min={1}
|
||||||
|
max={65535}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<Label>
|
||||||
|
<Trans>Interval (seconds)</Trans>
|
||||||
|
</Label>
|
||||||
|
<Input
|
||||||
|
type="number"
|
||||||
|
value={monitorInterval}
|
||||||
|
onChange={(e) => setMonitorInterval(e.target.value)}
|
||||||
|
min={1}
|
||||||
|
max={3600}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<DialogFooter>
|
||||||
|
{!isEditing && onOpenBulkAdd && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
onClick={() => onOpenBulkAdd(selectedSystemIds)}
|
||||||
|
disabled={loading}
|
||||||
|
className="me-auto"
|
||||||
|
>
|
||||||
|
<ListIcon className="size-4 me-2" />
|
||||||
|
<Trans>Bulk Add</Trans>
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
<Button
|
||||||
|
type="submit"
|
||||||
|
disabled={loading || (!systemId && (isEditing ? !selectedSystemId : !selectedSystemIds.size))}
|
||||||
|
>
|
||||||
|
{isEditing ? (
|
||||||
|
<Trans>Save {{ foo: t`Monitor` }}</Trans>
|
||||||
|
) : (
|
||||||
|
<Trans>Add {{ foo: t`Monitor` }}</Trans>
|
||||||
|
)}
|
||||||
|
</Button>
|
||||||
|
</DialogFooter>
|
||||||
|
</form>
|
||||||
|
</DialogContent>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -0,0 +1,411 @@
|
|||||||
|
import type { CellContext, Column, ColumnDef } from "@tanstack/react-table"
|
||||||
|
import { Button } from "@/components/ui/button"
|
||||||
|
import { cn, copyToClipboard, decimalString, formatMicroseconds, hourWithSeconds } from "@/lib/utils"
|
||||||
|
import {
|
||||||
|
GlobeIcon,
|
||||||
|
TimerIcon,
|
||||||
|
WifiOffIcon,
|
||||||
|
Trash2Icon,
|
||||||
|
ArrowLeftRightIcon,
|
||||||
|
MoreHorizontalIcon,
|
||||||
|
ServerIcon,
|
||||||
|
ClockIcon,
|
||||||
|
RefreshCwIcon,
|
||||||
|
PenBoxIcon,
|
||||||
|
PauseCircleIcon,
|
||||||
|
PlayCircleIcon,
|
||||||
|
CopyIcon,
|
||||||
|
CopyPlusIcon,
|
||||||
|
} from "lucide-react"
|
||||||
|
import { t } from "@lingui/core/macro"
|
||||||
|
import type { NetworkMonitorRecord, SystemRecord } from "@/types"
|
||||||
|
import {
|
||||||
|
DropdownMenu,
|
||||||
|
DropdownMenuContent,
|
||||||
|
DropdownMenuItem,
|
||||||
|
DropdownMenuSeparator,
|
||||||
|
DropdownMenuSub,
|
||||||
|
DropdownMenuSubContent,
|
||||||
|
DropdownMenuSubTrigger,
|
||||||
|
DropdownMenuTrigger,
|
||||||
|
} from "@/components/ui/dropdown-menu"
|
||||||
|
import { Trans } from "@lingui/react/macro"
|
||||||
|
import { $allSystemsById, $longestSystemName } from "@/lib/stores"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
import { SystemStatus } from "@/lib/enums"
|
||||||
|
import { Checkbox } from "@/components/ui/checkbox"
|
||||||
|
import { useMemo } from "react"
|
||||||
|
import { formatBulkMonitorLine } from "@/components/network-monitors-table/monitor-dialog"
|
||||||
|
import { Badge } from "../ui/badge"
|
||||||
|
import { getMonitorTarget } from "@/lib/network-monitor-utils"
|
||||||
|
import { pb } from "@/lib/api"
|
||||||
|
|
||||||
|
const protocolColors: Record<string, string> = {
|
||||||
|
icmp: "bg-blue-500/15! text-blue-600 dark:text-blue-400",
|
||||||
|
tcp: "bg-purple-500/15! text-purple-600 dark:text-purple-400",
|
||||||
|
http: "bg-green-500/15! text-green-700 dark:text-green-400",
|
||||||
|
dns: "bg-amber-500/15! text-amber-600 dark:text-amber-400",
|
||||||
|
}
|
||||||
|
|
||||||
|
const SYSTEM_STATUS_COLORS = {
|
||||||
|
[SystemStatus.Up]: "bg-green-500",
|
||||||
|
[SystemStatus.Down]: "bg-red-500",
|
||||||
|
[SystemStatus.Paused]: "bg-primary/40",
|
||||||
|
[SystemStatus.Pending]: "bg-yellow-500",
|
||||||
|
} as const
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A monitor is considered muted if it's disabled or if its associated system is not up.
|
||||||
|
*/
|
||||||
|
const isMuted = (record: NetworkMonitorRecord, systemRecord: SystemRecord | undefined) =>
|
||||||
|
!record.enabled || systemRecord?.status !== SystemStatus.Up
|
||||||
|
|
||||||
|
export function getMonitorColumns(
|
||||||
|
longestTarget = "",
|
||||||
|
{
|
||||||
|
onEdit,
|
||||||
|
onDelete,
|
||||||
|
onSetEnabled,
|
||||||
|
}: {
|
||||||
|
onEdit?: (monitor: NetworkMonitorRecord) => void
|
||||||
|
onDelete?: (monitors: NetworkMonitorRecord[]) => void | Promise<void>
|
||||||
|
onSetEnabled?: (monitors: NetworkMonitorRecord[], enabled: boolean) => void | Promise<void>
|
||||||
|
} = {}
|
||||||
|
): ColumnDef<NetworkMonitorRecord>[] {
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
id: "select",
|
||||||
|
header: ({ table }) => (
|
||||||
|
<Checkbox
|
||||||
|
className="ms-2"
|
||||||
|
checked={table.getIsAllRowsSelected() || (table.getIsSomeRowsSelected() && "indeterminate")}
|
||||||
|
onClick={(event) => event.stopPropagation()}
|
||||||
|
onCheckedChange={(value) => table.toggleAllRowsSelected(!!value)}
|
||||||
|
aria-label={t`Select all`}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
cell: ({ row }) => (
|
||||||
|
<Checkbox
|
||||||
|
checked={row.getIsSelected()}
|
||||||
|
onClick={(event) => event.stopPropagation()}
|
||||||
|
onCheckedChange={(value) => row.toggleSelected(!!value)}
|
||||||
|
aria-label={t`Select row`}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
enableSorting: false,
|
||||||
|
enableHiding: false,
|
||||||
|
size: 44,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "system",
|
||||||
|
accessorFn: (record) => record.system,
|
||||||
|
sortingFn: (a, b) => {
|
||||||
|
const allSystems = $allSystemsById.get()
|
||||||
|
const systemNameA = allSystems[a.original.system]?.name ?? ""
|
||||||
|
const systemNameB = allSystems[b.original.system]?.name ?? ""
|
||||||
|
const primary = systemNameA.localeCompare(systemNameB)
|
||||||
|
if (primary !== 0) {
|
||||||
|
return primary
|
||||||
|
}
|
||||||
|
return a.original.target.localeCompare(b.original.target)
|
||||||
|
},
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`System`} Icon={ServerIcon} />,
|
||||||
|
cell: ({ getValue }) => {
|
||||||
|
const system = useStore($allSystemsById)[getValue() as string] as SystemRecord | undefined
|
||||||
|
const longestSystemName = useStore($longestSystemName)
|
||||||
|
const name = system?.name
|
||||||
|
const status = system?.status as SystemStatus // undefined val is fine but makes lsp mad
|
||||||
|
|
||||||
|
return useMemo(
|
||||||
|
() => (
|
||||||
|
<div className="ms-1.5 max-w-44 flex gap-2 items-center tabular-nums">
|
||||||
|
<span className={cn("shrink-0 size-2 rounded-full", SYSTEM_STATUS_COLORS[status])} />
|
||||||
|
<div className="relative w-fit min-w-0 max-w-full">
|
||||||
|
<span className="invisible block whitespace-nowrap" aria-hidden="true">
|
||||||
|
{longestSystemName}
|
||||||
|
</span>
|
||||||
|
<span className="absolute inset-0 truncate">{name}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
),
|
||||||
|
[status, name]
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "target",
|
||||||
|
sortingFn: (a, b) => a.original.target.localeCompare(b.original.target),
|
||||||
|
accessorFn: (record) => getMonitorTarget(record),
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Target`} Icon={GlobeIcon} />,
|
||||||
|
cell: ({ row, getValue }) => {
|
||||||
|
const monitor = row.original
|
||||||
|
const { status } = useStore($allSystemsById)[monitor.system] || {}
|
||||||
|
|
||||||
|
let color = "bg-green-500"
|
||||||
|
if (!monitor.enabled || status === SystemStatus.Paused) {
|
||||||
|
color = "bg-primary/40"
|
||||||
|
} else if (status === SystemStatus.Down || status === SystemStatus.Pending) {
|
||||||
|
color = "bg-yellow-500"
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className="ms-1.5 max-w-64 flex gap-2 items-center tabular-nums">
|
||||||
|
<span className={cn("shrink-0 size-2 rounded-full", color)} />
|
||||||
|
<div className="relative w-fit min-w-0 max-w-full">
|
||||||
|
<span className="invisible block overflow-hidden whitespace-nowrap" aria-hidden="true">
|
||||||
|
{longestTarget}
|
||||||
|
</span>
|
||||||
|
<span className="absolute inset-0 truncate">{getValue() as string}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "protocol",
|
||||||
|
accessorFn: (record) => record.protocol,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Protocol`} Icon={ArrowLeftRightIcon} />,
|
||||||
|
cell: ({ getValue }) => {
|
||||||
|
const protocol = getValue() as string
|
||||||
|
return <Badge className={cn("uppercase", protocolColors[protocol])}>{protocol}</Badge>
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "interval",
|
||||||
|
accessorFn: (record) => record.interval,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Interval`} Icon={RefreshCwIcon} />,
|
||||||
|
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{getValue() as number}s</span>,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "res",
|
||||||
|
accessorFn: (record) => record.res,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Response`} Icon={TimerIcon} />,
|
||||||
|
cell: responseTimeCell,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "res1h",
|
||||||
|
accessorFn: (record) => record.resAvg1h,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Avg 1h`} Icon={TimerIcon} />,
|
||||||
|
cell: responseTimeCell,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "max1h",
|
||||||
|
accessorFn: (record) => record.resMax1h,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Max 1h`} Icon={TimerIcon} />,
|
||||||
|
cell: responseTimeCell,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "min1h",
|
||||||
|
accessorFn: (record) => record.resMin1h,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Min 1h`} Icon={TimerIcon} />,
|
||||||
|
cell: responseTimeCell,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "loss",
|
||||||
|
accessorFn: (record) => record.loss1h,
|
||||||
|
invertSorting: true,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Loss 1h`} Icon={WifiOffIcon} />,
|
||||||
|
cell: ({ row }) => {
|
||||||
|
const { loss1h, res, system } = row.original
|
||||||
|
const systemRecord = useStore($allSystemsById)[system]
|
||||||
|
|
||||||
|
if (loss1h === undefined || (!res && !loss1h)) {
|
||||||
|
return <span className="ms-1.5 text-muted-foreground">-</span>
|
||||||
|
}
|
||||||
|
|
||||||
|
const muted = isMuted(row.original, systemRecord)
|
||||||
|
let color = "bg-green-500"
|
||||||
|
if (muted) {
|
||||||
|
color = "bg-muted-foreground/50"
|
||||||
|
} else if (loss1h) {
|
||||||
|
color = loss1h > 20 ? "bg-red-500" : "bg-yellow-500"
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
|
||||||
|
<span className={cn("shrink-0 size-2 rounded-full", color)} />
|
||||||
|
{loss1h === 100 ? loss1h : decimalString(loss1h, loss1h >= 10 ? 1 : 2)}%
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "updated",
|
||||||
|
invertSorting: true,
|
||||||
|
accessorFn: (record) => record.updated,
|
||||||
|
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
|
||||||
|
cell: ({ getValue }) => {
|
||||||
|
const timestamp = getValue() as number
|
||||||
|
if (!timestamp) {
|
||||||
|
return <span className="ms-1.5 text-muted-foreground">-</span>
|
||||||
|
}
|
||||||
|
return <span className="ms-1.5 tabular-nums">{hourWithSeconds(timestamp)}</span>
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: "actions",
|
||||||
|
enableSorting: false,
|
||||||
|
enableHiding: false,
|
||||||
|
header: () => null,
|
||||||
|
size: 40,
|
||||||
|
cell: ({ row, table }) => {
|
||||||
|
const selectedRows = table.getSelectedRowModel().rows
|
||||||
|
const actionRows =
|
||||||
|
row.getIsSelected() && selectedRows.length > 1
|
||||||
|
? selectedRows.map((selectedRow) => selectedRow.original)
|
||||||
|
: [row.original]
|
||||||
|
const isBulkAction = actionRows.length > 1
|
||||||
|
const shouldPause = actionRows.some((monitor) => monitor.enabled)
|
||||||
|
const bulkCopyContent = actionRows.map((monitor) => formatBulkMonitorLine(monitor)).join("\n")
|
||||||
|
const allSystems = useStore($allSystemsById)
|
||||||
|
const otherSystems = useMemo(
|
||||||
|
() => Object.values(allSystems).filter((s) => !isBulkAction && s.id !== row.original.system),
|
||||||
|
[allSystems, isBulkAction]
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
<DropdownMenu>
|
||||||
|
<DropdownMenuTrigger asChild>
|
||||||
|
<Button variant="ghost" size="icon" className="size-10">
|
||||||
|
<span className="sr-only">
|
||||||
|
<Trans>Open menu</Trans>
|
||||||
|
</span>
|
||||||
|
<MoreHorizontalIcon className="w-5" />
|
||||||
|
</Button>
|
||||||
|
</DropdownMenuTrigger>
|
||||||
|
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
|
||||||
|
{!isBulkAction && (
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => {
|
||||||
|
onEdit?.(row.original)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<PenBoxIcon className="me-2.5 size-4" />
|
||||||
|
<Trans>Edit</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
|
)}
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => {
|
||||||
|
onSetEnabled?.(actionRows, !shouldPause)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{shouldPause ? (
|
||||||
|
<>
|
||||||
|
<PauseCircleIcon className="me-2.5 size-4" />
|
||||||
|
<Trans>Pause</Trans>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<PlayCircleIcon className="me-2.5 size-4" />
|
||||||
|
<Trans>Resume</Trans>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => {
|
||||||
|
copyToClipboard(bulkCopyContent)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<CopyIcon className="me-2.5 size-4" />
|
||||||
|
<Trans>Bulk copy</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
|
{!isBulkAction && otherSystems.length > 0 && (
|
||||||
|
<DropdownMenuSub>
|
||||||
|
<DropdownMenuSubTrigger>
|
||||||
|
<CopyPlusIcon className="me-2.5 size-4" />
|
||||||
|
<Trans>Copy to system</Trans>
|
||||||
|
</DropdownMenuSubTrigger>
|
||||||
|
<DropdownMenuSubContent className="max-h-[min(20rem,var(--radix-dropdown-menu-content-available-height))] overflow-y-auto">
|
||||||
|
{otherSystems.map((sys) => (
|
||||||
|
<DropdownMenuItem
|
||||||
|
key={sys.id}
|
||||||
|
onClick={() => {
|
||||||
|
const { id: _id, system: _system, ...rest } = row.original
|
||||||
|
pb.collection("network_monitors")
|
||||||
|
.create({ ...rest, system: sys.id })
|
||||||
|
.catch(() => {})
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{sys.name}
|
||||||
|
</DropdownMenuItem>
|
||||||
|
))}
|
||||||
|
</DropdownMenuSubContent>
|
||||||
|
</DropdownMenuSub>
|
||||||
|
)}
|
||||||
|
<DropdownMenuSeparator />
|
||||||
|
<DropdownMenuItem
|
||||||
|
onClick={() => {
|
||||||
|
onDelete?.(actionRows)
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Trash2Icon className="me-2.5 size-4" />
|
||||||
|
<Trans>Delete</Trans>
|
||||||
|
</DropdownMenuItem>
|
||||||
|
</DropdownMenuContent>
|
||||||
|
</DropdownMenu>
|
||||||
|
)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
const responseTimeThresholds = {
|
||||||
|
http: { warning: 800_000, critical: 3_000_000 },
|
||||||
|
tcp: { warning: 500_000, critical: 2_000_000 },
|
||||||
|
icmp: { warning: 100_000, critical: 500_000 },
|
||||||
|
dns: { warning: 150_000, critical: 800_000 },
|
||||||
|
}
|
||||||
|
|
||||||
|
function responseTimeCell(cell: CellContext<NetworkMonitorRecord, unknown>) {
|
||||||
|
const monitor = cell.row.original
|
||||||
|
const systemRecord = useStore($allSystemsById)[monitor.system]
|
||||||
|
const responseTime = cell.getValue() as number | undefined
|
||||||
|
|
||||||
|
if (!responseTime) {
|
||||||
|
return <span className="ms-1.5 text-muted-foreground">-</span>
|
||||||
|
}
|
||||||
|
|
||||||
|
const muted = isMuted(monitor, systemRecord)
|
||||||
|
let color = "bg-green-500"
|
||||||
|
if (muted) {
|
||||||
|
color = "bg-muted-foreground/50"
|
||||||
|
} else if (responseTime > responseTimeThresholds[monitor.protocol].warning) {
|
||||||
|
color = "bg-yellow-500"
|
||||||
|
}
|
||||||
|
if (!muted && responseTime > responseTimeThresholds[monitor.protocol].critical) {
|
||||||
|
color = "bg-red-500"
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<span className="ms-1.5 tabular-nums flex gap-2 items-center">
|
||||||
|
<span className={cn("shrink-0 size-2 rounded-full", color)} />
|
||||||
|
{formatMicroseconds(responseTime)}
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function HeaderButton({
|
||||||
|
column,
|
||||||
|
name,
|
||||||
|
Icon,
|
||||||
|
}: {
|
||||||
|
column: Column<NetworkMonitorRecord>
|
||||||
|
name: string
|
||||||
|
Icon: React.ElementType
|
||||||
|
}) {
|
||||||
|
const isSorted = column.getIsSorted()
|
||||||
|
return (
|
||||||
|
<Button
|
||||||
|
className={cn(
|
||||||
|
"h-9 px-3 flex items-center gap-2 duration-50",
|
||||||
|
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
|
||||||
|
)}
|
||||||
|
variant="ghost"
|
||||||
|
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
|
||||||
|
>
|
||||||
|
{Icon && <Icon className="size-4" />}
|
||||||
|
{name}
|
||||||
|
</Button>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -0,0 +1,574 @@
|
|||||||
|
import { getMonitorTarget } from "@/lib/network-monitor-utils"
|
||||||
|
import { t } from "@lingui/core/macro"
|
||||||
|
import { Trans } from "@lingui/react/macro"
|
||||||
|
import {
|
||||||
|
type ColumnFiltersState,
|
||||||
|
flexRender,
|
||||||
|
getCoreRowModel,
|
||||||
|
getFilteredRowModel,
|
||||||
|
getSortedRowModel,
|
||||||
|
type Row,
|
||||||
|
type RowSelectionState,
|
||||||
|
type SortingState,
|
||||||
|
type Table as TableType,
|
||||||
|
useReactTable,
|
||||||
|
type VisibilityState,
|
||||||
|
} from "@tanstack/react-table"
|
||||||
|
import { useVirtualizer, type VirtualItem } from "@tanstack/react-virtual"
|
||||||
|
import {
|
||||||
|
AlertDialog,
|
||||||
|
AlertDialogAction,
|
||||||
|
AlertDialogCancel,
|
||||||
|
AlertDialogContent,
|
||||||
|
AlertDialogDescription,
|
||||||
|
AlertDialogFooter,
|
||||||
|
AlertDialogHeader,
|
||||||
|
AlertDialogTitle,
|
||||||
|
} from "@/components/ui/alert-dialog"
|
||||||
|
import { Button, buttonVariants } from "@/components/ui/button"
|
||||||
|
import { memo, useCallback, useMemo, useRef, useState } from "react"
|
||||||
|
import { getMonitorColumns } from "@/components/network-monitors-table/network-monitors-columns"
|
||||||
|
import { Card, CardHeader, CardTitle } from "@/components/ui/card"
|
||||||
|
import { Input } from "@/components/ui/input"
|
||||||
|
import { TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
|
||||||
|
import { useToast } from "@/components/ui/use-toast"
|
||||||
|
import { isReadOnlyUser } from "@/lib/api"
|
||||||
|
import { pb } from "@/lib/api"
|
||||||
|
import { $allSystemsById, $direction, $userSettings } from "@/lib/stores"
|
||||||
|
import {
|
||||||
|
cn,
|
||||||
|
isVisuallyLonger,
|
||||||
|
matchesFilterGroups,
|
||||||
|
parseFilterGroups,
|
||||||
|
parseSemVer,
|
||||||
|
useBrowserStorage,
|
||||||
|
} from "@/lib/utils"
|
||||||
|
import type { ChartData, NetworkMonitorRecord } from "@/types"
|
||||||
|
import { AddMonitorDialog, EditMonitorDialog } from "./monitor-dialog"
|
||||||
|
import { ArrowLeftRightIcon, EthernetPortIcon, LoaderCircleIcon, ServerIcon, XIcon } from "lucide-react"
|
||||||
|
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
|
||||||
|
import ChartTimeSelect from "@/components/charts/chart-time-select"
|
||||||
|
import { LossChart, AvgMinMaxResponseChart } from "@/components/routes/system/charts/monitors-charts"
|
||||||
|
import { useNetworkMonitorStats } from "@/lib/use-network-monitors"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
import { atom } from "nanostores"
|
||||||
|
import { Separator } from "../ui/separator"
|
||||||
|
import { $router, Link } from "../router"
|
||||||
|
import { getPagePath } from "@nanostores/router"
|
||||||
|
|
||||||
|
export default function NetworkMonitorsTableNew({
|
||||||
|
systemId,
|
||||||
|
monitors,
|
||||||
|
isLoading,
|
||||||
|
}: {
|
||||||
|
systemId?: string
|
||||||
|
monitors: NetworkMonitorRecord[]
|
||||||
|
isLoading: boolean
|
||||||
|
}) {
|
||||||
|
const [sorting, setSorting] = useBrowserStorage<SortingState>(
|
||||||
|
`sort-np-target-${systemId ? 1 : 0}`,
|
||||||
|
[{ id: systemId ? "target" : "system", desc: false }],
|
||||||
|
sessionStorage
|
||||||
|
)
|
||||||
|
const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([])
|
||||||
|
const [columnVisibility, setColumnVisibility] = useState<VisibilityState>({})
|
||||||
|
const [rowSelection, setRowSelection] = useState<RowSelectionState>({})
|
||||||
|
const [globalFilter, setGlobalFilter] = useState("")
|
||||||
|
const [deleteOpen, setDeleteOpen] = useState(false)
|
||||||
|
const [pendingDeleteIds, setPendingDeleteIds] = useState<string[]>([])
|
||||||
|
const [editingMonitor, setEditingMonitor] = useState<NetworkMonitorRecord>()
|
||||||
|
|
||||||
|
const { toast } = useToast()
|
||||||
|
const canManageMonitors = !isReadOnlyUser()
|
||||||
|
|
||||||
|
const longestTarget = useMemo(() => {
|
||||||
|
let longestTarget = ""
|
||||||
|
for (const p of monitors) {
|
||||||
|
if (isVisuallyLonger(getMonitorTarget(p), longestTarget)) {
|
||||||
|
longestTarget = getMonitorTarget(p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return longestTarget
|
||||||
|
}, [monitors])
|
||||||
|
|
||||||
|
const runMonitorBatch = useCallback(
|
||||||
|
async (ids: string[], enqueue: (batch: ReturnType<typeof pb.createBatch>, id: string) => void) => {
|
||||||
|
let batch = pb.createBatch()
|
||||||
|
let inBatch = 0
|
||||||
|
for (const id of ids) {
|
||||||
|
enqueue(batch, id)
|
||||||
|
if (++inBatch >= 20) {
|
||||||
|
await batch.send()
|
||||||
|
batch = pb.createBatch()
|
||||||
|
inBatch = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (inBatch) {
|
||||||
|
await batch.send()
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[]
|
||||||
|
)
|
||||||
|
|
||||||
|
const handleDeleteRequest = useCallback(
|
||||||
|
async (monitorsToDelete: NetworkMonitorRecord[]) => {
|
||||||
|
if (!monitorsToDelete.length) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const ids = monitorsToDelete.map((monitor) => monitor.id)
|
||||||
|
if (ids.length === 1) {
|
||||||
|
try {
|
||||||
|
await pb.collection("network_monitors").delete(ids[0])
|
||||||
|
} catch (err: unknown) {
|
||||||
|
toast({
|
||||||
|
variant: "destructive",
|
||||||
|
title: t`Error`,
|
||||||
|
description: (err as Error)?.message || t`Failed to delete monitors.`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
setPendingDeleteIds(ids)
|
||||||
|
setDeleteOpen(true)
|
||||||
|
},
|
||||||
|
[toast]
|
||||||
|
)
|
||||||
|
|
||||||
|
const handleBulkDelete = async () => {
|
||||||
|
setDeleteOpen(false)
|
||||||
|
if (!pendingDeleteIds.length) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await runMonitorBatch(pendingDeleteIds, (batch, id) => batch.collection("network_monitors").delete(id))
|
||||||
|
setPendingDeleteIds([])
|
||||||
|
setRowSelection({})
|
||||||
|
} catch (err: unknown) {
|
||||||
|
toast({
|
||||||
|
variant: "destructive",
|
||||||
|
title: t`Error`,
|
||||||
|
description: (err as Error)?.message || t`Failed to delete monitors.`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const handleSetEnabled = useCallback(
|
||||||
|
async (monitorsToUpdate: NetworkMonitorRecord[], enabled: boolean) => {
|
||||||
|
if (!monitorsToUpdate.length) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const pendingUpdates = monitorsToUpdate.filter((monitor) => monitor.enabled !== enabled)
|
||||||
|
if (!pendingUpdates.length) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (pendingUpdates.length === 1) {
|
||||||
|
await pb.collection("network_monitors").update(pendingUpdates[0].id, { enabled })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await runMonitorBatch(
|
||||||
|
pendingUpdates.map((monitor) => monitor.id),
|
||||||
|
(batch, id) => batch.collection("network_monitors").update(id, { enabled })
|
||||||
|
)
|
||||||
|
if (monitorsToUpdate.length > 1) {
|
||||||
|
setRowSelection({})
|
||||||
|
}
|
||||||
|
} catch (err: unknown) {
|
||||||
|
toast({
|
||||||
|
variant: "destructive",
|
||||||
|
title: t`Error`,
|
||||||
|
description: (err as Error)?.message || t`Failed to update monitors.`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[runMonitorBatch, toast]
|
||||||
|
)
|
||||||
|
|
||||||
|
const columns = useMemo(() => {
|
||||||
|
let columns = getMonitorColumns(longestTarget, {
|
||||||
|
onEdit: setEditingMonitor,
|
||||||
|
onDelete: handleDeleteRequest,
|
||||||
|
onSetEnabled: handleSetEnabled,
|
||||||
|
})
|
||||||
|
columns = systemId ? columns.filter((col) => col.id !== "system") : columns
|
||||||
|
columns = canManageMonitors ? columns : columns.filter((col) => col.id !== "actions")
|
||||||
|
return columns
|
||||||
|
}, [canManageMonitors, handleDeleteRequest, handleSetEnabled, systemId, longestTarget])
|
||||||
|
|
||||||
|
const table = useReactTable({
|
||||||
|
data: monitors,
|
||||||
|
columns,
|
||||||
|
getRowId: (row) => row.id,
|
||||||
|
getCoreRowModel: getCoreRowModel(),
|
||||||
|
getSortedRowModel: getSortedRowModel(),
|
||||||
|
getFilteredRowModel: getFilteredRowModel(),
|
||||||
|
onSortingChange: setSorting,
|
||||||
|
onColumnFiltersChange: setColumnFilters,
|
||||||
|
onColumnVisibilityChange: setColumnVisibility,
|
||||||
|
onRowSelectionChange: setRowSelection,
|
||||||
|
defaultColumn: {
|
||||||
|
sortUndefined: "last",
|
||||||
|
size: 900,
|
||||||
|
minSize: 0,
|
||||||
|
},
|
||||||
|
state: {
|
||||||
|
sorting,
|
||||||
|
columnFilters,
|
||||||
|
columnVisibility,
|
||||||
|
rowSelection,
|
||||||
|
globalFilter,
|
||||||
|
},
|
||||||
|
onGlobalFilterChange: setGlobalFilter,
|
||||||
|
globalFilterFn: (row, _columnId, filterValue) => {
|
||||||
|
const value = (filterValue as string).trim()
|
||||||
|
if (!value) return true
|
||||||
|
const monitor = row.original
|
||||||
|
const systemName = $allSystemsById.get()[monitor.system]?.name ?? ""
|
||||||
|
const searchString = `${getMonitorTarget(monitor)}${monitor.protocol}${systemName}`.toLocaleLowerCase()
|
||||||
|
return matchesFilterGroups(searchString, parseFilterGroups(value))
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const rows = table.getRowModel().rows
|
||||||
|
const visibleColumns = table.getVisibleLeafColumns()
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
|
||||||
|
<CardHeader className="p-0 mb-3 sm:mb-4">
|
||||||
|
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
|
||||||
|
<div className="px-2 sm:px-1">
|
||||||
|
<CardTitle className="mb-2">
|
||||||
|
<Trans>Network Monitors</Trans>
|
||||||
|
</CardTitle>
|
||||||
|
<div className="text-sm text-muted-foreground flex items-center flex-wrap">
|
||||||
|
<Trans>Response time monitoring from agents.</Trans>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="md:ms-auto flex items-center gap-2">
|
||||||
|
{monitors.length > 0 && (
|
||||||
|
<div className="relative">
|
||||||
|
<Input
|
||||||
|
placeholder={t`Filter...`}
|
||||||
|
value={globalFilter}
|
||||||
|
onChange={(e) => setGlobalFilter(e.target.value)}
|
||||||
|
className="ms-auto px-4 w-full max-w-full md:w-50"
|
||||||
|
/>
|
||||||
|
{globalFilter && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="ghost"
|
||||||
|
size="icon"
|
||||||
|
aria-label={t`Clear`}
|
||||||
|
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
|
||||||
|
onClick={() => setGlobalFilter("")}
|
||||||
|
>
|
||||||
|
<XIcon className="h-4 w-4" />
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{canManageMonitors ? <AddMonitorDialog systemId={systemId} monitors={monitors} /> : null}
|
||||||
|
{canManageMonitors ? (
|
||||||
|
<EditMonitorDialog
|
||||||
|
systemId={systemId}
|
||||||
|
monitor={editingMonitor}
|
||||||
|
open={!!editingMonitor}
|
||||||
|
setOpen={(open) => {
|
||||||
|
if (!open) {
|
||||||
|
setEditingMonitor(undefined)
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
|
<AlertDialog
|
||||||
|
open={deleteOpen}
|
||||||
|
onOpenChange={(open) => {
|
||||||
|
setDeleteOpen(open)
|
||||||
|
if (!open) {
|
||||||
|
setPendingDeleteIds([])
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<AlertDialogContent>
|
||||||
|
<AlertDialogHeader>
|
||||||
|
<AlertDialogTitle>
|
||||||
|
<Trans>Are you sure?</Trans>
|
||||||
|
</AlertDialogTitle>
|
||||||
|
<AlertDialogDescription>
|
||||||
|
<Trans>This will permanently delete all selected records from the database.</Trans>
|
||||||
|
</AlertDialogDescription>
|
||||||
|
</AlertDialogHeader>
|
||||||
|
<AlertDialogFooter>
|
||||||
|
<AlertDialogCancel>
|
||||||
|
<Trans>Cancel</Trans>
|
||||||
|
</AlertDialogCancel>
|
||||||
|
<AlertDialogAction
|
||||||
|
className={cn(buttonVariants({ variant: "destructive" }))}
|
||||||
|
onClick={handleBulkDelete}
|
||||||
|
>
|
||||||
|
<Trans>Continue</Trans>
|
||||||
|
</AlertDialogAction>
|
||||||
|
</AlertDialogFooter>
|
||||||
|
</AlertDialogContent>
|
||||||
|
</AlertDialog>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</CardHeader>
|
||||||
|
<div className="rounded-md">
|
||||||
|
<NetworkMonitorsTable
|
||||||
|
table={table}
|
||||||
|
rows={rows}
|
||||||
|
colLength={visibleColumns.length}
|
||||||
|
rowSelection={rowSelection}
|
||||||
|
isLoading={isLoading}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</Card>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const NetworkMonitorsTable = memo(function NetworkMonitorTable({
|
||||||
|
table,
|
||||||
|
rows,
|
||||||
|
colLength,
|
||||||
|
rowSelection,
|
||||||
|
isLoading,
|
||||||
|
}: {
|
||||||
|
table: TableType<NetworkMonitorRecord>
|
||||||
|
rows: Row<NetworkMonitorRecord>[]
|
||||||
|
colLength: number
|
||||||
|
rowSelection: RowSelectionState
|
||||||
|
isLoading: boolean
|
||||||
|
}) {
|
||||||
|
const scrollRef = useRef<HTMLDivElement>(null)
|
||||||
|
const [sheetOpen, setSheetOpen] = useState(false)
|
||||||
|
const [activeMonitorId, setActiveMonitorId] = useState<string | null>(null)
|
||||||
|
const activeMonitor = activeMonitorId
|
||||||
|
? table.options.data.find((monitor) => monitor.id === activeMonitorId)
|
||||||
|
: undefined
|
||||||
|
const openSheet = useCallback((monitor: NetworkMonitorRecord) => {
|
||||||
|
setActiveMonitorId(monitor.id)
|
||||||
|
setSheetOpen(true)
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
const virtualizer = useVirtualizer<HTMLDivElement, HTMLTableRowElement>({
|
||||||
|
count: rows.length,
|
||||||
|
estimateSize: () => 54,
|
||||||
|
getScrollElement: () => scrollRef.current,
|
||||||
|
overscan: 5,
|
||||||
|
})
|
||||||
|
const virtualRows = virtualizer.getVirtualItems()
|
||||||
|
|
||||||
|
const paddingTop = Math.max(0, virtualRows[0]?.start ?? 0 - virtualizer.options.scrollMargin)
|
||||||
|
const paddingBottom = Math.max(0, virtualizer.getTotalSize() - (virtualRows[virtualRows.length - 1]?.end ?? 0))
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={cn(
|
||||||
|
"h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto border rounded-md",
|
||||||
|
(!rows.length || rows.length > 2) && "min-h-50"
|
||||||
|
)}
|
||||||
|
ref={scrollRef}
|
||||||
|
>
|
||||||
|
<div style={{ height: `${virtualizer.getTotalSize() + 48}px`, paddingTop, paddingBottom }}>
|
||||||
|
<table className="text-sm w-full h-full text-nowrap">
|
||||||
|
<NetworkMonitorTableHead table={table} />
|
||||||
|
<TableBody>
|
||||||
|
{rows.length ? (
|
||||||
|
virtualRows.map((virtualRow) => {
|
||||||
|
const row = rows[virtualRow.index]
|
||||||
|
return (
|
||||||
|
<NetworkMonitorTableRow
|
||||||
|
key={row.id}
|
||||||
|
row={row}
|
||||||
|
virtualRow={virtualRow}
|
||||||
|
isSelected={row.getIsSelected()}
|
||||||
|
rowSelection={rowSelection}
|
||||||
|
openSheet={openSheet}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
})
|
||||||
|
) : (
|
||||||
|
<TableRow>
|
||||||
|
<TableCell colSpan={colLength} className="h-37 text-center pointer-events-none">
|
||||||
|
{isLoading ? (
|
||||||
|
<LoaderCircleIcon className="animate-spin size-10 opacity-60 mx-auto" />
|
||||||
|
) : (
|
||||||
|
<Trans>No results.</Trans>
|
||||||
|
)}
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
)}
|
||||||
|
</TableBody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
<NetworkMonitorSheet
|
||||||
|
open={sheetOpen}
|
||||||
|
onOpenChange={(nextOpen) => {
|
||||||
|
setSheetOpen(nextOpen)
|
||||||
|
}}
|
||||||
|
monitor={activeMonitor}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
function NetworkMonitorTableHead({ table }: { table: TableType<NetworkMonitorRecord> }) {
|
||||||
|
return (
|
||||||
|
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
|
||||||
|
{table.getHeaderGroups().map((headerGroup) => (
|
||||||
|
<tr key={headerGroup.id}>
|
||||||
|
{headerGroup.headers.map((header) => {
|
||||||
|
return (
|
||||||
|
<TableHead className="px-2" key={header.id}>
|
||||||
|
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
|
||||||
|
</TableHead>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</TableHeader>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const NetworkMonitorTableRow = memo(function NetworkMonitorTableRow({
|
||||||
|
row,
|
||||||
|
virtualRow,
|
||||||
|
isSelected,
|
||||||
|
rowSelection: _rowSelection,
|
||||||
|
openSheet,
|
||||||
|
}: {
|
||||||
|
row: Row<NetworkMonitorRecord>
|
||||||
|
virtualRow: VirtualItem
|
||||||
|
isSelected: boolean
|
||||||
|
// Menus depend on the entire selection, including changes to other rows.
|
||||||
|
rowSelection: RowSelectionState
|
||||||
|
openSheet: (monitor: NetworkMonitorRecord) => void
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<TableRow
|
||||||
|
data-state={isSelected && "selected"}
|
||||||
|
className="cursor-pointer transition-opacity"
|
||||||
|
onClick={() => openSheet(row.original)}
|
||||||
|
>
|
||||||
|
{row.getVisibleCells().map((cell) => (
|
||||||
|
<TableCell
|
||||||
|
key={cell.id}
|
||||||
|
className="py-0"
|
||||||
|
style={{
|
||||||
|
width: `${cell.column.getSize()}px`,
|
||||||
|
height: virtualRow.size,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{flexRender(cell.column.columnDef.cell, cell.getContext())}
|
||||||
|
</TableCell>
|
||||||
|
))}
|
||||||
|
</TableRow>
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
function NetworkMonitorSheet({
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
monitor,
|
||||||
|
}: {
|
||||||
|
open: boolean
|
||||||
|
onOpenChange: (open: boolean) => void
|
||||||
|
monitor?: NetworkMonitorRecord
|
||||||
|
}) {
|
||||||
|
if (!monitor) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
return <NetworkMonitorSheetContent key={monitor.system} open={open} onOpenChange={onOpenChange} monitor={monitor} />
|
||||||
|
}
|
||||||
|
|
||||||
|
function NetworkMonitorSheetContent({
|
||||||
|
open,
|
||||||
|
onOpenChange,
|
||||||
|
monitor,
|
||||||
|
}: {
|
||||||
|
open: boolean
|
||||||
|
onOpenChange: (open: boolean) => void
|
||||||
|
monitor: NetworkMonitorRecord
|
||||||
|
}) {
|
||||||
|
// Keep monitor exploration independent of the system charts' time range.
|
||||||
|
const [chartTimeStore] = useState(() => {
|
||||||
|
const defaultTime = $userSettings.get().chartTime
|
||||||
|
return atom(defaultTime === "1m" ? "1h" : defaultTime)
|
||||||
|
})
|
||||||
|
const chartTime = useStore(chartTimeStore)
|
||||||
|
const direction = useStore($direction)
|
||||||
|
const system = useStore($allSystemsById)[monitor.system]
|
||||||
|
|
||||||
|
const monitorStats = useNetworkMonitorStats({
|
||||||
|
systemId: monitor.system,
|
||||||
|
monitorId: monitor.id,
|
||||||
|
chartTime,
|
||||||
|
enabled: open,
|
||||||
|
})
|
||||||
|
|
||||||
|
const chartData = useMemo<ChartData>(
|
||||||
|
() => ({
|
||||||
|
agentVersion: parseSemVer(system?.info?.v),
|
||||||
|
orientation: direction === "rtl" ? "right" : "left",
|
||||||
|
chartTime,
|
||||||
|
}),
|
||||||
|
[system?.info?.v, direction, chartTime]
|
||||||
|
)
|
||||||
|
const hasMonitorStats = monitorStats.some((record) => record.stats?.[monitor.id] != null)
|
||||||
|
const monitorLabel = getMonitorTarget(monitor)
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||||
|
<SheetContent className="w-full sm:max-w-220 overflow-auto p-4 sm:p-6">
|
||||||
|
<SheetHeader className="mb-0 border-b p-0 pb-4">
|
||||||
|
<SheetTitle>{monitorLabel}</SheetTitle>
|
||||||
|
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
|
||||||
|
<ServerIcon className="size-3.5 text-muted-foreground" />
|
||||||
|
<Link className="hover:underline" href={getPagePath($router, "system", { id: system?.id ?? "" })}>
|
||||||
|
{system?.name ?? ""}
|
||||||
|
</Link>
|
||||||
|
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||||
|
<ArrowLeftRightIcon className="size-3.5 text-muted-foreground" />
|
||||||
|
{monitor.protocol.toUpperCase()}
|
||||||
|
{monitor.protocol === "tcp" && monitor.port > 0 && (
|
||||||
|
<>
|
||||||
|
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||||
|
<EthernetPortIcon className="size-3.5 text-muted-foreground" />
|
||||||
|
<span>{monitor.port}</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</SheetDescription>
|
||||||
|
</SheetHeader>
|
||||||
|
<div className="grid gap-4">
|
||||||
|
<ChartTimeSelect
|
||||||
|
className="bg-card"
|
||||||
|
agentVersion={chartData.agentVersion}
|
||||||
|
chartTimeStore={chartTimeStore}
|
||||||
|
allowRealtime={false}
|
||||||
|
/>
|
||||||
|
<AvgMinMaxResponseChart
|
||||||
|
monitorStats={monitorStats}
|
||||||
|
monitor={monitor}
|
||||||
|
chartData={chartData}
|
||||||
|
empty={!hasMonitorStats}
|
||||||
|
/>
|
||||||
|
<LossChart
|
||||||
|
monitorStats={monitorStats}
|
||||||
|
grid={false}
|
||||||
|
monitors={[monitor]}
|
||||||
|
chartData={chartData}
|
||||||
|
empty={!hasMonitorStats}
|
||||||
|
showFilter={false}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</SheetContent>
|
||||||
|
</Sheet>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ const routes = {
|
|||||||
home: "/",
|
home: "/",
|
||||||
containers: "/containers",
|
containers: "/containers",
|
||||||
smart: "/smart",
|
smart: "/smart",
|
||||||
|
monitors: "/monitors",
|
||||||
system: `/system/:id`,
|
system: `/system/:id`,
|
||||||
settings: `/settings/:name?`,
|
settings: `/settings/:name?`,
|
||||||
forgot_password: `/forgot-password`,
|
forgot_password: `/forgot-password`,
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
import { useLingui } from "@lingui/react/macro"
|
||||||
|
import { memo, useEffect } from "react"
|
||||||
|
import NetworkMonitorsTableNew from "@/components/network-monitors-table/network-monitors-table"
|
||||||
|
import { ActiveAlerts } from "@/components/active-alerts"
|
||||||
|
import { FooterRepoLink } from "@/components/footer-repo-link"
|
||||||
|
import { useNetworkMonitors } from "@/lib/use-network-monitors"
|
||||||
|
import { $allSystemsById } from "@/lib/stores"
|
||||||
|
import { supportsNetworkMonitors } from "@/lib/utils"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
|
||||||
|
export default memo(() => {
|
||||||
|
const { t } = useLingui()
|
||||||
|
const { monitors, isLoading } = useNetworkMonitors({})
|
||||||
|
const systems = useStore($allSystemsById)
|
||||||
|
const visibleMonitors = monitors.filter((monitor) => {
|
||||||
|
const system = systems[monitor.system]
|
||||||
|
return !system || supportsNetworkMonitors(system)
|
||||||
|
})
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
document.title = `${t`Network Monitors`} / Beszel`
|
||||||
|
}, [t])
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div className="grid gap-4">
|
||||||
|
<ActiveAlerts />
|
||||||
|
<NetworkMonitorsTableNew monitors={visibleMonitors} isLoading={isLoading} />
|
||||||
|
</div>
|
||||||
|
<FooterRepoLink />
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
})
|
||||||
@@ -78,7 +78,7 @@ export default function AlertsHistoryDataTable() {
|
|||||||
let unsubscribe: (() => void) | undefined
|
let unsubscribe: (() => void) | undefined
|
||||||
const pbOptions = {
|
const pbOptions = {
|
||||||
expand: "system",
|
expand: "system",
|
||||||
fields: "id,name,value,state,created,resolved,expand.system.name",
|
fields: "id,name,monitor_name,value,state,created,resolved,expand.system.name",
|
||||||
}
|
}
|
||||||
// Initial load
|
// Initial load
|
||||||
pb.collection<AlertsHistoryRecord>("alerts_history")
|
pb.collection<AlertsHistoryRecord>("alerts_history")
|
||||||
@@ -199,7 +199,7 @@ export default function AlertsHistoryDataTable() {
|
|||||||
if (!selectedRows.length) return
|
if (!selectedRows.length) return
|
||||||
const cells: Record<string, (record: AlertsHistoryRecord) => string> = {
|
const cells: Record<string, (record: AlertsHistoryRecord) => string> = {
|
||||||
system: (record) => record.expand?.system?.name || record.system,
|
system: (record) => record.expand?.system?.name || record.system,
|
||||||
name: (record) => alertInfo[record.name]?.name() || record.name,
|
name: (record) => [alertInfo[record.name]?.name() || record.name, record.monitor_name].filter(Boolean).join(": "),
|
||||||
value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),
|
value: (record) => record.value + (alertInfo[record.name]?.unit ?? ""),
|
||||||
state: (record) => (record.resolved ? t`Resolved` : t`Active`),
|
state: (record) => (record.resolved ? t`Resolved` : t`Active`),
|
||||||
created: (record) => formatShortDate(record.created),
|
created: (record) => formatShortDate(record.created),
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { memo, useState } from "react"
|
import { memo, useState } from "react"
|
||||||
import { Trans } from "@lingui/react/macro"
|
import { Trans } from "@lingui/react/macro"
|
||||||
import { compareSemVer, parseSemVer } from "@/lib/utils"
|
import { compareSemVer, parseSemVer, supportsNetworkMonitors } from "@/lib/utils"
|
||||||
import type { GPUData } from "@/types"
|
import type { GPUData } from "@/types"
|
||||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
||||||
import InfoBar from "./system/info-bar"
|
import InfoBar from "./system/info-bar"
|
||||||
@@ -12,9 +12,15 @@ import { ZfsCharts } from "./system/charts/storage-pool-charts"
|
|||||||
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
|
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
|
||||||
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
|
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
|
||||||
import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"
|
import { GpuPowerChart, GpuCharts } from "./system/charts/gpu-charts"
|
||||||
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
|
import {
|
||||||
|
LazyContainersTable,
|
||||||
|
LazyNetworkMonitorsTable,
|
||||||
|
LazySmartTable,
|
||||||
|
LazySystemdTable,
|
||||||
|
LazyZfsTable,
|
||||||
|
} from "./system/lazy-tables"
|
||||||
import { LoadAverageChart } from "./system/charts/load-average-chart"
|
import { LoadAverageChart } from "./system/charts/load-average-chart"
|
||||||
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
|
import { ContainerIcon, CpuIcon, HardDriveIcon, NetworkIcon, TerminalSquareIcon } from "lucide-react"
|
||||||
import { GpuIcon } from "../ui/icons"
|
import { GpuIcon } from "../ui/icons"
|
||||||
import SystemdTable from "../systemd-table/systemd-table"
|
import SystemdTable from "../systemd-table/systemd-table"
|
||||||
import ContainersTable from "../containers-table/containers-table"
|
import ContainersTable from "../containers-table/containers-table"
|
||||||
@@ -65,9 +71,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
const hasSystemd = system.info.sv
|
const hasSystemd = system.info.sv
|
||||||
const hasGpu = hasGpuData || hasGpuPowerData
|
const hasGpu = hasGpuData || hasGpuPowerData
|
||||||
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
|
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
|
||||||
|
const hasNetworkMonitors = supportsNetworkMonitors(system)
|
||||||
|
|
||||||
// keep tabsRef in sync for keyboard navigation
|
// keep tabsRef in sync for keyboard navigation
|
||||||
const tabs = ["core", "disk"]
|
const tabs = ["core", "network", "disk"]
|
||||||
if (hasGpu) tabs.push("gpu")
|
if (hasGpu) tabs.push("gpu")
|
||||||
if (hasContainers) tabs.push("containers")
|
if (hasContainers) tabs.push("containers")
|
||||||
if (hasSystemd) tabs.push("services")
|
if (hasSystemd) tabs.push("services")
|
||||||
@@ -153,6 +160,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
|
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
|
||||||
|
|
||||||
{hasSystemd && <LazySystemdTable systemId={system.id} />}
|
{hasSystemd && <LazySystemdTable systemId={system.id} />}
|
||||||
|
|
||||||
|
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
|
||||||
</>
|
</>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -165,6 +174,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
<CpuIcon className="size-3.5" />
|
<CpuIcon className="size-3.5" />
|
||||||
<Trans context="Core system metrics">Core</Trans>
|
<Trans context="Core system metrics">Core</Trans>
|
||||||
</TabsTrigger>
|
</TabsTrigger>
|
||||||
|
<TabsTrigger value="network" className="w-full flex items-center gap-1.5">
|
||||||
|
<NetworkIcon className="size-3.5" />
|
||||||
|
<Trans>Network</Trans>
|
||||||
|
</TabsTrigger>
|
||||||
<TabsTrigger value="disk" className="w-full flex items-center gap-1.5">
|
<TabsTrigger value="disk" className="w-full flex items-center gap-1.5">
|
||||||
<HardDriveIcon className="size-3.5" />
|
<HardDriveIcon className="size-3.5" />
|
||||||
<Trans>Disk</Trans>
|
<Trans>Disk</Trans>
|
||||||
@@ -192,17 +205,27 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
|||||||
<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}>
|
<TabsContent value="core" forceMount className={activeTab === "core" ? "contents" : "hidden"}>
|
||||||
<div className="grid xl:grid-cols-2 gap-4">
|
<div className="grid xl:grid-cols-2 gap-4">
|
||||||
<CpuChart {...coreProps} />
|
<CpuChart {...coreProps} />
|
||||||
<MemoryChart {...coreProps} />
|
|
||||||
<LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />
|
<LoadAverageChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} />
|
||||||
<BandwidthChart {...coreProps} systemStats={systemStats} />
|
<MemoryChart {...coreProps} />
|
||||||
|
<SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
|
||||||
<TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} />
|
<TemperatureChart {...coreProps} setPageBottomExtraMargin={setPageBottomExtraMargin} />
|
||||||
<FanChart {...coreProps} />
|
<FanChart {...coreProps} />
|
||||||
<BatteryChart system={system} {...coreProps} />
|
<BatteryChart system={system} {...coreProps} />
|
||||||
<SwapChart chartData={chartData} grid={grid} dataEmpty={dataEmpty} systemStats={systemStats} />
|
|
||||||
{pageBottomExtraMargin > 0 && <div style={{ marginBottom: pageBottomExtraMargin }}></div>}
|
{pageBottomExtraMargin > 0 && <div style={{ marginBottom: pageBottomExtraMargin }}></div>}
|
||||||
</div>
|
</div>
|
||||||
</TabsContent>
|
</TabsContent>
|
||||||
|
|
||||||
|
<TabsContent value="network" forceMount className={activeTab === "network" ? "contents" : "hidden"}>
|
||||||
|
{mountedTabs.has("network") && (
|
||||||
|
<>
|
||||||
|
<div className="grid xl:grid-cols-2 gap-4">
|
||||||
|
<BandwidthChart {...coreProps} systemStats={systemStats} />
|
||||||
|
</div>
|
||||||
|
{hasNetworkMonitors && <LazyNetworkMonitorsTable systemId={system.id} />}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</TabsContent>
|
||||||
|
|
||||||
<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>
|
<TabsContent value="disk" forceMount className={activeTab === "disk" ? "contents" : "hidden"}>
|
||||||
{mountedTabs.has("disk") && (
|
{mountedTabs.has("disk") && (
|
||||||
<>
|
<>
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
import { timeTicks } from "d3-time"
|
|
||||||
import { getPbTimestamp, pb } from "@/lib/api"
|
import { getPbTimestamp, pb } from "@/lib/api"
|
||||||
import { chartTimeData } from "@/lib/utils"
|
import { chartTimeData } from "@/lib/utils"
|
||||||
import type { ChartData, ChartTimes, ContainerStatsRecord, SystemStatsRecord } from "@/types"
|
import type {
|
||||||
|
ChartData,
|
||||||
|
ChartDataContainer,
|
||||||
|
ChartTimes,
|
||||||
|
ContainerStatsRecord,
|
||||||
|
NetworkMonitorStatsRecord,
|
||||||
|
SystemStatsRecord,
|
||||||
|
} from "@/types"
|
||||||
|
|
||||||
type ChartTimeData = {
|
type ChartTimeData = {
|
||||||
time: number
|
time: number
|
||||||
@@ -17,31 +23,10 @@ export const cache = new Map<
|
|||||||
ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]
|
ChartTimeData | SystemStatsRecord[] | ContainerStatsRecord[] | ChartData["containerData"]
|
||||||
>()
|
>()
|
||||||
|
|
||||||
// create ticks and domain for charts
|
|
||||||
export function getTimeData(chartTime: ChartTimes, lastCreated: number) {
|
|
||||||
const cached = cache.get("td") as ChartTimeData | undefined
|
|
||||||
if (cached && cached.chartTime === chartTime) {
|
|
||||||
if (!lastCreated || cached.time >= lastCreated) {
|
|
||||||
return cached.data
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// const buffer = chartTime === "1m" ? 400 : 20_000
|
|
||||||
const now = new Date(Date.now())
|
|
||||||
const startTime = chartTimeData[chartTime].getOffset(now)
|
|
||||||
const ticks = timeTicks(startTime, now, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
|
|
||||||
const data = {
|
|
||||||
ticks,
|
|
||||||
domain: [chartTimeData[chartTime].getOffset(now).getTime(), now.getTime()],
|
|
||||||
}
|
|
||||||
cache.set("td", { time: now.getTime(), data, chartTime })
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.
|
/** Append new records onto prev with gap detection. Converts string `created` values to ms timestamps in place.
|
||||||
* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */
|
* Pass `maxLen` to cap the result length in one copy instead of slicing again after the call. */
|
||||||
export function appendData<T extends { created: string | number | null }>(
|
export function appendData<T extends { created: string | number | null }>(
|
||||||
prev: T[],
|
prev: T[] = [],
|
||||||
newRecords: T[],
|
newRecords: T[],
|
||||||
expectedInterval: number,
|
expectedInterval: number,
|
||||||
maxLen?: number
|
maxLen?: number
|
||||||
@@ -66,17 +51,18 @@ export function appendData<T extends { created: string | number | null }>(
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord>(
|
export async function getStats<T extends SystemStatsRecord | ContainerStatsRecord | NetworkMonitorStatsRecord>(
|
||||||
collection: string,
|
collection: string,
|
||||||
systemId: string,
|
systemId: string,
|
||||||
chartTime: ChartTimes
|
chartTime: ChartTimes,
|
||||||
|
cachedStats?: { created: string | number | null }[],
|
||||||
|
createdIsNumber?: boolean
|
||||||
): Promise<T[]> {
|
): Promise<T[]> {
|
||||||
const cachedStats = cache.get(`${systemId}_${chartTime}_${collection}`) as T[] | undefined
|
|
||||||
const lastCached = cachedStats?.at(-1)?.created as number
|
const lastCached = cachedStats?.at(-1)?.created as number
|
||||||
return await pb.collection<T>(collection).getFullList({
|
return await pb.collection<T>(collection).getFullList({
|
||||||
filter: pb.filter("system={:id} && created > {:created} && type={:type}", {
|
filter: pb.filter("system={:id} && created > {:created} && type={:type}", {
|
||||||
id: systemId,
|
id: systemId,
|
||||||
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined),
|
created: getPbTimestamp(chartTime, lastCached ? new Date(lastCached + 1000) : undefined, createdIsNumber),
|
||||||
type: chartTimeData[chartTime].type,
|
type: chartTimeData[chartTime].type,
|
||||||
}),
|
}),
|
||||||
fields: "created,stats",
|
fields: "created,stats",
|
||||||
@@ -84,11 +70,11 @@ export async function getStats<T extends SystemStatsRecord | ContainerStatsRecor
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function makeContainerData(containers: ContainerStatsRecord[]): ChartData["containerData"] {
|
export function makeContainerData(containers: ContainerStatsRecord[]): ChartDataContainer[] {
|
||||||
const result = [] as ChartData["containerData"]
|
const result = [] as ChartDataContainer[]
|
||||||
for (const { created, stats } of containers) {
|
for (const { created, stats } of containers) {
|
||||||
if (!created) {
|
if (!created) {
|
||||||
result.push({ created: null } as ChartData["containerData"][0])
|
result.push({ created: null } as ChartDataContainer)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
result.push(makeContainerPoint(new Date(created).getTime(), stats))
|
result.push(makeContainerPoint(new Date(created).getTime(), stats))
|
||||||
@@ -97,11 +83,8 @@ export function makeContainerData(containers: ContainerStatsRecord[]): ChartData
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Transform a single realtime container stats message into a ChartDataContainer point. */
|
/** Transform a single realtime container stats message into a ChartDataContainer point. */
|
||||||
export function makeContainerPoint(
|
export function makeContainerPoint(created: number, stats: ContainerStatsRecord["stats"]): ChartDataContainer {
|
||||||
created: number,
|
const point: ChartDataContainer = { created } as ChartDataContainer
|
||||||
stats: ContainerStatsRecord["stats"]
|
|
||||||
): ChartData["containerData"][0] {
|
|
||||||
const point: ChartData["containerData"][0] = { created } as ChartData["containerData"][0]
|
|
||||||
for (const container of stats) {
|
for (const container of stats) {
|
||||||
;(point as Record<string, unknown>)[container.n] = container
|
;(point as Record<string, unknown>)[container.n] = container
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,212 @@
|
|||||||
|
import { getMonitorTarget } from "@/lib/network-monitor-utils"
|
||||||
|
import LineChartDefault from "@/components/charts/line-chart"
|
||||||
|
import type { DataPoint } from "@/components/charts/line-chart"
|
||||||
|
import { decimalString, formatMicroseconds, matchesFilterGroups, parseFilterGroups, toFixedFloat } from "@/lib/utils"
|
||||||
|
import { $monitorFilter } from "@/lib/stores"
|
||||||
|
import { useLingui } from "@lingui/react/macro"
|
||||||
|
import { ChartCard, FilterBar } from "../chart-card"
|
||||||
|
import type { ChartData, MonitorStats, NetworkMonitorRecord, NetworkMonitorStatsRecord } from "@/types"
|
||||||
|
import { useMemo } from "react"
|
||||||
|
import { useStore } from "@nanostores/react"
|
||||||
|
|
||||||
|
type MonitorChartProps = {
|
||||||
|
monitorStats: NetworkMonitorStatsRecord[]
|
||||||
|
grid?: boolean
|
||||||
|
monitors: NetworkMonitorRecord[]
|
||||||
|
chartData: ChartData
|
||||||
|
empty: boolean
|
||||||
|
showFilter?: boolean
|
||||||
|
/** Prepended to the chart title, e.g. a target/system name (rendered as "{titlePrefix} — Response"). */
|
||||||
|
titlePrefix?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
type MonitorChartBaseProps = MonitorChartProps & {
|
||||||
|
metric: keyof MonitorStats
|
||||||
|
title: string
|
||||||
|
description: string
|
||||||
|
tickFormatter: (value: number) => string
|
||||||
|
contentFormatter: ({ value }: { value: number | string }) => string | number
|
||||||
|
domain?: [number | "auto", number | "auto"]
|
||||||
|
}
|
||||||
|
|
||||||
|
function MonitorChart({
|
||||||
|
monitorStats,
|
||||||
|
grid,
|
||||||
|
monitors,
|
||||||
|
chartData,
|
||||||
|
empty,
|
||||||
|
metric,
|
||||||
|
title,
|
||||||
|
description,
|
||||||
|
tickFormatter,
|
||||||
|
contentFormatter,
|
||||||
|
domain,
|
||||||
|
showFilter = monitors.length > 1,
|
||||||
|
}: MonitorChartBaseProps) {
|
||||||
|
const storedFilter = useStore($monitorFilter)
|
||||||
|
const filter = showFilter ? storedFilter : ""
|
||||||
|
|
||||||
|
const { dataPoints, visibleKeys } = useMemo(() => {
|
||||||
|
const sortedMonitors = [...monitors].sort((a, b) => b.resAvg1h - a.resAvg1h)
|
||||||
|
const count = sortedMonitors.length
|
||||||
|
const points: DataPoint<NetworkMonitorStatsRecord>[] = []
|
||||||
|
const visibleIDs: string[] = []
|
||||||
|
const filterGroups = parseFilterGroups(filter)
|
||||||
|
const dot = chartData.chartTime === "1m"
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const p = sortedMonitors[i]
|
||||||
|
const label = getMonitorTarget(p)
|
||||||
|
const labelLower = label.toLowerCase()
|
||||||
|
const filtered = filterGroups.length > 0 && !matchesFilterGroups(labelLower, filterGroups)
|
||||||
|
if (filtered) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
visibleIDs.push(p.id)
|
||||||
|
points.push({
|
||||||
|
order: i,
|
||||||
|
label,
|
||||||
|
dataKey: (record: NetworkMonitorStatsRecord) => record.stats?.[p.id]?.[metric] ?? null,
|
||||||
|
dot,
|
||||||
|
color: count <= 5 ? i + 1 : `hsl(${(i * 360) / count}, var(--chart-saturation), var(--chart-lightness))`,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return { dataPoints: points, visibleKeys: visibleIDs }
|
||||||
|
}, [monitors, filter, metric, chartData.chartTime])
|
||||||
|
|
||||||
|
const filteredMonitorStats = useMemo(() => {
|
||||||
|
if (!visibleKeys.length) return monitorStats
|
||||||
|
return monitorStats.filter((record) => visibleKeys.some((id) => record.stats?.[id] != null))
|
||||||
|
}, [monitorStats, visibleKeys])
|
||||||
|
|
||||||
|
const legend = dataPoints.length < 10 && showFilter
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ChartCard
|
||||||
|
legend={legend || !showFilter}
|
||||||
|
cornerEl={showFilter ? <FilterBar store={$monitorFilter} /> : undefined}
|
||||||
|
empty={empty}
|
||||||
|
title={title}
|
||||||
|
description={description}
|
||||||
|
grid={grid}
|
||||||
|
>
|
||||||
|
<LineChartDefault
|
||||||
|
truncate
|
||||||
|
chartData={chartData}
|
||||||
|
customData={filteredMonitorStats}
|
||||||
|
dataPoints={dataPoints}
|
||||||
|
domain={domain ?? ["auto", "auto"]}
|
||||||
|
connectNulls
|
||||||
|
tickFormatter={tickFormatter}
|
||||||
|
contentFormatter={contentFormatter}
|
||||||
|
legend={legend}
|
||||||
|
filter={filter}
|
||||||
|
/>
|
||||||
|
</ChartCard>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
interface AvgMinMaxResponseChartProps {
|
||||||
|
monitorStats: NetworkMonitorStatsRecord[]
|
||||||
|
monitor: NetworkMonitorRecord | null
|
||||||
|
chartData: ChartData
|
||||||
|
empty: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export function AvgMinMaxResponseChart({ monitorStats, monitor, chartData, empty }: AvgMinMaxResponseChartProps) {
|
||||||
|
const { t } = useLingui()
|
||||||
|
|
||||||
|
const { chartTime } = chartData
|
||||||
|
const hasLongInterval = (monitor?.interval ?? 61) > 60
|
||||||
|
|
||||||
|
// only one monitor is relevant for this chart
|
||||||
|
const dataPoints: DataPoint<NetworkMonitorStatsRecord>[] = useMemo(() => {
|
||||||
|
const dataFn = (metric: keyof MonitorStats) => (record: NetworkMonitorStatsRecord) =>
|
||||||
|
record.stats?.[monitor?.id ?? ""]?.[metric] ?? "-"
|
||||||
|
const avgPoint = {
|
||||||
|
label: "Avg",
|
||||||
|
dataKey: dataFn("res_avg"),
|
||||||
|
color: 1,
|
||||||
|
order: 0,
|
||||||
|
}
|
||||||
|
if (chartTime === "1m" || (hasLongInterval && chartTime === "1h")) {
|
||||||
|
// avg, min, max are all the same for 1m interval, so just show avg
|
||||||
|
return [avgPoint]
|
||||||
|
}
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
label: "Max",
|
||||||
|
dataKey: dataFn("res_max"),
|
||||||
|
color: 3,
|
||||||
|
order: 0,
|
||||||
|
},
|
||||||
|
avgPoint,
|
||||||
|
{
|
||||||
|
label: "Min",
|
||||||
|
dataKey: dataFn("res_min"),
|
||||||
|
color: 2,
|
||||||
|
order: 2,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}, [chartTime, hasLongInterval, monitor?.id])
|
||||||
|
|
||||||
|
const data = useMemo(() => {
|
||||||
|
if (!monitor) return []
|
||||||
|
return monitorStats.filter((record) => record.stats && monitor.id in record.stats)
|
||||||
|
}, [monitor, monitorStats])
|
||||||
|
|
||||||
|
const legend = dataPoints.length > 1
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ChartCard
|
||||||
|
legend={true}
|
||||||
|
empty={empty}
|
||||||
|
title={t`Response`}
|
||||||
|
description={t`Average, minimum, and maximum response time`}
|
||||||
|
grid={false}
|
||||||
|
>
|
||||||
|
<LineChartDefault
|
||||||
|
truncate
|
||||||
|
chartData={chartData}
|
||||||
|
customData={data}
|
||||||
|
dataPoints={dataPoints}
|
||||||
|
domain={["auto", "auto"]}
|
||||||
|
connectNulls
|
||||||
|
legend={legend}
|
||||||
|
tickFormatter={(value) => formatMicroseconds(value, false)}
|
||||||
|
contentFormatter={({ value }) => {
|
||||||
|
if (typeof value !== "number") {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return formatMicroseconds(value)
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</ChartCard>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function LossChart({ monitorStats, grid, monitors, chartData, empty, titlePrefix }: MonitorChartProps) {
|
||||||
|
const { t } = useLingui()
|
||||||
|
const lossTitle = t({ message: "Loss", context: "Packet loss" })
|
||||||
|
const title = titlePrefix ? `${titlePrefix} — ${lossTitle}` : lossTitle
|
||||||
|
|
||||||
|
return (
|
||||||
|
<MonitorChart
|
||||||
|
monitorStats={monitorStats}
|
||||||
|
grid={grid}
|
||||||
|
monitors={monitors}
|
||||||
|
chartData={chartData}
|
||||||
|
empty={empty}
|
||||||
|
metric="loss"
|
||||||
|
title={title}
|
||||||
|
description={t`Packet loss (%)`}
|
||||||
|
domain={[0, 100]}
|
||||||
|
tickFormatter={(value) => `${toFixedFloat(value, value >= 10 ? 0 : 1)}%`}
|
||||||
|
contentFormatter={({ value }) => {
|
||||||
|
if (typeof value !== "number") {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return `${decimalString(value, 2)}%`
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { lazy } from "react"
|
import { lazy } from "react"
|
||||||
import { useIntersectionObserver } from "@/lib/use-intersection-observer"
|
import { useIntersectionObserver } from "@/lib/use-intersection-observer"
|
||||||
import { cn } from "@/lib/utils"
|
import { cn } from "@/lib/utils"
|
||||||
|
import { useNetworkMonitors } from "@/lib/use-network-monitors"
|
||||||
|
|
||||||
const ContainersTable = lazy(() => import("../../containers-table/containers-table"))
|
const ContainersTable = lazy(() => import("../../containers-table/containers-table"))
|
||||||
|
|
||||||
@@ -45,3 +46,19 @@ export function LazySystemdTable({ systemId }: { systemId: string }) {
|
|||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const NetworkMonitorsTable = lazy(() => import("../../network-monitors-table/network-monitors-table"))
|
||||||
|
|
||||||
|
export function LazyNetworkMonitorsTable({ systemId }: { systemId: string }) {
|
||||||
|
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
|
||||||
|
return (
|
||||||
|
<div ref={ref} className={cn(isIntersecting && "contents")}>
|
||||||
|
{isIntersecting && <SystemNetworkMonitorsTable systemId={systemId} />}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function SystemNetworkMonitorsTable({ systemId }: { systemId: string }) {
|
||||||
|
const { monitors, isLoading } = useNetworkMonitors({ systemId })
|
||||||
|
return <NetworkMonitorsTable systemId={systemId} monitors={monitors} isLoading={isLoading} />
|
||||||
|
}
|
||||||
|
|||||||
@@ -43,7 +43,6 @@ import {
|
|||||||
toFixedFloat,
|
toFixedFloat,
|
||||||
formatTemperature,
|
formatTemperature,
|
||||||
cn,
|
cn,
|
||||||
getVisualStringWidth,
|
|
||||||
secondsToString,
|
secondsToString,
|
||||||
hourWithSeconds,
|
hourWithSeconds,
|
||||||
formatShortDate,
|
formatShortDate,
|
||||||
@@ -117,9 +116,9 @@ function formatDataUnits(units: number): string {
|
|||||||
const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"
|
const SMART_DEVICE_FIELDS = "id,system,name,model,state,capacity,temp,type,hours,cycles,updated"
|
||||||
|
|
||||||
export const createColumns = (
|
export const createColumns = (
|
||||||
longestName: number,
|
longestName: string,
|
||||||
longestModel: number,
|
longestModel: string,
|
||||||
longestDevice: number
|
longestDevice: string
|
||||||
): ColumnDef<SmartDeviceRecord>[] => [
|
): ColumnDef<SmartDeviceRecord>[] => [
|
||||||
{
|
{
|
||||||
id: "system",
|
id: "system",
|
||||||
@@ -134,8 +133,11 @@ export const createColumns = (
|
|||||||
cell: ({ getValue }) => {
|
cell: ({ getValue }) => {
|
||||||
const allSystems = useStore($allSystemsById)
|
const allSystems = useStore($allSystemsById)
|
||||||
return (
|
return (
|
||||||
<div className="ms-1.5 max-w-40 block truncate" style={{ width: `${longestName / 1.05}ch` }}>
|
<div className="ms-1.5 relative w-fit max-w-44">
|
||||||
{allSystems[getValue() as string]?.name ?? ""}
|
<span className="invisible block whitespace-nowrap" aria-hidden="true">
|
||||||
|
{longestName}
|
||||||
|
</span>
|
||||||
|
<span className="absolute inset-0 truncate">{allSystems[getValue() as string]?.name ?? ""}</span>
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
@@ -145,12 +147,11 @@ export const createColumns = (
|
|||||||
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
||||||
header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,
|
header: ({ column }) => <HeaderButton column={column} name={t`Device`} Icon={HardDrive} />,
|
||||||
cell: ({ getValue }) => (
|
cell: ({ getValue }) => (
|
||||||
<div
|
<div className="font-medium ms-1 relative w-fit max-w-44" title={getValue() as string}>
|
||||||
className="font-medium max-w-40 truncate ms-1"
|
<span className="invisible block whitespace-nowrap" aria-hidden="true">
|
||||||
title={getValue() as string}
|
{longestDevice}
|
||||||
style={{ width: `${longestDevice / 1.05}ch` }}
|
</span>
|
||||||
>
|
<span className="absolute inset-0 truncate">{getValue() as string}</span>
|
||||||
{getValue() as string}
|
|
||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -161,12 +162,11 @@ export const createColumns = (
|
|||||||
<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />
|
<HeaderButton column={column} name={t({ message: "Model", comment: "Device model" })} Icon={Box} />
|
||||||
),
|
),
|
||||||
cell: ({ getValue }) => (
|
cell: ({ getValue }) => (
|
||||||
<div
|
<div className="ms-1 relative w-fit max-w-44" title={getValue() as string}>
|
||||||
className="max-w-48 truncate ms-1"
|
<span className="invisible block whitespace-nowrap" aria-hidden="true">
|
||||||
title={getValue() as string}
|
{longestModel}
|
||||||
style={{ width: `${longestModel / 1.05}ch` }}
|
</span>
|
||||||
>
|
<span className="absolute inset-0 truncate">{getValue() as string}</span>
|
||||||
{getValue() as string}
|
|
||||||
</div>
|
</div>
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@@ -320,7 +320,7 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
|
|||||||
|
|
||||||
// Calculate the right width for the columns based on the longest strings among the displayed devices
|
// Calculate the right width for the columns based on the longest strings among the displayed devices
|
||||||
const { longestName, longestModel, longestDevice } = useMemo(() => {
|
const { longestName, longestModel, longestDevice } = useMemo(() => {
|
||||||
const result = { longestName: 0, longestModel: 0, longestDevice: 0 }
|
const result = { longestName: "", longestModel: "", longestDevice: "" }
|
||||||
if (!smartDevices || Object.keys(allSystems).length === 0) {
|
if (!smartDevices || Object.keys(allSystems).length === 0) {
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
@@ -329,10 +329,16 @@ export default function DisksTable({ systemId }: { systemId?: string }) {
|
|||||||
if (!systemId && !seenSystems.has(device.system)) {
|
if (!systemId && !seenSystems.has(device.system)) {
|
||||||
seenSystems.add(device.system)
|
seenSystems.add(device.system)
|
||||||
const name = allSystems[device.system]?.name ?? ""
|
const name = allSystems[device.system]?.name ?? ""
|
||||||
result.longestName = Math.max(result.longestName, getVisualStringWidth(name))
|
if (name.length > result.longestName.length) {
|
||||||
|
result.longestName = name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ((device.model ?? "").length > result.longestModel.length) {
|
||||||
|
result.longestModel = device.model ?? ""
|
||||||
|
}
|
||||||
|
if ((device.name ?? "").length > result.longestDevice.length) {
|
||||||
|
result.longestDevice = device.name ?? ""
|
||||||
}
|
}
|
||||||
result.longestModel = Math.max(result.longestModel, getVisualStringWidth(device.model ?? ""))
|
|
||||||
result.longestDevice = Math.max(result.longestDevice, getVisualStringWidth(device.name ?? ""))
|
|
||||||
}
|
}
|
||||||
return result
|
return result
|
||||||
}, [smartDevices, systemId, allSystems])
|
}, [smartDevices, systemId, allSystems])
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import type {
|
|||||||
SystemStatsRecord,
|
SystemStatsRecord,
|
||||||
} from "@/types"
|
} from "@/types"
|
||||||
import { $router, navigate } from "../../router"
|
import { $router, navigate } from "../../router"
|
||||||
import { appendData, cache, getStats, getTimeData, makeContainerData, makeContainerPoint } from "./chart-data"
|
import { appendData, cache, getStats, makeContainerData, makeContainerPoint } from "./chart-data"
|
||||||
|
|
||||||
export type SystemData = ReturnType<typeof useSystemData>
|
export type SystemData = ReturnType<typeof useSystemData>
|
||||||
|
|
||||||
@@ -185,16 +185,11 @@ export function useSystemData(id: string) {
|
|||||||
const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])
|
const agentVersion = useMemo(() => parseSemVer(system?.info?.v), [system?.info?.v])
|
||||||
|
|
||||||
const chartData: ChartData = useMemo(() => {
|
const chartData: ChartData = useMemo(() => {
|
||||||
const lastCreated = Math.max(
|
|
||||||
(systemStats.at(-1)?.created as number) ?? 0,
|
|
||||||
(containerData.at(-1)?.created as number) ?? 0
|
|
||||||
)
|
|
||||||
return {
|
return {
|
||||||
systemStats,
|
systemStats,
|
||||||
containerData,
|
containerData,
|
||||||
chartTime,
|
chartTime,
|
||||||
orientation: direction === "rtl" ? "right" : "left",
|
orientation: direction === "rtl" ? "right" : "left",
|
||||||
...getTimeData(chartTime, lastCreated),
|
|
||||||
agentVersion,
|
agentVersion,
|
||||||
}
|
}
|
||||||
}, [systemStats, containerData, direction])
|
}, [systemStats, containerData, direction])
|
||||||
@@ -234,8 +229,8 @@ export function useSystemData(id: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Promise.allSettled([
|
Promise.allSettled([
|
||||||
getStats<SystemStatsRecord>("system_stats", systemId, chartTime),
|
getStats<SystemStatsRecord>("system_stats", systemId, chartTime, cachedSystemStats),
|
||||||
getStats<ContainerStatsRecord>("container_stats", systemId, chartTime),
|
getStats<ContainerStatsRecord>("container_stats", systemId, chartTime, cachedContainerData),
|
||||||
]).then(([systemStats, containerStats]) => {
|
]).then(([systemStats, containerStats]) => {
|
||||||
// Ignore responses for a previous system or chart time
|
// Ignore responses for a previous system or chart time
|
||||||
if (requestId !== statsRequestId.current) {
|
if (requestId !== statsRequestId.current) {
|
||||||
@@ -327,7 +322,7 @@ export function useSystemData(id: string) {
|
|||||||
// derived values
|
// derived values
|
||||||
const isLongerChart = !["1m", "1h"].includes(chartTime)
|
const isLongerChart = !["1m", "1h"].includes(chartTime)
|
||||||
const showMax = maxValues && isLongerChart
|
const showMax = maxValues && isLongerChart
|
||||||
const dataEmpty = !chartLoading && chartData.systemStats.length === 0
|
const dataEmpty = !chartLoading && chartData.systemStats?.length === 0
|
||||||
const lastGpus = systemStats.at(-1)?.stats?.g
|
const lastGpus = systemStats.at(-1)?.stats?.g
|
||||||
const isPodman = details?.podman ?? system.info?.p ?? false
|
const isPodman = details?.podman ?? system.info?.p ?? false
|
||||||
|
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import { memo, useMemo, useRef, useState } from "react"
|
|||||||
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
|
import { Tooltip, TooltipContent, TooltipTrigger } from "../ui/tooltip"
|
||||||
import { isReadOnlyUser, pb } from "@/lib/api"
|
import { isReadOnlyUser, pb } from "@/lib/api"
|
||||||
import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"
|
import { BatteryState, ConnectionType, connectionTypeLabels, MeterState, SystemStatus } from "@/lib/enums"
|
||||||
import { $longestSystemNameLen, $userSettings } from "@/lib/stores"
|
import { $longestSystemName, $userSettings } from "@/lib/stores"
|
||||||
import {
|
import {
|
||||||
cn,
|
cn,
|
||||||
copyToClipboard,
|
copyToClipboard,
|
||||||
@@ -135,7 +135,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
|
|||||||
Icon: ServerIcon,
|
Icon: ServerIcon,
|
||||||
cell: (info) => {
|
cell: (info) => {
|
||||||
const { name, id } = info.row.original
|
const { name, id } = info.row.original
|
||||||
const longestName = useStore($longestSystemNameLen)
|
const longestName = useStore($longestSystemName)
|
||||||
const linkUrl = getPagePath($router, "system", { id })
|
const linkUrl = getPagePath($router, "system", { id })
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -145,8 +145,7 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
|
|||||||
<Link
|
<Link
|
||||||
href={linkUrl}
|
href={linkUrl}
|
||||||
tabIndex={-1}
|
tabIndex={-1}
|
||||||
className="truncate z-10 relative"
|
className="relative w-fit max-w-48 z-10"
|
||||||
style={{ width: `${longestName / 1.05}ch` }}
|
|
||||||
onMouseEnter={(e) => {
|
onMouseEnter={(e) => {
|
||||||
// set title on hover if text is truncated to show full name
|
// set title on hover if text is truncated to show full name
|
||||||
const a = e.currentTarget
|
const a = e.currentTarget
|
||||||
@@ -157,7 +156,10 @@ export function SystemsTableColumns(viewMode: "table" | "grid"): ColumnDef<Syste
|
|||||||
}
|
}
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{name}
|
<span className="invisible block" aria-hidden="true">
|
||||||
|
{longestName}
|
||||||
|
</span>
|
||||||
|
<span className="absolute inset-0 truncate">{name}</span>
|
||||||
</Link>
|
</Link>
|
||||||
</span>
|
</span>
|
||||||
<Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>
|
<Link href={linkUrl} className="inset-0 absolute size-full" aria-label={name}></Link>
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
import type { JSX } from "react"
|
|
||||||
import { useLingui } from "@lingui/react/macro"
|
import { useLingui } from "@lingui/react/macro"
|
||||||
import * as React from "react"
|
import * as React from "react"
|
||||||
import * as RechartsPrimitive from "recharts"
|
import * as RechartsPrimitive from "recharts"
|
||||||
import { chartTimeData, cn } from "@/lib/utils"
|
import { chartTimeData, cn } from "@/lib/utils"
|
||||||
import type { ChartData } from "@/types"
|
import type { ChartTimes } from "@/types"
|
||||||
import { Separator } from "./separator"
|
import { Separator } from "./separator"
|
||||||
import { AxisDomain } from "recharts/types/util/types"
|
import type { AxisDomain } from "recharts/types/util/types"
|
||||||
|
import { timeTicks } from "d3-time"
|
||||||
|
|
||||||
// Format: { THEME_NAME: CSS_SELECTOR }
|
// Format: { THEME_NAME: CSS_SELECTOR }
|
||||||
const THEMES = { light: "", dark: ".dark" } as const
|
const THEMES = { light: "", dark: ".dark" } as const
|
||||||
@@ -101,7 +101,7 @@ const ChartTooltipContent = React.forwardRef<
|
|||||||
labelKey?: string
|
labelKey?: string
|
||||||
unit?: string
|
unit?: string
|
||||||
filter?: string
|
filter?: string
|
||||||
contentFormatter?: (item: any, key: string) => React.ReactNode | string
|
contentFormatter?: (item: unknown, key: string) => React.ReactNode | string
|
||||||
truncate?: boolean
|
truncate?: boolean
|
||||||
showTotal?: boolean
|
showTotal?: boolean
|
||||||
totalLabel?: React.ReactNode
|
totalLabel?: React.ReactNode
|
||||||
@@ -175,7 +175,13 @@ const ChartTooltipContent = React.forwardRef<
|
|||||||
}
|
}
|
||||||
|
|
||||||
const totalKey = "__total__"
|
const totalKey = "__total__"
|
||||||
const totalItem: any = {
|
const totalItem: {
|
||||||
|
value: number
|
||||||
|
name: string
|
||||||
|
dataKey: string
|
||||||
|
color: string | undefined
|
||||||
|
payload?: unknown
|
||||||
|
} = {
|
||||||
value: totalValue,
|
value: totalValue,
|
||||||
name: totalName,
|
name: totalName,
|
||||||
dataKey: totalKey,
|
dataKey: totalKey,
|
||||||
@@ -222,6 +228,11 @@ const ChartTooltipContent = React.forwardRef<
|
|||||||
return null
|
return null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
payload = payload.filter((item) => item.value != null)
|
||||||
|
if (!payload.length) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
// const nestLabel = payload.length === 1 && indicator !== 'dot'
|
// const nestLabel = payload.length === 1 && indicator !== 'dot'
|
||||||
const nestLabel = false
|
const nestLabel = false
|
||||||
|
|
||||||
@@ -400,26 +411,57 @@ function getPayloadConfigFromPayload(config: ChartConfig, payload: unknown, key:
|
|||||||
return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]
|
return configLabelKey in config ? config[configLabelKey] : config[key as keyof typeof config]
|
||||||
}
|
}
|
||||||
|
|
||||||
let cachedAxis: JSX.Element
|
interface XAxisData {
|
||||||
const xAxis = ({ domain, ticks, chartTime }: ChartData) => {
|
el: React.ReactElement
|
||||||
if (cachedAxis && domain[0] === cachedAxis.props.domain[0]) {
|
domain: [number, number]
|
||||||
return cachedAxis
|
}
|
||||||
|
|
||||||
|
const xAxisCache = new Map<ChartTimes, XAxisData>()
|
||||||
|
|
||||||
|
function createXAxisData(chartTime: ChartTimes): XAxisData {
|
||||||
|
// console.log("Creating XAxis for", chartTime, new Date())
|
||||||
|
const axisEndTime = Date.now() + 500
|
||||||
|
const axisEndDate = new Date(axisEndTime)
|
||||||
|
const startTime = chartTimeData[chartTime].getOffset(axisEndDate)
|
||||||
|
const ticks = timeTicks(startTime, axisEndDate, chartTimeData[chartTime].ticks ?? 12).map((date) => date.getTime())
|
||||||
|
const domain: [number, number] = [startTime.getTime(), axisEndTime]
|
||||||
|
|
||||||
|
return {
|
||||||
|
domain,
|
||||||
|
el: (
|
||||||
|
<RechartsPrimitive.XAxis
|
||||||
|
dataKey="created"
|
||||||
|
domain={domain}
|
||||||
|
ticks={ticks}
|
||||||
|
allowDataOverflow
|
||||||
|
type="number"
|
||||||
|
scale="time"
|
||||||
|
minTickGap={12}
|
||||||
|
tickMargin={8}
|
||||||
|
axisLine={false}
|
||||||
|
tickFormatter={chartTimeData[chartTime].format}
|
||||||
|
/>
|
||||||
|
),
|
||||||
}
|
}
|
||||||
cachedAxis = (
|
}
|
||||||
<RechartsPrimitive.XAxis
|
|
||||||
dataKey="created"
|
function xAxis(chartTime: ChartTimes, lastCreated: number) {
|
||||||
domain={domain}
|
if (!lastCreated) {
|
||||||
ticks={ticks}
|
return null
|
||||||
allowDataOverflow
|
}
|
||||||
type="number"
|
const cachedAxis = xAxisCache.get(chartTime)
|
||||||
scale="time"
|
|
||||||
minTickGap={12}
|
const expectedInterval = chartTimeData[chartTime].expectedInterval
|
||||||
tickMargin={8}
|
const conservativeEndTime = Date.now() - expectedInterval / 2
|
||||||
axisLine={false}
|
const axisEndTime = Math.max(lastCreated, conservativeEndTime)
|
||||||
tickFormatter={chartTimeData[chartTime].format}
|
|
||||||
/>
|
if (cachedAxis && axisEndTime < cachedAxis.domain[1]) {
|
||||||
)
|
return cachedAxis.el
|
||||||
return cachedAxis
|
}
|
||||||
|
|
||||||
|
const axisData = createXAxisData(chartTime)
|
||||||
|
xAxisCache.set(chartTime, axisData)
|
||||||
|
return axisData.el
|
||||||
}
|
}
|
||||||
|
|
||||||
export {
|
export {
|
||||||
|
|||||||
@@ -90,8 +90,10 @@ DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName
|
|||||||
|
|
||||||
const DropdownMenuCheckboxItem = React.forwardRef<
|
const DropdownMenuCheckboxItem = React.forwardRef<
|
||||||
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
|
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
|
||||||
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>
|
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem> & {
|
||||||
>(({ className, children, checked, ...props }, ref) => (
|
indicatorClassName?: string
|
||||||
|
}
|
||||||
|
>(({ className, children, checked, indicatorClassName, ...props }, ref) => (
|
||||||
<DropdownMenuPrimitive.CheckboxItem
|
<DropdownMenuPrimitive.CheckboxItem
|
||||||
ref={ref}
|
ref={ref}
|
||||||
className={cn(
|
className={cn(
|
||||||
@@ -101,7 +103,7 @@ const DropdownMenuCheckboxItem = React.forwardRef<
|
|||||||
checked={checked}
|
checked={checked}
|
||||||
{...props}
|
{...props}
|
||||||
>
|
>
|
||||||
<span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
|
<span className={cn("absolute left-2 flex h-3.5 w-3.5 items-center justify-center", indicatorClassName)}>
|
||||||
<DropdownMenuPrimitive.ItemIndicator>
|
<DropdownMenuPrimitive.ItemIndicator>
|
||||||
<Check className="h-4 w-4" />
|
<Check className="h-4 w-4" />
|
||||||
</DropdownMenuPrimitive.ItemIndicator>
|
</DropdownMenuPrimitive.ItemIndicator>
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTML
|
|||||||
<tr
|
<tr
|
||||||
ref={ref}
|
ref={ref}
|
||||||
className={cn(
|
className={cn(
|
||||||
"border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted!",
|
"border-b border-border/60 hover:bg-muted/40 dark:hover:bg-muted/20 data-[state=selected]:bg-muted/40!",
|
||||||
className
|
className
|
||||||
)}
|
)}
|
||||||
{...props}
|
{...props}
|
||||||
|
|||||||
+204
-191
@@ -1,5 +1,5 @@
|
|||||||
import { t } from "@lingui/core/macro"
|
import { t } from "@lingui/core/macro"
|
||||||
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
|
import { ContainerIcon, CpuIcon, HardDriveIcon, MemoryStickIcon, NetworkIcon, ServerCrashIcon, ServerIcon } from "lucide-react"
|
||||||
import type { RecordSubscription } from "pocketbase"
|
import type { RecordSubscription } from "pocketbase"
|
||||||
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
|
import { EthernetIcon, GpuIcon } from "@/components/ui/icons"
|
||||||
import { $alerts } from "@/lib/stores"
|
import { $alerts } from "@/lib/stores"
|
||||||
@@ -9,210 +9,223 @@ import { ThermometerIcon, BatteryMediumIcon, HourglassIcon } from "@/components/
|
|||||||
|
|
||||||
/** Alert info for each alert type */
|
/** Alert info for each alert type */
|
||||||
export const alertInfo: Record<string, AlertInfo> = {
|
export const alertInfo: Record<string, AlertInfo> = {
|
||||||
Status: {
|
Status: {
|
||||||
name: () => t`Status`,
|
name: () => t`Status`,
|
||||||
unit: "",
|
unit: "",
|
||||||
icon: ServerIcon,
|
icon: ServerIcon,
|
||||||
desc: () => t`Triggers when status switches between up and down`,
|
desc: () => t`Triggers when status switches between up and down`,
|
||||||
/** "for x minutes" is appended to desc when only one value */
|
/** "for x minutes" is appended to desc when only one value */
|
||||||
singleDesc: () => `${t`System`} ${t`Down`}`,
|
singleDesc: () => `${t`System`} ${t`Down`}`,
|
||||||
},
|
},
|
||||||
CPU: {
|
CPU: {
|
||||||
name: () => t`CPU Usage`,
|
name: () => t`CPU Usage`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: CpuIcon,
|
icon: CpuIcon,
|
||||||
desc: () => t`Triggers when CPU usage exceeds a threshold`,
|
desc: () => t`Triggers when CPU usage exceeds a threshold`,
|
||||||
},
|
},
|
||||||
CPUIOWait: {
|
CPUIOWait: {
|
||||||
name: () => t`CPU I/O Wait`,
|
name: () => t`CPU I/O Wait`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: CpuIcon,
|
icon: CpuIcon,
|
||||||
desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
|
desc: () => t`Triggers when CPU I/O wait exceeds a threshold`,
|
||||||
},
|
},
|
||||||
CPUSteal: {
|
CPUSteal: {
|
||||||
name: () => t`CPU Steal Time`,
|
name: () => t`CPU Steal Time`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: CpuIcon,
|
icon: CpuIcon,
|
||||||
desc: () => t`Triggers when CPU steal time exceeds a threshold`,
|
desc: () => t`Triggers when CPU steal time exceeds a threshold`,
|
||||||
},
|
},
|
||||||
Memory: {
|
Memory: {
|
||||||
name: () => t`Memory Usage`,
|
name: () => t`Memory Usage`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: MemoryStickIcon,
|
icon: MemoryStickIcon,
|
||||||
desc: () => t`Triggers when memory usage exceeds a threshold`,
|
desc: () => t`Triggers when memory usage exceeds a threshold`,
|
||||||
},
|
},
|
||||||
Disk: {
|
Disk: {
|
||||||
name: () => t`Disk Usage`,
|
name: () => t`Disk Usage`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: HardDriveIcon,
|
icon: HardDriveIcon,
|
||||||
desc: () => t`Triggers when usage of any disk exceeds a threshold`,
|
desc: () => t`Triggers when usage of any disk exceeds a threshold`,
|
||||||
},
|
},
|
||||||
Bandwidth: {
|
Bandwidth: {
|
||||||
name: () => t`Bandwidth`,
|
name: () => t`Bandwidth`,
|
||||||
unit: " MB/s",
|
unit: " MB/s",
|
||||||
icon: EthernetIcon,
|
icon: EthernetIcon,
|
||||||
desc: () => t`Triggers when combined up/down exceeds a threshold`,
|
desc: () => t`Triggers when combined up/down exceeds a threshold`,
|
||||||
max: 250,
|
max: 250,
|
||||||
},
|
},
|
||||||
GPU: {
|
NetworkMonitorLoss: {
|
||||||
name: () => t`GPU Usage`,
|
name: () => t`Network Monitor Loss`,
|
||||||
unit: "%",
|
unit: "%",
|
||||||
icon: GpuIcon,
|
icon: NetworkIcon,
|
||||||
desc: () => t`Triggers when GPU usage exceeds a threshold`,
|
desc: () => t`Triggers when one hour loss exceeds a threshold`,
|
||||||
},
|
// note: () => t`Uses available history after three probes.`,
|
||||||
Temperature: {
|
noDuration: true,
|
||||||
name: () => t`Temperature`,
|
min: 0,
|
||||||
unit: "°C",
|
max: 99.9,
|
||||||
icon: ThermometerIcon,
|
step: 0.1,
|
||||||
desc: () => t`Triggers when any sensor exceeds a threshold`,
|
start: 5,
|
||||||
},
|
},
|
||||||
LoadAvg1: {
|
GPU: {
|
||||||
name: () => t`Load Average 1m`,
|
name: () => t`GPU Usage`,
|
||||||
unit: "",
|
unit: "%",
|
||||||
icon: HourglassIcon,
|
icon: GpuIcon,
|
||||||
max: 100,
|
desc: () => t`Triggers when GPU usage exceeds a threshold`,
|
||||||
min: 0.1,
|
},
|
||||||
start: 10,
|
Temperature: {
|
||||||
step: 0.1,
|
name: () => t`Temperature`,
|
||||||
desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
|
unit: "°C",
|
||||||
},
|
icon: ThermometerIcon,
|
||||||
LoadAvg5: {
|
desc: () => t`Triggers when any sensor exceeds a threshold`,
|
||||||
name: () => t`Load Average 5m`,
|
},
|
||||||
unit: "",
|
LoadAvg1: {
|
||||||
icon: HourglassIcon,
|
name: () => t`Load Average 1m`,
|
||||||
max: 100,
|
unit: "",
|
||||||
min: 0.1,
|
icon: HourglassIcon,
|
||||||
start: 10,
|
max: 100,
|
||||||
step: 0.1,
|
min: 0.1,
|
||||||
desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
|
start: 10,
|
||||||
},
|
step: 0.1,
|
||||||
LoadAvg15: {
|
desc: () => t`Triggers when 1 minute load average exceeds a threshold`,
|
||||||
name: () => t`Load Average 15m`,
|
},
|
||||||
unit: "",
|
LoadAvg5: {
|
||||||
icon: HourglassIcon,
|
name: () => t`Load Average 5m`,
|
||||||
min: 0.1,
|
unit: "",
|
||||||
max: 100,
|
icon: HourglassIcon,
|
||||||
start: 10,
|
max: 100,
|
||||||
step: 0.1,
|
min: 0.1,
|
||||||
desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
|
start: 10,
|
||||||
},
|
step: 0.1,
|
||||||
Battery: {
|
desc: () => t`Triggers when 5 minute load average exceeds a threshold`,
|
||||||
name: () => t`Battery`,
|
},
|
||||||
unit: "%",
|
LoadAvg15: {
|
||||||
icon: BatteryMediumIcon,
|
name: () => t`Load Average 15m`,
|
||||||
desc: () => t`Triggers when battery charge drops below a threshold`,
|
unit: "",
|
||||||
start: 20,
|
icon: HourglassIcon,
|
||||||
invert: true,
|
min: 0.1,
|
||||||
},
|
max: 100,
|
||||||
ContainerHealth: {
|
start: 10,
|
||||||
name: () => t`Container Health`,
|
step: 0.1,
|
||||||
unit: "",
|
desc: () => t`Triggers when 15 minute load average exceeds a threshold`,
|
||||||
icon: ContainerIcon,
|
},
|
||||||
desc: () => t`Triggers when a container's health check reports unhealthy`,
|
Battery: {
|
||||||
note: () =>
|
name: () => t`Battery`,
|
||||||
t`Notifications may include recent container log excerpts.`,
|
unit: "%",
|
||||||
triggeredDesc: () => t`One or more containers are unhealthy`,
|
icon: BatteryMediumIcon,
|
||||||
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
|
desc: () => t`Triggers when battery charge drops below a threshold`,
|
||||||
|
start: 20,
|
||||||
|
invert: true,
|
||||||
|
},
|
||||||
|
ContainerHealth: {
|
||||||
|
name: () => t`Container Health`,
|
||||||
|
unit: "",
|
||||||
|
icon: ContainerIcon,
|
||||||
|
desc: () => t`Triggers when a container's health check reports unhealthy`,
|
||||||
|
note: () =>
|
||||||
|
t`Notifications may include recent container log excerpts.`,
|
||||||
|
triggeredDesc: () => t`One or more containers are unhealthy`,
|
||||||
|
singleDesc: () => `${t`Container`} ${t`Unhealthy`}`,
|
||||||
|
|
||||||
},
|
},
|
||||||
SystemdFailed: {
|
SystemdFailed: {
|
||||||
name: () => t`Failed Services`,
|
name: () => t`Failed Services`,
|
||||||
unit: "",
|
unit: "",
|
||||||
icon: ServerCrashIcon,
|
icon: ServerCrashIcon,
|
||||||
desc: () => t`Triggers when any systemd service enters the failed state`,
|
desc: () => t`Triggers when any systemd service enters the failed state`,
|
||||||
triggeredDesc: () => t`One or more services are in a failed state`,
|
triggeredDesc: () => t`One or more services are in a failed state`,
|
||||||
/** Fires on first observation - the agent only polls systemd every 10 minutes */
|
/** Fires on first observation - the agent only polls systemd every 10 minutes */
|
||||||
noDuration: true,
|
noDuration: true,
|
||||||
},
|
noThreshold: true,
|
||||||
|
},
|
||||||
} as const
|
} as const
|
||||||
|
|
||||||
/** Helper to manage user alerts */
|
/** Helper to manage user alerts */
|
||||||
export const alertManager = (() => {
|
export const alertManager = (() => {
|
||||||
const collection = pb.collection<AlertRecord>("alerts")
|
const collection = pb.collection<AlertRecord>("alerts")
|
||||||
let unsub: () => void
|
let unsub: () => void
|
||||||
|
|
||||||
/** Fields to fetch from alerts collection */
|
/** Fields to fetch from alerts collection */
|
||||||
const fields = "id,name,system,value,min,triggered"
|
const fields = "id,name,system,value,min,triggered"
|
||||||
|
|
||||||
/** Fetch alerts from collection */
|
/** Fetch alerts from collection */
|
||||||
async function fetchAlerts(): Promise<AlertRecord[]> {
|
async function fetchAlerts(): Promise<AlertRecord[]> {
|
||||||
return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
|
return await collection.getFullList<AlertRecord>({ fields, sort: "updated" })
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Format alerts into a map of system id to alert name to alert record */
|
/** Format alerts into a map of system id to alert name to alert record */
|
||||||
function add(alerts: AlertRecord[]) {
|
function add(alerts: AlertRecord[]) {
|
||||||
for (const alert of alerts) {
|
for (const alert of alerts) {
|
||||||
const systemId = alert.system
|
const systemId = alert.system
|
||||||
const systemAlerts = $alerts.get()[systemId] ?? new Map()
|
const systemAlerts = $alerts.get()[systemId] ?? new Map()
|
||||||
const newAlerts = new Map(systemAlerts)
|
const newAlerts = new Map(systemAlerts)
|
||||||
newAlerts.set(alert.name, alert)
|
newAlerts.set(alert.name, alert)
|
||||||
$alerts.setKey(systemId, newAlerts)
|
$alerts.setKey(systemId, newAlerts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
|
function remove(alerts: Pick<AlertRecord, "name" | "system">[]) {
|
||||||
for (const alert of alerts) {
|
for (const alert of alerts) {
|
||||||
const systemId = alert.system
|
const systemId = alert.system
|
||||||
const systemAlerts = $alerts.get()[systemId]
|
const systemAlerts = $alerts.get()[systemId]
|
||||||
const newAlerts = new Map(systemAlerts)
|
const newAlerts = new Map(systemAlerts)
|
||||||
newAlerts.delete(alert.name)
|
newAlerts.delete(alert.name)
|
||||||
$alerts.setKey(systemId, newAlerts)
|
$alerts.setKey(systemId, newAlerts)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const actionFns = {
|
const actionFns = {
|
||||||
create: add,
|
create: add,
|
||||||
update: add,
|
update: add,
|
||||||
delete: remove,
|
delete: remove,
|
||||||
}
|
}
|
||||||
|
|
||||||
// batch alert updates to prevent unnecessary re-renders when adding many alerts at once
|
// batch alert updates to prevent unnecessary re-renders when adding many alerts at once
|
||||||
const batchUpdate = (() => {
|
const batchUpdate = (() => {
|
||||||
const batch = new Map<string, RecordSubscription<AlertRecord>>()
|
const batch = new Map<string, RecordSubscription<AlertRecord>>()
|
||||||
let timeout: ReturnType<typeof setTimeout>
|
let timeout: ReturnType<typeof setTimeout>
|
||||||
|
|
||||||
return (data: RecordSubscription<AlertRecord>) => {
|
return (data: RecordSubscription<AlertRecord>) => {
|
||||||
const { record } = data
|
const { record } = data
|
||||||
batch.set(`${record.system}${record.name}`, data)
|
batch.set(`${record.system}${record.name}`, data)
|
||||||
clearTimeout(timeout)
|
clearTimeout(timeout)
|
||||||
timeout = setTimeout(() => {
|
timeout = setTimeout(() => {
|
||||||
const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
|
const groups = { create: [], update: [], delete: [] } as Record<string, AlertRecord[]>
|
||||||
for (const { action, record } of batch.values()) {
|
for (const { action, record } of batch.values()) {
|
||||||
groups[action]?.push(record)
|
groups[action]?.push(record)
|
||||||
}
|
}
|
||||||
for (const key in groups) {
|
for (const key in groups) {
|
||||||
if (groups[key].length) {
|
if (groups[key].length) {
|
||||||
actionFns[key as keyof typeof actionFns]?.(groups[key])
|
actionFns[key as keyof typeof actionFns]?.(groups[key])
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
batch.clear()
|
batch.clear()
|
||||||
}, 50)
|
}, 50)
|
||||||
}
|
}
|
||||||
})()
|
})()
|
||||||
|
|
||||||
async function subscribe() {
|
async function subscribe() {
|
||||||
unsub = await collection.subscribe("*", batchUpdate, { fields })
|
unsub = await collection.subscribe("*", batchUpdate, { fields })
|
||||||
}
|
}
|
||||||
|
|
||||||
function unsubscribe() {
|
function unsubscribe() {
|
||||||
unsub?.()
|
unsub?.()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
const records = await fetchAlerts()
|
const records = await fetchAlerts()
|
||||||
add(records)
|
add(records)
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
/** Add alerts to store */
|
/** Add alerts to store */
|
||||||
add,
|
add,
|
||||||
/** Remove alerts from store */
|
/** Remove alerts from store */
|
||||||
remove,
|
remove,
|
||||||
/** Subscribe to alerts */
|
/** Subscribe to alerts */
|
||||||
subscribe,
|
subscribe,
|
||||||
/** Unsubscribe from alerts */
|
/** Unsubscribe from alerts */
|
||||||
unsubscribe,
|
unsubscribe,
|
||||||
/** Refresh alerts with latest data from hub */
|
/** Refresh alerts with latest data from hub */
|
||||||
refresh,
|
refresh,
|
||||||
}
|
}
|
||||||
})()
|
})()
|
||||||
|
|||||||
@@ -106,8 +106,11 @@ export async function updateUserSettings() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getPbTimestamp(timeString: ChartTimes, d?: Date) {
|
export function getPbTimestamp(timeString: ChartTimes, d?: Date, createdIsNumber?: boolean) {
|
||||||
d ||= chartTimeData[timeString].getOffset(new Date())
|
d ||= chartTimeData[timeString].getOffset(new Date())
|
||||||
|
if (createdIsNumber) {
|
||||||
|
return d.getTime()
|
||||||
|
}
|
||||||
const year = d.getUTCFullYear()
|
const year = d.getUTCFullYear()
|
||||||
const month = String(d.getUTCMonth() + 1).padStart(2, "0")
|
const month = String(d.getUTCMonth() + 1).padStart(2, "0")
|
||||||
const day = String(d.getUTCDate()).padStart(2, "0")
|
const day = String(d.getUTCDate()).padStart(2, "0")
|
||||||
|
|||||||
@@ -0,0 +1,17 @@
|
|||||||
|
import type { MonitorStats, NetworkMonitorRecord, RawMonitorStatsRecord } from "@/types"
|
||||||
|
|
||||||
|
/** Derive chart metrics from the counts and response sum stored at every retention tier. */
|
||||||
|
export function getMonitorStats(record: RawMonitorStatsRecord): MonitorStats {
|
||||||
|
return {
|
||||||
|
res_avg: record.success_count > 0 ? record.res_sum / record.success_count : 0,
|
||||||
|
res_min: record.res_min,
|
||||||
|
res_max: record.res_max,
|
||||||
|
loss: record.total_count > 0 ? ((record.total_count - record.success_count) / record.total_count) * 100 : 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getMonitorTarget(monitor: Pick<NetworkMonitorRecord, "target" | "protocol" | "port">) {
|
||||||
|
if (monitor.protocol !== "tcp") return monitor.target
|
||||||
|
const host = monitor.target.includes(":") && !monitor.target.startsWith("[") ? `[${monitor.target}]` : monitor.target
|
||||||
|
return `${host}:${monitor.port}`
|
||||||
|
}
|
||||||
@@ -64,6 +64,9 @@ export const $containerFilter = atom("")
|
|||||||
/** Temperature chart filter */
|
/** Temperature chart filter */
|
||||||
export const $temperatureFilter = atom("")
|
export const $temperatureFilter = atom("")
|
||||||
|
|
||||||
|
/** Filter for network monitor charts (compare page and per-system monitor charts) */
|
||||||
|
export const $monitorFilter = atom("")
|
||||||
|
|
||||||
/** Fan-speed chart filter */
|
/** Fan-speed chart filter */
|
||||||
export const $fanFilter = atom("")
|
export const $fanFilter = atom("")
|
||||||
|
|
||||||
@@ -73,7 +76,5 @@ export const $copyContent = atom("")
|
|||||||
/** Direction for localization */
|
/** Direction for localization */
|
||||||
export const $direction = atom<"ltr" | "rtl">("ltr")
|
export const $direction = atom<"ltr" | "rtl">("ltr")
|
||||||
|
|
||||||
/** Longest system name length. Used to set table column width. I know this
|
/** Longest system name string. Used to reserve width in virtualized tables. */
|
||||||
* is stupid but the table is virtualized and I know this will work.
|
export const $longestSystemName = atom("")
|
||||||
*/
|
|
||||||
export const $longestSystemNameLen = atom(8)
|
|
||||||
|
|||||||
@@ -5,20 +5,17 @@ import {
|
|||||||
$allSystemsById,
|
$allSystemsById,
|
||||||
$allSystemsByName,
|
$allSystemsByName,
|
||||||
$downSystems,
|
$downSystems,
|
||||||
$longestSystemNameLen,
|
$longestSystemName,
|
||||||
$pausedSystems,
|
$pausedSystems,
|
||||||
$upSystems,
|
$upSystems,
|
||||||
} from "@/lib/stores"
|
} from "@/lib/stores"
|
||||||
import { getVisualStringWidth, updateFavicon } from "@/lib/utils"
|
import { isVisuallyLonger, updateFavicon } from "@/lib/utils"
|
||||||
import type { SystemRecord } from "@/types"
|
import type { SystemRecord } from "@/types"
|
||||||
import { SystemStatus } from "./enums"
|
import { SystemStatus } from "./enums"
|
||||||
|
|
||||||
const COLLECTION = pb.collection<SystemRecord>("systems")
|
const COLLECTION = pb.collection<SystemRecord>("systems")
|
||||||
const FIELDS_DEFAULT = "id,name,host,port,info,status"
|
const FIELDS_DEFAULT = "id,name,host,port,info,status"
|
||||||
|
|
||||||
/** Maximum system name length for display purposes */
|
|
||||||
const MAX_SYSTEM_NAME_LENGTH = 22
|
|
||||||
|
|
||||||
let initialized = false
|
let initialized = false
|
||||||
// biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks
|
// biome-ignore lint/suspicious/noConfusingVoidType: typescript rocks
|
||||||
let unsub: (() => void) | undefined | void
|
let unsub: (() => void) | undefined | void
|
||||||
@@ -44,7 +41,7 @@ export function init() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!newSystem) {
|
if (!newSystem) {
|
||||||
onSystemsChanged(newSystems, undefined)
|
onSystemsChanged(newSystems, newSystem, oldSystem)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,20 +65,28 @@ export function init() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// run things that need to be done when systems change
|
// run things that need to be done when systems change
|
||||||
onSystemsChanged(newSystems, newSystem)
|
onSystemsChanged(newSystems, newSystem, oldSystem)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Update the longest system name length and favicon based on system status */
|
/** Update the longest system name string and favicon based on system status */
|
||||||
function onSystemsChanged(_: Record<string, SystemRecord>, changedSystem: SystemRecord | undefined) {
|
function onSystemsChanged(systems: Record<string, SystemRecord>, newSystem?: SystemRecord, oldSystem?: SystemRecord) {
|
||||||
const downSystemsStore = $downSystems.get()
|
const downSystemsStore = $downSystems.get()
|
||||||
const downSystems = Object.values(downSystemsStore)
|
const downSystems = Object.values(downSystemsStore)
|
||||||
|
|
||||||
// Update longest system name length
|
// if the old system's old name was the longest, we need to find the new longest name
|
||||||
const longestName = $longestSystemNameLen.get()
|
// otherwise, if the changed system's new name is longer than the current longest, update it
|
||||||
const nameLen = Math.min(MAX_SYSTEM_NAME_LENGTH, getVisualStringWidth(changedSystem?.name || ""))
|
const longestName = $longestSystemName.get()
|
||||||
if (nameLen > longestName) {
|
if (oldSystem?.name === longestName && oldSystem.name !== newSystem?.name) {
|
||||||
$longestSystemNameLen.set(nameLen)
|
let newLongest = ""
|
||||||
|
for (const id in systems) {
|
||||||
|
if (isVisuallyLonger(systems[id].name, newLongest)) {
|
||||||
|
newLongest = systems[id].name
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$longestSystemName.set(newLongest)
|
||||||
|
} else if (newSystem && newSystem.name !== longestName && isVisuallyLonger(newSystem.name, longestName)) {
|
||||||
|
$longestSystemName.set(newSystem.name)
|
||||||
}
|
}
|
||||||
|
|
||||||
updateFavicon(downSystems.length)
|
updateFavicon(downSystems.length)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user