mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-18 22:14:28 +00:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f861f11a4 |
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
# Node.js dependencies
|
# Node.js dependencies
|
||||||
node_modules/
|
node_modules
|
||||||
**/node_modules/
|
internalsite/node_modules
|
||||||
|
|
||||||
# Go build artifacts and binaries
|
# Go build artifacts and binaries
|
||||||
build
|
build
|
||||||
|
|||||||
+2
-4
@@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
## Reporting a Vulnerability
|
## Reporting a Vulnerability
|
||||||
|
|
||||||
**PLEASE ONLY USE SECURITY ADVISORIES FOR REAL HIGH SEVERITY VULNERABILITIES.**
|
If you find a vulnerability in the latest version, please [submit a private advisory](https://github.com/henrygd/beszel/security/advisories/new).
|
||||||
|
|
||||||
If you find a vulnerability in the latest version, and it is not high severity, open an issue instead of an advisory.
|
If it's low severity (use best judgement) you may open an issue instead of an advisory.
|
||||||
|
|
||||||
I am overwhelmed with advisories, often erroneous, which are clearly found and written by AI. I don't have the capacity to review all of them.
|
|
||||||
|
|||||||
+4
-13
@@ -48,8 +48,7 @@ type Agent struct {
|
|||||||
keys []gossh.PublicKey // SSH public keys
|
keys []gossh.PublicKey // SSH public keys
|
||||||
smartManager *SmartManager // Manages SMART data
|
smartManager *SmartManager // Manages SMART data
|
||||||
systemdManager *systemdManager // Manages systemd services
|
systemdManager *systemdManager // Manages systemd services
|
||||||
monitorManager *MonitorManager // Manages network monitors
|
zfsManager *ZfsManager // Manages ZFS pool and dataset data
|
||||||
storagePoolManager *StoragePoolManager // Manages storage pool and dataset data
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewAgent creates a new agent with the given data directory for persisting data.
|
// NewAgent creates a new agent with the given data directory for persisting data.
|
||||||
@@ -123,15 +122,12 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
|
|||||||
// initialize handler registry
|
// initialize handler registry
|
||||||
agent.handlerRegistry = NewHandlerRegistry()
|
agent.handlerRegistry = NewHandlerRegistry()
|
||||||
|
|
||||||
// initialize monitor manager
|
agent.zfsManager = newZfsManager()
|
||||||
agent.monitorManager = newMonitorManager()
|
|
||||||
|
|
||||||
agent.storagePoolManager = newStoragePoolManager()
|
// ZFS_INTERVAL env var to update ZFS detail data at this interval
|
||||||
|
|
||||||
// Retain ZFS_INTERVAL for the shared storage pool detail refresh interval.
|
|
||||||
if zfsIntervalEnv, exists := utils.GetEnv("ZFS_INTERVAL"); exists {
|
if zfsIntervalEnv, exists := utils.GetEnv("ZFS_INTERVAL"); exists {
|
||||||
if duration, err := time.ParseDuration(zfsIntervalEnv); err == nil && duration > 0 {
|
if duration, err := time.ParseDuration(zfsIntervalEnv); err == nil && duration > 0 {
|
||||||
agent.storagePoolManager.detailInterval = duration
|
agent.zfsManager.detailInterval = duration
|
||||||
agent.systemDetails.ZfsInterval = duration
|
agent.systemDetails.ZfsInterval = duration
|
||||||
slog.Info("ZFS_INTERVAL", "duration", duration)
|
slog.Info("ZFS_INTERVAL", "duration", duration)
|
||||||
} else {
|
} else {
|
||||||
@@ -196,11 +192,6 @@ func (a *Agent) gatherStats(options common.DataRequestOptions) *system.CombinedD
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if a.monitorManager != nil {
|
|
||||||
data.Monitors = a.monitorManager.GetResults(cacheTimeMs)
|
|
||||||
slog.Debug("Monitors", "data", data.Monitors)
|
|
||||||
}
|
|
||||||
|
|
||||||
// skip updating systemd services if cache time is not the default 60sec interval
|
// skip updating systemd services if cache time is not the default 60sec interval
|
||||||
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
if a.systemdManager != nil && cacheTimeMs == defaultDataCacheTimeMs {
|
||||||
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
totalCount := uint16(a.systemdManager.getServiceStatsCount())
|
||||||
|
|||||||
@@ -1,26 +0,0 @@
|
|||||||
// Package btrfs reads btrfs filesystem state from sysfs.
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
// Filesystem is a mounted btrfs filesystem read from /sys/fs/btrfs/<uuid>.
|
|
||||||
type Filesystem struct {
|
|
||||||
UUID string // stable filesystem UUID from sysfs
|
|
||||||
MountID string // kernel filesystem identity for matching monitored mounts
|
|
||||||
IODevice string // sole member block-device name, empty for multi-device/unknown pools
|
|
||||||
Name string // label, else first mountpoint, else UUID
|
|
||||||
Size uint64 // effective usable capacity, or raw member capacity when Raw
|
|
||||||
Raw bool // capacity and usage are physical bytes, unsuitable for disk alerts
|
|
||||||
Alloc uint64 // raw bytes allocated to data, metadata and system chunks
|
|
||||||
Health string // ONLINE, or DEGRADED when a device is missing
|
|
||||||
NRead uint64 // cumulative bytes read across member devices
|
|
||||||
NWrite uint64 // cumulative bytes written across member devices
|
|
||||||
Devices []Device
|
|
||||||
}
|
|
||||||
|
|
||||||
// Device is one member device (devinfo/<devid>) with its error counters.
|
|
||||||
type Device struct {
|
|
||||||
Name string // "devid N"; sysfs does not expose the block device path
|
|
||||||
State string // ONLINE or MISSING
|
|
||||||
ReadErrs uint64
|
|
||||||
WriteErrs uint64
|
|
||||||
CorruptionErrs uint64
|
|
||||||
}
|
|
||||||
@@ -1,285 +0,0 @@
|
|||||||
//go:build linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"unsafe"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
sysfsPath = "/sys/fs/btrfs"
|
|
||||||
mountsPath = "/proc/self/mounts"
|
|
||||||
mountinfoPath = "/proc/self/mountinfo"
|
|
||||||
mountUUID = MountID
|
|
||||||
deviceSize = ioctlDeviceSize
|
|
||||||
filesystemUsage = statfsUsage
|
|
||||||
)
|
|
||||||
|
|
||||||
// Filesystems returns all mounted btrfs filesystems, or nil when there are none.
|
|
||||||
func Filesystems() ([]Filesystem, error) {
|
|
||||||
entries, err := os.ReadDir(sysfsPath)
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
mounts := mountpointsByDevice()
|
|
||||||
var filesystems []Filesystem
|
|
||||||
for _, entry := range entries {
|
|
||||||
if !entry.IsDir() || entry.Name() == "features" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
fs, err := readFilesystem(filepath.Join(sysfsPath, entry.Name()), mounts)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("btrfs %s: %w", entry.Name(), err)
|
|
||||||
}
|
|
||||||
filesystems = append(filesystems, fs)
|
|
||||||
}
|
|
||||||
return filesystems, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func readFilesystem(dir string, mounts map[string]string) (Filesystem, error) {
|
|
||||||
fs := Filesystem{UUID: filepath.Base(dir), Name: utils.ReadStringFile(filepath.Join(dir, "label")), Health: "UNKNOWN"}
|
|
||||||
for _, kind := range []string{"data", "metadata", "system"} {
|
|
||||||
if value, ok := utils.ReadUintFile(filepath.Join(dir, "allocation", kind, "disk_used")); ok {
|
|
||||||
fs.Alloc += value
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// devices/<name> links to the block device's sysfs directory.
|
|
||||||
devices, err := os.ReadDir(filepath.Join(dir, "devices"))
|
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
mountpoint := mounts["uuid:"+fs.UUID]
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = mountpoint
|
|
||||||
}
|
|
||||||
var backingSize uint64
|
|
||||||
for _, dev := range devices {
|
|
||||||
if mountpoint == "" {
|
|
||||||
mountpoint = mounts[dev.Name()]
|
|
||||||
}
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = mountpoint
|
|
||||||
}
|
|
||||||
devDir := filepath.Join(dir, "devices", dev.Name())
|
|
||||||
if size, ok := utils.ReadUintFile(filepath.Join(devDir, "size")); ok {
|
|
||||||
backingSize += size * 512
|
|
||||||
}
|
|
||||||
if stat := strings.Fields(utils.ReadStringFile(filepath.Join(devDir, "stat"))); len(stat) >= 7 {
|
|
||||||
fs.NRead += parseUint(stat[2]) * 512
|
|
||||||
fs.NWrite += parseUint(stat[6]) * 512
|
|
||||||
}
|
|
||||||
}
|
|
||||||
devids, err := os.ReadDir(filepath.Join(dir, "devinfo"))
|
|
||||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
capacityAvailable := len(devids) > 0
|
|
||||||
healthKnown := len(devids) > 0
|
|
||||||
for _, devid := range devids {
|
|
||||||
devDir := filepath.Join(dir, "devinfo", devid.Name())
|
|
||||||
// Replacement targets do not add filesystem capacity.
|
|
||||||
replaceTarget, _ := utils.ReadUintFile(filepath.Join(devDir, "replace_target"))
|
|
||||||
if replaceTarget != 1 {
|
|
||||||
devid, err := strconv.ParseUint(devid.Name(), 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return fs, err
|
|
||||||
}
|
|
||||||
size, err := deviceSize(mountpoint, devid)
|
|
||||||
if err != nil {
|
|
||||||
capacityAvailable = false
|
|
||||||
}
|
|
||||||
fs.Size += size
|
|
||||||
}
|
|
||||||
dev := Device{Name: "devid " + devid.Name(), State: "ONLINE"}
|
|
||||||
missing := utils.ReadStringFile(filepath.Join(devDir, "missing"))
|
|
||||||
if missing != "0" && missing != "1" {
|
|
||||||
healthKnown = false
|
|
||||||
dev.State = "UNKNOWN"
|
|
||||||
}
|
|
||||||
if missing == "1" {
|
|
||||||
dev.State = "MISSING"
|
|
||||||
fs.Health = "DEGRADED"
|
|
||||||
}
|
|
||||||
for line := range strings.Lines(utils.ReadStringFile(filepath.Join(devDir, "error_stats"))) {
|
|
||||||
if fields := strings.Fields(line); len(fields) == 2 {
|
|
||||||
switch fields[0] {
|
|
||||||
case "read_errs":
|
|
||||||
dev.ReadErrs = parseUint(fields[1])
|
|
||||||
case "write_errs":
|
|
||||||
dev.WriteErrs = parseUint(fields[1])
|
|
||||||
case "corruption_errs":
|
|
||||||
dev.CorruptionErrs = parseUint(fields[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs.Devices = append(fs.Devices, dev)
|
|
||||||
}
|
|
||||||
// Use one capacity source for the whole filesystem: device IDs cannot be
|
|
||||||
// reliably matched to block-device names in sysfs. A partial ioctl result
|
|
||||||
// must not be added to the complete backing-device total.
|
|
||||||
if !capacityAvailable {
|
|
||||||
fs.Size = backingSize
|
|
||||||
}
|
|
||||||
if fs.Health != "DEGRADED" && healthKnown {
|
|
||||||
fs.Health = "ONLINE"
|
|
||||||
}
|
|
||||||
fs.MountID = mountUUID(mountpoint)
|
|
||||||
if len(devices) == 1 && len(devids) == 1 && fs.Health == "ONLINE" {
|
|
||||||
fs.IODevice = devices[0].Name()
|
|
||||||
}
|
|
||||||
fs.Raw = true
|
|
||||||
if used, available, err := filesystemUsage(mountpoint); err == nil {
|
|
||||||
// Effective capacity excludes reserved/unavailable space, so Size-Alloc
|
|
||||||
// is available to applications and the usage ratio matches df.
|
|
||||||
fs.Size, fs.Alloc, fs.Raw = used+available, used, false
|
|
||||||
}
|
|
||||||
if fs.Name == "" {
|
|
||||||
fs.Name = filepath.Base(dir)
|
|
||||||
}
|
|
||||||
return fs, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// mountpointsByDevice prefers UUID matches from mountinfo and retains source
|
|
||||||
// device names as a fallback for environments where FS_INFO is unavailable.
|
|
||||||
func mountpointsByDevice() map[string]string {
|
|
||||||
mounts := mountpointsByUUID(utils.ReadStringFile(mountinfoPath), mountUUID)
|
|
||||||
for line := range strings.Lines(utils.ReadStringFile(mountsPath)) {
|
|
||||||
fields := strings.Fields(line)
|
|
||||||
if len(fields) < 3 || fields[2] != "btrfs" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
device := fields[0]
|
|
||||||
if resolved, err := filepath.EvalSymlinks(device); err == nil {
|
|
||||||
device = resolved
|
|
||||||
}
|
|
||||||
if _, seen := mounts[filepath.Base(device)]; !seen {
|
|
||||||
mounts[filepath.Base(device)] = unescapeMountPath(fields[1])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return mounts
|
|
||||||
}
|
|
||||||
|
|
||||||
func parseUint(s string) uint64 {
|
|
||||||
n, _ := strconv.ParseUint(s, 10, 64)
|
|
||||||
return n
|
|
||||||
}
|
|
||||||
|
|
||||||
// ioctlDeviceSize reads Btrfs's recorded device size, which can be smaller
|
|
||||||
// than the block device after a filesystem resize. BTRFS_IOC_DEV_INFO is
|
|
||||||
// _IOWR(0x94, 30, struct btrfs_ioctl_dev_info_args), a 4096-byte ABI structure.
|
|
||||||
func ioctlDeviceSize(mountpoint string, devid uint64) (uint64, error) {
|
|
||||||
if mountpoint == "" {
|
|
||||||
return 0, errors.New("no accessible mountpoint")
|
|
||||||
}
|
|
||||||
f, err := os.Open(mountpoint)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
args := struct {
|
|
||||||
Devid uint64
|
|
||||||
UUID [16]byte
|
|
||||||
BytesUsed uint64
|
|
||||||
TotalBytes uint64
|
|
||||||
Reserved [4096 - 40]byte
|
|
||||||
}{Devid: devid}
|
|
||||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), 0xd000941e, uintptr(unsafe.Pointer(&args)))
|
|
||||||
if errno != 0 {
|
|
||||||
return 0, errno
|
|
||||||
}
|
|
||||||
return args.TotalBytes, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// The filesystem magic is unsigned even when Statfs_t.Type is int32.
|
|
||||||
func isBtrfs(stat *unix.Statfs_t) bool {
|
|
||||||
return uint32(stat.Type) == unix.BTRFS_SUPER_MAGIC
|
|
||||||
}
|
|
||||||
|
|
||||||
func statfsUsage(path string) (used, available uint64, err error) {
|
|
||||||
if path == "" {
|
|
||||||
return 0, 0, errors.New("no accessible mountpoint")
|
|
||||||
}
|
|
||||||
var stat unix.Statfs_t
|
|
||||||
if err = unix.Statfs(path, &stat); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !isBtrfs(&stat) {
|
|
||||||
return 0, 0, errors.New("mountpoint is not Btrfs")
|
|
||||||
}
|
|
||||||
blockSize := uint64(stat.Bsize)
|
|
||||||
return (stat.Blocks - min(stat.Blocks, stat.Bfree)) * blockSize, min(stat.Blocks, stat.Bavail) * blockSize, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// MountID returns the filesystem UUID via BTRFS_IOC_FS_INFO. Unlike statfs
|
|
||||||
// f_fsid, this identity is shared by all subvolumes and bind mounts.
|
|
||||||
func MountID(path string) string {
|
|
||||||
if path == "" {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
var stat unix.Statfs_t
|
|
||||||
if unix.Statfs(path, &stat) != nil || !isBtrfs(&stat) {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
f, err := os.Open(path)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
args := struct {
|
|
||||||
MaxID uint64
|
|
||||||
NumDevices uint64
|
|
||||||
FSID [16]byte
|
|
||||||
Reserved [992]byte
|
|
||||||
}{}
|
|
||||||
// _IOR(0x94, 31, 1024). Reuse the platform's read-direction bits;
|
|
||||||
// MIPS/PowerPC use a different encoding than asm-generic.
|
|
||||||
request := uintptr(unix.FS_IOC_GETFLAGS&0xe0000000) | 0x0400941f
|
|
||||||
_, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), request, uintptr(unsafe.Pointer(&args)))
|
|
||||||
if errno != 0 {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
id := args.FSID
|
|
||||||
return fmt.Sprintf("%x-%x-%x-%x-%x", id[:4], id[4:6], id[6:8], id[8:10], id[10:])
|
|
||||||
}
|
|
||||||
|
|
||||||
// Btrfs mountinfo device numbers can be virtual (0:N), so query the UUID
|
|
||||||
// through the mount instead of comparing those numbers with sysfs block devs.
|
|
||||||
// Retry another path when a bind mount is inaccessible. Once resolved, reuse
|
|
||||||
// the result for that mount device to avoid opening every Docker bind mount.
|
|
||||||
func mountpointsByUUID(mountinfo string, identify func(string) string) map[string]string {
|
|
||||||
mounts := make(map[string]string)
|
|
||||||
resolved := make(map[string]bool)
|
|
||||||
for line := range strings.Lines(mountinfo) {
|
|
||||||
before, after, ok := strings.Cut(line, " - ")
|
|
||||||
fields, fs := strings.Fields(before), strings.Fields(after)
|
|
||||||
if !ok || len(fields) < 6 || len(fs) < 3 || fs[0] != "btrfs" || resolved[fields[2]] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
path := unescapeMountPath(fields[4])
|
|
||||||
uuid := identify(path)
|
|
||||||
if uuid == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
resolved[fields[2]] = true
|
|
||||||
if mounts["uuid:"+uuid] == "" {
|
|
||||||
mounts["uuid:"+uuid] = path
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return mounts
|
|
||||||
}
|
|
||||||
|
|
||||||
func unescapeMountPath(path string) string {
|
|
||||||
return strings.NewReplacer(`\040`, " ", `\011`, "\t", `\012`, "\n", `\134`, `\`).Replace(path)
|
|
||||||
}
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
//go:build testing && linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestFilesystems(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldSysfs, oldMounts := sysfsPath, mountsPath
|
|
||||||
sysfsPath, mountsPath = root, filepath.Join(root, "mounts")
|
|
||||||
t.Cleanup(func() { sysfsPath, mountsPath = oldSysfs, oldMounts })
|
|
||||||
|
|
||||||
fsDir := filepath.Join(root, "1b2c3d4e-0000-0000-0000-000000000000")
|
|
||||||
write := func(rel, content string) {
|
|
||||||
path := filepath.Join(fsDir, rel)
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755))
|
|
||||||
require.NoError(t, os.WriteFile(path, []byte(content), 0o644))
|
|
||||||
}
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "features"), 0o755))
|
|
||||||
oldUsage := filesystemUsage
|
|
||||||
filesystemUsage = func(string) (uint64, uint64, error) { return 0, 0, os.ErrNotExist }
|
|
||||||
t.Cleanup(func() { filesystemUsage = oldUsage })
|
|
||||||
oldDeviceSize := deviceSize
|
|
||||||
t.Cleanup(func() { deviceSize = oldDeviceSize })
|
|
||||||
deviceSize = func(_ string, devid uint64) (uint64, error) {
|
|
||||||
value, _ := utils.ReadUintFile(filepath.Join(fsDir, "recorded-size", strconv.FormatUint(devid, 10)))
|
|
||||||
return value, nil
|
|
||||||
}
|
|
||||||
// Recorded member capacities differ from the unchanged backing devices.
|
|
||||||
write("recorded-size/1", "256000\n")
|
|
||||||
write("recorded-size/2", "128000\n")
|
|
||||||
write("label", "tank\n")
|
|
||||||
write("allocation/data/disk_used", "4096\n")
|
|
||||||
write("allocation/metadata/disk_used", "2048\n")
|
|
||||||
write("allocation/system/disk_used", "1024\n")
|
|
||||||
write("devices/sda/size", "1000\n")
|
|
||||||
write("devices/sda/stat", "10 0 200 0 20 0 400 0 0 0 0\n")
|
|
||||||
write("devices/sdb/size", "1000\n")
|
|
||||||
write("devices/sdb/stat", "10 0 100 0 20 0 100 0 0 0 0\n")
|
|
||||||
write("devinfo/1/missing", "0\n")
|
|
||||||
write("devinfo/1/error_stats", "write_errs 1\nread_errs 2\nflush_errs 0\ncorruption_errs 3\ngeneration_errs 0\n")
|
|
||||||
write("devinfo/2/missing", "1\n")
|
|
||||||
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, Filesystem{
|
|
||||||
UUID: "1b2c3d4e-0000-0000-0000-000000000000", Raw: true, Name: "tank", Size: 384000, Alloc: 7168, Health: "DEGRADED", NRead: 153600, NWrite: 256000,
|
|
||||||
Devices: []Device{
|
|
||||||
{Name: "devid 1", State: "ONLINE", ReadErrs: 2, WriteErrs: 1, CorruptionErrs: 3},
|
|
||||||
{Name: "devid 2", State: "MISSING"},
|
|
||||||
},
|
|
||||||
}, filesystems[0])
|
|
||||||
|
|
||||||
// Unlabeled filesystems fall back to the first mountpoint, then the UUID.
|
|
||||||
write("label", "\n")
|
|
||||||
require.NoError(t, os.WriteFile(mountsPath, []byte(
|
|
||||||
"/dev/sdz1 /other btrfs rw 0 0\n/dev/sdb /mnt/storage btrfs rw 0 0\n/dev/sdb /mnt/storage/sub btrfs rw,subvol=/sub 0 0\n",
|
|
||||||
), 0o644))
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "/mnt/storage", filesystems[0].Name)
|
|
||||||
|
|
||||||
require.NoError(t, os.Remove(mountsPath))
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "1b2c3d4e-0000-0000-0000-000000000000", filesystems[0].Name)
|
|
||||||
write("devinfo/3/replace_target", "1\n")
|
|
||||||
write("recorded-size/3", "512000\n")
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(384000), filesystems[0].Size, "replacement target must not inflate capacity")
|
|
||||||
|
|
||||||
deviceSize = func(string, uint64) (uint64, error) { return 0, os.ErrPermission }
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
assert.Equal(t, "DEGRADED", filesystems[0].Health)
|
|
||||||
assert.Equal(t, uint64(153600), filesystems[0].NRead)
|
|
||||||
|
|
||||||
// A partial ioctl result must not be mixed with the backing-device total.
|
|
||||||
deviceSize = func(_ string, devid uint64) (uint64, error) {
|
|
||||||
if devid == 2 {
|
|
||||||
return 0, os.ErrPermission
|
|
||||||
}
|
|
||||||
return 256000, nil
|
|
||||||
}
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
|
|
||||||
// With no mount visible (e.g. Docker), the real lookup falls back too.
|
|
||||||
deviceSize = ioctlDeviceSize
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, filesystems, 1)
|
|
||||||
assert.Equal(t, uint64(1024000), filesystems[0].Size)
|
|
||||||
|
|
||||||
filesystemUsage = func(string) (uint64, uint64, error) { return 100, 900, nil }
|
|
||||||
filesystems, err = Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(1000), filesystems[0].Size)
|
|
||||||
assert.Equal(t, uint64(100), filesystems[0].Alloc)
|
|
||||||
assert.False(t, filesystems[0].Raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesystemsNoBtrfs(t *testing.T) {
|
|
||||||
oldPath := sysfsPath
|
|
||||||
sysfsPath = filepath.Join(t.TempDir(), "missing")
|
|
||||||
t.Cleanup(func() { sysfsPath = oldPath })
|
|
||||||
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, filesystems)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIoctlDeviceSizeFailure(t *testing.T) {
|
|
||||||
_, err := ioctlDeviceSize("", 1)
|
|
||||||
require.Error(t, err)
|
|
||||||
_, err = ioctlDeviceSize(t.TempDir(), 1)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.ErrorIs(t, err, unix.ENOTTY)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountpointsDecodeEscapes(t *testing.T) {
|
|
||||||
oldMounts := mountsPath
|
|
||||||
mountsPath = filepath.Join(t.TempDir(), "mounts")
|
|
||||||
t.Cleanup(func() { mountsPath = oldMounts })
|
|
||||||
require.NoError(t, os.WriteFile(mountsPath, []byte("/dev/test-btrfs /mnt/my\\040data btrfs rw 0 0\n"), 0o644))
|
|
||||||
assert.Equal(t, "/mnt/my data", mountpointsByDevice()["test-btrfs"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFilesystemWithoutDevinfo(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "devices", "sda"), 0755))
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(root, "devices", "sda", "size"), []byte("1000"), 0644))
|
|
||||||
fs, err := readFilesystem(root, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, uint64(512000), fs.Size)
|
|
||||||
assert.True(t, fs.Raw)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
assert.Empty(t, fs.Devices)
|
|
||||||
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Join(root, "devinfo", "1"), 0755))
|
|
||||||
fs, err = readFilesystem(root, nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
require.Len(t, fs.Devices, 1)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Devices[0].State)
|
|
||||||
|
|
||||||
// Some older interfaces lack the devices directory too.
|
|
||||||
fs, err = readFilesystem(t.TempDir(), nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "UNKNOWN", fs.Health)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLocalBtrfsUsage(t *testing.T) {
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for read-only live validation")
|
|
||||||
}
|
|
||||||
used, available, err := statfsUsage(path)
|
|
||||||
require.NoError(t, err)
|
|
||||||
filesystems, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, fs := range filesystems {
|
|
||||||
if !fs.Raw && fs.Alloc == used && fs.Size == used+available {
|
|
||||||
t.Logf("pool=%s used=%d available=%d effective_capacity=%d", fs.Name, used, available, fs.Size)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Fatal("collector did not report the mounted filesystem's usable capacity")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountID(t *testing.T) {
|
|
||||||
assert.Empty(t, MountID(""))
|
|
||||||
assert.Empty(t, MountID(filepath.Join(t.TempDir(), "missing")))
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for live identity validation")
|
|
||||||
}
|
|
||||||
id := MountID(path)
|
|
||||||
require.NotEmpty(t, id)
|
|
||||||
assert.Equal(t, id, MountID(filepath.Join(path, ".")))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMountinfoUUIDLookup(t *testing.T) {
|
|
||||||
info := `1 0 0:40 /@ /inaccessible ro shared:1 - btrfs /dev/mapper/unavailable rw
|
|
||||||
2 0 0:40 /@/docker/hosts /etc/hosts ro - btrfs /dev/mapper/unavailable rw
|
|
||||||
3 0 0:40 /@/docker/hostname /etc/hostname ro - btrfs /dev/mapper/unavailable rw
|
|
||||||
4 0 0:41 /subvol /extra-filesystems/my\040disk ro master:2 - btrfs /dev/missing rw
|
|
||||||
5 0 0:42 / /ext4 ro - ext4 /dev/mapper/unavailable rw
|
|
||||||
malformed
|
|
||||||
6 0 0:43 / /bad ro - btrfs
|
|
||||||
`
|
|
||||||
var calls []string
|
|
||||||
mounts := mountpointsByUUID(info, func(path string) string {
|
|
||||||
calls = append(calls, path)
|
|
||||||
switch path {
|
|
||||||
case "/etc/hosts":
|
|
||||||
return "root-uuid"
|
|
||||||
case "/extra-filesystems/my disk":
|
|
||||||
return "extra-uuid"
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
})
|
|
||||||
assert.Equal(t, map[string]string{"uuid:root-uuid": "/etc/hosts", "uuid:extra-uuid": "/extra-filesystems/my disk"}, mounts)
|
|
||||||
assert.Equal(t, []string{"/inaccessible", "/etc/hosts", "/extra-filesystems/my disk"}, calls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDockerFilesystemWithoutDeviceNodes(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldSysfs, oldMounts, oldInfo, oldUUID, oldUsage := sysfsPath, mountsPath, mountinfoPath, mountUUID, filesystemUsage
|
|
||||||
t.Cleanup(func() {
|
|
||||||
sysfsPath, mountsPath, mountinfoPath, mountUUID, filesystemUsage = oldSysfs, oldMounts, oldInfo, oldUUID, oldUsage
|
|
||||||
})
|
|
||||||
sysfsPath = filepath.Join(root, "sysfs")
|
|
||||||
mountsPath = filepath.Join(root, "missing-mounts")
|
|
||||||
mountinfoPath = filepath.Join(root, "mountinfo")
|
|
||||||
uuid := "11111111-1111-4111-8111-111111111111"
|
|
||||||
dir := filepath.Join(sysfsPath, uuid)
|
|
||||||
for path, content := range map[string]string{"devices/dm-0/size": "1000", "devinfo/1/missing": "0"} {
|
|
||||||
target := filepath.Join(dir, path)
|
|
||||||
require.NoError(t, os.MkdirAll(filepath.Dir(target), 0755))
|
|
||||||
require.NoError(t, os.WriteFile(target, []byte(content), 0644))
|
|
||||||
}
|
|
||||||
require.NoError(t, os.WriteFile(mountinfoPath, []byte("2 1 0:40 /@/docker/hosts /etc/hosts ro - btrfs /dev/mapper/not-in-container rw\n"), 0644))
|
|
||||||
mountUUID = func(path string) string {
|
|
||||||
if path == "/etc/hosts" {
|
|
||||||
return uuid
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
filesystemUsage = func(path string) (uint64, uint64, error) { require.Equal(t, "/etc/hosts", path); return 100, 900, nil }
|
|
||||||
fs, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, fs, 1)
|
|
||||||
assert.Equal(t, uuid, fs[0].MountID)
|
|
||||||
assert.Equal(t, "dm-0", fs[0].IODevice)
|
|
||||||
assert.False(t, fs[0].Raw)
|
|
||||||
assert.Equal(t, uint64(1000), fs[0].Size)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLivePoolMountIdentity(t *testing.T) {
|
|
||||||
path := os.Getenv("BESZEL_TEST_BTRFS_MOUNT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set BESZEL_TEST_BTRFS_MOUNT for live validation")
|
|
||||||
}
|
|
||||||
id := MountID(path)
|
|
||||||
require.NotEmpty(t, id)
|
|
||||||
pools, err := Filesystems()
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, pool := range pools {
|
|
||||||
if pool.UUID != id {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
assert.Equal(t, id, pool.MountID)
|
|
||||||
assert.False(t, pool.Raw)
|
|
||||||
t.Logf("uuid=%s mount_identity=%s io_device=%s raw=%v", pool.UUID, pool.MountID, pool.IODevice, pool.Raw)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
t.Fatal("mounted Btrfs filesystem was not discovered")
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
//go:build !linux
|
|
||||||
|
|
||||||
package btrfs
|
|
||||||
|
|
||||||
import "errors"
|
|
||||||
|
|
||||||
func Filesystems() ([]Filesystem, error) {
|
|
||||||
return nil, errors.ErrUnsupported
|
|
||||||
}
|
|
||||||
|
|
||||||
func MountID(string) string { return "" }
|
|
||||||
+2
-9
@@ -25,16 +25,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// Keep the connection alive long enough for a slow collection cycle to
|
wsDeadline = 70 * time.Second
|
||||||
// finish before the hub considers the agent disconnected.
|
|
||||||
wsDeadline = 120 * time.Second
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// errNoHubURL is returned when HUB_URL is unset. This is not a failure
|
|
||||||
// condition: an agent configured with only a public key runs in SSH-only mode,
|
|
||||||
// where the hub dials the agent and no outbound WebSocket client is expected.
|
|
||||||
var errNoHubURL = errors.New("HUB_URL environment variable not set")
|
|
||||||
|
|
||||||
type caCertFileError struct {
|
type caCertFileError struct {
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
@@ -68,7 +61,7 @@ type WebSocketClient struct {
|
|||||||
func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
|
func newWebSocketClient(agent *Agent) (client *WebSocketClient, err error) {
|
||||||
hubURLStr, exists := utils.GetEnv("HUB_URL")
|
hubURLStr, exists := utils.GetEnv("HUB_URL")
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, errNoHubURL
|
return nil, errors.New("HUB_URL environment variable not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
client = &WebSocketClient{}
|
client = &WebSocketClient{}
|
||||||
|
|||||||
@@ -32,28 +32,6 @@ import (
|
|||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TestNewWebSocketClientNoHubURL verifies that an unset HUB_URL returns the
|
|
||||||
// errNoHubURL sentinel rather than an opaque error. Callers rely on this to
|
|
||||||
// distinguish SSH-only mode -- a supported configuration in which the hub dials
|
|
||||||
// the agent -- from an actual misconfiguration.
|
|
||||||
func TestNewWebSocketClientNoHubURL(t *testing.T) {
|
|
||||||
agent := createTestAgent(t)
|
|
||||||
|
|
||||||
// t.Setenv registers restoration of the original value; unset afterwards so
|
|
||||||
// GetEnv's LookupEnv reports the variable as absent rather than empty.
|
|
||||||
t.Setenv("BESZEL_AGENT_HUB_URL", "")
|
|
||||||
os.Unsetenv("BESZEL_AGENT_HUB_URL")
|
|
||||||
t.Setenv("HUB_URL", "")
|
|
||||||
os.Unsetenv("HUB_URL")
|
|
||||||
t.Setenv("BESZEL_AGENT_TOKEN", "test-token")
|
|
||||||
|
|
||||||
client, err := newWebSocketClient(agent)
|
|
||||||
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Nil(t, client)
|
|
||||||
assert.ErrorIs(t, err, errNoHubURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestNewWebSocketClient tests WebSocket client creation
|
// TestNewWebSocketClient tests WebSocket client creation
|
||||||
func TestNewWebSocketClient(t *testing.T) {
|
func TestNewWebSocketClient(t *testing.T) {
|
||||||
agent := createTestAgent(t)
|
agent := createTestAgent(t)
|
||||||
@@ -722,11 +700,3 @@ func TestGetToken(t *testing.T) {
|
|||||||
assert.Equal(t, expectedToken, token, "Whitespace should be stripped from token file content")
|
assert.Equal(t, expectedToken, token, "Whitespace should be stripped from token file content")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWebSocketDeadlineCoversSlowCollection(t *testing.T) {
|
|
||||||
const minimumDeadline = 120 * time.Second
|
|
||||||
|
|
||||||
if wsDeadline < minimumDeadline {
|
|
||||||
t.Fatalf("WebSocket deadline %s is shorter than the slow-collection window of %s", wsDeadline, minimumDeadline)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -91,15 +91,7 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
|
|||||||
if errors.As(err, &caCertErr) {
|
if errors.As(err, &caCertErr) {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
disableSSH, _ := utils.GetEnv("DISABLE_SSH")
|
slog.Warn("Error creating WebSocket client", "err", err)
|
||||||
if errors.Is(err, errNoHubURL) && disableSSH != "true" {
|
|
||||||
// SSH-only mode: the hub dials the agent, so there is nothing to warn
|
|
||||||
// about. With SSH also disabled there is no connection method at all,
|
|
||||||
// so that case still warns.
|
|
||||||
slog.Debug("WebSocket client not configured", "err", err)
|
|
||||||
} else {
|
|
||||||
slog.Warn("Error creating WebSocket client", "err", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
c.wsClient = wsClient
|
c.wsClient = wsClient
|
||||||
|
|
||||||
@@ -153,7 +145,6 @@ func (c *ConnectionManager) Start(serverOptions ServerOptions) error {
|
|||||||
// }
|
// }
|
||||||
func (c *ConnectionManager) stop() error {
|
func (c *ConnectionManager) stop() error {
|
||||||
_ = c.agent.StopServer()
|
_ = c.agent.StopServer()
|
||||||
c.agent.monitorManager.Stop()
|
|
||||||
c.closeWebSocket()
|
c.closeWebSocket()
|
||||||
return health.CleanUp()
|
return health.CleanUp()
|
||||||
}
|
}
|
||||||
|
|||||||
+14
-14
@@ -18,11 +18,11 @@ import (
|
|||||||
// fsRegistrationContext holds the shared lookup state needed to resolve a
|
// fsRegistrationContext holds the shared lookup state needed to resolve a
|
||||||
// filesystem into the tracked fsStats key and metadata.
|
// filesystem into the tracked fsStats key and metadata.
|
||||||
type fsRegistrationContext struct {
|
type fsRegistrationContext struct {
|
||||||
filesystem string // device part of optional FILESYSTEM env var
|
filesystem string // device part of optional FILESYSTEM env var
|
||||||
filesystemName string // optional custom name from FILESYSTEM=device__name
|
filesystemName string // optional custom name from FILESYSTEM=device__name
|
||||||
isWindows bool
|
isWindows bool
|
||||||
efPath string // path to extra filesystems (default "/extra-filesystems")
|
efPath string // path to extra filesystems (default "/extra-filesystems")
|
||||||
diskIoCounters map[string]disk.IOCountersStat
|
diskIoCounters map[string]disk.IOCountersStat
|
||||||
}
|
}
|
||||||
|
|
||||||
// diskDiscovery groups the transient state for a single initializeDiskInfo run so
|
// diskDiscovery groups the transient state for a single initializeDiskInfo run so
|
||||||
@@ -325,11 +325,11 @@ func (a *Agent) initializeDiskInfo() {
|
|||||||
}
|
}
|
||||||
slog.Debug("Disk I/O", "diskstats", diskIoCounters)
|
slog.Debug("Disk I/O", "diskstats", diskIoCounters)
|
||||||
ctx := fsRegistrationContext{
|
ctx := fsRegistrationContext{
|
||||||
filesystem: filesystem,
|
filesystem: filesystem,
|
||||||
filesystemName: filesystemName,
|
filesystemName: filesystemName,
|
||||||
isWindows: isWindows,
|
isWindows: isWindows,
|
||||||
diskIoCounters: diskIoCounters,
|
diskIoCounters: diskIoCounters,
|
||||||
efPath: "/extra-filesystems",
|
efPath: "/extra-filesystems",
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get the appropriate root mount point for this system
|
// Get the appropriate root mount point for this system
|
||||||
@@ -540,8 +540,8 @@ func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersS
|
|||||||
// ZFS datasets have no /proc/diskstats entry, so they are excluded from
|
// ZFS datasets have no /proc/diskstats entry, so they are excluded from
|
||||||
// I/O tracking instead of warning about a missing device (#1541).
|
// I/O tracking instead of warning about a missing device (#1541).
|
||||||
var zfsMountpoints map[string]bool
|
var zfsMountpoints map[string]bool
|
||||||
if a.storagePoolManager != nil {
|
if a.zfsManager != nil {
|
||||||
zfsMountpoints = a.storagePoolManager.ZfsMountpoints()
|
zfsMountpoints = a.zfsManager.ZfsMountpoints()
|
||||||
}
|
}
|
||||||
for device, stats := range a.fsStats {
|
for device, stats := range a.fsStats {
|
||||||
if zfsMountpoints[stats.Mountpoint] {
|
if zfsMountpoints[stats.Mountpoint] {
|
||||||
@@ -574,8 +574,8 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
|
|||||||
// ZFS dataset mountpoints use `zfs list` values because statfs(2) reports
|
// ZFS dataset mountpoints use `zfs list` values because statfs(2) reports
|
||||||
// dataset-level usage that excludes child datasets (#1541).
|
// dataset-level usage that excludes child datasets (#1541).
|
||||||
var zfsUsage map[string]zfsDatasetUsage
|
var zfsUsage map[string]zfsDatasetUsage
|
||||||
if a.storagePoolManager != nil {
|
if a.zfsManager != nil {
|
||||||
zfsUsage = a.storagePoolManager.DatasetUsage()
|
zfsUsage = a.zfsManager.DatasetUsage()
|
||||||
}
|
}
|
||||||
|
|
||||||
// disk usage
|
// disk usage
|
||||||
|
|||||||
+10
-11
@@ -4,7 +4,6 @@ package agent
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
@@ -17,8 +16,8 @@ import (
|
|||||||
// is a ZFS dataset reports `zfs list` usage (which includes child datasets)
|
// is a ZFS dataset reports `zfs list` usage (which includes child datasets)
|
||||||
// instead of the dataset-scoped statfs values (#1541).
|
// instead of the dataset-scoped statfs values (#1541).
|
||||||
func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
zm := &ZfsManager{}
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
return []zfs.Dataset{
|
return []zfs.Dataset{
|
||||||
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -27,7 +26,7 @@ func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
|||||||
fsStats: map[string]*system.FsStats{
|
fsStats: map[string]*system.FsStats{
|
||||||
"tank": {Root: false, Mountpoint: "/tank"},
|
"tank": {Root: false, Mountpoint: "/tank"},
|
||||||
},
|
},
|
||||||
storagePoolManager: zm,
|
zfsManager: zm,
|
||||||
}
|
}
|
||||||
|
|
||||||
var stats system.Stats
|
var stats system.Stats
|
||||||
@@ -44,8 +43,8 @@ func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
|
|||||||
// TestUpdateDiskUsageZfsRootPopulatesSystemStats verifies the root disk values
|
// TestUpdateDiskUsageZfsRootPopulatesSystemStats verifies the root disk values
|
||||||
// are derived from ZFS usage when the root mountpoint is a ZFS dataset.
|
// are derived from ZFS usage when the root mountpoint is a ZFS dataset.
|
||||||
func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
|
func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
zm := &ZfsManager{}
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
return []zfs.Dataset{
|
return []zfs.Dataset{
|
||||||
{Name: "rpool/ROOT/pve-1", Used: 900000000000, Avail: 300000000000, Mountpoint: "/"},
|
{Name: "rpool/ROOT/pve-1", Used: 900000000000, Avail: 300000000000, Mountpoint: "/"},
|
||||||
}, nil
|
}, nil
|
||||||
@@ -54,7 +53,7 @@ func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
|
|||||||
fsStats: map[string]*system.FsStats{
|
fsStats: map[string]*system.FsStats{
|
||||||
"rpool/ROOT/pve-1": {Root: true, Mountpoint: "/"},
|
"rpool/ROOT/pve-1": {Root: true, Mountpoint: "/"},
|
||||||
},
|
},
|
||||||
storagePoolManager: zm,
|
zfsManager: zm,
|
||||||
}
|
}
|
||||||
|
|
||||||
var stats system.Stats
|
var stats system.Stats
|
||||||
@@ -86,8 +85,8 @@ func TestUpdateDiskUsageWithoutZfsManager(t *testing.T) {
|
|||||||
// TestInitializeDiskIoStatsSkipsZfsMountpoints verifies ZFS filesystems are
|
// TestInitializeDiskIoStatsSkipsZfsMountpoints verifies ZFS filesystems are
|
||||||
// excluded from diskstats I/O tracking instead of warning about a missing device.
|
// excluded from diskstats I/O tracking instead of warning about a missing device.
|
||||||
func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
|
func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
zm := &ZfsManager{}
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
return []zfs.Dataset{{Name: "tank", Mountpoint: "/tank"}}, nil
|
return []zfs.Dataset{{Name: "tank", Mountpoint: "/tank"}}, nil
|
||||||
}
|
}
|
||||||
agent := &Agent{
|
agent := &Agent{
|
||||||
@@ -95,8 +94,8 @@ func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
|
|||||||
"tank": {Root: false, Mountpoint: "/tank"},
|
"tank": {Root: false, Mountpoint: "/tank"},
|
||||||
"sda1": {Root: false, Mountpoint: "/mnt/data"},
|
"sda1": {Root: false, Mountpoint: "/mnt/data"},
|
||||||
},
|
},
|
||||||
storagePoolManager: zm,
|
zfsManager: zm,
|
||||||
diskPrev: make(map[uint16]map[string]prevDisk),
|
diskPrev: make(map[uint16]map[string]prevDisk),
|
||||||
}
|
}
|
||||||
|
|
||||||
agent.initializeDiskIoStats(map[string]disk.IOCountersStat{
|
agent.initializeDiskIoStats(map[string]disk.IOCountersStat{
|
||||||
|
|||||||
+8
-22
@@ -65,14 +65,10 @@ type dockerManager struct {
|
|||||||
dockerVersionChecked bool // Whether a version probe has completed successfully
|
dockerVersionChecked bool // Whether a version probe has completed successfully
|
||||||
isWindows bool // Whether the Docker Engine API is running on Windows
|
isWindows bool // Whether the Docker Engine API is running on Windows
|
||||||
buf *bytes.Buffer // Buffer to store and read response bodies
|
buf *bytes.Buffer // Buffer to store and read response bodies
|
||||||
|
apiStats *container.ApiStats // Reusable API stats object
|
||||||
excludeContainers []string // Patterns to exclude containers by name
|
excludeContainers []string // Patterns to exclude containers by name
|
||||||
usingPodman bool // Whether the Docker Engine API is running on Podman
|
usingPodman bool // Whether the Docker Engine API is running on Podman
|
||||||
|
|
||||||
registryClient *http.Client // Client for registry requests; nil uses a client with a 10-second timeout
|
|
||||||
imageUpdatesMutex sync.RWMutex // Protects imageUpdates, its entries, and imageUpdatesRunning
|
|
||||||
imageUpdates map[string]*imageUpdateStatus // Shared update status keyed by normalized image reference
|
|
||||||
imageUpdatesRunning bool // Whether a background image-update batch is in progress
|
|
||||||
|
|
||||||
// Cache-time-aware tracking for CPU stats (similar to cpu.go)
|
// Cache-time-aware tracking for CPU stats (similar to cpu.go)
|
||||||
// Maps cache time intervals to container-specific CPU usage tracking
|
// Maps cache time intervals to container-specific CPU usage tracking
|
||||||
lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage
|
lastCpuContainer map[uint16]map[string]uint64 // cacheTimeMs -> containerId -> last cpu container usage
|
||||||
@@ -165,9 +161,6 @@ func (dm *dockerManager) getDockerStats(cacheTimeMs uint16) ([]*container.Stats,
|
|||||||
clear(dm.validIds)
|
clear(dm.validIds)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only schedule auxiliary work here; metrics never wait for image discovery.
|
|
||||||
dm.refreshImageUpdates(dm.apiContainerList, time.Now())
|
|
||||||
|
|
||||||
var failedContainers []*container.ApiInfo
|
var failedContainers []*container.ApiInfo
|
||||||
|
|
||||||
for _, ctr := range dm.apiContainerList {
|
for _, ctr := range dm.apiContainerList {
|
||||||
@@ -513,17 +506,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Read and decode the response before locking shared stats to avoid blocking
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return fmt.Errorf("container stats request failed: %s", resp.Status)
|
|
||||||
}
|
|
||||||
res := &container.ApiStats{}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(res); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
updateAvailable := dm.cachedImageUpdate(ctr.Image)
|
|
||||||
|
|
||||||
dm.containerStatsMutex.Lock()
|
dm.containerStatsMutex.Lock()
|
||||||
defer dm.containerStatsMutex.Unlock()
|
defer dm.containerStatsMutex.Unlock()
|
||||||
|
|
||||||
@@ -538,9 +520,6 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
stats.Status = statusText
|
stats.Status = statusText
|
||||||
stats.Health = health
|
stats.Health = health
|
||||||
|
|
||||||
stats.Image = ctr.Image
|
|
||||||
stats.UpdateAvailable = updateAvailable
|
|
||||||
|
|
||||||
if len(ctr.Ports) > 0 {
|
if len(ctr.Ports) > 0 {
|
||||||
stats.Ports = convertContainerPortsToString(ctr)
|
stats.Ports = convertContainerPortsToString(ctr)
|
||||||
}
|
}
|
||||||
@@ -553,6 +532,12 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM
|
|||||||
stats.NetworkSent = 0
|
stats.NetworkSent = 0
|
||||||
stats.NetworkRecv = 0
|
stats.NetworkRecv = 0
|
||||||
|
|
||||||
|
res := dm.apiStats
|
||||||
|
res.Networks = nil
|
||||||
|
if err := dm.decode(resp, res); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize CPU tracking for this cache time interval
|
// Initialize CPU tracking for this cache time interval
|
||||||
dm.initializeCpuTracking(cacheTimeMs)
|
dm.initializeCpuTracking(cacheTimeMs)
|
||||||
|
|
||||||
@@ -710,6 +695,7 @@ func newDockerManager(agent *Agent) *dockerManager {
|
|||||||
containerStatsMap: make(map[string]*container.Stats),
|
containerStatsMap: make(map[string]*container.Stats),
|
||||||
sem: make(chan struct{}, 5),
|
sem: make(chan struct{}, 5),
|
||||||
apiContainerList: []*container.ApiInfo{},
|
apiContainerList: []*container.ApiInfo{},
|
||||||
|
apiStats: &container.ApiStats{},
|
||||||
excludeContainers: excludeContainers,
|
excludeContainers: excludeContainers,
|
||||||
|
|
||||||
// Initialize cache-time-aware tracking structures
|
// Initialize cache-time-aware tracking structures
|
||||||
|
|||||||
@@ -1,105 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"log/slog"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/distribution/reference"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
)
|
|
||||||
|
|
||||||
const imageUpdateInterval = time.Hour
|
|
||||||
|
|
||||||
type imageUpdateStatus struct {
|
|
||||||
available bool
|
|
||||||
checkedAt time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
func normalizedImageReference(image string) string {
|
|
||||||
named, err := reference.ParseNormalizedNamed(image)
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
// Digest-pinned references cannot move to a new version.
|
|
||||||
if _, pinned := named.(reference.Digested); pinned {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return reference.TagNameOnly(named).String()
|
|
||||||
}
|
|
||||||
|
|
||||||
// refreshImageUpdates starts at most one background batch. Neither its network
|
|
||||||
// work nor its completion is part of the container metrics wait group.
|
|
||||||
func (dm *dockerManager) refreshImageUpdates(containers []*container.ApiInfo, now time.Time) {
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
defer dm.imageUpdatesMutex.Unlock()
|
|
||||||
if dm.imageUpdatesRunning {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if dm.imageUpdates == nil {
|
|
||||||
dm.imageUpdates = make(map[string]*imageUpdateStatus)
|
|
||||||
}
|
|
||||||
active := make(map[string]struct{}, len(containers))
|
|
||||||
pending := make(map[string]*imageUpdateStatus)
|
|
||||||
for _, ctr := range containers {
|
|
||||||
if len(ctr.Names) > 0 && dm.shouldExcludeContainer(ctr.Names[0][1:]) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key := normalizedImageReference(ctr.Image)
|
|
||||||
if key == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
active[key] = struct{}{}
|
|
||||||
entry := dm.imageUpdates[key]
|
|
||||||
if entry == nil {
|
|
||||||
entry = &imageUpdateStatus{}
|
|
||||||
dm.imageUpdates[key] = entry
|
|
||||||
}
|
|
||||||
if entry.checkedAt.IsZero() || now.Sub(entry.checkedAt) >= imageUpdateInterval {
|
|
||||||
pending[key] = entry
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for key := range dm.imageUpdates {
|
|
||||||
if _, ok := active[key]; !ok {
|
|
||||||
delete(dm.imageUpdates, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(pending) == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
dm.imageUpdatesRunning = true
|
|
||||||
go func() {
|
|
||||||
// Limit auxiliary requests even on hosts running many different images.
|
|
||||||
sem := make(chan struct{}, 2)
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for key, entry := range pending {
|
|
||||||
sem <- struct{}{}
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
defer func() { <-sem }()
|
|
||||||
available, err := dm.checkImageUpdate(key)
|
|
||||||
if err != nil {
|
|
||||||
available = false
|
|
||||||
slog.Debug("Image update check failed", "image", key, "err", err)
|
|
||||||
}
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
entry.available = available
|
|
||||||
entry.checkedAt = time.Now()
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdatesRunning = false
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) cachedImageUpdate(image string) bool {
|
|
||||||
key := normalizedImageReference(image)
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
defer dm.imageUpdatesMutex.RUnlock()
|
|
||||||
entry := dm.imageUpdates[key]
|
|
||||||
return entry != nil && entry.available
|
|
||||||
}
|
|
||||||
@@ -1,225 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func waitForImageUpdates(t *testing.T, dm *dockerManager) {
|
|
||||||
t.Helper()
|
|
||||||
require.Eventually(t, func() bool {
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
defer dm.imageUpdatesMutex.RUnlock()
|
|
||||||
return !dm.imageUpdatesRunning
|
|
||||||
}, time.Second*3, time.Millisecond)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateCacheAndStats(t *testing.T) {
|
|
||||||
local := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
remote := "sha256:" + strings.Repeat("b", 64)
|
|
||||||
var inspections, lookups atomic.Int32
|
|
||||||
var fail atomic.Bool
|
|
||||||
var upToDate atomic.Bool
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
switch {
|
|
||||||
case strings.HasPrefix(r.URL.Path, "/images/"):
|
|
||||||
inspections.Add(1)
|
|
||||||
fmt.Fprintf(w, `{"RepoDigests":["docker.io/library/nginx@%s"]}`, local)
|
|
||||||
case r.URL.Path == "/containers/json":
|
|
||||||
fmt.Fprint(w, `[{"Id":"aaaaaaaaaaaa","Names":["/one"],"Image":"nginx","Status":"Up 2 hours"},{"Id":"bbbbbbbbbbbb","Names":["/two"],"Image":"docker.io/library/nginx:latest","Status":"Up 2 hours"}]`)
|
|
||||||
case strings.Contains(r.URL.Path, "/stats"):
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576},"cpu_stats":{},"networks":{}}`)
|
|
||||||
default:
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
dm.dockerVersionChecked = true
|
|
||||||
dm.registryClient = &http.Client{Timeout: time.Second, Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
|
||||||
if fail.Load() {
|
|
||||||
return nil, fmt.Errorf("registry unavailable")
|
|
||||||
}
|
|
||||||
response := &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"token":"test"}`))}
|
|
||||||
if r.Method == http.MethodHead {
|
|
||||||
lookups.Add(1)
|
|
||||||
digest := remote
|
|
||||||
if upToDate.Load() {
|
|
||||||
digest = local
|
|
||||||
}
|
|
||||||
response.Header.Set("Docker-Content-Digest", digest)
|
|
||||||
}
|
|
||||||
return response, nil
|
|
||||||
})}
|
|
||||||
stats, err := dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 2)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
require.EqualValues(t, 1, lookups.Load())
|
|
||||||
require.EqualValues(t, 1, inspections.Load())
|
|
||||||
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, stat := range stats {
|
|
||||||
require.True(t, stat.UpdateAvailable)
|
|
||||||
if stat.Id == "aaaaaaaaaaaa" {
|
|
||||||
require.Equal(t, "nginx", stat.Image)
|
|
||||||
} else {
|
|
||||||
require.Equal(t, "docker.io/library/nginx:latest", stat.Image)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.EqualValues(t, 1, lookups.Load())
|
|
||||||
|
|
||||||
expire := func() {
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdates["docker.io/library/nginx:latest"].checkedAt = time.Now().Add(-imageUpdateInterval)
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
}
|
|
||||||
upToDate.Store(true)
|
|
||||||
expire()
|
|
||||||
_, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
require.EqualValues(t, 2, lookups.Load())
|
|
||||||
require.False(t, dm.cachedImageUpdate("nginx:latest"))
|
|
||||||
|
|
||||||
// An expired positive result is cleared on failure, and the failure itself
|
|
||||||
// is cached so realtime stats do not retry a broken registry every second.
|
|
||||||
dm.imageUpdatesMutex.Lock()
|
|
||||||
dm.imageUpdates["docker.io/library/nginx:latest"].available = true
|
|
||||||
dm.imageUpdatesMutex.Unlock()
|
|
||||||
fail.Store(true)
|
|
||||||
expire()
|
|
||||||
_, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
waitForImageUpdates(t, dm)
|
|
||||||
failedInspections := inspections.Load()
|
|
||||||
stats, err = dm.getDockerStats(defaultCacheTimeMs)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, stats, 2)
|
|
||||||
require.Equal(t, failedInspections, inspections.Load())
|
|
||||||
for _, stat := range stats {
|
|
||||||
require.False(t, stat.UpdateAvailable)
|
|
||||||
require.Equal(t, 1.0, stat.Mem)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageDiscoveryDoesNotBlockStats(t *testing.T) {
|
|
||||||
started := make(chan struct{}, 1)
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
fmt.Fprintf(w, `{"RepoDigests":["example.com/app@sha256:%s"]}`, strings.Repeat("a", 64))
|
|
||||||
} else {
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
defer func() { close(release); waitForImageUpdates(t, dm) }()
|
|
||||||
dm.registryClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return nil, fmt.Errorf("timeout")
|
|
||||||
})}
|
|
||||||
ctr := &container.ApiInfo{IdShort: "aaaaaaaaaaaa", Image: "example.com/app", Names: []string{"/one"}}
|
|
||||||
dm.refreshImageUpdates([]*container.ApiInfo{ctr}, time.Now())
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("check did not start")
|
|
||||||
}
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- dm.updateContainerStats(ctr, defaultCacheTimeMs) }()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.NoError(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("registry blocked stats")
|
|
||||||
}
|
|
||||||
dm.imageUpdatesMutex.RLock()
|
|
||||||
require.True(t, dm.imageUpdatesRunning)
|
|
||||||
dm.imageUpdatesMutex.RUnlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNormalizeImageUpdateReferences(t *testing.T) {
|
|
||||||
require.Equal(t, normalizedImageReference("nginx"), normalizedImageReference("docker.io/library/nginx:latest"))
|
|
||||||
require.Empty(t, normalizedImageReference("bad reference"))
|
|
||||||
require.Empty(t, normalizedImageReference("nginx@sha256:"+strings.Repeat("a", 64)))
|
|
||||||
}
|
|
||||||
|
|
||||||
// A stats request can return headers promptly and then stall while reading its
|
|
||||||
// body. The stats-map mutex must remain available during that read.
|
|
||||||
func TestStatsResponseBodyDoesNotHoldStatsLock(t *testing.T) {
|
|
||||||
started := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusOK)
|
|
||||||
w.(http.Flusher).Flush()
|
|
||||||
close(started)
|
|
||||||
<-release
|
|
||||||
fmt.Fprint(w, `{"memory_stats":{"usage":1048576}}`)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
dm := newDockerManagerForVersionTest(server)
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- dm.updateContainerStats(&container.ApiInfo{IdShort: "aaaaaaaaaaaa", Names: []string{"/one"}, Image: "nginx"}, defaultCacheTimeMs)
|
|
||||||
}()
|
|
||||||
<-started
|
|
||||||
locked := make(chan struct{})
|
|
||||||
go func() { dm.containerStatsMutex.Lock(); dm.containerStatsMutex.Unlock(); close(locked) }()
|
|
||||||
select {
|
|
||||||
case <-locked:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
close(release)
|
|
||||||
<-done
|
|
||||||
t.Fatal("Docker response body held the stats mutex")
|
|
||||||
}
|
|
||||||
close(release)
|
|
||||||
require.NoError(t, <-done)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateStatsEncoding(t *testing.T) {
|
|
||||||
original := container.Stats{Image: "nginx:latest", UpdateAvailable: true}
|
|
||||||
encoded, err := cbor.Marshal(original)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var fields map[int]any
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &fields))
|
|
||||||
require.Equal(t, true, fields[11])
|
|
||||||
require.Equal(t, "nginx:latest", fields[8])
|
|
||||||
var decoded container.Stats
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
|
|
||||||
require.True(t, decoded.UpdateAvailable)
|
|
||||||
require.Equal(t, original.Image, decoded.Image)
|
|
||||||
encoded, err = json.Marshal(original)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Contains(t, string(encoded), `"u":true`)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestImageUpdateCacheExpiryBoundaryAndPruning(t *testing.T) {
|
|
||||||
now := time.Now()
|
|
||||||
key := normalizedImageReference("nginx")
|
|
||||||
dm := &dockerManager{imageUpdates: map[string]*imageUpdateStatus{
|
|
||||||
key: {available: true, checkedAt: now},
|
|
||||||
"unused.example/image:latest": {checkedAt: now},
|
|
||||||
}}
|
|
||||||
dm.refreshImageUpdates([]*container.ApiInfo{{Image: "nginx"}}, now.Add(imageUpdateInterval-time.Nanosecond))
|
|
||||||
require.False(t, dm.imageUpdatesRunning)
|
|
||||||
require.Len(t, dm.imageUpdates, 1)
|
|
||||||
require.True(t, dm.cachedImageUpdate("nginx:latest"))
|
|
||||||
dm.refreshImageUpdates(nil, now)
|
|
||||||
require.Empty(t, dm.imageUpdates)
|
|
||||||
}
|
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
_ "crypto/sha256"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"net/url"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/distribution/reference"
|
|
||||||
"github.com/opencontainers/go-digest"
|
|
||||||
)
|
|
||||||
|
|
||||||
const imageRegistryTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
const imageManifestAccept = "application/vnd.docker.distribution.manifest.list.v2+json, " +
|
|
||||||
"application/vnd.docker.distribution.manifest.v2+json, " +
|
|
||||||
"application/vnd.oci.image.manifest.v1+json, " +
|
|
||||||
"application/vnd.oci.image.index.v1+json"
|
|
||||||
|
|
||||||
// checkImageUpdate compares the digest recorded by Docker for image with the
|
|
||||||
// digest currently advertised by its registry. A digest-pinned reference is
|
|
||||||
// immutable and therefore never has an update available.
|
|
||||||
func (dm *dockerManager) checkImageUpdate(image string) (bool, error) {
|
|
||||||
named, err := reference.ParseNormalizedNamed(image)
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("parse image reference %q: %w", image, err)
|
|
||||||
}
|
|
||||||
if _, pinned := named.(reference.Digested); pinned {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
named = reference.TagNameOnly(named)
|
|
||||||
|
|
||||||
registry := reference.Domain(named)
|
|
||||||
repository := reference.Path(named)
|
|
||||||
tag := named.(reference.Tagged).Tag()
|
|
||||||
|
|
||||||
localDigest, err := dm.inspectImageDigest(image, registry, repository)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
remoteDigest, err := dm.registryImageDigest(registry, repository, tag)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return remoteDigest != localDigest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// inspectImageDigest reads Docker's image metadata without using dm.decode.
|
|
||||||
// The checker runs in the image-discovery goroutine, so it must not hold any
|
|
||||||
// of the container statistics locks while waiting on the Docker API.
|
|
||||||
func (dm *dockerManager) inspectImageDigest(image, registry, repository string) (string, error) {
|
|
||||||
if dm.client == nil {
|
|
||||||
return "", fmt.Errorf("inspect image %q: Docker client is unavailable", image)
|
|
||||||
}
|
|
||||||
|
|
||||||
endpoint := "http://localhost/images/" + url.PathEscape(image) + "/json"
|
|
||||||
resp, err := dm.client.Get(endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("inspect image %q: %w", image, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("inspect image %q failed: %s", image, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
var inspect struct {
|
|
||||||
RepoDigests []string `json:"RepoDigests"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&inspect); err != nil {
|
|
||||||
return "", fmt.Errorf("decode image inspect %q: %w", image, err)
|
|
||||||
}
|
|
||||||
if len(inspect.RepoDigests) == 0 {
|
|
||||||
return "", fmt.Errorf("inspect image %q returned no repository digests", image)
|
|
||||||
}
|
|
||||||
|
|
||||||
localDigest, ok := matchingRepositoryDigest(inspect.RepoDigests, registry, repository)
|
|
||||||
if !ok {
|
|
||||||
return "", fmt.Errorf("inspect image %q returned no valid digest for %s/%s", image, registry, repository)
|
|
||||||
}
|
|
||||||
return localDigest, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// matchingRepositoryDigest returns a valid digest belonging to the requested
|
|
||||||
// repository. Docker can return multiple RepoDigests for one local image; an
|
|
||||||
// unrelated first entry must never be used for the comparison.
|
|
||||||
func matchingRepositoryDigest(repoDigests []string, registry, repository string) (string, bool) {
|
|
||||||
for _, repoDigest := range repoDigests {
|
|
||||||
repoDigest = strings.TrimSpace(repoDigest)
|
|
||||||
at := strings.LastIndexByte(repoDigest, '@')
|
|
||||||
if at <= 0 || at == len(repoDigest)-1 || strings.Contains(repoDigest[:at], "@") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
repoRef, err := reference.ParseNormalizedNamed(repoDigest[:at])
|
|
||||||
if err != nil || reference.Path(repoRef) != repository || !sameRegistry(reference.Domain(repoRef), registry) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, hasTag := repoRef.(reference.Tagged); hasTag {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
d, err := digest.Parse(repoDigest[at+1:])
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return d.String(), true
|
|
||||||
}
|
|
||||||
return "", false
|
|
||||||
}
|
|
||||||
|
|
||||||
func sameRegistry(left, right string) bool {
|
|
||||||
left = canonicalRegistry(left)
|
|
||||||
right = canonicalRegistry(right)
|
|
||||||
return left == right ||
|
|
||||||
(left == "ghcr.io" && right == "lscr.io") ||
|
|
||||||
(left == "lscr.io" && right == "ghcr.io")
|
|
||||||
}
|
|
||||||
|
|
||||||
func canonicalRegistry(registry string) string {
|
|
||||||
if registry == "index.docker.io" {
|
|
||||||
return "docker.io"
|
|
||||||
}
|
|
||||||
return registry
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) registryImageDigest(registry, repository, tag string) (string, error) {
|
|
||||||
client := dm.registryClient
|
|
||||||
if client == nil {
|
|
||||||
client = &http.Client{Timeout: imageRegistryTimeout}
|
|
||||||
}
|
|
||||||
|
|
||||||
token, err := dm.registryToken(client, registry, repository)
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
host := registry
|
|
||||||
if registry == "docker.io" {
|
|
||||||
host = "registry-1.docker.io"
|
|
||||||
}
|
|
||||||
manifestURL := "https://" + host + "/v2/" + repository + "/manifests/" + url.PathEscape(tag)
|
|
||||||
req, err := http.NewRequest(http.MethodHead, manifestURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("create manifest request: %w", err)
|
|
||||||
}
|
|
||||||
req.Header.Set("Accept", imageManifestAccept)
|
|
||||||
if token != "" {
|
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("fetch manifest %s:%s: %w", registry, repository, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("manifest request for %s:%s failed: %s", repository, tag, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
remote := strings.TrimSpace(resp.Header.Get("Docker-Content-Digest"))
|
|
||||||
d, err := digest.Parse(remote)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("manifest request for %s:%s returned invalid digest: %w", repository, tag, err)
|
|
||||||
}
|
|
||||||
return d.String(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (dm *dockerManager) registryToken(client *http.Client, registry, repository string) (string, error) {
|
|
||||||
var authURL string
|
|
||||||
switch registry {
|
|
||||||
case "docker.io":
|
|
||||||
authURL = "https://auth.docker.io/token?service=registry.docker.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
|
|
||||||
case "ghcr.io", "lscr.io":
|
|
||||||
// lscr.io is the LinuxServer alias for its GHCR-backed images.
|
|
||||||
authURL = "https://ghcr.io/token?service=ghcr.io&scope=" + url.QueryEscape("repository:"+repository+":pull")
|
|
||||||
default:
|
|
||||||
// Anonymous registries remain supported, as they were before the
|
|
||||||
// authenticated Docker Hub and GHCR paths were added.
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
req, err := http.NewRequest(http.MethodGet, authURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("create registry auth request: %w", err)
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("fetch registry auth token for %s: %w", repository, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return "", fmt.Errorf("registry auth request for %s failed: %s", repository, responseStatus(resp))
|
|
||||||
}
|
|
||||||
|
|
||||||
var tokenResponse struct {
|
|
||||||
Token string `json:"token"`
|
|
||||||
AccessToken string `json:"access_token"`
|
|
||||||
}
|
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&tokenResponse); err != nil {
|
|
||||||
return "", fmt.Errorf("decode registry auth response for %s: %w", repository, err)
|
|
||||||
}
|
|
||||||
token := strings.TrimSpace(tokenResponse.Token)
|
|
||||||
if token == "" {
|
|
||||||
token = strings.TrimSpace(tokenResponse.AccessToken)
|
|
||||||
}
|
|
||||||
if token == "" {
|
|
||||||
return "", fmt.Errorf("registry auth response for %s contained no token", repository)
|
|
||||||
}
|
|
||||||
return token, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func responseStatus(resp *http.Response) string {
|
|
||||||
if resp.Status != "" {
|
|
||||||
return resp.Status
|
|
||||||
}
|
|
||||||
return http.StatusText(resp.StatusCode)
|
|
||||||
}
|
|
||||||
@@ -1,204 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type registryTransportFunc func(*http.Request) (*http.Response, error)
|
|
||||||
|
|
||||||
func (fn registryTransportFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
||||||
return fn(req)
|
|
||||||
}
|
|
||||||
|
|
||||||
func registryResponse(status int, body string) *http.Response {
|
|
||||||
return &http.Response{
|
|
||||||
StatusCode: status,
|
|
||||||
Status: fmt.Sprintf("%d %s", status, http.StatusText(status)),
|
|
||||||
Header: make(http.Header),
|
|
||||||
Body: io.NopCloser(strings.NewReader(body)),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func registryDigest(fill byte) string {
|
|
||||||
return "sha256:" + strings.Repeat(string(fill), 64)
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRegistryChecker(t *testing.T, inspectBody string, transport http.RoundTripper) *dockerManager {
|
|
||||||
t.Helper()
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
_, _ = io.WriteString(w, inspectBody)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
http.NotFound(w, r)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
return &dockerManager{
|
|
||||||
client: newDockerManagerForVersionTest(server).client,
|
|
||||||
registryClient: &http.Client{Transport: transport},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateUsesInspectAndManifestDigests(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
remote := registryDigest('b')
|
|
||||||
var authCalls, manifestCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
switch {
|
|
||||||
case req.Method == http.MethodGet && req.URL.Host == "auth.docker.io":
|
|
||||||
authCalls.Add(1)
|
|
||||||
require.Equal(t, "/token", req.URL.Path)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"test-token"}`), nil
|
|
||||||
case req.Method == http.MethodHead && req.URL.Host == "registry-1.docker.io":
|
|
||||||
manifestCalls.Add(1)
|
|
||||||
require.Equal(t, "/v2/library/alpine/manifests/latest", req.URL.Path)
|
|
||||||
require.Equal(t, "Bearer test-token", req.Header.Get("Authorization"))
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", remote)
|
|
||||||
return resp, nil
|
|
||||||
default:
|
|
||||||
return registryResponse(http.StatusNotFound, ""), nil
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
|
|
||||||
available, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.True(t, available)
|
|
||||||
require.EqualValues(t, 1, authCalls.Load())
|
|
||||||
require.EqualValues(t, 1, manifestCalls.Load())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateReportsUnknownInspectState(t *testing.T) {
|
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
body string
|
|
||||||
}{
|
|
||||||
{name: "missing field", body: `{}`},
|
|
||||||
{name: "empty field", body: `{"RepoDigests":[]}`},
|
|
||||||
{name: "malformed reference", body: `{"RepoDigests":["not-a-repo-digest"]}`},
|
|
||||||
{name: "wrong repository", body: `{"RepoDigests":["docker.io/library/busybox@` + registryDigest('a') + `"]}`},
|
|
||||||
{name: "malformed digest", body: `{"RepoDigests":["docker.io/library/alpine@sha256:not-a-digest"]}`},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
var registryCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, test.body, registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
registryCalls.Add(1)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"unexpected"}`), nil
|
|
||||||
}))
|
|
||||||
|
|
||||||
available, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.Error(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 0, registryCalls.Load(), "invalid local state must not query a registry")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateChecksInspectAuthAndManifestStatuses(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
validInspect := fmt.Sprintf(`{"RepoDigests":["docker.io/library/alpine@%s"]}`, local)
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
inspectCode int
|
|
||||||
authCode int
|
|
||||||
manifestCode int
|
|
||||||
remote string
|
|
||||||
want string
|
|
||||||
}{
|
|
||||||
{name: "inspect status", inspectCode: http.StatusNotFound, want: "inspect image"},
|
|
||||||
{name: "auth status", inspectCode: http.StatusOK, authCode: http.StatusUnauthorized, want: "registry auth"},
|
|
||||||
{name: "manifest status", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusNotFound, remote: local, want: "manifest request"},
|
|
||||||
{name: "missing digest", inspectCode: http.StatusOK, authCode: http.StatusOK, manifestCode: http.StatusOK, want: "invalid digest"},
|
|
||||||
}
|
|
||||||
for _, test := range tests {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
if test.inspectCode != http.StatusOK && strings.HasPrefix(r.URL.Path, "/images/") {
|
|
||||||
w.WriteHeader(test.inspectCode)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
_, _ = io.WriteString(w, validInspect)
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
|
|
||||||
calls := 0
|
|
||||||
dm := &dockerManager{client: newDockerManagerForVersionTest(server).client, registryClient: &http.Client{Transport: registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
calls++
|
|
||||||
if req.Method == http.MethodGet {
|
|
||||||
return registryResponse(test.authCode, `{"token":"test"}`), nil
|
|
||||||
}
|
|
||||||
response := registryResponse(test.manifestCode, "")
|
|
||||||
response.Header.Set("Docker-Content-Digest", test.remote)
|
|
||||||
return response, nil
|
|
||||||
})}}
|
|
||||||
|
|
||||||
_, err := dm.checkImageUpdate("alpine")
|
|
||||||
require.Error(t, err)
|
|
||||||
require.Contains(t, err.Error(), test.want)
|
|
||||||
if test.inspectCode != http.StatusOK {
|
|
||||||
require.Zero(t, calls)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateSupportsAnonymousAndLSCRRegistries(t *testing.T) {
|
|
||||||
t.Run("anonymous registry", func(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
var calls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["example.com/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
require.Equal(t, http.MethodHead, req.Method)
|
|
||||||
require.Equal(t, "example.com", req.URL.Host)
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", local)
|
|
||||||
return resp, nil
|
|
||||||
}))
|
|
||||||
available, err := dm.checkImageUpdate("example.com/app")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 1, calls.Load())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lscr ghcr alias", func(t *testing.T) {
|
|
||||||
local := registryDigest('a')
|
|
||||||
var authCalls, manifestCalls atomic.Int32
|
|
||||||
dm := newRegistryChecker(t, fmt.Sprintf(`{"RepoDigests":["ghcr.io/linuxserver/app@%s"]}`, local), registryTransportFunc(func(req *http.Request) (*http.Response, error) {
|
|
||||||
if req.Method == http.MethodGet {
|
|
||||||
authCalls.Add(1)
|
|
||||||
return registryResponse(http.StatusOK, `{"token":"test"}`), nil
|
|
||||||
}
|
|
||||||
manifestCalls.Add(1)
|
|
||||||
require.Equal(t, "lscr.io", req.URL.Host)
|
|
||||||
resp := registryResponse(http.StatusOK, "")
|
|
||||||
resp.Header.Set("Docker-Content-Digest", local)
|
|
||||||
return resp, nil
|
|
||||||
}))
|
|
||||||
available, err := dm.checkImageUpdate("lscr.io/linuxserver/app")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
require.EqualValues(t, 1, authCalls.Load())
|
|
||||||
require.EqualValues(t, 1, manifestCalls.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCheckImageUpdateSkipsPinnedDigest(t *testing.T) {
|
|
||||||
image := "docker.io/library/alpine@" + registryDigest('a')
|
|
||||||
dm := &dockerManager{}
|
|
||||||
available, err := dm.checkImageUpdate(image)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.False(t, available)
|
|
||||||
}
|
|
||||||
@@ -1184,6 +1184,7 @@ func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})},
|
})},
|
||||||
containerStatsMap: make(map[string]*container.Stats),
|
containerStatsMap: make(map[string]*container.Stats),
|
||||||
|
apiStats: &container.ApiStats{},
|
||||||
usingPodman: true,
|
usingPodman: true,
|
||||||
lastCpuContainer: map[uint16]map[string]uint64{
|
lastCpuContainer: map[uint16]map[string]uint64{
|
||||||
defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
|
defaultCacheTimeMs: {"0123456789ab": prevCpuUsage},
|
||||||
@@ -1675,6 +1676,7 @@ func TestUpdateContainerStatsUsesPodmanInspectHealthFallback(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})},
|
})},
|
||||||
containerStatsMap: make(map[string]*container.Stats),
|
containerStatsMap: make(map[string]*container.Stats),
|
||||||
|
apiStats: &container.ApiStats{},
|
||||||
usingPodman: true,
|
usingPodman: true,
|
||||||
lastCpuContainer: make(map[uint16]map[string]uint64),
|
lastCpuContainer: make(map[uint16]map[string]uint64),
|
||||||
lastCpuSystem: make(map[uint16]map[string]uint64),
|
lastCpuSystem: make(map[uint16]map[string]uint64),
|
||||||
|
|||||||
+2
-1
@@ -1119,6 +1119,7 @@ func TestCalculateGPUAverage(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
func TestGPUCapabilitiesAndLegacyPriority(t *testing.T) {
|
||||||
|
// Save original PATH
|
||||||
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
hasAmdSysfs := (&GPUManager{}).hasAmdSysfs()
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
@@ -1212,7 +1213,7 @@ echo "[]"`
|
|||||||
{
|
{
|
||||||
name: "no gpu tools available",
|
name: "no gpu tools available",
|
||||||
setupCommands: func(_ string) error {
|
setupCommands: func(_ string) error {
|
||||||
// The subtest already restricts PATH to its empty temporary directory.
|
t.Setenv("PATH", "")
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
|
|||||||
+2
-22
@@ -7,7 +7,6 @@ import (
|
|||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
"github.com/fxamacker/cbor/v2"
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
|
|
||||||
"log/slog"
|
"log/slog"
|
||||||
@@ -52,7 +51,6 @@ func NewHandlerRegistry() *HandlerRegistry {
|
|||||||
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
|
||||||
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
|
||||||
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
|
||||||
registry.Register(common.SyncNetworkMonitors, &SyncNetworkMonitorsHandler{})
|
|
||||||
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
|
||||||
|
|
||||||
return registry
|
return registry
|
||||||
@@ -188,14 +186,14 @@ func (h *GetSmartDataHandler) Handle(hctx *HandlerContext) error {
|
|||||||
type GetZfsDataHandler struct{}
|
type GetZfsDataHandler struct{}
|
||||||
|
|
||||||
func (h *GetZfsDataHandler) Handle(hctx *HandlerContext) error {
|
func (h *GetZfsDataHandler) Handle(hctx *HandlerContext) error {
|
||||||
if hctx.Agent.storagePoolManager == nil {
|
if hctx.Agent.zfsManager == nil {
|
||||||
return hctx.SendResponse(nil, hctx.RequestID)
|
return hctx.SendResponse(nil, hctx.RequestID)
|
||||||
}
|
}
|
||||||
var req common.ZfsDataRequest
|
var req common.ZfsDataRequest
|
||||||
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return hctx.SendResponse(hctx.Agent.storagePoolManager.GetDetail(req.Force), hctx.RequestID)
|
return hctx.SendResponse(hctx.Agent.zfsManager.GetDetail(req.Force), hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
////////////////////////////////////////////////////////////////////////////
|
||||||
@@ -225,21 +223,3 @@ func (h *GetSystemdInfoHandler) Handle(hctx *HandlerContext) error {
|
|||||||
|
|
||||||
return hctx.SendResponse(details, hctx.RequestID)
|
return hctx.SendResponse(details, hctx.RequestID)
|
||||||
}
|
}
|
||||||
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
////////////////////////////////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// SyncNetworkMonitorsHandler handles monitor configuration sync from hub
|
|
||||||
type SyncNetworkMonitorsHandler struct{}
|
|
||||||
|
|
||||||
func (h *SyncNetworkMonitorsHandler) Handle(hctx *HandlerContext) error {
|
|
||||||
var req monitor.SyncRequest
|
|
||||||
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
resp, err := hctx.Agent.monitorManager.HandleSyncRequest(req)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return hctx.SendResponse(resp, hctx.RequestID)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -34,19 +34,19 @@ func TestNewAgentResponseSmartData(t *testing.T) {
|
|||||||
|
|
||||||
func TestGetZfsDataHandlerForceRefresh(t *testing.T) {
|
func TestGetZfsDataHandlerForceRefresh(t *testing.T) {
|
||||||
poolCalls := 0
|
poolCalls := 0
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
poolCalls++
|
poolCalls++
|
||||||
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
||||||
}
|
}
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
zm.GetDetail(false)
|
zm.GetDetail(false)
|
||||||
|
|
||||||
requestData, err := cbor.Marshal(common.ZfsDataRequest{Force: true})
|
requestData, err := cbor.Marshal(common.ZfsDataRequest{Force: true})
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
ctx := &HandlerContext{
|
ctx := &HandlerContext{
|
||||||
Agent: &Agent{storagePoolManager: zm},
|
Agent: &Agent{zfsManager: zm},
|
||||||
Request: &common.HubRequest[cbor.RawMessage]{
|
Request: &common.HubRequest[cbor.RawMessage]{
|
||||||
Action: common.GetZfsData,
|
Action: common.GetZfsData,
|
||||||
Data: requestData,
|
Data: requestData,
|
||||||
|
|||||||
@@ -1,176 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net/http"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// MonitorManager manages network monitor configurations and task lifetimes.
|
|
||||||
type MonitorManager struct {
|
|
||||||
mu sync.RWMutex
|
|
||||||
monitors map[string]*monitorTask // keyed by monitor ID
|
|
||||||
probe monitorProbe
|
|
||||||
resumeGuard monitorResumeGuard
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorManager() *MonitorManager {
|
|
||||||
return newMonitorManagerWithProbe(networkMonitorProbe(&http.Client{Timeout: monitor.MaxProbeTimeout}))
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorManagerWithProbe(probe monitorProbe) *MonitorManager {
|
|
||||||
return &MonitorManager{monitors: make(map[string]*monitorTask), probe: probe}
|
|
||||||
}
|
|
||||||
|
|
||||||
// SyncMonitors replaces all monitor tasks with the given configs.
|
|
||||||
func (pm *MonitorManager) SyncMonitors(configs []monitor.Config) {
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
|
|
||||||
// Build set of new keys
|
|
||||||
newKeys := make(map[string]monitor.Config, len(configs))
|
|
||||||
for _, cfg := range configs {
|
|
||||||
if cfg.ID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
newKeys[cfg.ID] = cfg
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop removed monitors
|
|
||||||
for key, task := range pm.monitors {
|
|
||||||
if _, exists := newKeys[key]; !exists {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, key)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start new monitors and restart tasks whose config changed.
|
|
||||||
for key, cfg := range newKeys {
|
|
||||||
task, exists := pm.monitors[key]
|
|
||||||
if exists && task.config == cfg {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
task.cancel()
|
|
||||||
}
|
|
||||||
task = newMonitorTaskFromExisting(cfg, task)
|
|
||||||
task.resumeGuard = &pm.resumeGuard
|
|
||||||
pm.resumeGuard.start()
|
|
||||||
pm.monitors[key] = task
|
|
||||||
pm.startMonitor(task)
|
|
||||||
}
|
|
||||||
if len(pm.monitors) == 0 {
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleSyncRequest applies a full or incremental monitor sync request.
|
|
||||||
func (pm *MonitorManager) HandleSyncRequest(req monitor.SyncRequest) (monitor.SyncResponse, error) {
|
|
||||||
switch req.Action {
|
|
||||||
case monitor.SyncActionReplace:
|
|
||||||
pm.SyncMonitors(req.Configs)
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
case monitor.SyncActionUpsert:
|
|
||||||
result, err := pm.UpsertMonitor(req.Config, req.RunNow)
|
|
||||||
if err != nil {
|
|
||||||
return monitor.SyncResponse{}, err
|
|
||||||
}
|
|
||||||
if result == nil {
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
}
|
|
||||||
return monitor.SyncResponse{Result: *result}, nil
|
|
||||||
case monitor.SyncActionDelete:
|
|
||||||
if req.Config.ID == "" {
|
|
||||||
return monitor.SyncResponse{}, errors.New("missing monitor ID for delete")
|
|
||||||
}
|
|
||||||
pm.DeleteMonitor(req.Config.ID)
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
default:
|
|
||||||
return monitor.SyncResponse{}, fmt.Errorf("unknown monitor sync action: %d", req.Action)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpsertMonitor creates or replaces a single monitor task.
|
|
||||||
func (pm *MonitorManager) UpsertMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
|
|
||||||
if config.ID == "" {
|
|
||||||
return nil, errors.New("missing monitor ID")
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.mu.Lock()
|
|
||||||
task, exists := pm.monitors[config.ID]
|
|
||||||
if exists && task.config == config {
|
|
||||||
pm.mu.Unlock()
|
|
||||||
if !runNow {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return task.runProbe(pm.probe), nil
|
|
||||||
}
|
|
||||||
if exists {
|
|
||||||
task.cancel()
|
|
||||||
}
|
|
||||||
task = newMonitorTaskFromExisting(config, task)
|
|
||||||
task.resumeGuard = &pm.resumeGuard
|
|
||||||
pm.resumeGuard.start()
|
|
||||||
pm.monitors[config.ID] = task
|
|
||||||
pm.mu.Unlock()
|
|
||||||
|
|
||||||
if runNow {
|
|
||||||
result := task.runProbe(pm.probe)
|
|
||||||
pm.startMonitor(task)
|
|
||||||
return result, nil
|
|
||||||
}
|
|
||||||
pm.startMonitor(task)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteMonitor stops and removes a single monitor task.
|
|
||||||
func (pm *MonitorManager) DeleteMonitor(id string) {
|
|
||||||
if id == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
if task, exists := pm.monitors[id]; exists {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, id)
|
|
||||||
}
|
|
||||||
if len(pm.monitors) == 0 {
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetResults returns aggregated results for all monitors over the last supplied duration in ms.
|
|
||||||
func (pm *MonitorManager) GetResults(durationMs uint16) map[string]monitor.Result {
|
|
||||||
pm.mu.RLock()
|
|
||||||
defer pm.mu.RUnlock()
|
|
||||||
|
|
||||||
results := make(map[string]monitor.Result, len(pm.monitors))
|
|
||||||
now := time.Now()
|
|
||||||
duration := time.Duration(durationMs) * time.Millisecond
|
|
||||||
|
|
||||||
for _, task := range pm.monitors {
|
|
||||||
result, ok := task.history.result(duration, now)
|
|
||||||
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
results[task.config.ID] = result
|
|
||||||
}
|
|
||||||
|
|
||||||
return results
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stop stops all monitor tasks.
|
|
||||||
func (pm *MonitorManager) Stop() {
|
|
||||||
pm.mu.Lock()
|
|
||||||
defer pm.mu.Unlock()
|
|
||||||
for key, task := range pm.monitors {
|
|
||||||
task.cancel()
|
|
||||||
delete(pm.monitors, key)
|
|
||||||
}
|
|
||||||
pm.resumeGuard.shutdown()
|
|
||||||
}
|
|
||||||
@@ -1,274 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Monitors run at user-defined intervals (e.g., every 10s).
|
|
||||||
// To keep memory usage low and constant, data is stored in two layers:
|
|
||||||
// 1. Raw samples: The most recent individual results (kept for monitorRawRetention).
|
|
||||||
// 2. Minute buckets: A ring buffer of 61 buckets, each representing one
|
|
||||||
// wall-clock minute. Samples collected within the same minute are aggregated
|
|
||||||
// (sum, min, max, count) into a single bucket.
|
|
||||||
//
|
|
||||||
// Short-term requests (<= 61s) use raw samples.
|
|
||||||
// Long-term requests (up to 1h) use the minute buckets to avoid storing thousands
|
|
||||||
// of individual data points.
|
|
||||||
|
|
||||||
const (
|
|
||||||
// monitorRawRetention is the duration to keep individual samples
|
|
||||||
monitorRawRetention = 61 * time.Second
|
|
||||||
// monitorMinuteBucketLen is the number of 1-minute buckets to keep (1 hour + 1 for partials)
|
|
||||||
monitorMinuteBucketLen int32 = 61
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorHistory owns retention and aggregation, independently of probe execution.
|
|
||||||
type monitorHistory struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
sampleCount int64
|
|
||||||
samples []monitorSample
|
|
||||||
buckets [monitorMinuteBucketLen]monitorBucket
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorHistory() *monitorHistory {
|
|
||||||
// Start small for typical intervals; append grows the buffer for faster probes.
|
|
||||||
return &monitorHistory{samples: make([]monitorSample, 0, 4)}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) clone() *monitorHistory {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
cloned := newMonitorHistory()
|
|
||||||
cloned.samples = append(cloned.samples, h.samples...)
|
|
||||||
cloned.buckets = h.buckets
|
|
||||||
cloned.sampleCount = h.sampleCount
|
|
||||||
return cloned
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) result(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
return h.resultLocked(duration, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorHistory) record(sample monitorSample) monitor.Result {
|
|
||||||
h.mu.Lock()
|
|
||||||
defer h.mu.Unlock()
|
|
||||||
h.addSampleLocked(sample)
|
|
||||||
result, _ := h.resultLocked(time.Minute, sample.timestamp)
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorSample stores one monitor attempt and its collection time.
|
|
||||||
type monitorSample struct {
|
|
||||||
responseUs int64 // -1 means loss
|
|
||||||
timestamp time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorBucket stores one minute of aggregated monitor data.
|
|
||||||
type monitorBucket struct {
|
|
||||||
minute int32
|
|
||||||
filled bool
|
|
||||||
stats monitorAggregate
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorAggregate accumulates successful response stats and total sample counts.
|
|
||||||
type monitorAggregate struct {
|
|
||||||
sumUs int64
|
|
||||||
minUs int64
|
|
||||||
maxUs int64
|
|
||||||
totalCount int64
|
|
||||||
successCount int64
|
|
||||||
}
|
|
||||||
|
|
||||||
// newMonitorAggregate initializes an aggregate with an unset minimum value.
|
|
||||||
func newMonitorAggregate() monitorAggregate {
|
|
||||||
return monitorAggregate{minUs: math.MaxInt64}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addResponse folds a single monitor sample into the aggregate.
|
|
||||||
func (agg *monitorAggregate) addResponse(responseUs int64) {
|
|
||||||
agg.totalCount++
|
|
||||||
if responseUs < 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
agg.successCount++
|
|
||||||
agg.sumUs += responseUs
|
|
||||||
if responseUs < agg.minUs {
|
|
||||||
agg.minUs = responseUs
|
|
||||||
}
|
|
||||||
if responseUs > agg.maxUs {
|
|
||||||
agg.maxUs = responseUs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// addAggregate merges another aggregate into this one.
|
|
||||||
func (agg *monitorAggregate) addAggregate(other monitorAggregate) {
|
|
||||||
if other.totalCount == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
agg.totalCount += other.totalCount
|
|
||||||
agg.successCount += other.successCount
|
|
||||||
agg.sumUs += other.sumUs
|
|
||||||
if other.successCount == 0 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if agg.minUs == math.MaxInt64 || other.minUs < agg.minUs {
|
|
||||||
agg.minUs = other.minUs
|
|
||||||
}
|
|
||||||
if other.maxUs > agg.maxUs {
|
|
||||||
agg.maxUs = other.maxUs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// hasData reports whether the aggregate contains any samples.
|
|
||||||
func (agg monitorAggregate) hasData() bool {
|
|
||||||
return agg.totalCount > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
// result converts the aggregate into the monitor result format.
|
|
||||||
func (agg monitorAggregate) result() monitor.Result {
|
|
||||||
avg := agg.avgResponse()
|
|
||||||
result := monitor.Result{
|
|
||||||
AvgResponse: avg,
|
|
||||||
MinResponse: agg.minUs,
|
|
||||||
MaxResponse: agg.maxUs,
|
|
||||||
PacketLoss: agg.lossPercentage(),
|
|
||||||
TotalCount: agg.totalCount,
|
|
||||||
SuccessCount: agg.successCount,
|
|
||||||
ResponseSum: agg.sumUs,
|
|
||||||
}
|
|
||||||
if agg.successCount == 0 {
|
|
||||||
result.MinResponse, result.MaxResponse = 0, 0
|
|
||||||
}
|
|
||||||
return result
|
|
||||||
}
|
|
||||||
|
|
||||||
// avgResponse returns the rounded average of successful samples.
|
|
||||||
func (agg monitorAggregate) avgResponse() int64 {
|
|
||||||
if agg.successCount == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return agg.sumUs / agg.successCount
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// lossPercentage returns the rounded failure rate for the aggregate.
|
|
||||||
func (agg monitorAggregate) lossPercentage() float64 {
|
|
||||||
if agg.totalCount == 0 {
|
|
||||||
return 0
|
|
||||||
}
|
|
||||||
return math.Round(float64(agg.totalCount-agg.successCount)/float64(agg.totalCount)*10000) / 100
|
|
||||||
}
|
|
||||||
|
|
||||||
// resultLocked returns the aggregated monitor result for the requested duration along with a bool indicating whether any data was available.
|
|
||||||
func (h *monitorHistory) resultLocked(duration time.Duration, now time.Time) (monitor.Result, bool) {
|
|
||||||
agg := h.aggregateLocked(duration, now)
|
|
||||||
if !agg.hasData() {
|
|
||||||
// short realtime windows (e.g. the 1s window used for 1m/realtime charts) often fall
|
|
||||||
// between monitor samples since monitors run at longer, user-defined intervals; fall back to
|
|
||||||
// the most recent sample so realtime requests still report current status.
|
|
||||||
agg = h.latestSampleAggregateLocked()
|
|
||||||
}
|
|
||||||
hourAgg := h.aggregateLocked(time.Hour, now)
|
|
||||||
if !agg.hasData() {
|
|
||||||
return monitor.Result{}, false
|
|
||||||
}
|
|
||||||
|
|
||||||
result := agg.result()
|
|
||||||
if len(h.samples) > 0 {
|
|
||||||
result.LastProbeAt = h.samples[len(h.samples)-1].timestamp.UnixMilli()
|
|
||||||
}
|
|
||||||
|
|
||||||
result.AvgResponse1h = hourAgg.avgResponse()
|
|
||||||
result.MinResponse1h = hourAgg.minUs
|
|
||||||
result.MaxResponse1h = hourAgg.maxUs
|
|
||||||
result.PacketLoss1h = hourAgg.lossPercentage()
|
|
||||||
result.SampleCount = h.sampleCount
|
|
||||||
|
|
||||||
if hourAgg.successCount == 0 {
|
|
||||||
result.MinResponse1h, result.MaxResponse1h = 0, 0
|
|
||||||
}
|
|
||||||
return result, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// latestSampleAggregateLocked returns an aggregate containing only the most recent sample, if any.
|
|
||||||
func (h *monitorHistory) latestSampleAggregateLocked() monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
if len(h.samples) == 0 {
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
agg.addResponse(h.samples[len(h.samples)-1].responseUs)
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateLocked collects monitor data for the requested time window.
|
|
||||||
func (h *monitorHistory) aggregateLocked(duration time.Duration, now time.Time) monitorAggregate {
|
|
||||||
cutoff := now.Add(-duration)
|
|
||||||
// Keep short windows exact; longer windows read from minute buckets to avoid raw-sample retention.
|
|
||||||
if duration <= monitorRawRetention {
|
|
||||||
return aggregateSamplesSince(h.samples, cutoff)
|
|
||||||
}
|
|
||||||
return aggregateBucketsSince(h.buckets[:], cutoff, now)
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateSamplesSince aggregates raw samples newer than the cutoff.
|
|
||||||
func aggregateSamplesSince(samples []monitorSample, cutoff time.Time) monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
for _, sample := range samples {
|
|
||||||
if sample.timestamp.Before(cutoff) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
agg.addResponse(sample.responseUs)
|
|
||||||
}
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// aggregateBucketsSince aggregates minute buckets overlapping the requested window.
|
|
||||||
func aggregateBucketsSince(buckets []monitorBucket, cutoff, now time.Time) monitorAggregate {
|
|
||||||
agg := newMonitorAggregate()
|
|
||||||
startMinute := int32(cutoff.Unix() / 60)
|
|
||||||
endMinute := int32(now.Unix() / 60)
|
|
||||||
for _, bucket := range buckets {
|
|
||||||
if !bucket.filled || bucket.minute < startMinute || bucket.minute > endMinute {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
agg.addAggregate(bucket.stats)
|
|
||||||
}
|
|
||||||
return agg
|
|
||||||
}
|
|
||||||
|
|
||||||
// addSampleLocked stores a fresh sample in both raw and per-minute retention buffers.
|
|
||||||
func (h *monitorHistory) addSampleLocked(sample monitorSample) {
|
|
||||||
h.sampleCount++
|
|
||||||
cutoff := sample.timestamp.Add(-monitorRawRetention)
|
|
||||||
start := 0
|
|
||||||
for i := range h.samples {
|
|
||||||
if !h.samples[i].timestamp.Before(cutoff) {
|
|
||||||
start = i
|
|
||||||
break
|
|
||||||
}
|
|
||||||
if i == len(h.samples)-1 {
|
|
||||||
start = len(h.samples)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if start > 0 {
|
|
||||||
size := copy(h.samples, h.samples[start:])
|
|
||||||
h.samples = h.samples[:size]
|
|
||||||
}
|
|
||||||
h.samples = append(h.samples, sample)
|
|
||||||
|
|
||||||
minute := int32(sample.timestamp.Unix() / 60)
|
|
||||||
// Each slot stores one wall-clock minute, so the ring stays fixed-size at ~1h per monitor.
|
|
||||||
bucket := &h.buckets[minute%monitorMinuteBucketLen]
|
|
||||||
if !bucket.filled || bucket.minute != minute {
|
|
||||||
bucket.minute = minute
|
|
||||||
bucket.filled = true
|
|
||||||
bucket.stats = newMonitorAggregate()
|
|
||||||
}
|
|
||||||
bucket.stats.addResponse(sample.responseUs)
|
|
||||||
}
|
|
||||||
@@ -1,154 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorHistoryWindowCounts(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
now := time.Now()
|
|
||||||
// This older success counts toward lifetime warm-up, but not this window.
|
|
||||||
history.record(monitorSample{responseUs: 1000, timestamp: now.Add(-2 * time.Minute)})
|
|
||||||
history.record(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
history.record(monitorSample{responseUs: 21, timestamp: now.Add(-20 * time.Second)})
|
|
||||||
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
result, ok := history.result(time.Minute, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 4, result.SampleCount)
|
|
||||||
assert.EqualValues(t, 3, result.TotalCount)
|
|
||||||
assert.EqualValues(t, 2, result.SuccessCount)
|
|
||||||
assert.EqualValues(t, 31, result.ResponseSum, "preserve the sum before average rounding")
|
|
||||||
assert.EqualValues(t, 15, result.AvgResponse)
|
|
||||||
assert.Equal(t, 33.33, result.PacketLoss)
|
|
||||||
|
|
||||||
encoded, err := cbor.Marshal(result)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var decoded monitor.Result
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &decoded))
|
|
||||||
assert.Equal(t, result, decoded)
|
|
||||||
stats := monitor.Stats{}.FromResult(decoded)
|
|
||||||
assert.Equal(t, result.TotalCount, stats.TotalCount)
|
|
||||||
assert.Equal(t, result.SuccessCount, stats.SuccessCount)
|
|
||||||
assert.Equal(t, result.ResponseSum, stats.ResponseSum)
|
|
||||||
|
|
||||||
// Reads do not consume samples. A short window's latest-sample fallback
|
|
||||||
// carries the count for that single failure, not the minute or lifetime count.
|
|
||||||
repeated, _ := history.result(time.Minute, now)
|
|
||||||
assert.Equal(t, result, repeated)
|
|
||||||
fallback, ok := history.result(time.Second, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 1, fallback.TotalCount)
|
|
||||||
assert.Zero(t, fallback.SuccessCount)
|
|
||||||
assert.Zero(t, fallback.ResponseSum)
|
|
||||||
assert.Equal(t, 100.0, fallback.PacketLoss)
|
|
||||||
assert.EqualValues(t, 4, fallback.SampleCount)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAggregateLockedUsesRawSamplesForShortWindows(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-90 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(1), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MaxResponse)
|
|
||||||
assert.Equal(t, 50.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAggregateLockedUsesMinuteBucketsForLongWindows(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 30, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-11 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-90 * time.Second)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(10*time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(4), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(3), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(30), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(40), result.MaxResponse)
|
|
||||||
assert.Equal(t, 25.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryAddSampleLockedTrimsRawSamplesButKeepsBucketHistory(t *testing.T) {
|
|
||||||
now := time.Date(2026, time.April, 21, 12, 0, 0, 0, time.UTC)
|
|
||||||
history := newMonitorHistory()
|
|
||||||
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-10 * time.Minute)})
|
|
||||||
history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now})
|
|
||||||
|
|
||||||
require.Len(t, history.samples, 1)
|
|
||||||
assert.Equal(t, int64(20), history.samples[0].responseUs)
|
|
||||||
|
|
||||||
agg := history.aggregateLocked(10*time.Minute, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(2), agg.successCount)
|
|
||||||
result := agg.result()
|
|
||||||
assert.Equal(t, int64(15), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(10), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(20), result.MaxResponse)
|
|
||||||
assert.Equal(t, 0.0, result.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistoryProbeTimestamp(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
start := time.Date(2026, time.September, 14, 12, 0, 0, 0, time.UTC)
|
|
||||||
_, ok := history.result(time.Minute, start)
|
|
||||||
require.False(t, ok)
|
|
||||||
first := history.record(monitorSample{responseUs: 20, timestamp: start})
|
|
||||||
assert.Equal(t, start.UnixMilli(), first.LastProbeAt)
|
|
||||||
for minute := 0; minute < 5; minute++ {
|
|
||||||
now := start.Add(time.Duration(minute)*time.Minute + time.Second)
|
|
||||||
// Realtime reads must not consume freshness for the persistence request.
|
|
||||||
for _, window := range []time.Duration{time.Second, time.Minute} {
|
|
||||||
result, ok := history.result(window, now)
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, first.LastProbeAt, result.LastProbeAt)
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next := start.Add(5 * time.Minute)
|
|
||||||
failed := history.record(monitorSample{responseUs: -1, timestamp: next})
|
|
||||||
assert.Equal(t, next.UnixMilli(), failed.LastProbeAt)
|
|
||||||
assert.Equal(t, float64(100), failed.PacketLoss)
|
|
||||||
repeated, ok := history.result(time.Minute, next.Add(2*time.Minute))
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, failed.LastProbeAt, repeated.LastProbeAt)
|
|
||||||
assert.Equal(t, float64(100), repeated.PacketLoss)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHistorySampleCount(t *testing.T) {
|
|
||||||
history := newMonitorHistory()
|
|
||||||
now := time.Now()
|
|
||||||
// Both failed and successful probes count, including older samples so
|
|
||||||
// monitors with hourly intervals can finish warming up.
|
|
||||||
history.record(monitorSample{responseUs: -1, timestamp: now.Add(-2 * time.Hour)})
|
|
||||||
for i, response := range []int64{10, -1, 20} {
|
|
||||||
result := history.record(monitorSample{responseUs: response, timestamp: now.Add(time.Duration(i) * time.Second)})
|
|
||||||
assert.EqualValues(t, i+2, result.SampleCount)
|
|
||||||
}
|
|
||||||
result, ok := history.clone().result(time.Minute, now.Add(3*time.Second))
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.EqualValues(t, 4, result.SampleCount)
|
|
||||||
}
|
|
||||||
@@ -1,312 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"regexp"
|
|
||||||
"runtime"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"golang.org/x/net/icmp"
|
|
||||||
"golang.org/x/net/ipv4"
|
|
||||||
"golang.org/x/net/ipv6"
|
|
||||||
|
|
||||||
"log/slog"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Match the numeric RTT independently of the localized label used by Windows.
|
|
||||||
var pingTimeRegex = regexp.MustCompile(`(?i)[=<]\s*([0-9]+(?:[.,][0-9]+)?)\s*ms\b`)
|
|
||||||
|
|
||||||
var icmpSequence atomic.Uint32
|
|
||||||
|
|
||||||
type icmpPacketConn interface {
|
|
||||||
Close() error
|
|
||||||
}
|
|
||||||
|
|
||||||
// icmpMethod tracks which ICMP approach to use. Once a method succeeds or
|
|
||||||
// all native methods fail, the choice is cached so subsequent monitors skip
|
|
||||||
// the trial-and-error overhead.
|
|
||||||
type icmpMethod uint8
|
|
||||||
|
|
||||||
const (
|
|
||||||
icmpUntried icmpMethod = iota // haven't tried yet
|
|
||||||
icmpRaw // privileged raw socket
|
|
||||||
icmpDatagram // unprivileged datagram socket
|
|
||||||
icmpExecFallback // shell out to system ping command
|
|
||||||
)
|
|
||||||
|
|
||||||
// icmpFamily holds the network parameters and cached detection result for one address family.
|
|
||||||
type icmpFamily struct {
|
|
||||||
rawNetwork string // e.g. "ip4:icmp" or "ip6:ipv6-icmp"
|
|
||||||
dgramNetwork string // e.g. "udp4" or "udp6"
|
|
||||||
listenAddr string // "0.0.0.0" or "::"
|
|
||||||
echoType icmp.Type // outgoing echo request type
|
|
||||||
replyType icmp.Type // expected echo reply type
|
|
||||||
proto int // IANA protocol number for parsing replies
|
|
||||||
isIPv6 bool
|
|
||||||
mode icmpMethod // cached detection result (guarded by icmpModeMu)
|
|
||||||
}
|
|
||||||
|
|
||||||
var (
|
|
||||||
icmpV4 = icmpFamily{
|
|
||||||
rawNetwork: "ip4:icmp",
|
|
||||||
dgramNetwork: "udp4",
|
|
||||||
listenAddr: "0.0.0.0",
|
|
||||||
echoType: ipv4.ICMPTypeEcho,
|
|
||||||
replyType: ipv4.ICMPTypeEchoReply,
|
|
||||||
proto: 1,
|
|
||||||
}
|
|
||||||
icmpV6 = icmpFamily{
|
|
||||||
rawNetwork: "ip6:ipv6-icmp",
|
|
||||||
dgramNetwork: "udp6",
|
|
||||||
listenAddr: "::",
|
|
||||||
echoType: ipv6.ICMPTypeEchoRequest,
|
|
||||||
replyType: ipv6.ICMPTypeEchoReply,
|
|
||||||
proto: 58,
|
|
||||||
isIPv6: true,
|
|
||||||
}
|
|
||||||
icmpModeMu sync.Mutex
|
|
||||||
icmpListen = func(network, listenAddr string) (icmpPacketConn, error) {
|
|
||||||
return icmp.ListenPacket(network, listenAddr)
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorICMP sends an ICMP echo request and measures round-trip response.
|
|
||||||
// Supports both IPv4 and IPv6 targets. The ICMP method (raw socket,
|
|
||||||
// unprivileged datagram, or exec fallback) is detected once per address
|
|
||||||
// family and cached for subsequent monitors.
|
|
||||||
// Returns response in microseconds, or -1 and an error on failure.
|
|
||||||
func monitorICMP(ctx context.Context, target string) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
family, ip, err := resolveICMPTarget(ctx, target)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
icmpModeMu.Lock()
|
|
||||||
if family.mode == icmpUntried {
|
|
||||||
family.mode = detectICMPMode(family, icmpListen)
|
|
||||||
}
|
|
||||||
mode := family.mode
|
|
||||||
icmpModeMu.Unlock()
|
|
||||||
|
|
||||||
switch mode {
|
|
||||||
case icmpRaw:
|
|
||||||
return monitorICMPNative(ctx, family.rawNetwork, family, &net.IPAddr{IP: ip})
|
|
||||||
case icmpDatagram:
|
|
||||||
return monitorICMPNative(ctx, family.dgramNetwork, family, &net.UDPAddr{IP: ip})
|
|
||||||
case icmpExecFallback:
|
|
||||||
return monitorICMPExec(ctx, ip.String(), family.isIPv6)
|
|
||||||
default:
|
|
||||||
return -1, errors.New("unsupported ICMP mode")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// resolveICMPTarget resolves a target hostname or IP to determine the address
|
|
||||||
// family and concrete IP address. Prefers IPv4 for dual-stack hostnames.
|
|
||||||
func resolveICMPTarget(ctx context.Context, target string) (*icmpFamily, net.IP, error) {
|
|
||||||
if ip := net.ParseIP(target); ip != nil {
|
|
||||||
if ip.To4() != nil {
|
|
||||||
return &icmpV4, ip.To4(), nil
|
|
||||||
}
|
|
||||||
return &icmpV6, ip, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
ips, err := net.DefaultResolver.LookupIP(ctx, "ip", target)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
return nil, nil, err
|
|
||||||
}
|
|
||||||
for _, ip := range ips {
|
|
||||||
if v4 := ip.To4(); v4 != nil {
|
|
||||||
return &icmpV4, v4, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return &icmpV6, ips[0], nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func detectICMPMode(family *icmpFamily, listen func(network, listenAddr string) (icmpPacketConn, error)) icmpMethod {
|
|
||||||
label := "IPv4"
|
|
||||||
if family.isIPv6 {
|
|
||||||
label = "IPv6"
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err := listen(family.rawNetwork, family.listenAddr)
|
|
||||||
slog.Debug("ICMP raw socket test", "family", label, "err", err)
|
|
||||||
if err == nil {
|
|
||||||
conn.Close()
|
|
||||||
return icmpRaw
|
|
||||||
}
|
|
||||||
|
|
||||||
conn, err = listen(family.dgramNetwork, family.listenAddr)
|
|
||||||
slog.Debug("ICMP datagram socket test", "family", label, "err", err)
|
|
||||||
if err == nil {
|
|
||||||
conn.Close()
|
|
||||||
return icmpDatagram
|
|
||||||
}
|
|
||||||
|
|
||||||
return icmpExecFallback
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorICMPNative sends an ICMP echo request using Go's x/net/icmp package.
|
|
||||||
func monitorICMPNative(ctx context.Context, network string, family *icmpFamily, dst net.Addr) (int64, error) {
|
|
||||||
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
|
|
||||||
return monitorICMPPacket(ctx, conn, family, dst)
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorICMPPacket(ctx context.Context, conn net.PacketConn, family *icmpFamily, dst net.Addr) (int64, error) {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
// Closing the socket interrupts both reads and writes on cancellation.
|
|
||||||
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
// Prepare correlation data before starting the round-trip timer. The token
|
|
||||||
// also distinguishes delayed replies after the 16-bit sequence wraps.
|
|
||||||
token := make([]byte, 16)
|
|
||||||
if _, err := rand.Read(token); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
echo := &icmp.Echo{
|
|
||||||
ID: os.Getpid() & 0xffff,
|
|
||||||
Seq: int(icmpSequence.Add(1) & 0xffff),
|
|
||||||
Data: token,
|
|
||||||
}
|
|
||||||
// Linux ping sockets replace the Echo ID with their bound port. Darwin
|
|
||||||
// datagram sockets and raw sockets preserve the supplied ID.
|
|
||||||
if local, ok := conn.LocalAddr().(*net.UDPAddr); ok && runtime.GOOS == "linux" {
|
|
||||||
echo.ID = local.Port
|
|
||||||
}
|
|
||||||
targetIP := icmpAddrIP(dst)
|
|
||||||
msg := &icmp.Message{
|
|
||||||
Type: family.echoType,
|
|
||||||
Code: 0,
|
|
||||||
Body: echo,
|
|
||||||
}
|
|
||||||
msgBytes, err := msg.Marshal(nil)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set deadline before sending
|
|
||||||
if err := conn.SetDeadline(time.Now().Add(3 * time.Second)); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
buf := make([]byte, 1500)
|
|
||||||
start := time.Now()
|
|
||||||
if _, err := conn.WriteTo(msgBytes, dst); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read reply
|
|
||||||
for {
|
|
||||||
n, peer, err := conn.ReadFrom(buf)
|
|
||||||
received := time.Now()
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
if !targetIP.Equal(icmpAddrIP(peer)) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
reply, err := icmp.ParseMessage(family.proto, buf[:n])
|
|
||||||
if err != nil || reply.Type != family.replyType || reply.Code != 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
body, ok := reply.Body.(*icmp.Echo)
|
|
||||||
if ok && body.ID == echo.ID && body.Seq == echo.Seq && bytes.Equal(body.Data, echo.Data) {
|
|
||||||
return received.Sub(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
// Keep waiting for our reply without extending the original deadline.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func icmpAddrIP(addr net.Addr) net.IP {
|
|
||||||
switch addr := addr.(type) {
|
|
||||||
case *net.IPAddr:
|
|
||||||
return addr.IP
|
|
||||||
case *net.UDPAddr:
|
|
||||||
return addr.IP
|
|
||||||
default:
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// pingCommand selects the executable and arguments for the supported agent platforms.
|
|
||||||
// The context deadline enforces the timeout: -W has incompatible meanings across
|
|
||||||
// Linux, BSD IPv4 ping, and macOS ping6.
|
|
||||||
func pingCommand(goos, target string, isIPv6 bool) (string, []string, error) {
|
|
||||||
family := "-4"
|
|
||||||
if isIPv6 {
|
|
||||||
family = "-6"
|
|
||||||
}
|
|
||||||
switch goos {
|
|
||||||
case "windows":
|
|
||||||
return "ping", []string{family, "-n", "1", "-w", "3000", target}, nil
|
|
||||||
case "linux":
|
|
||||||
return "ping", []string{family, "-n", "-c", "1", target}, nil
|
|
||||||
case "darwin", "freebsd", "openbsd":
|
|
||||||
command := "ping"
|
|
||||||
if isIPv6 {
|
|
||||||
command = "ping6"
|
|
||||||
}
|
|
||||||
return command, []string{"-n", "-c", "1", target}, nil
|
|
||||||
default:
|
|
||||||
return "", nil, fmt.Errorf("ping fallback is unsupported on %s", goos)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorICMPExec falls back to the system ping command. Returns -1 and an error on failure.
|
|
||||||
func monitorICMPExec(ctx context.Context, target string, isIPv6 bool) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
name, args, err := pingCommand(runtime.GOOS, target, isIPv6)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
cmd := exec.CommandContext(ctx, name, args...)
|
|
||||||
// Keep Unix output and decimal formatting stable. Windows ignores LC_ALL.
|
|
||||||
cmd.Env = append(os.Environ(), "LC_ALL=C")
|
|
||||||
output, err := cmd.Output()
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return -1, ctx.Err()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return -1, fmt.Errorf("%s failed: %w", name, err)
|
|
||||||
}
|
|
||||||
return parsePingResponse(output)
|
|
||||||
}
|
|
||||||
|
|
||||||
// parsePingResponse returns the reported RTT, never subprocess execution time.
|
|
||||||
// For a bounded value such as Windows' time<1ms, retain the reported upper bound.
|
|
||||||
func parsePingResponse(output []byte) (int64, error) {
|
|
||||||
matches := pingTimeRegex.FindSubmatch(output)
|
|
||||||
if len(matches) < 2 {
|
|
||||||
return -1, errors.New("ping output contains no round-trip time")
|
|
||||||
}
|
|
||||||
ms, err := strconv.ParseFloat(strings.ReplaceAll(string(matches[1]), ",", "."), 64)
|
|
||||||
if err != nil || math.IsInf(ms, 0) || ms >= float64(math.MaxInt64)/1000 {
|
|
||||||
return -1, errors.New("invalid round-trip time in ping output")
|
|
||||||
}
|
|
||||||
return int64(math.Round(ms * 1000)), nil
|
|
||||||
}
|
|
||||||
@@ -1,433 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/net/icmp"
|
|
||||||
)
|
|
||||||
|
|
||||||
type testICMPPacketConn struct{}
|
|
||||||
|
|
||||||
func (testICMPPacketConn) Close() error { return nil }
|
|
||||||
|
|
||||||
type blockingICMPConn struct {
|
|
||||||
net.PacketConn
|
|
||||||
reading chan struct{}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *blockingICMPConn) WriteTo(p []byte, addr net.Addr) (int, error) {
|
|
||||||
return len(p), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *blockingICMPConn) ReadFrom(p []byte) (int, net.Addr, error) {
|
|
||||||
close(c.reading)
|
|
||||||
return c.PacketConn.ReadFrom(p)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPPacketCancellation(t *testing.T) {
|
|
||||||
conn, err := net.ListenPacket("udp4", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer conn.Close()
|
|
||||||
blocking := &blockingICMPConn{PacketConn: conn, reading: make(chan struct{})}
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := monitorICMPPacket(ctx, blocking, &icmpV4, conn.LocalAddr())
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-blocking.reading:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("probe did not begin reading")
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.Error(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("cancellation did not interrupt the socket read")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPExecCancellation(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("test uses a POSIX shell stub for ping")
|
|
||||||
}
|
|
||||||
dir := t.TempDir()
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte("#!/bin/sh\nexec sleep 30\n"), 0o755))
|
|
||||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := monitorICMPExec(ctx, "127.0.0.1", false)
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.ErrorIs(t, err, context.DeadlineExceeded)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("cancellation did not terminate ping")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPingCommand(t *testing.T) {
|
|
||||||
for _, goos := range []string{"linux", "windows", "darwin", "freebsd", "openbsd"} {
|
|
||||||
for _, ipv6 := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("%s/ipv6=%t", goos, ipv6), func(t *testing.T) {
|
|
||||||
target, family := "192.0.2.1", "-4"
|
|
||||||
if ipv6 {
|
|
||||||
target, family = "2001:db8::1", "-6"
|
|
||||||
}
|
|
||||||
name, args, err := pingCommand(goos, target, ipv6)
|
|
||||||
require.NoError(t, err)
|
|
||||||
wantName := "ping"
|
|
||||||
wantArgs := []string{"-n", "-c", "1", target}
|
|
||||||
switch goos {
|
|
||||||
case "windows":
|
|
||||||
wantArgs = []string{family, "-n", "1", "-w", "3000", target}
|
|
||||||
case "linux":
|
|
||||||
wantArgs = append([]string{family}, wantArgs...)
|
|
||||||
default:
|
|
||||||
if ipv6 {
|
|
||||||
wantName = "ping6"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
assert.Equal(t, wantName, name)
|
|
||||||
assert.Equal(t, wantArgs, args)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_, _, err := pingCommand("unsupported", "192.0.2.1", false)
|
|
||||||
require.Error(t, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestParsePingResponse(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
wantUs int64
|
|
||||||
}{
|
|
||||||
{"linux", "64 bytes from 192.0.2.1: icmp_seq=1 ttl=64 time=12.345 ms", 12345},
|
|
||||||
{"bsd", "64 bytes from 192.0.2.1: icmp_seq=0 ttl=64 time=0.023 ms", 23},
|
|
||||||
{"ipv6", "64 bytes from 2001:db8::1: icmp_seq=0 hlim=64 time=1.234 ms", 1234},
|
|
||||||
{"windows", "Reply from 192.0.2.1: bytes=32 time=12ms TTL=128", 12000},
|
|
||||||
{"windows submillisecond", "Reply from ::1: time<1ms", 1000},
|
|
||||||
{"localized windows", "Antwort von 192.0.2.1: Bytes=32 Zeit=12ms TTL=128", 12000},
|
|
||||||
{"decimal comma", "64 bytes from 192.0.2.1: time=1,234 ms", 1234},
|
|
||||||
{"rounding", "time=0.1236 ms", 124},
|
|
||||||
{"empty", "", -1},
|
|
||||||
{"timeout", "Request timed out.", -1},
|
|
||||||
{"unreachable", "Reply from 192.0.2.1: Destination host unreachable.", -1},
|
|
||||||
{"malformed", "time=oops ms", -1},
|
|
||||||
{"negative", "time=-1 ms", -1},
|
|
||||||
{"overflow", "time=999999999999999999999 ms", -1},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
responseUs, err := parsePingResponse([]byte(tc.output))
|
|
||||||
if tc.wantUs < 0 {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.wantUs, responseUs)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPExecOutput(t *testing.T) {
|
|
||||||
if runtime.GOOS == "windows" {
|
|
||||||
t.Skip("test uses a POSIX shell stub for ping")
|
|
||||||
}
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
output string
|
|
||||||
exit int
|
|
||||||
wantUs int64
|
|
||||||
}{
|
|
||||||
{"success", "time=1.234 ms", 0, 1234},
|
|
||||||
{"missing RTT", "unrecognized output", 0, -1},
|
|
||||||
{"failed command with RTT", "time=1.234 ms", 1, -1},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
// Also verify an inherited locale cannot override the C locale.
|
|
||||||
script := fmt.Sprintf("#!/bin/sh\n[ \"$LC_ALL\" = C ] || exit 2\nprintf '%%s\\n' '%s'\nexit %d\n", tc.output, tc.exit)
|
|
||||||
require.NoError(t, os.WriteFile(filepath.Join(dir, "ping"), []byte(script), 0o755))
|
|
||||||
t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH"))
|
|
||||||
t.Setenv("LC_ALL", "de_DE.UTF-8")
|
|
||||||
responseUs, err := monitorICMPExec(t.Context(), "127.0.0.1", false)
|
|
||||||
if tc.wantUs < 0 {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.wantUs, responseUs)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type icmpTestReply struct {
|
|
||||||
data []byte
|
|
||||||
peer net.Addr
|
|
||||||
}
|
|
||||||
|
|
||||||
type scriptedICMPConn struct {
|
|
||||||
net.PacketConn
|
|
||||||
local net.Addr
|
|
||||||
onWrite func([]byte, net.Addr)
|
|
||||||
replies []icmpTestReply
|
|
||||||
reads int
|
|
||||||
deadlineSets int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) LocalAddr() net.Addr { return c.local }
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) SetDeadline(deadline time.Time) error {
|
|
||||||
c.deadlineSets++
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) WriteTo(data []byte, dst net.Addr) (int, error) {
|
|
||||||
c.onWrite(data, dst)
|
|
||||||
return len(data), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *scriptedICMPConn) ReadFrom(buf []byte) (int, net.Addr, error) {
|
|
||||||
c.reads++
|
|
||||||
if len(c.replies) == 0 {
|
|
||||||
return 0, nil, os.ErrDeadlineExceeded
|
|
||||||
}
|
|
||||||
reply := c.replies[0]
|
|
||||||
c.replies = c.replies[1:]
|
|
||||||
return copy(buf, reply.data), reply.peer, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPReplyCorrelation(t *testing.T) {
|
|
||||||
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
|
||||||
for _, datagram := range []bool{false, true} {
|
|
||||||
network := family.rawNetwork
|
|
||||||
ip, other := net.ParseIP("192.0.2.1"), net.ParseIP("192.0.2.2")
|
|
||||||
if family.isIPv6 {
|
|
||||||
ip, other = net.ParseIP("2001:db8::1"), net.ParseIP("2001:db8::2")
|
|
||||||
}
|
|
||||||
var dst net.Addr = &net.IPAddr{IP: ip}
|
|
||||||
var wrongPeer net.Addr = &net.IPAddr{IP: other}
|
|
||||||
if datagram {
|
|
||||||
network = family.dgramNetwork
|
|
||||||
dst = &net.UDPAddr{IP: ip}
|
|
||||||
wrongPeer = &net.UDPAddr{IP: other}
|
|
||||||
}
|
|
||||||
for _, mismatch := range []string{"source", "id", "sequence", "payload", "type", "code", "malformed"} {
|
|
||||||
for _, eventuallyMatches := range []bool{false, true} {
|
|
||||||
ending := "timeout"
|
|
||||||
if eventuallyMatches {
|
|
||||||
ending = "success"
|
|
||||||
}
|
|
||||||
t.Run(network+"/"+mismatch+"/"+ending, func(t *testing.T) {
|
|
||||||
conn := &scriptedICMPConn{local: &net.IPAddr{IP: net.IPv4zero}}
|
|
||||||
if datagram {
|
|
||||||
conn.local = &net.UDPAddr{Port: 12345}
|
|
||||||
if runtime.GOOS == "linux" {
|
|
||||||
// Deliberately differ from the process ID.
|
|
||||||
conn.local = &net.UDPAddr{Port: (os.Getpid() % 65534) + 1}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
conn.onWrite = func(data []byte, target net.Addr) {
|
|
||||||
require.Equal(t, dst, target)
|
|
||||||
request, err := icmp.ParseMessage(family.proto, data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
echo := request.Body.(*icmp.Echo)
|
|
||||||
expectedID := os.Getpid() & 0xffff
|
|
||||||
if datagram && runtime.GOOS == "linux" {
|
|
||||||
expectedID = conn.local.(*net.UDPAddr).Port
|
|
||||||
}
|
|
||||||
require.Equal(t, expectedID, echo.ID)
|
|
||||||
reply := &icmp.Message{Type: family.replyType, Body: echo}
|
|
||||||
valid, err := reply.Marshal(nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
peer := dst
|
|
||||||
switch mismatch {
|
|
||||||
case "source":
|
|
||||||
peer = wrongPeer
|
|
||||||
case "id":
|
|
||||||
echo.ID ^= 1
|
|
||||||
case "sequence":
|
|
||||||
echo.Seq ^= 1
|
|
||||||
case "payload":
|
|
||||||
echo.Data[0] ^= 1
|
|
||||||
case "type":
|
|
||||||
reply.Type = family.echoType
|
|
||||||
case "code":
|
|
||||||
reply.Code = 1
|
|
||||||
}
|
|
||||||
invalid, err := reply.Marshal(nil)
|
|
||||||
require.NoError(t, err)
|
|
||||||
if mismatch == "malformed" {
|
|
||||||
invalid = invalid[:2]
|
|
||||||
}
|
|
||||||
conn.replies = []icmpTestReply{{invalid, peer}}
|
|
||||||
if eventuallyMatches {
|
|
||||||
conn.replies = append(conn.replies, icmpTestReply{valid, dst})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
|
||||||
if eventuallyMatches {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, elapsed, int64(0))
|
|
||||||
} else {
|
|
||||||
require.ErrorIs(t, err, os.ErrDeadlineExceeded)
|
|
||||||
assert.Equal(t, int64(-1), elapsed)
|
|
||||||
}
|
|
||||||
assert.Equal(t, 2, conn.reads)
|
|
||||||
assert.Equal(t, 1, conn.deadlineSets)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorICMPLoopback(t *testing.T) {
|
|
||||||
for _, family := range []*icmpFamily{&icmpV4, &icmpV6} {
|
|
||||||
for _, network := range []string{family.rawNetwork, family.dgramNetwork} {
|
|
||||||
t.Run(network, func(t *testing.T) {
|
|
||||||
conn, err := icmp.ListenPacket(network, family.listenAddr)
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("ICMP socket unavailable: %v", err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
ip := net.ParseIP("127.0.0.1")
|
|
||||||
if family.isIPv6 {
|
|
||||||
ip = net.ParseIP("::1")
|
|
||||||
}
|
|
||||||
var dst net.Addr = &net.IPAddr{IP: ip}
|
|
||||||
if network == family.dgramNetwork {
|
|
||||||
dst = &net.UDPAddr{IP: ip}
|
|
||||||
}
|
|
||||||
elapsed, err := monitorICMPPacket(context.Background(), conn, family, dst)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, elapsed, int64(0))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDetectICMPMode(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
family *icmpFamily
|
|
||||||
rawErr error
|
|
||||||
udpErr error
|
|
||||||
want icmpMethod
|
|
||||||
wantNetworks []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "IPv4 prefers raw socket when available",
|
|
||||||
family: &icmpV4,
|
|
||||||
want: icmpRaw,
|
|
||||||
wantNetworks: []string{"ip4:icmp"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv4 uses datagram when raw unavailable",
|
|
||||||
family: &icmpV4,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
want: icmpDatagram,
|
|
||||||
wantNetworks: []string{"ip4:icmp", "udp4"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv4 falls back to exec when both unavailable",
|
|
||||||
family: &icmpV4,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
udpErr: errors.New("protocol not supported"),
|
|
||||||
want: icmpExecFallback,
|
|
||||||
wantNetworks: []string{"ip4:icmp", "udp4"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 prefers raw socket when available",
|
|
||||||
family: &icmpV6,
|
|
||||||
want: icmpRaw,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 uses datagram when raw unavailable",
|
|
||||||
family: &icmpV6,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
want: icmpDatagram,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "IPv6 falls back to exec when both unavailable",
|
|
||||||
family: &icmpV6,
|
|
||||||
rawErr: errors.New("operation not permitted"),
|
|
||||||
udpErr: errors.New("protocol not supported"),
|
|
||||||
want: icmpExecFallback,
|
|
||||||
wantNetworks: []string{"ip6:ipv6-icmp", "udp6"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
calls := make([]string, 0, 2)
|
|
||||||
listen := func(network, listenAddr string) (icmpPacketConn, error) {
|
|
||||||
require.Equal(t, tt.family.listenAddr, listenAddr)
|
|
||||||
calls = append(calls, network)
|
|
||||||
switch network {
|
|
||||||
case tt.family.rawNetwork:
|
|
||||||
if tt.rawErr != nil {
|
|
||||||
return nil, tt.rawErr
|
|
||||||
}
|
|
||||||
case tt.family.dgramNetwork:
|
|
||||||
if tt.udpErr != nil {
|
|
||||||
return nil, tt.udpErr
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected network %q", network)
|
|
||||||
}
|
|
||||||
return testICMPPacketConn{}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
assert.Equal(t, tt.want, detectICMPMode(tt.family, listen))
|
|
||||||
assert.Equal(t, tt.wantNetworks, calls)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestResolveICMPTarget(t *testing.T) {
|
|
||||||
t.Run("IPv4 literal", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "127.0.0.1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.False(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "127.0.0.1", ip.String())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("IPv6 literal", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "::1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.True(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "::1", ip.String())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("IPv4-mapped IPv6 resolves as IPv4", func(t *testing.T) {
|
|
||||||
family, ip, err := resolveICMPTarget(context.Background(), "::ffff:127.0.0.1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, family)
|
|
||||||
assert.False(t, family.isIPv6)
|
|
||||||
assert.Equal(t, "127.0.0.1", ip.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,105 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorProbe performs one check. Errors are recorded as loss by the task runner.
|
|
||||||
// Implementations must honor cancellation and bound their execution time.
|
|
||||||
type monitorProbe func(context.Context, monitor.Config) (int64, error)
|
|
||||||
|
|
||||||
func networkMonitorProbe(client *http.Client) monitorProbe {
|
|
||||||
return func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
switch config.Protocol {
|
|
||||||
case "icmp":
|
|
||||||
return monitorICMP(ctx, config.Target)
|
|
||||||
case "tcp":
|
|
||||||
return monitorTCP(ctx, config.Target, config.Port)
|
|
||||||
case "http":
|
|
||||||
return monitorHTTP(ctx, client, config.Target)
|
|
||||||
case "dns":
|
|
||||||
return monitorDNS(ctx, config.Target)
|
|
||||||
default:
|
|
||||||
return -1, fmt.Errorf("unknown monitor protocol: %s", config.Protocol)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorTCP measures connection establishment time, including address fallback
|
|
||||||
// but excluding DNS resolution.
|
|
||||||
// Returns -1 and an error on failure.
|
|
||||||
func monitorTCP(ctx context.Context, target string, port uint16) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// Resolve DNS first, outside the timing window but within the probe deadline.
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
if len(ips) == 0 {
|
|
||||||
return -1, errors.New("no addresses resolved for TCP monitor")
|
|
||||||
}
|
|
||||||
portString := fmt.Sprintf("%d", port)
|
|
||||||
deadline, _ := ctx.Deadline()
|
|
||||||
|
|
||||||
// Share the remaining probe budget across addresses so an unresponsive
|
|
||||||
// first address cannot consume all the time available for alternatives.
|
|
||||||
start := time.Now()
|
|
||||||
for i, ip := range ips {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
dialer := net.Dialer{Timeout: time.Until(deadline) / time.Duration(len(ips)-i)}
|
|
||||||
var conn net.Conn
|
|
||||||
conn, err = dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, portString))
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
responseUs := time.Since(start).Microseconds()
|
|
||||||
conn.Close()
|
|
||||||
return responseUs, nil
|
|
||||||
}
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorDNS measures DNS resolution response time in microseconds. Returns -1 and an error on failure.
|
|
||||||
func monitorDNS(ctx context.Context, target string) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(ctx, target)
|
|
||||||
if err != nil || len(ips) == 0 {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
return time.Since(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorHTTP measures HTTP GET request response in microseconds. Returns -1 and an error on failure.
|
|
||||||
func monitorHTTP(ctx context.Context, client *http.Client, url string) (int64, error) {
|
|
||||||
if client == nil {
|
|
||||||
client = http.DefaultClient
|
|
||||||
}
|
|
||||||
start := time.Now()
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return -1, err
|
|
||||||
}
|
|
||||||
resp.Body.Close()
|
|
||||||
if resp.StatusCode >= 400 {
|
|
||||||
return -1, fmt.Errorf("HTTP error: %s", resp.Status)
|
|
||||||
}
|
|
||||||
return time.Since(start).Microseconds(), nil
|
|
||||||
}
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
monitorResumeHeartbeat = 10 * time.Second
|
|
||||||
// Allow scheduling jitter without mistaking an ordinary tick for resume.
|
|
||||||
monitorResumeGap = 2 * monitorResumeHeartbeat
|
|
||||||
monitorResumePause = 10 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// monitorResumeGuard detects likely suspend/resume using wall time. A long
|
|
||||||
// process stall or forward clock adjustment can also trigger the bounded pause.
|
|
||||||
// One heartbeat is shared by all configured monitors.
|
|
||||||
type monitorResumeGuard struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
stop chan struct{}
|
|
||||||
lastTick time.Time
|
|
||||||
pauseUntil time.Time
|
|
||||||
generation uint32
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *monitorResumeGuard) start() {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
stop := make(chan struct{})
|
|
||||||
g.stop = stop
|
|
||||||
g.lastTick = time.Now().Round(0)
|
|
||||||
g.pauseUntil = time.Time{}
|
|
||||||
go func() {
|
|
||||||
ticker := time.NewTicker(monitorResumeHeartbeat)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
g.mu.Lock()
|
|
||||||
if g.stop == stop {
|
|
||||||
g.observe(time.Now())
|
|
||||||
}
|
|
||||||
g.mu.Unlock()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (g *monitorResumeGuard) shutdown() {
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop != nil {
|
|
||||||
close(g.stop)
|
|
||||||
g.stop = nil
|
|
||||||
g.generation++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// observe requires mu. Strip the monotonic component because it can stop during
|
|
||||||
// suspend. Read the current time rather than the ticker's queued timestamp.
|
|
||||||
func (g *monitorResumeGuard) observe(now time.Time) {
|
|
||||||
now = now.Round(0)
|
|
||||||
if now.Sub(g.lastTick) > monitorResumeGap {
|
|
||||||
g.pauseUntil = now.Add(monitorResumePause)
|
|
||||||
g.generation++
|
|
||||||
}
|
|
||||||
g.lastTick = now
|
|
||||||
}
|
|
||||||
|
|
||||||
// snapshot also observes time so a probe waking before the heartbeat detects
|
|
||||||
// resume itself. A changed generation invalidates probes spanning suspend.
|
|
||||||
func (g *monitorResumeGuard) snapshot() (generation uint32, allowed bool) {
|
|
||||||
if g == nil {
|
|
||||||
return 0, true
|
|
||||||
}
|
|
||||||
g.mu.Lock()
|
|
||||||
defer g.mu.Unlock()
|
|
||||||
if g.stop == nil {
|
|
||||||
return g.generation, true
|
|
||||||
}
|
|
||||||
g.observe(time.Now())
|
|
||||||
return g.generation, !g.lastTick.Before(g.pauseUntil)
|
|
||||||
}
|
|
||||||
@@ -1,121 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func simulateMonitorSleep(g *monitorResumeGuard) {
|
|
||||||
g.mu.Lock()
|
|
||||||
g.lastTick = time.Now().Add(-time.Hour).Round(0)
|
|
||||||
g.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumePause(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var g monitorResumeGuard
|
|
||||||
g.start()
|
|
||||||
defer g.shutdown()
|
|
||||||
generation, allowed := g.snapshot()
|
|
||||||
require.True(t, allowed)
|
|
||||||
// Heartbeats alone must keep the guard current between infrequent probes.
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
steadyGeneration, allowed := g.snapshot()
|
|
||||||
require.True(t, allowed)
|
|
||||||
require.Equal(t, generation, steadyGeneration)
|
|
||||||
// The probe, rather than the heartbeat, must detect this gap.
|
|
||||||
simulateMonitorSleep(&g)
|
|
||||||
next, allowed := g.snapshot()
|
|
||||||
assert.False(t, allowed)
|
|
||||||
assert.NotEqual(t, generation, next)
|
|
||||||
time.Sleep(9 * time.Second)
|
|
||||||
_, allowed = g.snapshot()
|
|
||||||
assert.False(t, allowed)
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
end, allowed := g.snapshot()
|
|
||||||
assert.True(t, allowed)
|
|
||||||
assert.Equal(t, next, end)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeGuardLifecycle(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) { return 1, nil })
|
|
||||||
defer pm.Stop()
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
pm.SyncMonitors([]monitor.Config{{ID: "a", Interval: 3600}, {ID: "b", Interval: 3600}})
|
|
||||||
stop := pm.resumeGuard.stop
|
|
||||||
require.NotNil(t, stop)
|
|
||||||
pm.DeleteMonitor("a")
|
|
||||||
assert.Equal(t, stop, pm.resumeGuard.stop)
|
|
||||||
pm.DeleteMonitor("b")
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
select {
|
|
||||||
case <-stop:
|
|
||||||
default:
|
|
||||||
t.Fatal("heartbeat was not stopped")
|
|
||||||
}
|
|
||||||
time.Sleep(time.Hour)
|
|
||||||
_, err := pm.UpsertMonitor(monitor.Config{ID: "c", Interval: 3600}, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, allowed := pm.resumeGuard.snapshot()
|
|
||||||
assert.True(t, allowed, "idle time must not trigger a resume pause")
|
|
||||||
pm.SyncMonitors(nil)
|
|
||||||
assert.Nil(t, pm.resumeGuard.stop)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeDiscardsInflightProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var g monitorResumeGuard
|
|
||||||
g.start()
|
|
||||||
defer g.shutdown()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "test"})
|
|
||||||
defer task.cancel()
|
|
||||||
task.resumeGuard = &g
|
|
||||||
result := task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
simulateMonitorSleep(&g)
|
|
||||||
return 0, errors.New("network not ready")
|
|
||||||
})
|
|
||||||
assert.Nil(t, result)
|
|
||||||
assert.Empty(t, task.history.samples)
|
|
||||||
// Explicit requests may still run during the pause and record real failures.
|
|
||||||
result = task.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
return 0, errors.New("unreachable")
|
|
||||||
})
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResumeSkipsScheduledProbes(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
pm := newMonitorManagerWithProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
return 1, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
pm.SyncMonitors([]monitor.Config{{ID: "test", Interval: 1}})
|
|
||||||
simulateMonitorSleep(&pm.resumeGuard)
|
|
||||||
pm.resumeGuard.snapshot()
|
|
||||||
time.Sleep(9 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Zero(t, calls.Load())
|
|
||||||
assert.Empty(t, pm.GetResults(1000))
|
|
||||||
time.Sleep(2 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Positive(t, calls.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,60 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"math/rand"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (pm *MonitorManager) startMonitor(task *monitorTask) {
|
|
||||||
interval := time.Duration(task.config.Interval) * time.Second
|
|
||||||
if interval < time.Second {
|
|
||||||
interval = 30 * time.Second
|
|
||||||
}
|
|
||||||
delay := getStagger(interval.Milliseconds())
|
|
||||||
slog.Debug("starting monitor task", "target", task.config.Target, "delay", delay, "interval", interval)
|
|
||||||
go runMonitorSchedule(task.ctx, interval, delay, func() {
|
|
||||||
if _, allowed := task.resumeGuard.snapshot(); allowed {
|
|
||||||
task.runProbe(pm.probe)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// runMonitorSchedule owns only timing. Checks run serially, and slow checks
|
|
||||||
// naturally drop missed ticks rather than building an execution backlog.
|
|
||||||
func runMonitorSchedule(ctx context.Context, interval, delay time.Duration, run func()) {
|
|
||||||
timer := time.NewTimer(delay)
|
|
||||||
defer timer.Stop()
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
run()
|
|
||||||
ticker := time.NewTicker(interval)
|
|
||||||
defer ticker.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return
|
|
||||||
case <-ticker.C:
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
run()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// getStagger returns an initial delay between half an interval and one interval.
|
|
||||||
func getStagger(intervalMilli int64) time.Duration {
|
|
||||||
delay := rand.Intn(int(intervalMilli))
|
|
||||||
if delay < int(intervalMilli)/2 {
|
|
||||||
delay += int(intervalMilli) / 2
|
|
||||||
}
|
|
||||||
return time.Duration(delay) * time.Millisecond
|
|
||||||
}
|
|
||||||
@@ -1,167 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorScheduleTiming(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
defer cancel()
|
|
||||||
var calls atomic.Int32
|
|
||||||
go runMonitorSchedule(ctx, 10*time.Second, 5*time.Second, func() { calls.Add(1) })
|
|
||||||
synctest.Wait()
|
|
||||||
time.Sleep(4 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 0, int(calls.Load()))
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
time.Sleep(10 * time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
cancel()
|
|
||||||
synctest.Wait()
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorScheduleSlowProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
ctx, cancel := context.WithCancel(t.Context())
|
|
||||||
defer cancel()
|
|
||||||
var calls atomic.Int32
|
|
||||||
release := make(chan struct{})
|
|
||||||
go runMonitorSchedule(ctx, time.Second, 0, func() {
|
|
||||||
calls.Add(1)
|
|
||||||
select {
|
|
||||||
case <-release:
|
|
||||||
case <-ctx.Done():
|
|
||||||
}
|
|
||||||
})
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
time.Sleep(time.Minute)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()), "a slow probe must not spawn overlapping checks")
|
|
||||||
close(release)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()), "missed intervals must not accumulate a backlog")
|
|
||||||
time.Sleep(time.Second)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
cancel()
|
|
||||||
synctest.Wait()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorScheduledAndImmediateRequestsShareProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
release := make(chan struct{})
|
|
||||||
cfg := monitor.Config{ID: "test", Interval: 10}
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
assert.Equal(t, cfg, config)
|
|
||||||
calls.Add(1)
|
|
||||||
<-release
|
|
||||||
return 42, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
go runMonitorSchedule(task.ctx, 10*time.Second, 0, func() { task.runProbe(pm.probe) })
|
|
||||||
synctest.Wait()
|
|
||||||
results := make(chan *monitor.Result, 2)
|
|
||||||
for range 2 {
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
results <- result
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Equal(t, 1, int(calls.Load()))
|
|
||||||
assert.Empty(t, pm.GetResults(1000), "reading history must not wait for network I/O")
|
|
||||||
close(release)
|
|
||||||
synctest.Wait()
|
|
||||||
first, second := <-results, <-results
|
|
||||||
require.NotNil(t, first)
|
|
||||||
require.NotNil(t, second)
|
|
||||||
assert.Equal(t, int64(42), first.AvgResponse)
|
|
||||||
assert.Equal(t, first, second)
|
|
||||||
assert.NotSame(t, first, second, "callers must not share mutable result pointers")
|
|
||||||
assert.Len(t, task.history.samples, 1)
|
|
||||||
// A later explicit request must still perform a fresh probe.
|
|
||||||
_, err := pm.UpsertMonitor(cfg, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 2, int(calls.Load()))
|
|
||||||
assert.Len(t, task.history.samples, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorReplacementCancelsSharedProbe(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
cfg := monitor.Config{ID: "test", Interval: 10}
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, config monitor.Config) (int64, error) {
|
|
||||||
if config.Interval == 10 {
|
|
||||||
<-ctx.Done()
|
|
||||||
return 0, ctx.Err()
|
|
||||||
}
|
|
||||||
return 30, nil
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
task.history.record(monitorSample{responseUs: 10, timestamp: time.Now()})
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
results := make(chan *monitor.Result, 2)
|
|
||||||
for range 2 {
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
results <- result
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
synctest.Wait()
|
|
||||||
updated := cfg
|
|
||||||
updated.Interval = 20
|
|
||||||
result, err := pm.UpsertMonitor(updated, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, int64(20), result.AvgResponse)
|
|
||||||
assert.Zero(t, result.PacketLoss)
|
|
||||||
synctest.Wait()
|
|
||||||
assert.Nil(t, <-results)
|
|
||||||
assert.Nil(t, <-results)
|
|
||||||
assert.Len(t, task.history.samples, 1)
|
|
||||||
assert.Len(t, pm.monitors[cfg.ID].history.samples, 2)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorInjectedProbeTimeoutRecordsLoss(t *testing.T) {
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
pm := newMonitorManagerWithProbe(func(ctx context.Context, _ monitor.Config) (int64, error) {
|
|
||||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
<-ctx.Done()
|
|
||||||
return 0, ctx.Err()
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
start := time.Now()
|
|
||||||
result, err := pm.UpsertMonitor(monitor.Config{ID: "test", Interval: 3600}, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 3*time.Second, time.Since(start))
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.NoError(t, pm.monitors["test"].ctx.Err())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,116 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"log/slog"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
const monitorFailureLogInterval = 5 * time.Minute
|
|
||||||
|
|
||||||
// monitorTask coordinates a probe and its history for one immutable configuration.
|
|
||||||
type monitorTask struct {
|
|
||||||
config monitor.Config
|
|
||||||
ctx context.Context
|
|
||||||
cancel context.CancelFunc
|
|
||||||
history *monitorHistory
|
|
||||||
resumeGuard *monitorResumeGuard
|
|
||||||
runMu sync.Mutex
|
|
||||||
inflight *monitorRun
|
|
||||||
lastFailureLog int64 // Unix nanoseconds
|
|
||||||
}
|
|
||||||
|
|
||||||
type monitorRun struct {
|
|
||||||
done chan struct{}
|
|
||||||
result *monitor.Result // published by closing done; never mutated afterwards
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorTask(config monitor.Config) *monitorTask {
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
task := &monitorTask{config: config, ctx: ctx, history: newMonitorHistory()}
|
|
||||||
// Serialize cancellation with publication, so canceled probes cannot enter
|
|
||||||
// history copied into a replacement task.
|
|
||||||
task.cancel = func() {
|
|
||||||
task.runMu.Lock()
|
|
||||||
cancel()
|
|
||||||
task.runMu.Unlock()
|
|
||||||
}
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
func newMonitorTaskFromExisting(config monitor.Config, existing *monitorTask) *monitorTask {
|
|
||||||
task := newMonitorTask(config)
|
|
||||||
if existing != nil {
|
|
||||||
task.history = existing.history.clone()
|
|
||||||
}
|
|
||||||
return task
|
|
||||||
}
|
|
||||||
|
|
||||||
// runProbe shares an in-flight check between scheduled and immediate requests.
|
|
||||||
// Every completed check contributes exactly one sample, regardless of how many
|
|
||||||
// callers were waiting for it. No task or history lock is held during network I/O.
|
|
||||||
func (task *monitorTask) runProbe(probe monitorProbe) *monitor.Result {
|
|
||||||
task.runMu.Lock()
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
task.runMu.Unlock()
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if run := task.inflight; run != nil {
|
|
||||||
task.runMu.Unlock()
|
|
||||||
select {
|
|
||||||
case <-task.ctx.Done():
|
|
||||||
return nil
|
|
||||||
case <-run.done:
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return copyMonitorResult(run.result)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
run := &monitorRun{done: make(chan struct{})}
|
|
||||||
task.inflight = run
|
|
||||||
task.runMu.Unlock()
|
|
||||||
|
|
||||||
generation, _ := task.resumeGuard.snapshot()
|
|
||||||
responseUs, err := probe(task.ctx, task.config)
|
|
||||||
var logFailure bool
|
|
||||||
task.runMu.Lock()
|
|
||||||
currentGeneration, _ := task.resumeGuard.snapshot()
|
|
||||||
if task.ctx.Err() == nil && generation == currentGeneration {
|
|
||||||
now := time.Now()
|
|
||||||
if err != nil {
|
|
||||||
responseUs = -1
|
|
||||||
logAt := now.UnixNano()
|
|
||||||
if task.lastFailureLog == 0 || logAt < task.lastFailureLog || logAt-task.lastFailureLog >= int64(monitorFailureLogInterval) {
|
|
||||||
logFailure = true
|
|
||||||
task.lastFailureLog = logAt
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
task.lastFailureLog = 0
|
|
||||||
}
|
|
||||||
result := task.history.record(monitorSample{responseUs: responseUs, timestamp: now})
|
|
||||||
run.result = &result
|
|
||||||
}
|
|
||||||
|
|
||||||
task.inflight = nil
|
|
||||||
close(run.done)
|
|
||||||
task.runMu.Unlock()
|
|
||||||
if logFailure {
|
|
||||||
slog.Warn("monitor failed", "err", err, "target", task.config.Target, "protocol", task.config.Protocol)
|
|
||||||
}
|
|
||||||
if task.ctx.Err() != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return copyMonitorResult(run.result)
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyMonitorResult(result *monitor.Result) *monitor.Result {
|
|
||||||
if result == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
copy := *result
|
|
||||||
return ©
|
|
||||||
}
|
|
||||||
@@ -1,79 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"testing"
|
|
||||||
"testing/synctest"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorFailureLogCooldown(t *testing.T) {
|
|
||||||
var logs bytes.Buffer
|
|
||||||
previous := slog.Default()
|
|
||||||
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, nil)))
|
|
||||||
t.Cleanup(func() { slog.SetDefault(previous) })
|
|
||||||
|
|
||||||
synctest.Test(t, func(t *testing.T) {
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "test", Target: "example.test", Protocol: "tcp"})
|
|
||||||
defer task.cancel()
|
|
||||||
failure := errors.New("connection refused")
|
|
||||||
probe := func(context.Context, monitor.Config) (int64, error) { return 42, failure }
|
|
||||||
var samples int64
|
|
||||||
check := func(wantLog bool) {
|
|
||||||
t.Helper()
|
|
||||||
logs.Reset()
|
|
||||||
result := task.runProbe(probe)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
samples++
|
|
||||||
assert.Equal(t, samples, result.SampleCount, "suppressed warnings must still record samples")
|
|
||||||
if !wantLog {
|
|
||||||
assert.Empty(t, logs.String())
|
|
||||||
} else {
|
|
||||||
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
|
||||||
assert.Equal(t, 1, bytes.Count(logs.Bytes(), []byte("\n")))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(5*time.Minute - time.Nanosecond)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(time.Nanosecond)
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
time.Sleep(5 * time.Minute)
|
|
||||||
check(true)
|
|
||||||
check(false)
|
|
||||||
|
|
||||||
// Recovery clears the cooldown.
|
|
||||||
failure = nil
|
|
||||||
check(false)
|
|
||||||
failure = errors.New("connection refused again")
|
|
||||||
check(true)
|
|
||||||
|
|
||||||
// Another monitor has its own cooldown.
|
|
||||||
other := newMonitorTask(task.config)
|
|
||||||
defer other.cancel()
|
|
||||||
logs.Reset()
|
|
||||||
require.NotNil(t, other.runProbe(probe))
|
|
||||||
assert.Contains(t, logs.String(), `msg="monitor failed"`)
|
|
||||||
|
|
||||||
// A canceled probe must not publish a failure or emit a warning.
|
|
||||||
logs.Reset()
|
|
||||||
result := other.runProbe(func(context.Context, monitor.Config) (int64, error) {
|
|
||||||
other.cancel()
|
|
||||||
return -1, context.Canceled
|
|
||||||
})
|
|
||||||
assert.Nil(t, result)
|
|
||||||
assert.Empty(t, logs.String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,524 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
"golang.org/x/net/dns/dnsmessage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestMonitorManagerGetResultsIncludesHourResponseRange(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 10, timestamp: now.Add(-30 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 20, timestamp: now.Add(-9 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 40, timestamp: now.Add(-5 * time.Minute)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 30, timestamp: now.Add(-50 * time.Second)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
|
||||||
|
|
||||||
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
|
||||||
result, ok := results["monitor-1"]
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, int64(30), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(25), result.AvgResponse1h)
|
|
||||||
assert.Equal(t, int64(30), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(10), result.MinResponse1h)
|
|
||||||
assert.Equal(t, int64(30), result.MaxResponse)
|
|
||||||
assert.Equal(t, int64(40), result.MaxResponse1h)
|
|
||||||
assert.Equal(t, 50.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 20.0, result.PacketLoss1h)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerGetResultsIncludesLossOnlyHourData(t *testing.T) {
|
|
||||||
now := time.Now().UTC()
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "monitor-1"})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: -1, timestamp: now.Add(-10 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{"icmp:example.com": task}
|
|
||||||
|
|
||||||
results := pm.GetResults(uint16(time.Minute / time.Millisecond))
|
|
||||||
result, ok := results["monitor-1"]
|
|
||||||
require.True(t, ok)
|
|
||||||
assert.Equal(t, int64(0), result.AvgResponse)
|
|
||||||
assert.Equal(t, int64(0), result.AvgResponse1h)
|
|
||||||
assert.Equal(t, int64(0), result.MinResponse)
|
|
||||||
assert.Equal(t, int64(0), result.MinResponse1h)
|
|
||||||
assert.Equal(t, int64(0), result.MaxResponse)
|
|
||||||
assert.Equal(t, int64(0), result.MaxResponse1h)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss1h)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorConfigResultKeyUsesSyncedID(t *testing.T) {
|
|
||||||
cfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
assert.Equal(t, "monitor-1", cfg.ID)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsSkipsConfigsWithoutStableID(t *testing.T) {
|
|
||||||
validCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
invalidCfg := monitor.Config{Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.SyncMonitors([]monitor.Config{validCfg, invalidCfg})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
_, validExists := pm.monitors[validCfg.ID]
|
|
||||||
_, invalidExists := pm.monitors[invalidCfg.ID]
|
|
||||||
assert.True(t, validExists)
|
|
||||||
assert.False(t, invalidExists)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsStopsRemovedTasksButKeepsExisting(t *testing.T) {
|
|
||||||
keepCfg := monitor.Config{ID: "monitor-1", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
removeCfg := monitor.Config{ID: "monitor-2", Target: "ignored", Protocol: "noop", Interval: 10}
|
|
||||||
|
|
||||||
keptTask := newMonitorTask(keepCfg)
|
|
||||||
removedTask := newMonitorTask(removeCfg)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{
|
|
||||||
keepCfg.ID: keptTask,
|
|
||||||
removeCfg.ID: removedTask,
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.SyncMonitors([]monitor.Config{keepCfg})
|
|
||||||
|
|
||||||
assert.Same(t, keptTask, pm.monitors[keepCfg.ID])
|
|
||||||
_, exists := pm.monitors[removeCfg.ID]
|
|
||||||
assert.False(t, exists)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-removedTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected removed monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-keptTask.ctx.Done():
|
|
||||||
t.Fatal("expected existing monitor task to remain active")
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerSyncMonitorsRestartsChangedConfig(t *testing.T) {
|
|
||||||
originalCfg := monitor.Config{ID: "monitor-1", Target: "ignored-a", Protocol: "noop", Interval: 10}
|
|
||||||
updatedCfg := monitor.Config{ID: "monitor-1", Target: "ignored-b", Protocol: "noop", Interval: 10}
|
|
||||||
originalTask := newMonitorTask(originalCfg)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{
|
|
||||||
originalCfg.ID: originalTask,
|
|
||||||
}
|
|
||||||
|
|
||||||
pm.SyncMonitors([]monitor.Config{updatedCfg})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
restartedTask := pm.monitors[updatedCfg.ID]
|
|
||||||
assert.NotSame(t, originalTask, restartedTask)
|
|
||||||
assert.Equal(t, updatedCfg, restartedTask.config)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-originalTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected changed monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerApplySyncUpsertRunsImmediatelyAndReturnsResult(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
pm := &MonitorManager{
|
|
||||||
monitors: make(map[string]*monitorTask),
|
|
||||||
probe: networkMonitorProbe(server.Client()),
|
|
||||||
}
|
|
||||||
|
|
||||||
resp, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionUpsert,
|
|
||||||
Config: monitor.Config{ID: "monitor-1", Target: server.URL, Protocol: "http", Interval: 10},
|
|
||||||
RunNow: true,
|
|
||||||
})
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, resp.Result.AvgResponse, int64(0))
|
|
||||||
assert.Equal(t, 0.0, resp.Result.PacketLoss)
|
|
||||||
assert.Equal(t, 0.0, resp.Result.PacketLoss1h)
|
|
||||||
|
|
||||||
task := pm.monitors["monitor-1"]
|
|
||||||
require.NotNil(t, task)
|
|
||||||
task.history.mu.Lock()
|
|
||||||
defer task.history.mu.Unlock()
|
|
||||||
require.Len(t, task.history.samples, 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerUpsertMonitorKeepsHistoryWhenOnlyIntervalChanges(t *testing.T) {
|
|
||||||
originalCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
updatedCfg := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 30}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
|
|
||||||
existingTask := newMonitorTask(originalCfg)
|
|
||||||
existingTask.history.addSampleLocked(monitorSample{responseUs: 12, timestamp: now.Add(-50 * time.Minute)})
|
|
||||||
existingTask.history.addSampleLocked(monitorSample{responseUs: 24, timestamp: now.Add(-30 * time.Second)})
|
|
||||||
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{originalCfg.ID: existingTask}
|
|
||||||
|
|
||||||
result, err := pm.UpsertMonitor(updatedCfg, false)
|
|
||||||
defer pm.Stop()
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, result)
|
|
||||||
|
|
||||||
updatedTask := pm.monitors[updatedCfg.ID]
|
|
||||||
require.NotNil(t, updatedTask)
|
|
||||||
assert.NotSame(t, existingTask, updatedTask)
|
|
||||||
assert.Equal(t, updatedCfg, updatedTask.config)
|
|
||||||
|
|
||||||
updatedTask.history.mu.Lock()
|
|
||||||
defer updatedTask.history.mu.Unlock()
|
|
||||||
require.Len(t, updatedTask.history.samples, 1)
|
|
||||||
assert.Equal(t, int64(24), updatedTask.history.samples[0].responseUs)
|
|
||||||
|
|
||||||
agg := updatedTask.history.aggregateLocked(time.Hour, now)
|
|
||||||
require.True(t, agg.hasData())
|
|
||||||
assert.Equal(t, int64(2), agg.totalCount)
|
|
||||||
assert.Equal(t, int64(2), agg.successCount)
|
|
||||||
assert.Equal(t, int64(18), agg.avgResponse())
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-existingTask.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected original monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerApplySyncDeleteRemovesTask(t *testing.T) {
|
|
||||||
config := monitor.Config{ID: "monitor-1", Target: "1.1.1.1", Protocol: "icmp", Interval: 10}
|
|
||||||
task := newMonitorTask(config)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.monitors = map[string]*monitorTask{config.ID: task}
|
|
||||||
|
|
||||||
_, err := pm.HandleSyncRequest(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionDelete,
|
|
||||||
Config: monitor.Config{ID: config.ID},
|
|
||||||
})
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, exists := pm.monitors[config.ID]
|
|
||||||
assert.False(t, exists)
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-task.ctx.Done():
|
|
||||||
default:
|
|
||||||
t.Fatal("expected deleted monitor task to be cancelled")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerGetRandomDelay(t *testing.T) {
|
|
||||||
for i := 1000; i < 360_000; i += 1000 {
|
|
||||||
delay := getStagger(int64(i))
|
|
||||||
assert.GreaterOrEqual(t, delay, time.Duration(i/2)*time.Millisecond)
|
|
||||||
assert.LessOrEqual(t, delay, time.Duration(i)*time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorHTTP(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
w.WriteHeader(http.StatusNoContent)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("server error", func(t *testing.T) {
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
http.Error(w, "boom", http.StatusInternalServerError)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
responseUs, err := monitorHTTP(context.Background(), server.Client(), server.URL)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorTCP(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer listener.Close()
|
|
||||||
|
|
||||||
accepted := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
defer close(accepted)
|
|
||||||
conn, err := listener.Accept()
|
|
||||||
if err == nil {
|
|
||||||
_ = conn.Close()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
|
||||||
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
<-accepted
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("connection failure", func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
|
|
||||||
port := uint16(listener.Addr().(*net.TCPAddr).Port)
|
|
||||||
require.NoError(t, listener.Close())
|
|
||||||
|
|
||||||
responseUs, err := monitorTCP(context.Background(), "127.0.0.1", port)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorTCPAddressFallback(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
ips []string
|
|
||||||
loss bool
|
|
||||||
}{
|
|
||||||
{"first address fails", []string{"127.0.0.2", "127.0.0.1"}, false},
|
|
||||||
{"first address succeeds", []string{"127.0.0.1", "127.0.0.2"}, false},
|
|
||||||
{"all addresses fail", []string{"127.0.0.2", "127.0.0.3"}, true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
listener, err := net.Listen("tcp4", "127.0.0.1:0")
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer listener.Close()
|
|
||||||
original := net.DefaultResolver
|
|
||||||
net.DefaultResolver = tcpMonitorTestResolver(tc.ips)
|
|
||||||
defer func() { net.DefaultResolver = original }()
|
|
||||||
|
|
||||||
// Verify the resolver preserves the intended order, so success cannot
|
|
||||||
// accidentally bypass the failed first address in the regression case.
|
|
||||||
ips, err := net.DefaultResolver.LookupHost(t.Context(), "tcp-monitor.invalid.")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, tc.ips, ips)
|
|
||||||
responseUs, err := monitorTCP(t.Context(), "tcp-monitor.invalid.", uint16(listener.Addr().(*net.TCPAddr).Port))
|
|
||||||
if tc.loss {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// tcpMonitorTestResolver supplies multiple A records without external DNS.
|
|
||||||
func tcpMonitorTestResolver(ips []string) *net.Resolver {
|
|
||||||
return &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
client, server := net.Pipe()
|
|
||||||
go func() {
|
|
||||||
defer server.Close()
|
|
||||||
// net.Resolver uses TCP framing when its connection is not a PacketConn.
|
|
||||||
var size uint16
|
|
||||||
if err := binary.Read(server, binary.BigEndian, &size); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
packet := make([]byte, size)
|
|
||||||
if _, err := io.ReadFull(server, packet); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var msg dnsmessage.Message
|
|
||||||
if err := msg.Unpack(packet); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
msg.Header.Response = true
|
|
||||||
msg.Header.RecursionAvailable = true
|
|
||||||
for _, question := range msg.Questions {
|
|
||||||
if question.Type != dnsmessage.TypeA {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for _, ip := range ips {
|
|
||||||
msg.Answers = append(msg.Answers, dnsmessage.Resource{
|
|
||||||
Header: dnsmessage.ResourceHeader{Name: question.Name, Type: dnsmessage.TypeA, Class: dnsmessage.ClassINET},
|
|
||||||
Body: &dnsmessage.AResource{A: [4]byte(net.ParseIP(ip).To4())},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
packet, err := msg.Pack()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
response := binary.BigEndian.AppendUint16(nil, uint16(len(packet)))
|
|
||||||
_, _ = server.Write(append(response, packet...))
|
|
||||||
}()
|
|
||||||
return client, nil
|
|
||||||
}}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorDNS(t *testing.T) {
|
|
||||||
t.Run("success", func(t *testing.T) {
|
|
||||||
responseUs, err := monitorDNS(context.Background(), "localhost")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.GreaterOrEqual(t, responseUs, int64(0))
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("lookup failure", func(t *testing.T) {
|
|
||||||
responseUs, err := monitorDNS(context.Background(), "")
|
|
||||||
assert.Equal(t, int64(-1), responseUs)
|
|
||||||
require.Error(t, err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorManagerCancelsActiveProbe(t *testing.T) {
|
|
||||||
for _, action := range []string{"stop", "delete", "upsert", "sync replace", "sync remove"} {
|
|
||||||
t.Run(action, func(t *testing.T) {
|
|
||||||
started := make(chan struct{})
|
|
||||||
canceled := make(chan struct{})
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
close(started)
|
|
||||||
select {
|
|
||||||
case <-r.Context().Done():
|
|
||||||
close(canceled)
|
|
||||||
case <-release:
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
defer close(release)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
defer pm.Stop()
|
|
||||||
cfg := monitor.Config{ID: "test", Protocol: "http", Target: server.URL, Interval: 3600}
|
|
||||||
task := newMonitorTask(cfg)
|
|
||||||
// Seed history to ensure a canceled RunNow does not return an old result.
|
|
||||||
task.history.addSampleLocked(monitorSample{responseUs: 123, timestamp: time.Now()})
|
|
||||||
pm.monitors[cfg.ID] = task
|
|
||||||
done := make(chan *monitor.Result, 1)
|
|
||||||
go func() {
|
|
||||||
result, _ := pm.UpsertMonitor(cfg, true)
|
|
||||||
done <- result
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("probe did not start")
|
|
||||||
}
|
|
||||||
updated := cfg
|
|
||||||
updated.Interval--
|
|
||||||
switch action {
|
|
||||||
case "stop":
|
|
||||||
pm.Stop()
|
|
||||||
case "delete":
|
|
||||||
pm.DeleteMonitor(cfg.ID)
|
|
||||||
case "upsert":
|
|
||||||
_, err := pm.UpsertMonitor(updated, false)
|
|
||||||
require.NoError(t, err)
|
|
||||||
case "sync replace":
|
|
||||||
pm.SyncMonitors([]monitor.Config{updated})
|
|
||||||
case "sync remove":
|
|
||||||
pm.SyncMonitors(nil)
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-canceled:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("active HTTP request was not canceled")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case result := <-done:
|
|
||||||
assert.Nil(t, result)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("RunNow did not return after cancellation")
|
|
||||||
}
|
|
||||||
task.history.mu.Lock()
|
|
||||||
assert.Len(t, task.history.samples, 1, "cancellation must not record packet loss")
|
|
||||||
task.history.mu.Unlock()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorResolutionCancellation(t *testing.T) {
|
|
||||||
for _, protocol := range []string{"tcp", "dns", "icmp"} {
|
|
||||||
t.Run(protocol, func(t *testing.T) {
|
|
||||||
started := make(chan struct{}, 1)
|
|
||||||
original := net.DefaultResolver
|
|
||||||
net.DefaultResolver = &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
select {
|
|
||||||
case started <- struct{}{}:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
<-ctx.Done()
|
|
||||||
return nil, ctx.Err()
|
|
||||||
}}
|
|
||||||
defer func() { net.DefaultResolver = original }()
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
defer cancel()
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
var err error
|
|
||||||
switch protocol {
|
|
||||||
case "tcp":
|
|
||||||
_, err = monitorTCP(ctx, "monitor-cancellation.invalid.", 80)
|
|
||||||
case "dns":
|
|
||||||
_, err = monitorDNS(ctx, "monitor-cancellation.invalid.")
|
|
||||||
case "icmp":
|
|
||||||
_, err = monitorICMP(ctx, "monitor-cancellation.invalid.")
|
|
||||||
}
|
|
||||||
done <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("lookup did not start")
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
select {
|
|
||||||
case err := <-done:
|
|
||||||
require.Error(t, err)
|
|
||||||
case <-time.After(time.Second):
|
|
||||||
t.Fatal("lookup did not cancel")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMonitorProbeTimeoutRecordsLoss(t *testing.T) {
|
|
||||||
release := make(chan struct{})
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
select {
|
|
||||||
case <-r.Context().Done():
|
|
||||||
case <-release:
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
defer close(release)
|
|
||||||
pm := newMonitorManager()
|
|
||||||
pm.probe = networkMonitorProbe(&http.Client{Timeout: 20 * time.Millisecond})
|
|
||||||
task := newMonitorTask(monitor.Config{ID: "timeout", Protocol: "http", Target: server.URL})
|
|
||||||
defer task.cancel()
|
|
||||||
|
|
||||||
result := task.runProbe(pm.probe)
|
|
||||||
require.NotNil(t, result)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss)
|
|
||||||
assert.Equal(t, 100.0, result.PacketLoss1h)
|
|
||||||
require.Len(t, task.history.samples, 1)
|
|
||||||
assert.Equal(t, int64(-1), task.history.samples[0].responseUs)
|
|
||||||
assert.NoError(t, task.ctx.Err(), "a probe timeout must not cancel the task")
|
|
||||||
}
|
|
||||||
@@ -931,6 +931,9 @@ func (sm *SmartManager) parseSmartForSata(output []byte, deviceType string) (boo
|
|||||||
if parsed, ok := smart.ParseSmartRawValueString(attr.Raw.String); ok {
|
if parsed, ok := smart.ParseSmartRawValueString(attr.Raw.String); ok {
|
||||||
rawValue = parsed
|
rawValue = parsed
|
||||||
}
|
}
|
||||||
|
if smartData.SmartStatus == "PASSED" && rawValue > 0 && (attr.ID == 5 || attr.ID == 197 || attr.ID == 198) {
|
||||||
|
smartData.SmartStatus = "WARNING"
|
||||||
|
}
|
||||||
smartAttr := &smart.SmartAttribute{
|
smartAttr := &smart.SmartAttribute{
|
||||||
ID: attr.ID,
|
ID: attr.ID,
|
||||||
Name: attr.Name,
|
Name: attr.Name,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
@@ -89,6 +90,27 @@ func TestParseSmartForSata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseSmartForSataWarnsForCriticalAttributes(t *testing.T) {
|
||||||
|
for _, attrID := range []int{5, 197, 198} {
|
||||||
|
t.Run("attribute "+strconv.Itoa(attrID), func(t *testing.T) {
|
||||||
|
jsonPayload := []byte(fmt.Sprintf(`{
|
||||||
|
"smartctl": {"exit_status": 0},
|
||||||
|
"device": {"name": "/dev/sda", "type": "sat"},
|
||||||
|
"model_name": "Example",
|
||||||
|
"serial_number": "WARNING%d",
|
||||||
|
"smart_status": {"passed": true},
|
||||||
|
"temperature": {"current": 30},
|
||||||
|
"ata_smart_attributes": {"table": [{"id": %d, "raw": {"value": 1, "string": "1"}}]}
|
||||||
|
}`, attrID, attrID))
|
||||||
|
|
||||||
|
sm := &SmartManager{SmartDataMap: make(map[string]*smart.SmartData)}
|
||||||
|
hasData, _ := sm.parseSmartForSata(jsonPayload, "")
|
||||||
|
require.True(t, hasData)
|
||||||
|
assert.Equal(t, "WARNING", sm.SmartDataMap[fmt.Sprintf("WARNING%d", attrID)].SmartStatus)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseSmartForSataPreservesFailedAndUnknownStatus(t *testing.T) {
|
func TestParseSmartForSataPreservesFailedAndUnknownStatus(t *testing.T) {
|
||||||
for _, test := range []struct {
|
for _, test := range []struct {
|
||||||
name string
|
name string
|
||||||
|
|||||||
@@ -1,462 +0,0 @@
|
|||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"log/slog"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
|
|
||||||
type zfsDatasetUsage struct {
|
|
||||||
used uint64
|
|
||||||
avail uint64
|
|
||||||
}
|
|
||||||
|
|
||||||
// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
|
|
||||||
// mountpoint usage map. Dataset inventory changes rarely.
|
|
||||||
const datasetUsageRefreshInterval = 5 * time.Minute
|
|
||||||
|
|
||||||
// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
|
|
||||||
// capacity. Health and I/O are read from procfs on Linux, so the utility only
|
|
||||||
// needs to refresh slow-moving space accounting.
|
|
||||||
const poolStatsRefreshInterval = time.Minute
|
|
||||||
|
|
||||||
// btrfsFilesystems is the btrfs source; overridable in tests.
|
|
||||||
var btrfsFilesystems = btrfs.Filesystems
|
|
||||||
|
|
||||||
type poolKernelSample struct {
|
|
||||||
nread uint64
|
|
||||||
nwrite uint64
|
|
||||||
at time.Time
|
|
||||||
}
|
|
||||||
|
|
||||||
// StoragePoolManager combines independent backend inventories. Metrics and
|
|
||||||
// dataset usage require the agent lock; GetDetail is safe for concurrent calls.
|
|
||||||
type StoragePoolManager struct {
|
|
||||||
backends []*poolBackend
|
|
||||||
detailInterval time.Duration
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolBackend owns one backend's collectors and caches. Collector functions
|
|
||||||
// are immutable after construction and may run concurrently for metrics/details.
|
|
||||||
type poolBackend struct {
|
|
||||||
name string
|
|
||||||
poolStatsFn func() ([]zfs.PoolStat, error) // capacity/health source
|
|
||||||
datasetsFn func() ([]zfs.Dataset, error) // dataset inventory source
|
|
||||||
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
|
|
||||||
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
|
|
||||||
|
|
||||||
poolData []zfs.PoolStat // cached pool inventory (TTL below)
|
|
||||||
lastPoolStats time.Time
|
|
||||||
kernelSamples map[string]poolKernelSample
|
|
||||||
|
|
||||||
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
|
|
||||||
lastUsageRefresh time.Time
|
|
||||||
|
|
||||||
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
|
|
||||||
// an interval. Accessed from handler goroutines, so it is mutex-protected.
|
|
||||||
detailMu sync.Mutex
|
|
||||||
detail *zfsentity.ZfsData
|
|
||||||
lastDetailRefresh time.Time
|
|
||||||
detailFailed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newStoragePoolManager() *StoragePoolManager {
|
|
||||||
return &StoragePoolManager{
|
|
||||||
backends: []*poolBackend{newZfsBackend(), newBtrfsBackend()},
|
|
||||||
detailInterval: time.Hour,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newZfsBackend() *poolBackend {
|
|
||||||
return &poolBackend{
|
|
||||||
name: "zfs",
|
|
||||||
poolStatsFn: optionalPoolSource(zfs.PoolStats),
|
|
||||||
datasetsFn: optionalPoolSource(zfs.Datasets),
|
|
||||||
kernelStatsFn: optionalPoolSource(zfs.PoolKernelStats),
|
|
||||||
poolStatusesFn: optionalPoolSource(zfs.PoolStatuses),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func newBtrfsBackend() *poolBackend {
|
|
||||||
return &poolBackend{
|
|
||||||
name: "btrfs",
|
|
||||||
poolStatsFn: btrfsSource(btrfsPoolStats),
|
|
||||||
kernelStatsFn: btrfsSource(btrfsKernelStats),
|
|
||||||
poolStatusesFn: btrfsSource(btrfsPoolStatuses),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// datasets is optional: only backends that expose datasets provide a collector.
|
|
||||||
func (b *poolBackend) datasets() ([]zfs.Dataset, error) {
|
|
||||||
if b.datasetsFn == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return b.datasetsFn()
|
|
||||||
}
|
|
||||||
|
|
||||||
// A missing utility/interface is a successfully observed absent backend.
|
|
||||||
func optionalPoolSource[T any](source func() ([]T, error)) func() ([]T, error) {
|
|
||||||
return func() ([]T, error) {
|
|
||||||
items, err := source()
|
|
||||||
if errors.Is(err, zfs.ErrNoZfs) || errors.Is(err, exec.ErrNotFound) || errors.Is(err, errors.ErrUnsupported) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return items, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsSource[T any](convert func(btrfs.Filesystem) T) func() ([]T, error) {
|
|
||||||
return func() ([]T, error) {
|
|
||||||
filesystems, err := optionalPoolSource(btrfsFilesystems)()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
items := make([]T, 0, len(filesystems))
|
|
||||||
for _, fs := range filesystems {
|
|
||||||
items = append(items, convert(fs))
|
|
||||||
}
|
|
||||||
return items, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
|
|
||||||
// throughput and health come from inexpensive kernel kstats on Linux. Pool
|
|
||||||
// capacity and dataset usage come from separately cached utility calls. The
|
|
||||||
// pool map is empty when both backends are absent.
|
|
||||||
func (m *StoragePoolManager) Update(systemStats *system.Stats) {
|
|
||||||
// Rebuild the combined map so successful pool removals clear old samples.
|
|
||||||
systemStats.ZfsPools = nil
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
backend.updateBackendStats(systemStats)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *poolBackend) updateBackendStats(systemStats *system.Stats) {
|
|
||||||
pools := b.poolStats()
|
|
||||||
if len(pools) == 0 {
|
|
||||||
b.kernelSamples = nil
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
kernelStats, ioRates := b.kernelStats()
|
|
||||||
|
|
||||||
if systemStats.ZfsPools == nil {
|
|
||||||
systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
|
|
||||||
}
|
|
||||||
for i := range pools {
|
|
||||||
pool := &pools[i]
|
|
||||||
// Full precision, matching the dataset values below; the frontend
|
|
||||||
// formats any magnitude.
|
|
||||||
stats := &system.ZfsPool{
|
|
||||||
DisplayName: pool.DisplayName,
|
|
||||||
Raw: pool.Raw,
|
|
||||||
Total: float64(pool.Size) / (1024 * 1024 * 1024),
|
|
||||||
Used: float64(pool.Alloc) / (1024 * 1024 * 1024),
|
|
||||||
Health: pool.Health,
|
|
||||||
}
|
|
||||||
if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
|
|
||||||
stats.Health = kernel.Health
|
|
||||||
}
|
|
||||||
if io, exists := ioRates[pool.Name]; exists {
|
|
||||||
stats.ReadBytes = io.NRead
|
|
||||||
stats.WriteBytes = io.NWrite
|
|
||||||
}
|
|
||||||
slog.Debug("Storage pool sample", "backend", b.name, "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
|
|
||||||
systemStats.ZfsPools[pool.Name] = stats
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolStats returns the cached pool inventory, calling its collector at most
|
|
||||||
// every poolStatsRefreshInterval. On failure the previous inventory is
|
|
||||||
// retained and the refresh is retried on the next cadence.
|
|
||||||
func (b *poolBackend) poolStats() []zfs.PoolStat {
|
|
||||||
if b.lastPoolStats.IsZero() || time.Since(b.lastPoolStats) >= poolStatsRefreshInterval {
|
|
||||||
pools, err := b.poolStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool stats unavailable", "backend", b.name, "err", err)
|
|
||||||
} else {
|
|
||||||
b.poolData = pools
|
|
||||||
}
|
|
||||||
b.lastPoolStats = time.Now()
|
|
||||||
}
|
|
||||||
return b.poolData
|
|
||||||
}
|
|
||||||
|
|
||||||
// kernelStats reads cumulative pool counters and converts them to per-second
|
|
||||||
// rates. Counter decreases indicate a pool export/import and reset the
|
|
||||||
// baseline instead of producing an underflow spike.
|
|
||||||
func (b *poolBackend) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
|
|
||||||
if b.kernelStatsFn == nil {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
stats, err := b.kernelStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool kernel stats unavailable", "backend", b.name, "err", err)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
byName := make(map[string]zfs.PoolKernelStat, len(stats))
|
|
||||||
rates := make(map[string]zfs.PoolIoStats, len(stats))
|
|
||||||
nextSamples := make(map[string]poolKernelSample, len(stats))
|
|
||||||
for _, stat := range stats {
|
|
||||||
byName[stat.Name] = stat
|
|
||||||
if previous, ok := b.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
|
|
||||||
stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
|
|
||||||
seconds := now.Sub(previous.at).Seconds()
|
|
||||||
rates[stat.Name] = zfs.PoolIoStats{
|
|
||||||
NRead: uint64(float64(stat.NRead-previous.nread) / seconds),
|
|
||||||
NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
|
|
||||||
}
|
|
||||||
b.kernelSamples = nextSamples
|
|
||||||
return byName, rates
|
|
||||||
}
|
|
||||||
|
|
||||||
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
|
|
||||||
// and rebuilds the mountpoint-keyed usage map.
|
|
||||||
func (b *poolBackend) refreshDatasetUsage() {
|
|
||||||
if !b.lastUsageRefresh.IsZero() && time.Since(b.lastUsageRefresh) < datasetUsageRefreshInterval {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
datasets, err := b.datasets()
|
|
||||||
if err != nil {
|
|
||||||
slog.Debug("Storage pool dataset usage unavailable", "backend", b.name, "err", err)
|
|
||||||
} else {
|
|
||||||
usage := make(map[string]zfsDatasetUsage, len(datasets))
|
|
||||||
for _, ds := range datasets {
|
|
||||||
if ds.Mountpoint != "" && ds.Mountpoint != "-" {
|
|
||||||
usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.datasetUsage = usage
|
|
||||||
}
|
|
||||||
b.lastUsageRefresh = time.Now()
|
|
||||||
}
|
|
||||||
|
|
||||||
// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
|
|
||||||
// most every datasetUsageRefreshInterval. On failure the previous map is
|
|
||||||
// retained and a debug log is emitted.
|
|
||||||
func (m *StoragePoolManager) DatasetUsage() map[string]zfsDatasetUsage {
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
if backend.name == "zfs" {
|
|
||||||
backend.refreshDatasetUsage()
|
|
||||||
return backend.datasetUsage
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetDetail combines backend snapshots, identifying successful inventories so
|
|
||||||
// the hub can accept partial updates without deleting failed backend records.
|
|
||||||
func (m *StoragePoolManager) GetDetail(force bool) *zfsentity.ZfsData {
|
|
||||||
data := &zfsentity.ZfsData{Complete: true}
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
snapshot := backend.getBackendDetail(force, m.detailInterval)
|
|
||||||
data.Pools = append(data.Pools, snapshot.Pools...)
|
|
||||||
if snapshot.Complete {
|
|
||||||
data.CompleteBackends = append(data.CompleteBackends, backend.name)
|
|
||||||
} else {
|
|
||||||
data.Complete = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func (b *poolBackend) getBackendDetail(force bool, interval time.Duration) *zfsentity.ZfsData {
|
|
||||||
b.detailMu.Lock()
|
|
||||||
defer b.detailMu.Unlock()
|
|
||||||
|
|
||||||
if force || b.detailFailed || b.detail == nil || time.Since(b.lastDetailRefresh) >= interval {
|
|
||||||
if data, err := b.collectDetail(b.detail); err != nil {
|
|
||||||
b.detailFailed = true
|
|
||||||
slog.Debug("Storage pool detail collection failed", "backend", b.name, "err", err)
|
|
||||||
if b.detail == nil {
|
|
||||||
return &zfsentity.ZfsData{}
|
|
||||||
}
|
|
||||||
return &zfsentity.ZfsData{Pools: b.detail.Pools}
|
|
||||||
} else {
|
|
||||||
b.detailFailed = false
|
|
||||||
b.detail = data
|
|
||||||
b.lastDetailRefresh = time.Now()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if b.detail == nil {
|
|
||||||
return &zfsentity.ZfsData{}
|
|
||||||
}
|
|
||||||
return b.detail
|
|
||||||
}
|
|
||||||
|
|
||||||
// collectDetail builds a ZfsData payload from the current system state.
|
|
||||||
func (b *poolBackend) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
|
|
||||||
pools, err := b.poolStatsFn()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if len(pools) == 0 {
|
|
||||||
return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
statuses, statusErr := b.poolStatusesFn()
|
|
||||||
if statusErr != nil {
|
|
||||||
slog.Debug("Storage pool status unavailable", "backend", b.name, "err", statusErr)
|
|
||||||
}
|
|
||||||
datasets, datasetsErr := b.datasets()
|
|
||||||
if datasetsErr != nil {
|
|
||||||
slog.Debug("Storage pool datasets unavailable", "backend", b.name, "err", datasetsErr)
|
|
||||||
}
|
|
||||||
|
|
||||||
statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
|
|
||||||
for _, st := range statuses {
|
|
||||||
statusByPool[st.Name] = st
|
|
||||||
}
|
|
||||||
|
|
||||||
previousByPool := make(map[string]*zfsentity.PoolDetail)
|
|
||||||
if previous != nil {
|
|
||||||
for _, pool := range previous.Pools {
|
|
||||||
if pool != nil {
|
|
||||||
previousByPool[pool.Name] = pool
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
|
|
||||||
for i := range pools {
|
|
||||||
p := &pools[i]
|
|
||||||
detail := &zfsentity.PoolDetail{
|
|
||||||
DisplayName: p.DisplayName,
|
|
||||||
Raw: p.Raw,
|
|
||||||
Name: p.Name,
|
|
||||||
Health: p.Health,
|
|
||||||
Size: p.Size,
|
|
||||||
Alloc: p.Alloc,
|
|
||||||
Free: p.Free,
|
|
||||||
}
|
|
||||||
if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
|
|
||||||
if st.Scrub.State != "" && st.Scrub.State != "NONE" {
|
|
||||||
detail.Scrub = &zfsentity.Scrub{
|
|
||||||
State: st.Scrub.State,
|
|
||||||
Progress: st.Scrub.Progress,
|
|
||||||
Errors: st.Scrub.Errors,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, v := range st.Vdevs {
|
|
||||||
detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
|
|
||||||
Name: v.Name,
|
|
||||||
State: v.State,
|
|
||||||
ReadErrs: v.ReadErrs,
|
|
||||||
WriteErrs: v.WriteErrs,
|
|
||||||
ChecksumErrs: v.ChecksumErrs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Scrub = cached.Scrub
|
|
||||||
detail.Vdevs = cached.Vdevs
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if datasetsErr == nil {
|
|
||||||
foundDataset := false
|
|
||||||
for _, ds := range datasets {
|
|
||||||
if poolOfDataset(ds.Name) == p.Name {
|
|
||||||
foundDataset = true
|
|
||||||
detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
|
|
||||||
Name: ds.Name,
|
|
||||||
Used: ds.Used,
|
|
||||||
Avail: ds.Avail,
|
|
||||||
Mountpoint: ds.Mountpoint,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !foundDataset {
|
|
||||||
if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Datasets = cached.Datasets
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else if cached := previousByPool[p.Name]; cached != nil {
|
|
||||||
detail.Datasets = cached.Datasets
|
|
||||||
}
|
|
||||||
data.Pools = append(data.Pools, detail)
|
|
||||||
}
|
|
||||||
return data, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// poolOfDataset returns the pool name for a dataset name (everything before
|
|
||||||
// the first '/'). Datasets without a separator belong to a pool of the same
|
|
||||||
// name.
|
|
||||||
func poolOfDataset(name string) string {
|
|
||||||
if idx := strings.IndexByte(name, '/'); idx >= 0 {
|
|
||||||
return name[:idx]
|
|
||||||
}
|
|
||||||
return name
|
|
||||||
}
|
|
||||||
|
|
||||||
// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
|
|
||||||
func (m *StoragePoolManager) ZfsMountpoints() map[string]bool {
|
|
||||||
usage := m.DatasetUsage()
|
|
||||||
mountpoints := make(map[string]bool, len(usage))
|
|
||||||
for mountpoint := range usage {
|
|
||||||
mountpoints[mountpoint] = true
|
|
||||||
}
|
|
||||||
return mountpoints
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsPoolStats(fs btrfs.Filesystem) zfs.PoolStat {
|
|
||||||
return zfs.PoolStat{MountID: fs.MountID, IODevice: fs.IODevice, Raw: fs.Raw, DisplayName: fs.Name, Name: "b:" + fs.UUID, Size: fs.Size, Alloc: fs.Alloc, Free: fs.Size - min(fs.Alloc, fs.Size), Health: fs.Health}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsKernelStats(fs btrfs.Filesystem) zfs.PoolKernelStat {
|
|
||||||
return zfs.PoolKernelStat{Name: "b:" + fs.UUID, Health: fs.Health, NRead: fs.NRead, NWrite: fs.NWrite}
|
|
||||||
}
|
|
||||||
|
|
||||||
func btrfsPoolStatuses(fs btrfs.Filesystem) zfs.PoolStatus {
|
|
||||||
status := zfs.PoolStatus{Name: "b:" + fs.UUID, State: fs.Health, Scrub: zfs.ScrubStatus{State: "NONE"}}
|
|
||||||
for _, dev := range fs.Devices {
|
|
||||||
status.Vdevs = append(status.Vdevs, zfs.VdevStatus{
|
|
||||||
Name: dev.Name, State: dev.State,
|
|
||||||
ReadErrs: dev.ReadErrs, WriteErrs: dev.WriteErrs, ChecksumErrs: dev.CorruptionErrs,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return status
|
|
||||||
}
|
|
||||||
|
|
||||||
// markDuplicateCharts leaves pool telemetry and detail intact, but tells the
|
|
||||||
// hub which charts already have a filesystem equivalent. Only exact kernel
|
|
||||||
// filesystem and I/O-device matches qualify; labels are never used.
|
|
||||||
func (m *StoragePoolManager) markDuplicateCharts(stats *system.Stats, filesystems map[string]*system.FsStats, mountID func(string) string) {
|
|
||||||
identities := make(map[string]string, len(filesystems))
|
|
||||||
for device, fs := range filesystems {
|
|
||||||
if fs.DiskTotal > 0 {
|
|
||||||
identities[device] = mountID(fs.Mountpoint)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, backend := range m.backends {
|
|
||||||
for _, pool := range backend.poolData {
|
|
||||||
sample := stats.ZfsPools[pool.Name]
|
|
||||||
if sample == nil || pool.MountID == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
for device, identity := range identities {
|
|
||||||
if identity != pool.MountID {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Raw physical usage is not equivalent to a filesystem usage chart.
|
|
||||||
sample.HideUsage = !pool.Raw
|
|
||||||
if pool.IODevice != "" && pool.IODevice == device {
|
|
||||||
sample.HideIO = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,520 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package agent
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"os/exec"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestOptionalPoolSource(t *testing.T) {
|
|
||||||
failure := errors.New("timeout")
|
|
||||||
for _, err := range []error{nil, zfs.ErrNoZfs, fmt.Errorf("zpool: %w", exec.ErrNotFound), errors.ErrUnsupported, failure} {
|
|
||||||
_, got := optionalPoolSource(func() ([]zfs.PoolStat, error) { return nil, err })()
|
|
||||||
if err == failure {
|
|
||||||
assert.ErrorIs(t, got, failure)
|
|
||||||
} else {
|
|
||||||
assert.NoError(t, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type poolTestBackend struct {
|
|
||||||
name string
|
|
||||||
err error
|
|
||||||
alloc uint64
|
|
||||||
read uint64
|
|
||||||
empty bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (state *poolTestBackend) backend() *poolBackend {
|
|
||||||
name := "zfs"
|
|
||||||
if strings.HasPrefix(state.name, "b:") {
|
|
||||||
name = "btrfs"
|
|
||||||
}
|
|
||||||
return &poolBackend{
|
|
||||||
name: name,
|
|
||||||
poolStatsFn: func() ([]zfs.PoolStat, error) {
|
|
||||||
if state.empty {
|
|
||||||
return nil, state.err
|
|
||||||
}
|
|
||||||
return []zfs.PoolStat{{Name: state.name, Size: 100, Alloc: state.alloc}}, state.err
|
|
||||||
},
|
|
||||||
kernelStatsFn: func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return []zfs.PoolKernelStat{{Name: state.name, NRead: state.read}}, state.err
|
|
||||||
},
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return nil, nil },
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndependentPoolBackendCaches(t *testing.T) {
|
|
||||||
for _, failed := range []int{0, 1} {
|
|
||||||
t.Run([]string{"zfs", "btrfs"}[failed], func(t *testing.T) {
|
|
||||||
states := []*poolTestBackend{{name: "tank", alloc: 10}, {name: "b:uuid", alloc: 10}}
|
|
||||||
managers := []*poolBackend{states[0].backend(), states[1].backend()}
|
|
||||||
zm := &StoragePoolManager{backends: managers, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 2)
|
|
||||||
require.True(t, zm.GetDetail(true).Complete)
|
|
||||||
baseline := poolKernelSample{at: time.Now().Add(-time.Second)}
|
|
||||||
for i, m := range managers {
|
|
||||||
m.lastPoolStats = time.Time{}
|
|
||||||
m.kernelSamples[states[i].name] = baseline
|
|
||||||
states[i].alloc = 20
|
|
||||||
states[i].read = 100
|
|
||||||
}
|
|
||||||
states[failed].err = errors.New("collection failed")
|
|
||||||
zm.Update(&stats)
|
|
||||||
healthy := 1 - failed
|
|
||||||
assert.Equal(t, uint64(10), managers[failed].poolData[0].Alloc)
|
|
||||||
assert.Equal(t, uint64(20), managers[healthy].poolData[0].Alloc)
|
|
||||||
assert.Equal(t, baseline, managers[failed].kernelSamples[states[failed].name])
|
|
||||||
assert.Zero(t, stats.ZfsPools[states[failed].name].ReadBytes)
|
|
||||||
assert.Positive(t, stats.ZfsPools[states[healthy].name].ReadBytes)
|
|
||||||
partial := zm.GetDetail(true)
|
|
||||||
assert.False(t, partial.Complete)
|
|
||||||
assert.False(t, partial.CanRefreshPool(states[failed].name))
|
|
||||||
assert.True(t, partial.CanRefreshPool(states[healthy].name))
|
|
||||||
assert.Equal(t, uint64(10), partial.Pools[failed].Alloc)
|
|
||||||
assert.Equal(t, uint64(20), partial.Pools[healthy].Alloc)
|
|
||||||
assert.False(t, zm.GetDetail(false).Complete, "a failed forced refresh must not become complete from cache")
|
|
||||||
|
|
||||||
// Successful empty inventory removes only the healthy backend's pool.
|
|
||||||
states[healthy].empty = true
|
|
||||||
managers[healthy].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 1)
|
|
||||||
assert.Contains(t, stats.ZfsPools, states[failed].name)
|
|
||||||
partial = zm.GetDetail(true)
|
|
||||||
require.Len(t, partial.Pools, 1)
|
|
||||||
assert.True(t, partial.CanRefreshPool(states[healthy].name))
|
|
||||||
|
|
||||||
// Recovery uses the retained I/O baseline, then normal removal works.
|
|
||||||
states[failed].err = nil
|
|
||||||
managers[failed].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Positive(t, stats.ZfsPools[states[failed].name].ReadBytes)
|
|
||||||
assert.True(t, zm.GetDetail(true).Complete)
|
|
||||||
states[failed].empty = true
|
|
||||||
managers[failed].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Empty(t, stats.ZfsPools)
|
|
||||||
assert.Empty(t, zm.GetDetail(true).Pools)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIndependentBackendsWithoutCache(t *testing.T) {
|
|
||||||
z := &poolTestBackend{name: "tank", err: errors.New("ZFS failure")}
|
|
||||||
b := &poolTestBackend{name: "b:uuid", alloc: 20}
|
|
||||||
zm := &StoragePoolManager{backends: []*poolBackend{z.backend(), b.backend()}, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 1)
|
|
||||||
assert.Contains(t, stats.ZfsPools, "b:uuid")
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.Len(t, detail.Pools, 1)
|
|
||||||
assert.False(t, detail.Complete)
|
|
||||||
assert.Equal(t, []string{"btrfs"}, detail.CompleteBackends)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestConcurrentBackendDetailsAndMetrics(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{backends: []*poolBackend{(&poolTestBackend{name: "tank"}).backend(), (&poolTestBackend{name: "b:uuid"}).backend()}, detailInterval: time.Hour}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for i := 0; i < 3; i++ {
|
|
||||||
wg.Add(1)
|
|
||||||
go func(metrics bool) {
|
|
||||||
defer wg.Done()
|
|
||||||
for j := 0; j < 10; j++ {
|
|
||||||
if metrics {
|
|
||||||
zm.Update(&system.Stats{})
|
|
||||||
} else {
|
|
||||||
zm.GetDetail(true)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}(i == 0)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestStoragePoolBackendOrder(t *testing.T) {
|
|
||||||
z := (&poolTestBackend{name: "tank"}).backend()
|
|
||||||
b := (&poolTestBackend{name: "b:uuid"}).backend()
|
|
||||||
b.datasetsFn = nil // Btrfs does not expose datasets.
|
|
||||||
z.datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank/data", Mountpoint: "/tank", Used: 10}}, nil
|
|
||||||
}
|
|
||||||
m := &StoragePoolManager{backends: []*poolBackend{b, z}, detailInterval: time.Hour}
|
|
||||||
var stats system.Stats
|
|
||||||
m.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 2)
|
|
||||||
detail := m.GetDetail(true)
|
|
||||||
require.True(t, detail.Complete)
|
|
||||||
assert.Equal(t, []string{"btrfs", "zfs"}, detail.CompleteBackends)
|
|
||||||
assert.Empty(t, detail.Pools[0].Datasets)
|
|
||||||
assert.Len(t, detail.Pools[1].Datasets, 1)
|
|
||||||
assert.Equal(t, uint64(10), m.DatasetUsage()["/tank"].used)
|
|
||||||
|
|
||||||
b.poolData[0].MountID = "uuid"
|
|
||||||
b.poolData[0].IODevice = "sda"
|
|
||||||
calls := 0
|
|
||||||
m.markDuplicateCharts(&stats, map[string]*system.FsStats{
|
|
||||||
"sda": {Mountpoint: "/", DiskTotal: 100},
|
|
||||||
}, func(string) string { calls++; return "uuid" })
|
|
||||||
assert.Equal(t, 1, calls, "resolve each filesystem once across all backends")
|
|
||||||
assert.True(t, stats.ZfsPools["b:uuid"].HideUsage)
|
|
||||||
assert.True(t, stats.ZfsPools["b:uuid"].HideIO)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdatePopulatesZfsPools(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
|
||||||
{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
|
|
||||||
// Small zvol (Proxmox VM EFI disk): must not round to zero.
|
|
||||||
{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
var kernelCalls int
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
kernelCalls++
|
|
||||||
return []zfs.PoolKernelStat{{
|
|
||||||
Name: "tank", Health: "ONLINE",
|
|
||||||
NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
|
|
||||||
}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
// The first kernel sample establishes the cumulative-counter baseline.
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.backends[0].kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.NotNil(t, stats.ZfsPools)
|
|
||||||
require.Contains(t, stats.ZfsPools, "tank")
|
|
||||||
assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
|
|
||||||
assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001) // Alloc in GiB
|
|
||||||
assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
|
|
||||||
assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
|
|
||||||
assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
|
|
||||||
// I/O instead of erroring.
|
|
||||||
func TestUpdateKernelStatsMissing(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.NotNil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateKernelCounterReset(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
zm.backends[0].kernelSamples = map[string]poolKernelSample{
|
|
||||||
"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
|
|
||||||
}
|
|
||||||
zm.backends[0].kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
|
||||||
return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
|
||||||
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateNoZfs(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
calls++
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Nil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateEmptyPools(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
calls++
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
zm.Update(&stats)
|
|
||||||
assert.Nil(t, stats.ZfsPools)
|
|
||||||
assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsage(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
calls := 0
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
calls++
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
|
||||||
{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
|
||||||
{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
usage := zm.DatasetUsage()
|
|
||||||
require.Len(t, usage, 2)
|
|
||||||
assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
|
|
||||||
assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
|
|
||||||
assert.Equal(t, 1, calls)
|
|
||||||
|
|
||||||
// Second call within the refresh window must not re-run the collector.
|
|
||||||
zm.DatasetUsage()
|
|
||||||
assert.Equal(t, 1, calls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
|
|
||||||
}
|
|
||||||
assert.Len(t, zm.DatasetUsage(), 1)
|
|
||||||
|
|
||||||
// Force refresh window expiry, then a failing collector.
|
|
||||||
zm.backends[0].lastUsageRefresh = time.Now().Add(-10 * time.Minute)
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
}
|
|
||||||
usage := zm.DatasetUsage()
|
|
||||||
assert.Len(t, usage, 1, "previous usage should be retained on error")
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetUsageClearsAbsentBackend(t *testing.T) {
|
|
||||||
b := newZfsBackend()
|
|
||||||
b.datasetUsage = map[string]zfsDatasetUsage{"/tank": {used: 1, avail: 1}}
|
|
||||||
b.datasetsFn = optionalPoolSource(func() ([]zfs.Dataset, error) {
|
|
||||||
return nil, zfs.ErrNoZfs
|
|
||||||
})
|
|
||||||
|
|
||||||
datasets, err := b.datasets()
|
|
||||||
require.NoError(t, err, "an absent backend must not produce an error to log")
|
|
||||||
assert.Empty(t, datasets)
|
|
||||||
b.refreshDatasetUsage()
|
|
||||||
assert.Empty(t, b.datasetUsage)
|
|
||||||
assert.False(t, b.lastUsageRefresh.IsZero())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailForceRefresh(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
poolCalls := 0
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
poolCalls++
|
|
||||||
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
|
|
||||||
first := zm.GetDetail(false)
|
|
||||||
assert.True(t, first.Complete)
|
|
||||||
require.Len(t, first.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(1), first.Pools[0].Alloc)
|
|
||||||
|
|
||||||
cached := zm.GetDetail(false)
|
|
||||||
require.Len(t, cached.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
|
|
||||||
assert.Equal(t, 1, poolCalls)
|
|
||||||
|
|
||||||
refreshed := zm.GetDetail(true)
|
|
||||||
assert.True(t, refreshed.Complete)
|
|
||||||
require.Len(t, refreshed.Pools, 1)
|
|
||||||
assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
|
|
||||||
assert.Equal(t, 2, poolCalls)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
|
||||||
|
|
||||||
require.Len(t, zm.GetDetail(false).Pools, 1)
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
|
|
||||||
empty := zm.GetDetail(true)
|
|
||||||
assert.True(t, empty.Complete)
|
|
||||||
assert.Empty(t, empty.Pools)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{{Name: "tank"}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) {
|
|
||||||
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
|
|
||||||
}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{{Name: "tank/data"}}, nil
|
|
||||||
}
|
|
||||||
first := zm.GetDetail(false)
|
|
||||||
require.True(t, first.Complete)
|
|
||||||
require.Len(t, first.Pools[0].Vdevs, 1)
|
|
||||||
require.Len(t, first.Pools[0].Datasets, 1)
|
|
||||||
|
|
||||||
zm.backends[0].poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
partial := zm.GetDetail(true)
|
|
||||||
require.True(t, partial.Complete)
|
|
||||||
require.Len(t, partial.Pools[0].Vdevs, 1)
|
|
||||||
require.Len(t, partial.Pools[0].Datasets, 1)
|
|
||||||
|
|
||||||
zm.backends[0].poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
|
|
||||||
lastSuccessfulRefresh := zm.backends[0].lastDetailRefresh
|
|
||||||
failed := zm.GetDetail(true)
|
|
||||||
assert.False(t, failed.Complete)
|
|
||||||
require.Len(t, failed.Pools, 1)
|
|
||||||
assert.Equal(t, "tank", failed.Pools[0].Name)
|
|
||||||
assert.Equal(t, lastSuccessfulRefresh, zm.backends[0].lastDetailRefresh)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestZfsMountpoints(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs"}}}
|
|
||||||
zm.backends[0].datasetsFn = func() ([]zfs.Dataset, error) {
|
|
||||||
return []zfs.Dataset{
|
|
||||||
{Name: "tank", Mountpoint: "/tank"},
|
|
||||||
{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
mountpoints := zm.ZfsMountpoints()
|
|
||||||
assert.Len(t, mountpoints, 2)
|
|
||||||
assert.True(t, mountpoints["/tank"])
|
|
||||||
assert.True(t, mountpoints["/"])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBtrfsRawCapacityPropagates(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) {
|
|
||||||
return []zfs.PoolStat{btrfsPoolStats(btrfs.Filesystem{UUID: "raw", Name: "raw", Size: 200, Alloc: 100, Raw: true})}, nil
|
|
||||||
},
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) { return nil, nil },
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return nil, nil }}}}
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.True(t, stats.ZfsPools["b:raw"].Raw)
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.True(t, detail.Complete)
|
|
||||||
require.Len(t, detail.Pools, 1)
|
|
||||||
assert.True(t, detail.Pools[0].Raw)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMarkDuplicatePoolCharts(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name, poolID, device string
|
|
||||||
raw bool
|
|
||||||
diskTotal float64
|
|
||||||
wantUsage, wantIO bool
|
|
||||||
}{
|
|
||||||
{"single device root", "fs1", "dm-0", false, 100, true, true},
|
|
||||||
{"multi device", "fs1", "", false, 100, true, false},
|
|
||||||
{"different IO device", "fs1", "nvme0n1", false, 100, true, false},
|
|
||||||
{"different filesystem", "fs2", "dm-0", false, 100, false, false},
|
|
||||||
{"unknown identity", "", "dm-0", false, 100, false, false},
|
|
||||||
{"raw usage", "fs1", "dm-0", true, 100, false, true},
|
|
||||||
{"failed disk collection", "fs1", "dm-0", false, 0, false, false},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolData: []zfs.PoolStat{{Name: "arbitrary label", MountID: tc.poolID, IODevice: tc.device, Raw: tc.raw}}}}}
|
|
||||||
stats := &system.Stats{ZfsPools: map[string]*system.ZfsPool{"arbitrary label": {}}}
|
|
||||||
fs := map[string]*system.FsStats{"dm-0": {Root: true, Mountpoint: "/", DiskTotal: tc.diskTotal}}
|
|
||||||
zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
|
|
||||||
assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
|
|
||||||
assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
|
|
||||||
// Bind mounts and custom extra-filesystem names have the same identity.
|
|
||||||
fs["dm-0"].Root = false
|
|
||||||
fs["dm-0"].Mountpoint = "/extra-filesystems/storage"
|
|
||||||
fs["dm-0"].Name = "custom name"
|
|
||||||
stats.ZfsPools["arbitrary label"] = &system.ZfsPool{}
|
|
||||||
zm.markDuplicateCharts(stats, fs, func(string) string { return "fs1" })
|
|
||||||
assert.Equal(t, tc.wantUsage, stats.ZfsPools["arbitrary label"].HideUsage)
|
|
||||||
assert.Equal(t, tc.wantIO, stats.ZfsPools["arbitrary label"].HideIO)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBtrfsPoolIdentities(t *testing.T) {
|
|
||||||
old := btrfsFilesystems
|
|
||||||
t.Cleanup(func() { btrfsFilesystems = old })
|
|
||||||
label := "tank"
|
|
||||||
btrfsFilesystems = func() ([]btrfs.Filesystem, error) {
|
|
||||||
return []btrfs.Filesystem{
|
|
||||||
{UUID: "11111111-1111-4111-8111-111111111111", Name: label, Size: 100, Health: "ONLINE", NRead: 100, Devices: []btrfs.Device{{Name: "first"}}},
|
|
||||||
{UUID: "22222222-2222-4222-8222-222222222222", Name: "tank", Size: 200, Health: "DEGRADED", NRead: 200, Devices: []btrfs.Device{{Name: "second"}}},
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
zm := &StoragePoolManager{detailInterval: time.Hour, backends: []*poolBackend{{name: "zfs", poolStatsFn: func() ([]zfs.PoolStat, error) { return []zfs.PoolStat{{Name: "tank", Size: 300}}, nil },
|
|
||||||
kernelStatsFn: func() ([]zfs.PoolKernelStat, error) { return []zfs.PoolKernelStat{{Name: "tank", NRead: 300}}, nil },
|
|
||||||
poolStatusesFn: func() ([]zfs.PoolStatus, error) {
|
|
||||||
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "zfs-device"}}}}, nil
|
|
||||||
},
|
|
||||||
|
|
||||||
datasetsFn: func() ([]zfs.Dataset, error) { return []zfs.Dataset{{Name: "tank/data"}}, nil }}, newBtrfsBackend()}}
|
|
||||||
first := "b:11111111-1111-4111-8111-111111111111"
|
|
||||||
second := "b:22222222-2222-4222-8222-222222222222"
|
|
||||||
var stats system.Stats
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 3)
|
|
||||||
assert.Contains(t, stats.ZfsPools, "tank")
|
|
||||||
assert.Equal(t, "ONLINE", stats.ZfsPools[first].Health)
|
|
||||||
assert.Equal(t, "DEGRADED", stats.ZfsPools[second].Health)
|
|
||||||
assert.Equal(t, uint64(100), zm.backends[1].kernelSamples[first].nread)
|
|
||||||
assert.Equal(t, uint64(200), zm.backends[1].kernelSamples[second].nread)
|
|
||||||
detail := zm.GetDetail(true)
|
|
||||||
require.Len(t, detail.Pools, 3)
|
|
||||||
assert.Equal(t, "zfs-device", detail.Pools[0].Vdevs[0].Name)
|
|
||||||
assert.Len(t, detail.Pools[0].Datasets, 1)
|
|
||||||
assert.Equal(t, "first", detail.Pools[1].Vdevs[0].Name)
|
|
||||||
assert.Empty(t, detail.Pools[1].Datasets)
|
|
||||||
assert.Equal(t, "second", detail.Pools[2].Vdevs[0].Name)
|
|
||||||
label = "renamed"
|
|
||||||
zm.backends[0].lastPoolStats = time.Time{}
|
|
||||||
zm.backends[1].lastPoolStats = time.Time{}
|
|
||||||
zm.Update(&stats)
|
|
||||||
require.Len(t, stats.ZfsPools, 3)
|
|
||||||
assert.Equal(t, "renamed", stats.ZfsPools[first].DisplayName)
|
|
||||||
assert.Equal(t, first, zm.GetDetail(true).Pools[1].Name)
|
|
||||||
assert.Equal(t, "renamed", zm.GetDetail(true).Pools[1].DisplayName)
|
|
||||||
}
|
|
||||||
+2
-4
@@ -12,7 +12,6 @@ import (
|
|||||||
|
|
||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
"github.com/henrygd/beszel/agent/battery"
|
"github.com/henrygd/beszel/agent/battery"
|
||||||
"github.com/henrygd/beszel/agent/btrfs"
|
|
||||||
"github.com/henrygd/beszel/agent/utils"
|
"github.com/henrygd/beszel/agent/utils"
|
||||||
"github.com/henrygd/beszel/agent/zfs"
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
@@ -220,9 +219,8 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
|
|||||||
// disk i/o (cache-aware per interval)
|
// disk i/o (cache-aware per interval)
|
||||||
a.updateDiskIo(cacheTimeMs, &systemStats)
|
a.updateDiskIo(cacheTimeMs, &systemStats)
|
||||||
|
|
||||||
// storage pool stats
|
// zfs pool stats
|
||||||
a.storagePoolManager.Update(&systemStats)
|
a.zfsManager.Update(&systemStats)
|
||||||
a.storagePoolManager.markDuplicateCharts(&systemStats, a.fsStats, btrfs.MountID)
|
|
||||||
|
|
||||||
// network stats (per cache interval)
|
// network stats (per cache interval)
|
||||||
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
a.updateNetworkStats(cacheTimeMs, &systemStats)
|
||||||
|
|||||||
+5
-15
@@ -33,15 +33,11 @@ var ErrNoZfs = errors.New("zfs utilities unavailable")
|
|||||||
|
|
||||||
// PoolStat is a snapshot of a ZFS pool's capacity and health.
|
// PoolStat is a snapshot of a ZFS pool's capacity and health.
|
||||||
type PoolStat struct {
|
type PoolStat struct {
|
||||||
DisplayName string // optional friendly name; Name remains the stable key
|
Name string
|
||||||
MountID string // Btrfs filesystem identity, empty for other backends
|
Size uint64 // total capacity in bytes
|
||||||
IODevice string // sole Btrfs member device, if known
|
Alloc uint64 // allocated bytes
|
||||||
Raw bool // physical accounting rather than usable filesystem space
|
Free uint64 // free bytes
|
||||||
Name string
|
Health string // ONLINE, DEGRADED, FAULTED, ...
|
||||||
Size uint64 // total capacity in bytes
|
|
||||||
Alloc uint64 // allocated bytes
|
|
||||||
Free uint64 // free bytes
|
|
||||||
Health string // ONLINE, DEGRADED, FAULTED, ...
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PoolKernelStat is the inexpensive pool telemetry exposed by the ZFS kernel.
|
// PoolKernelStat is the inexpensive pool telemetry exposed by the ZFS kernel.
|
||||||
@@ -70,9 +66,6 @@ type Dataset struct {
|
|||||||
// PoolStats returns capacity and health for all pools on the system using
|
// PoolStats returns capacity and health for all pools on the system using
|
||||||
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
|
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
|
||||||
func PoolStats() ([]PoolStat, error) {
|
func PoolStats() ([]PoolStat, error) {
|
||||||
if err := checkZfsDevice(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
|
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var exitErr *exec.ExitError
|
var exitErr *exec.ExitError
|
||||||
@@ -87,9 +80,6 @@ func PoolStats() ([]PoolStat, error) {
|
|||||||
// Datasets returns all datasets on the system with usage and mountpoint
|
// Datasets returns all datasets on the system with usage and mountpoint
|
||||||
// information using `zfs list` (recursive by default).
|
// information using `zfs list` (recursive by default).
|
||||||
func Datasets() ([]Dataset, error) {
|
func Datasets() ([]Dataset, error) {
|
||||||
if err := checkZfsDevice(); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
|
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("zfs list: %w", err)
|
return nil, fmt.Errorf("zfs list: %w", err)
|
||||||
|
|||||||
+1
-17
@@ -13,10 +13,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var procZfsPath = "/proc/spl/kstat/zfs"
|
||||||
procZfsPath = "/proc/spl/kstat/zfs"
|
|
||||||
devZfsPath = "/dev/zfs"
|
|
||||||
)
|
|
||||||
|
|
||||||
func ARCSize() (uint64, error) {
|
func ARCSize() (uint64, error) {
|
||||||
file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
|
file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
|
||||||
@@ -43,19 +40,6 @@ func ARCSize() (uint64, error) {
|
|||||||
return 0, fmt.Errorf("size field not found in arcstats")
|
return 0, fmt.Errorf("size field not found in arcstats")
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkZfsDevice lets containers without /dev/zfs fail fast instead of
|
|
||||||
// waiting for ZFS utility commands to time out.
|
|
||||||
func checkZfsDevice() error {
|
|
||||||
_, err := os.Stat(devZfsPath)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, os.ErrNotExist) {
|
|
||||||
return ErrNoZfs
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// PoolKernelStats reads pool state and cumulative I/O counters directly from
|
// PoolKernelStats reads pool state and cumulative I/O counters directly from
|
||||||
// procfs. These kstats are the same interfaces used by node_exporter's Linux
|
// procfs. These kstats are the same interfaces used by node_exporter's Linux
|
||||||
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive.
|
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive.
|
||||||
|
|||||||
@@ -88,66 +88,3 @@ func TestReadObjsetIORequiresAllCounters(t *testing.T) {
|
|||||||
_, _, err := readObjsetIO(path)
|
_, _, err := readObjsetIO(path)
|
||||||
require.Error(t, err)
|
require.Error(t, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestCollectorsSkipCommandsWhenDevZfsMissing(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = filepath.Join(root, "missing")
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
t.Fatalf("unexpected %s call with %v", name, args)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
_, err := PoolStats()
|
|
||||||
assert.ErrorIs(t, err, ErrNoZfs)
|
|
||||||
_, err = Datasets()
|
|
||||||
assert.ErrorIs(t, err, ErrNoZfs)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDatasetsDelegatesWhenDevZfsPresent(t *testing.T) {
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = filepath.Join(t.TempDir(), "zfs")
|
|
||||||
require.NoError(t, os.WriteFile(devZfsPath, nil, 0o644))
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
assert.Equal(t, "zfs", name)
|
|
||||||
assert.Equal(t, []string{"list", "-Hp", "-o", "name,used,avail,mountpoint"}, args)
|
|
||||||
return []byte("tank\t50\t50\t/tank\n"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
datasets, err := Datasets()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPoolStatsDelegatesToZpoolWhenDevZfsPresent(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
devFile := filepath.Join(root, "zfs")
|
|
||||||
require.NoError(t, os.WriteFile(devFile, []byte(""), 0o644))
|
|
||||||
|
|
||||||
oldDevZfsPath := devZfsPath
|
|
||||||
devZfsPath = devFile
|
|
||||||
t.Cleanup(func() { devZfsPath = oldDevZfsPath })
|
|
||||||
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
called := false
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
called = true
|
|
||||||
assert.Equal(t, "zpool", name)
|
|
||||||
assert.Equal(t, []string{"list", "-Hp", "-o", "name,size,alloc,free,health"}, args)
|
|
||||||
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
pools, err := PoolStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, called)
|
|
||||||
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
//go:build !linux
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
// The /dev/zfs probe is Linux-specific. Other platforms detect availability
|
|
||||||
// through the ZFS utilities themselves.
|
|
||||||
func checkZfsDevice() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
//go:build testing && !linux
|
|
||||||
|
|
||||||
package zfs
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCollectorsUseUtilitiesOnNonLinux(t *testing.T) {
|
|
||||||
oldCommandOutput := commandOutput
|
|
||||||
commandOutput = func(name string, args ...string) ([]byte, error) {
|
|
||||||
switch name {
|
|
||||||
case "zpool":
|
|
||||||
return []byte("tank\t100\t50\t50\tONLINE\n"), nil
|
|
||||||
case "zfs":
|
|
||||||
return []byte("tank\t50\t50\t/tank\n"), nil
|
|
||||||
default:
|
|
||||||
t.Fatalf("unexpected command %s", name)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { commandOutput = oldCommandOutput })
|
|
||||||
|
|
||||||
pools, err := PoolStats()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []PoolStat{{Name: "tank", Size: 100, Alloc: 50, Free: 50, Health: "ONLINE"}}, pools)
|
|
||||||
datasets, err := Datasets()
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, []Dataset{{Name: "tank", Used: 50, Avail: 50, Mountpoint: "/tank"}}, datasets)
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,320 @@
|
|||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log/slog"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
|
||||||
|
type zfsDatasetUsage struct {
|
||||||
|
used uint64
|
||||||
|
avail uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
|
||||||
|
// mountpoint usage map. Dataset inventory changes rarely.
|
||||||
|
const datasetUsageRefreshInterval = 5 * time.Minute
|
||||||
|
|
||||||
|
// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
|
||||||
|
// capacity. Health and I/O are read from procfs on Linux, so the utility only
|
||||||
|
// needs to refresh slow-moving space accounting.
|
||||||
|
const poolStatsRefreshInterval = time.Minute
|
||||||
|
|
||||||
|
type poolKernelSample struct {
|
||||||
|
nread uint64
|
||||||
|
nwrite uint64
|
||||||
|
at time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZfsManager collects ZFS pool and dataset statistics. Collection functions
|
||||||
|
// are fields so unit tests can substitute them (same pattern as
|
||||||
|
// diskDiscovery.usageFn). It is safe for concurrent use by a single goroutine
|
||||||
|
// only; callers must hold the agent lock like updateDiskUsage does.
|
||||||
|
type ZfsManager struct {
|
||||||
|
poolStatsFn func() ([]zfs.PoolStat, error) // capacity/health source
|
||||||
|
datasetsFn func() ([]zfs.Dataset, error) // dataset inventory source
|
||||||
|
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
|
||||||
|
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
|
||||||
|
|
||||||
|
poolData []zfs.PoolStat // cached pool inventory (TTL below)
|
||||||
|
lastPoolStats time.Time
|
||||||
|
kernelSamples map[string]poolKernelSample
|
||||||
|
|
||||||
|
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
|
||||||
|
lastUsageRefresh time.Time
|
||||||
|
|
||||||
|
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
|
||||||
|
// an interval. Accessed from handler goroutines, so it is mutex-protected.
|
||||||
|
detailMu sync.Mutex
|
||||||
|
detail *zfsentity.ZfsData
|
||||||
|
lastDetailRefresh time.Time
|
||||||
|
detailInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// newZfsManager creates a ZfsManager wired to the system's ZFS utilities.
|
||||||
|
func newZfsManager() *ZfsManager {
|
||||||
|
return &ZfsManager{
|
||||||
|
poolStatsFn: zfs.PoolStats,
|
||||||
|
datasetsFn: zfs.Datasets,
|
||||||
|
kernelStatsFn: zfs.PoolKernelStats,
|
||||||
|
poolStatusesFn: zfs.PoolStatuses,
|
||||||
|
detailInterval: time.Hour,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
|
||||||
|
// throughput and health come from inexpensive kernel kstats on Linux. Pool
|
||||||
|
// capacity and dataset usage come from separately cached utility calls. It is
|
||||||
|
// a no-op when ZFS is absent.
|
||||||
|
func (zm *ZfsManager) Update(systemStats *system.Stats) {
|
||||||
|
pools := zm.poolStats()
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
kernelStats, ioRates := zm.kernelStats()
|
||||||
|
|
||||||
|
if systemStats.ZfsPools == nil {
|
||||||
|
systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
|
||||||
|
}
|
||||||
|
for i := range pools {
|
||||||
|
pool := &pools[i]
|
||||||
|
// Full precision, matching the dataset values below; the frontend
|
||||||
|
// formats any magnitude.
|
||||||
|
stats := &system.ZfsPool{
|
||||||
|
Total: float64(pool.Size) / (1024 * 1024 * 1024),
|
||||||
|
Used: float64(pool.Alloc) / (1024 * 1024 * 1024),
|
||||||
|
Health: pool.Health,
|
||||||
|
}
|
||||||
|
if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
|
||||||
|
stats.Health = kernel.Health
|
||||||
|
}
|
||||||
|
if io, exists := ioRates[pool.Name]; exists {
|
||||||
|
stats.ReadBytes = io.NRead
|
||||||
|
stats.WriteBytes = io.NWrite
|
||||||
|
}
|
||||||
|
slog.Debug("ZFS pool sample", "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
|
||||||
|
systemStats.ZfsPools[pool.Name] = stats
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// poolStats returns the cached pool inventory, re-running `zpool list` at most
|
||||||
|
// every poolStatsRefreshInterval. On failure the previous inventory is
|
||||||
|
// retained and the refresh is retried on the next cadence.
|
||||||
|
func (zm *ZfsManager) poolStats() []zfs.PoolStat {
|
||||||
|
if zm.lastPoolStats.IsZero() || time.Since(zm.lastPoolStats) >= poolStatsRefreshInterval {
|
||||||
|
pools, err := zm.poolStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS pool stats unavailable", "err", err)
|
||||||
|
} else {
|
||||||
|
zm.poolData = pools
|
||||||
|
}
|
||||||
|
zm.lastPoolStats = time.Now()
|
||||||
|
}
|
||||||
|
return zm.poolData
|
||||||
|
}
|
||||||
|
|
||||||
|
// kernelStats reads cumulative pool counters and converts them to per-second
|
||||||
|
// rates. Counter decreases indicate a pool export/import and reset the
|
||||||
|
// baseline instead of producing an underflow spike.
|
||||||
|
func (zm *ZfsManager) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
|
||||||
|
if zm.kernelStatsFn == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
stats, err := zm.kernelStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS kernel stats unavailable", "err", err)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
byName := make(map[string]zfs.PoolKernelStat, len(stats))
|
||||||
|
rates := make(map[string]zfs.PoolIoStats, len(stats))
|
||||||
|
nextSamples := make(map[string]poolKernelSample, len(stats))
|
||||||
|
for _, stat := range stats {
|
||||||
|
byName[stat.Name] = stat
|
||||||
|
if previous, ok := zm.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
|
||||||
|
stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
|
||||||
|
seconds := now.Sub(previous.at).Seconds()
|
||||||
|
rates[stat.Name] = zfs.PoolIoStats{
|
||||||
|
NRead: uint64(float64(stat.NRead-previous.nread) / seconds),
|
||||||
|
NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
|
||||||
|
}
|
||||||
|
zm.kernelSamples = nextSamples
|
||||||
|
return byName, rates
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
|
||||||
|
// and rebuilds the mountpoint-keyed usage map.
|
||||||
|
func (zm *ZfsManager) refreshDatasetUsage() {
|
||||||
|
if !zm.lastUsageRefresh.IsZero() && time.Since(zm.lastUsageRefresh) < datasetUsageRefreshInterval {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
datasets, err := zm.datasetsFn()
|
||||||
|
if err != nil {
|
||||||
|
slog.Debug("ZFS dataset usage unavailable", "err", err)
|
||||||
|
} else {
|
||||||
|
usage := make(map[string]zfsDatasetUsage, len(datasets))
|
||||||
|
for _, ds := range datasets {
|
||||||
|
if ds.Mountpoint != "" && ds.Mountpoint != "-" {
|
||||||
|
usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
zm.datasetUsage = usage
|
||||||
|
}
|
||||||
|
zm.lastUsageRefresh = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
|
||||||
|
// most every datasetUsageRefreshInterval. On failure the previous map is
|
||||||
|
// retained and a debug log is emitted.
|
||||||
|
func (zm *ZfsManager) DatasetUsage() map[string]zfsDatasetUsage {
|
||||||
|
zm.refreshDatasetUsage()
|
||||||
|
return zm.datasetUsage
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDetail returns ZFS detail data (pool health, scrub, vdevs, datasets).
|
||||||
|
// Scheduled requests use the cached snapshot until stale; manual requests can
|
||||||
|
// force collection. On failure the previous snapshot is retained.
|
||||||
|
func (zm *ZfsManager) GetDetail(force bool) *zfsentity.ZfsData {
|
||||||
|
zm.detailMu.Lock()
|
||||||
|
defer zm.detailMu.Unlock()
|
||||||
|
|
||||||
|
if force || zm.detail == nil || time.Since(zm.lastDetailRefresh) >= zm.detailInterval {
|
||||||
|
if data, err := zm.collectDetail(zm.detail); err != nil {
|
||||||
|
slog.Debug("ZFS detail collection failed", "err", err)
|
||||||
|
if zm.detail == nil {
|
||||||
|
return &zfsentity.ZfsData{}
|
||||||
|
}
|
||||||
|
return &zfsentity.ZfsData{Pools: zm.detail.Pools}
|
||||||
|
} else {
|
||||||
|
zm.detail = data
|
||||||
|
zm.lastDetailRefresh = time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if zm.detail == nil {
|
||||||
|
return &zfsentity.ZfsData{}
|
||||||
|
}
|
||||||
|
return zm.detail
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectDetail builds a ZfsData payload from the current system state.
|
||||||
|
func (zm *ZfsManager) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
|
||||||
|
pools, err := zm.poolStatsFn()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(pools) == 0 {
|
||||||
|
return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
statuses, statusErr := zm.poolStatusesFn()
|
||||||
|
if statusErr != nil {
|
||||||
|
slog.Debug("ZFS pool status unavailable", "err", statusErr)
|
||||||
|
}
|
||||||
|
datasets, datasetsErr := zm.datasetsFn()
|
||||||
|
if datasetsErr != nil {
|
||||||
|
slog.Debug("ZFS datasets unavailable", "err", datasetsErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
|
||||||
|
for _, st := range statuses {
|
||||||
|
statusByPool[st.Name] = st
|
||||||
|
}
|
||||||
|
|
||||||
|
previousByPool := make(map[string]*zfsentity.PoolDetail)
|
||||||
|
if previous != nil {
|
||||||
|
for _, pool := range previous.Pools {
|
||||||
|
if pool != nil {
|
||||||
|
previousByPool[pool.Name] = pool
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
|
||||||
|
for i := range pools {
|
||||||
|
p := &pools[i]
|
||||||
|
detail := &zfsentity.PoolDetail{
|
||||||
|
Name: p.Name,
|
||||||
|
Health: p.Health,
|
||||||
|
Size: p.Size,
|
||||||
|
Alloc: p.Alloc,
|
||||||
|
Free: p.Free,
|
||||||
|
}
|
||||||
|
if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
|
||||||
|
if st.Scrub.State != "" && st.Scrub.State != "NONE" {
|
||||||
|
detail.Scrub = &zfsentity.Scrub{
|
||||||
|
State: st.Scrub.State,
|
||||||
|
Progress: st.Scrub.Progress,
|
||||||
|
Errors: st.Scrub.Errors,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, v := range st.Vdevs {
|
||||||
|
detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
|
||||||
|
Name: v.Name,
|
||||||
|
State: v.State,
|
||||||
|
ReadErrs: v.ReadErrs,
|
||||||
|
WriteErrs: v.WriteErrs,
|
||||||
|
ChecksumErrs: v.ChecksumErrs,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Scrub = cached.Scrub
|
||||||
|
detail.Vdevs = cached.Vdevs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if datasetsErr == nil {
|
||||||
|
foundDataset := false
|
||||||
|
for _, ds := range datasets {
|
||||||
|
if poolOfDataset(ds.Name) == p.Name {
|
||||||
|
foundDataset = true
|
||||||
|
detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
|
||||||
|
Name: ds.Name,
|
||||||
|
Used: ds.Used,
|
||||||
|
Avail: ds.Avail,
|
||||||
|
Mountpoint: ds.Mountpoint,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !foundDataset {
|
||||||
|
if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Datasets = cached.Datasets
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if cached := previousByPool[p.Name]; cached != nil {
|
||||||
|
detail.Datasets = cached.Datasets
|
||||||
|
}
|
||||||
|
data.Pools = append(data.Pools, detail)
|
||||||
|
}
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// poolOfDataset returns the pool name for a dataset name (everything before
|
||||||
|
// the first '/'). Datasets without a separator belong to a pool of the same
|
||||||
|
// name.
|
||||||
|
func poolOfDataset(name string) string {
|
||||||
|
if idx := strings.IndexByte(name, '/'); idx >= 0 {
|
||||||
|
return name[:idx]
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
|
||||||
|
func (zm *ZfsManager) ZfsMountpoints() map[string]bool {
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
mountpoints := make(map[string]bool, len(usage))
|
||||||
|
for mountpoint := range usage {
|
||||||
|
mountpoints[mountpoint] = true
|
||||||
|
}
|
||||||
|
return mountpoints
|
||||||
|
}
|
||||||
@@ -0,0 +1,245 @@
|
|||||||
|
//go:build testing
|
||||||
|
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/agent/zfs"
|
||||||
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpdatePopulatesZfsPools(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
||||||
|
{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
|
||||||
|
// Small zvol (Proxmox VM EFI disk): must not round to zero.
|
||||||
|
{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
var kernelCalls int
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
kernelCalls++
|
||||||
|
return []zfs.PoolKernelStat{{
|
||||||
|
Name: "tank", Health: "ONLINE",
|
||||||
|
NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
|
||||||
|
}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
// The first kernel sample establishes the cumulative-counter baseline.
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
|
||||||
|
zm.Update(&stats)
|
||||||
|
require.NotNil(t, stats.ZfsPools)
|
||||||
|
require.Contains(t, stats.ZfsPools, "tank")
|
||||||
|
assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
|
||||||
|
assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001) // Alloc in GiB
|
||||||
|
assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
|
||||||
|
assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
|
||||||
|
assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
|
||||||
|
// I/O instead of erroring.
|
||||||
|
func TestUpdateKernelStatsMissing(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
require.NotNil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateKernelCounterReset(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
zm.kernelSamples = map[string]poolKernelSample{
|
||||||
|
"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
|
||||||
|
}
|
||||||
|
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
|
||||||
|
return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
|
||||||
|
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateNoZfs(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
calls++
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Nil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateEmptyPools(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
calls++
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats system.Stats
|
||||||
|
zm.Update(&stats)
|
||||||
|
zm.Update(&stats)
|
||||||
|
assert.Nil(t, stats.ZfsPools)
|
||||||
|
assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetUsage(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
calls := 0
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
calls++
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
|
||||||
|
{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
|
||||||
|
{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
require.Len(t, usage, 2)
|
||||||
|
assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
|
||||||
|
assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
|
||||||
|
assert.Equal(t, 1, calls)
|
||||||
|
|
||||||
|
// Second call within the refresh window must not re-run the collector.
|
||||||
|
zm.DatasetUsage()
|
||||||
|
assert.Equal(t, 1, calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
|
||||||
|
}
|
||||||
|
assert.Len(t, zm.DatasetUsage(), 1)
|
||||||
|
|
||||||
|
// Force refresh window expiry, then a failing collector.
|
||||||
|
zm.lastUsageRefresh = time.Now().Add(-10 * time.Minute)
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return nil, zfs.ErrNoZfs
|
||||||
|
}
|
||||||
|
usage := zm.DatasetUsage()
|
||||||
|
assert.Len(t, usage, 1, "previous usage should be retained on error")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailForceRefresh(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
poolCalls := 0
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
poolCalls++
|
||||||
|
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
|
||||||
|
first := zm.GetDetail(false)
|
||||||
|
assert.True(t, first.Complete)
|
||||||
|
require.Len(t, first.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(1), first.Pools[0].Alloc)
|
||||||
|
|
||||||
|
cached := zm.GetDetail(false)
|
||||||
|
require.Len(t, cached.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
|
||||||
|
assert.Equal(t, 1, poolCalls)
|
||||||
|
|
||||||
|
refreshed := zm.GetDetail(true)
|
||||||
|
assert.True(t, refreshed.Complete)
|
||||||
|
require.Len(t, refreshed.Pools, 1)
|
||||||
|
assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
|
||||||
|
assert.Equal(t, 2, poolCalls)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank"}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
|
||||||
|
|
||||||
|
require.Len(t, zm.GetDetail(false).Pools, 1)
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
|
||||||
|
empty := zm.GetDetail(true)
|
||||||
|
assert.True(t, empty.Complete)
|
||||||
|
assert.Empty(t, empty.Pools)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
|
||||||
|
zm := &ZfsManager{detailInterval: time.Hour}
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
|
||||||
|
return []zfs.PoolStat{{Name: "tank"}}, nil
|
||||||
|
}
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) {
|
||||||
|
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
|
||||||
|
}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{{Name: "tank/data"}}, nil
|
||||||
|
}
|
||||||
|
first := zm.GetDetail(false)
|
||||||
|
require.True(t, first.Complete)
|
||||||
|
require.Len(t, first.Pools[0].Vdevs, 1)
|
||||||
|
require.Len(t, first.Pools[0].Datasets, 1)
|
||||||
|
|
||||||
|
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
partial := zm.GetDetail(true)
|
||||||
|
require.True(t, partial.Complete)
|
||||||
|
require.Len(t, partial.Pools[0].Vdevs, 1)
|
||||||
|
require.Len(t, partial.Pools[0].Datasets, 1)
|
||||||
|
|
||||||
|
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
|
||||||
|
lastSuccessfulRefresh := zm.lastDetailRefresh
|
||||||
|
failed := zm.GetDetail(true)
|
||||||
|
assert.False(t, failed.Complete)
|
||||||
|
require.Len(t, failed.Pools, 1)
|
||||||
|
assert.Equal(t, "tank", failed.Pools[0].Name)
|
||||||
|
assert.Equal(t, lastSuccessfulRefresh, zm.lastDetailRefresh)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestZfsMountpoints(t *testing.T) {
|
||||||
|
zm := &ZfsManager{}
|
||||||
|
zm.datasetsFn = func() ([]zfs.Dataset, error) {
|
||||||
|
return []zfs.Dataset{
|
||||||
|
{Name: "tank", Mountpoint: "/tank"},
|
||||||
|
{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
mountpoints := zm.ZfsMountpoints()
|
||||||
|
assert.Len(t, mountpoints, 2)
|
||||||
|
assert.True(t, mountpoints["/tank"])
|
||||||
|
assert.True(t, mountpoints["/"])
|
||||||
|
}
|
||||||
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
// Version is the current version of the application.
|
// Version is the current version of the application.
|
||||||
Version = "0.20.0"
|
Version = "0.19.0"
|
||||||
// AppName is the name of the application.
|
// AppName is the name of the application.
|
||||||
AppName = "beszel"
|
AppName = "beszel"
|
||||||
)
|
)
|
||||||
@@ -19,6 +19,3 @@ var MinVersionAgentResponse = semver.MustParse("0.13.0")
|
|||||||
|
|
||||||
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
|
||||||
var MinVersionZfsData = semver.MustParse("0.18.9")
|
var MinVersionZfsData = semver.MustParse("0.18.9")
|
||||||
|
|
||||||
// MinVersionNetworkMonitors is the minimum agent version that supports network monitor sync.
|
|
||||||
var MinVersionNetworkMonitors = semver.MustParse("0.20.0")
|
|
||||||
|
|||||||
@@ -5,25 +5,23 @@ go 1.27.1
|
|||||||
require (
|
require (
|
||||||
github.com/blang/semver v3.5.1+incompatible
|
github.com/blang/semver v3.5.1+incompatible
|
||||||
github.com/coreos/go-systemd/v22 v22.7.0
|
github.com/coreos/go-systemd/v22 v22.7.0
|
||||||
github.com/distribution/reference v0.6.0
|
|
||||||
github.com/ebitengine/purego v0.11.0
|
github.com/ebitengine/purego v0.11.0
|
||||||
github.com/fxamacker/cbor/v2 v2.9.4
|
github.com/fxamacker/cbor/v2 v2.9.3
|
||||||
github.com/gliderlabs/ssh v0.3.8
|
github.com/gliderlabs/ssh v0.3.8
|
||||||
github.com/lxzan/gws v1.10.2
|
github.com/google/uuid v1.6.0
|
||||||
github.com/nicholas-fedor/shoutrrr v0.21.0
|
github.com/lxzan/gws v1.10.1
|
||||||
github.com/opencontainers/go-digest v1.0.0
|
github.com/nicholas-fedor/shoutrrr v0.19.0
|
||||||
github.com/pocketbase/dbx v1.12.0
|
github.com/pocketbase/dbx v1.12.0
|
||||||
github.com/pocketbase/pocketbase v0.40.4
|
github.com/pocketbase/pocketbase v0.40.2
|
||||||
github.com/shirou/gopsutil/v4 v4.26.8
|
github.com/shirou/gopsutil/v4 v4.26.8
|
||||||
github.com/spf13/cast v1.10.0
|
github.com/spf13/cast v1.10.0
|
||||||
github.com/spf13/cobra v1.10.2
|
github.com/spf13/cobra v1.10.2
|
||||||
github.com/spf13/pflag v1.0.10
|
github.com/spf13/pflag v1.0.10
|
||||||
github.com/stretchr/testify v1.12.1
|
github.com/stretchr/testify v1.12.1
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.56.0
|
||||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
|
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa
|
||||||
golang.org/x/net v0.59.0
|
golang.org/x/net v0.58.0
|
||||||
golang.org/x/oauth2 v0.37.0
|
golang.org/x/sys v0.47.0
|
||||||
golang.org/x/sys v0.48.0
|
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
howett.net/plist v1.0.1
|
howett.net/plist v1.0.1
|
||||||
)
|
)
|
||||||
@@ -33,7 +31,7 @@ require (
|
|||||||
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
|
||||||
github.com/disintegration/imaging v1.6.2 // indirect
|
github.com/disintegration/imaging v1.6.2 // indirect
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
github.com/domodwyer/mailyak/v3 v3.6.2 // indirect
|
||||||
github.com/dustin/go-humanize v1.1.0 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
github.com/eclipse/paho.golang v0.23.0 // indirect
|
github.com/eclipse/paho.golang v0.23.0 // indirect
|
||||||
github.com/fatih/color v1.19.0 // indirect
|
github.com/fatih/color v1.19.0 // indirect
|
||||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||||
@@ -43,7 +41,6 @@ require (
|
|||||||
github.com/go-sql-driver/mysql v1.9.1 // indirect
|
github.com/go-sql-driver/mysql v1.9.1 // indirect
|
||||||
github.com/godbus/dbus/v5 v5.2.2 // indirect
|
github.com/godbus/dbus/v5 v5.2.2 // indirect
|
||||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||||
github.com/google/uuid v1.6.0 // indirect
|
|
||||||
github.com/gorilla/websocket v1.5.3 // indirect
|
github.com/gorilla/websocket v1.5.3 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/klauspost/compress v1.20.0 // indirect
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
@@ -52,23 +49,18 @@ require (
|
|||||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 // indirect
|
||||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 // indirect
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 // indirect
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||||
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
github.com/tklauser/go-sysconf v0.4.0 // indirect
|
||||||
github.com/tklauser/numcpus v0.12.0 // indirect
|
github.com/tklauser/numcpus v0.12.0 // indirect
|
||||||
github.com/x448/float16 v0.8.4 // indirect
|
github.com/x448/float16 v0.8.4 // indirect
|
||||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||||
golang.org/x/image v0.46.0 // indirect
|
golang.org/x/image v0.45.0 // indirect
|
||||||
golang.org/x/mod v0.41.0 // indirect
|
golang.org/x/oauth2 v0.36.0 // indirect
|
||||||
golang.org/x/sync v0.23.0 // indirect
|
golang.org/x/sync v0.22.0 // indirect
|
||||||
golang.org/x/term v0.46.0 // indirect
|
golang.org/x/term v0.45.0 // indirect
|
||||||
golang.org/x/text v0.42.0 // indirect
|
golang.org/x/text v0.41.0 // indirect
|
||||||
golang.org/x/tools v0.50.0 // indirect
|
|
||||||
mellium.im/reader v0.1.0 // indirect
|
|
||||||
mellium.im/sasl v0.3.2 // indirect
|
|
||||||
mellium.im/xmlstream v0.15.4 // indirect
|
|
||||||
mellium.im/xmpp v0.23.0 // indirect
|
|
||||||
modernc.org/libc v1.74.4 // indirect
|
modernc.org/libc v1.74.4 // indirect
|
||||||
modernc.org/mathutil v1.7.1 // indirect
|
modernc.org/mathutil v1.7.1 // indirect
|
||||||
modernc.org/memory v1.12.1 // indirect
|
modernc.org/memory v1.12.1 // indirect
|
||||||
|
|||||||
@@ -15,12 +15,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6N
|
|||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||||
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
|
|
||||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8=
|
||||||
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c=
|
||||||
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
github.com/ebitengine/purego v0.11.0 h1:jhp/D+Nyv7UUW8HAcmcjt2N2rYrYi9m3SL21k0Ua/NI=
|
||||||
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
github.com/ebitengine/purego v0.11.0/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ=
|
||||||
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
github.com/eclipse/paho.golang v0.23.0 h1:KHgl2wz6EJo7cMBmkuhpt7C576vP+kpPv7jjvSyR6Mk=
|
||||||
@@ -31,8 +29,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.4 h1:xwjVlxEMR3S605oUlgBjKLTTeGFciYPGYCtF/35LKGo=
|
github.com/fxamacker/cbor/v2 v2.9.3 h1:oQBnFATpNdY8gJHTndDDv5Xl4QqNaz51G5LLEPhng3Q=
|
||||||
github.com/fxamacker/cbor/v2 v2.9.4/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
github.com/fxamacker/cbor/v2 v2.9.3/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
github.com/gabriel-vasile/mimetype v1.4.15/go.mod h1:azpTcoLcDZRNgFou5j+APrqQx9HqVPWa6ijYQIIVswQ=
|
||||||
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
github.com/ganigeorgiev/fexpr v0.6.0 h1:Fza3O/QMBKEudUvxV862qe6GjxM60GJjjKytdp+VQus=
|
||||||
@@ -56,8 +54,8 @@ github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArs
|
|||||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe h1:QAinXoAFJdGQYztXn3VpFey7KCwpedbZ/EkzbplQ0cY=
|
github.com/google/pprof v0.0.0-20260902005441-ca85771921e4 h1:/6mPXfWmhv8eKck12I0YNIcIjwHtxP3YRIMKiEgTjWg=
|
||||||
github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
github.com/google/pprof v0.0.0-20260902005441-ca85771921e4/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
@@ -77,31 +75,29 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
|||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
|
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5 h1:eveIIGn4BGM3qknO74omf6HYr30/exH+eVUTuAgwjZ0=
|
||||||
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
github.com/lufia/plan9stats v0.0.0-20260802145828-341c2f0c90b5/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg=
|
||||||
github.com/lxzan/gws v1.10.2 h1:htReTvcY89iMk1ScVtUbk6J96kIZWaafj6r/lasK/NA=
|
github.com/lxzan/gws v1.10.1 h1:1xG+tDOV0lgDeVPf0wNT74u3cn0K3LpcavRrTPTrMwQ=
|
||||||
github.com/lxzan/gws v1.10.2/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
|
github.com/lxzan/gws v1.10.1/go.mod h1:gXHSCPmTGryWJ4icuqy8Yho32E4YIMHH0fkDRYJRbdc=
|
||||||
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY=
|
||||||
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.21.0 h1:as/mEwdaZMijCVu0FkTUEXashhvC3Y7C5g9dsXMcmQc=
|
github.com/nicholas-fedor/shoutrrr v0.19.0 h1:Rl6bpK3DXuR2Trtx2JV8t+wjUwkHdRHrc8nBKoEpHr0=
|
||||||
github.com/nicholas-fedor/shoutrrr v0.21.0/go.mod h1:dgg4kJv9K0tLXBH/1TXiSibNbM2hcd4SK6xb0sglyU4=
|
github.com/nicholas-fedor/shoutrrr v0.19.0/go.mod h1:Glfdi8AGTbnEn2k2+hW62n8oL0i9vqRVFtXaUIthNks=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.2 h1:2o6vyFvR6snrJWgRVztC+OwuqqPEMI1UzYl2s2iU7Cg=
|
github.com/onsi/ginkgo/v2 v2.32.1 h1:6tlvcDm/3sE8lGJbZ4+d4mO3RLy24/tQWOFzVSQNIfw=
|
||||||
github.com/onsi/ginkgo/v2 v2.32.2/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
github.com/onsi/ginkgo/v2 v2.32.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
|
||||||
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
github.com/onsi/gomega v1.43.0 h1:VlG/1FxqNxhSO+lq/OHBNaaqwiBK/mO8JbVkX9Y+FeU=
|
||||||
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
github.com/onsi/gomega v1.43.0/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg=
|
||||||
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
|
|
||||||
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
|
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
|
github.com/pocketbase/dbx v1.12.0 h1:/oLErM+A0b4xI0PWTGPqSDVjzix48PqI/bng2l0PzoA=
|
||||||
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
github.com/pocketbase/dbx v1.12.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0 h1:uKBDVma7bZqgR2a6AwE+k9hkuDFfiZMpBHQdZ1z3iQs=
|
||||||
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
github.com/pocketbase/ozzo-validation/v4 v4.3.0/go.mod h1:6XNjSTw/Jb2F8LOkKO3oyzIWExbrGiYoS4uVxVwz90g=
|
||||||
github.com/pocketbase/pocketbase v0.40.4 h1:0SvSUreR3NhUMCs9LchE59oEG53efZ3cKiMGyAGBN9U=
|
github.com/pocketbase/pocketbase v0.40.2 h1:7gTqvt3bmilkphyZZ1QNhX19g3BXHqT7ynDyU81RVT4=
|
||||||
github.com/pocketbase/pocketbase v0.40.4/go.mod h1:2mU+80FLiY1fb13WZRg8Xx/lKg4nTjgKVJMigyRH6k0=
|
github.com/pocketbase/pocketbase v0.40.2/go.mod h1:jc3YuyToy+ZXM4CeO7uSCN/htgR8yv+tjSE3eJZ8eh8=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0 h1:XA01Vk/wv9YikCi1V51yRzIHPMT5of9+cMpZoZvDn/M=
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6 h1:jL3a8soXdzuTCcRnKhOmtcsVOObdDTFf4O2B403HPRU=
|
||||||
github.com/power-devops/perfstat v0.0.0-20260916203055-22a1a467d9f0/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
github.com/power-devops/perfstat v0.0.0-20260805114148-88456608a4f6/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||||
@@ -136,37 +132,37 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
|||||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
|
||||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
|
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk=
|
||||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
|
golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM=
|
||||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
golang.org/x/image v0.46.0 h1:b1+oYj0Jbp6K5MDT4i4/eZpYlk3V8SJhhDKh6LBHAyQ=
|
golang.org/x/image v0.45.0 h1:FMb1nTbH5H9vF55SriQHgFw5GnNL9Jg6L25BwXKzhB0=
|
||||||
golang.org/x/image v0.46.0/go.mod h1:3B3W05VGVQyuXucLINLjXKrqISASfi4Xj+iCVkLMwew=
|
golang.org/x/image v0.45.0/go.mod h1:n62x/7RqlwXDvGsSU4u6IUTUf6KghUZ9Bt7cG/T9Fx4=
|
||||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs=
|
||||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE=
|
||||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
|
||||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
|
||||||
golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98=
|
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||||
golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/58=
|
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||||
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
|
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||||
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||||
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
|
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||||
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
|
||||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
|
golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI=
|
||||||
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
|
golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo=
|
||||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
@@ -176,14 +172,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
|||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
|
||||||
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
howett.net/plist v1.0.1/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
|
||||||
mellium.im/reader v0.1.0 h1:UUEMev16gdvaxxZC7fC08j7IzuDKh310nB6BlwnxTww=
|
|
||||||
mellium.im/reader v0.1.0/go.mod h1:F+X5HXpkIfJ9EE1zHQG9lM/hO946iYAmU7xjg5dsQHI=
|
|
||||||
mellium.im/sasl v0.3.2 h1:PT6Xp7ccn9XaXAnJ03FcEjmAn7kK1x7aoXV6F+Vmrl0=
|
|
||||||
mellium.im/sasl v0.3.2/go.mod h1:NKXDi1zkr+BlMHLQjY3ofYuU4KSPFxknb8mfEu6SveY=
|
|
||||||
mellium.im/xmlstream v0.15.4 h1:gLKxcWl4rLMUpKgtzrTBvr4OexPeO/edYus+uK3F6ZI=
|
|
||||||
mellium.im/xmlstream v0.15.4/go.mod h1:yXaCW2++fmVO4L9piKVkyLDqnCmictVYF7FDQW8prb4=
|
|
||||||
mellium.im/xmpp v0.23.0 h1:rvKvOvMdIURCLaAWEJN8J0QpO3AJYCDKjxLsqtTPSjY=
|
|
||||||
mellium.im/xmpp v0.23.0/go.mod h1:GHDKlKKQe0LNmD9YqExyxnFEEBiz84KGqnfiA2VNzb8=
|
|
||||||
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
|
||||||
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||||
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
|
||||||
|
|||||||
@@ -20,11 +20,10 @@ type hubLike interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type AlertManager struct {
|
type AlertManager struct {
|
||||||
hub hubLike
|
hub hubLike
|
||||||
stopOnce sync.Once
|
stopOnce sync.Once
|
||||||
pendingAlerts sync.Map
|
pendingAlerts sync.Map
|
||||||
alertsCache *AlertsCache
|
alertsCache *AlertsCache
|
||||||
networkMonitors *networkMonitorCache
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type AlertMessageData struct {
|
type AlertMessageData struct {
|
||||||
@@ -67,7 +66,6 @@ type SystemAlertGPUData struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type SystemAlertZfsPool struct {
|
type SystemAlertZfsPool struct {
|
||||||
Raw bool `json:"raw,omitempty"`
|
|
||||||
Total float64 `json:"d"`
|
Total float64 `json:"d"`
|
||||||
Used float64 `json:"du"`
|
Used float64 `json:"du"`
|
||||||
}
|
}
|
||||||
@@ -108,9 +106,8 @@ var supportsTitle = map[string]struct{}{
|
|||||||
// NewAlertManager creates a new AlertManager instance.
|
// NewAlertManager creates a new AlertManager instance.
|
||||||
func NewAlertManager(app hubLike) *AlertManager {
|
func NewAlertManager(app hubLike) *AlertManager {
|
||||||
am := &AlertManager{
|
am := &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
networkMonitors: newNetworkMonitorCache(app),
|
|
||||||
}
|
}
|
||||||
am.bindEvents()
|
am.bindEvents()
|
||||||
return am
|
return am
|
||||||
@@ -118,7 +115,6 @@ func NewAlertManager(app hubLike) *AlertManager {
|
|||||||
|
|
||||||
// Bind events to the alerts collection lifecycle
|
// Bind events to the alerts collection lifecycle
|
||||||
func (am *AlertManager) bindEvents() {
|
func (am *AlertManager) bindEvents() {
|
||||||
am.bindNetworkMonitorAlertEvents()
|
|
||||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||||
@@ -235,20 +231,8 @@ func (am *AlertManager) SendAlert(data AlertMessageData) error {
|
|||||||
am.hub.Logger().Error("Failed to unmarshal user settings", "err", err)
|
am.hub.Logger().Error("Failed to unmarshal user settings", "err", err)
|
||||||
}
|
}
|
||||||
// send alerts via webhooks
|
// send alerts via webhooks
|
||||||
send := sendPublicNotification
|
|
||||||
if len(userAlertSettings.Webhooks) > 0 {
|
|
||||||
// Read the owner's current role at delivery time, including for URLs
|
|
||||||
// saved before an admin was demoted. Never fall back on lookup failure.
|
|
||||||
owner, err := am.hub.FindRecordById("users", data.UserID)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("load notification owner: %w", err)
|
|
||||||
}
|
|
||||||
if owner.GetString("role") == "admin" {
|
|
||||||
send = shoutrrr.Send
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for _, webhook := range userAlertSettings.Webhooks {
|
for _, webhook := range userAlertSettings.Webhooks {
|
||||||
if err := am.sendShoutrrrAlert(webhook, data.Title, data.Message, data.Link, data.LinkText, send); err != nil {
|
if err := am.SendShoutrrrAlert(webhook, data.Title, data.Message, data.Link, data.LinkText); err != nil {
|
||||||
am.hub.Logger().Error("Failed to send shoutrrr alert", "err", err)
|
am.hub.Logger().Error("Failed to send shoutrrr alert", "err", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -279,10 +263,6 @@ func (am *AlertManager) SendAlert(data AlertMessageData) error {
|
|||||||
|
|
||||||
// SendShoutrrrAlert sends an alert via a Shoutrrr URL
|
// SendShoutrrrAlert sends an alert via a Shoutrrr URL
|
||||||
func (am *AlertManager) SendShoutrrrAlert(notificationUrl, title, message, link, linkText string) error {
|
func (am *AlertManager) SendShoutrrrAlert(notificationUrl, title, message, link, linkText string) error {
|
||||||
return am.sendShoutrrrAlert(notificationUrl, title, message, link, linkText, shoutrrr.Send)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) sendShoutrrrAlert(notificationUrl, title, message, link, linkText string, send func(string, string) error) error {
|
|
||||||
// Parse the URL
|
// Parse the URL
|
||||||
parsedURL, err := url.Parse(notificationUrl)
|
parsedURL, err := url.Parse(notificationUrl)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -325,7 +305,7 @@ func (am *AlertManager) sendShoutrrrAlert(notificationUrl, title, message, link,
|
|||||||
parsedURL.RawQuery = queryParams.Encode()
|
parsedURL.RawQuery = queryParams.Encode()
|
||||||
// log.Println("URL after modification:", parsedURL.String())
|
// log.Println("URL after modification:", parsedURL.String())
|
||||||
|
|
||||||
err = send(parsedURL.String(), message)
|
err = shoutrrr.Send(parsedURL.String(), message)
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
am.hub.Logger().Info("Sent shoutrrr alert", "title", title)
|
am.hub.Logger().Info("Sent shoutrrr alert", "title", title)
|
||||||
|
|||||||
@@ -3,11 +3,13 @@ package alerts
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"slices"
|
"slices"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/hub/utils"
|
"github.com/henrygd/beszel/internal/hub/utils"
|
||||||
"github.com/nicholas-fedor/shoutrrr"
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
)
|
)
|
||||||
@@ -29,13 +31,6 @@ func UpsertUserAlerts(e *core.RequestEvent) error {
|
|||||||
return e.BadRequestError("Bad data", err)
|
return e.BadRequestError("Bad data", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if reqData.Name == alertNameNetworkMonitorLoss {
|
|
||||||
if reqData.Value < 0 || reqData.Value >= 100 {
|
|
||||||
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
|
||||||
}
|
|
||||||
reqData.Min = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
alertsCollection, err := e.App.FindCachedCollectionByNameOrId("alerts")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -152,16 +147,72 @@ func (am *AlertManager) SendTestNotification(e *core.RequestEvent) error {
|
|||||||
if err != nil || data.URL == "" {
|
if err != nil || data.URL == "" {
|
||||||
return e.BadRequestError("URL is required", err)
|
return e.BadRequestError("URL is required", err)
|
||||||
}
|
}
|
||||||
send := shoutrrr.Send
|
// Only allow admins to send test notifications to internal URLs
|
||||||
if !e.Auth.IsSuperuser() && e.Auth.GetString("role") != "admin" {
|
if !e.Auth.IsSuperuser() && e.Auth.GetString("role") != "admin" {
|
||||||
send = sendPublicNotification
|
internalURL, err := isInternalURL(data.URL)
|
||||||
}
|
if err != nil {
|
||||||
err = am.sendShoutrrrAlert(data.URL, "Test Alert", "This is a notification from Beszel.", am.hub.Settings().Meta.AppURL, "View Beszel", send)
|
return e.BadRequestError(err.Error(), nil)
|
||||||
if errors.Is(err, errInternalDestination) || errors.Is(err, errUnrestrictedService) {
|
}
|
||||||
return e.ForbiddenError(err.Error(), nil)
|
if internalURL {
|
||||||
|
return e.ForbiddenError("Only admins can send to internal destinations", nil)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
err = am.SendShoutrrrAlert(data.URL, "Test Alert", "This is a notification from Beszel.", am.hub.Settings().Meta.AppURL, "View Beszel")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return e.JSON(200, map[string]string{"err": err.Error()})
|
return e.JSON(200, map[string]string{"err": err.Error()})
|
||||||
}
|
}
|
||||||
return e.JSON(200, map[string]bool{"err": false})
|
return e.JSON(200, map[string]bool{"err": false})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// isInternalURL checks if the given shoutrrr URL points to an internal destination (localhost or private IP)
|
||||||
|
func isInternalURL(rawURL string) (bool, error) {
|
||||||
|
parsedURL, err := url.Parse(rawURL)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
|
||||||
|
host := parsedURL.Hostname()
|
||||||
|
if host == "" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.EqualFold(host, "localhost") {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if ip := net.ParseIP(host); ip != nil {
|
||||||
|
return isInternalIP(ip), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Some Shoutrrr URLs use the host position for service identifiers rather than a
|
||||||
|
// network hostname (for example, discord://token@webhookid). Restrict DNS lookups
|
||||||
|
// to names that look like actual hostnames so valid service URLs keep working.
|
||||||
|
if !strings.Contains(host, ".") {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ips, err := net.LookupIP(host)
|
||||||
|
if err != nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if slices.ContainsFunc(ips, isInternalIP) {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var cgnatNetwork = &net.IPNet{
|
||||||
|
IP: net.IPv4(100, 64, 0, 0),
|
||||||
|
Mask: net.CIDRMask(10, 32),
|
||||||
|
}
|
||||||
|
|
||||||
|
func isInternalIP(ip net.IP) bool {
|
||||||
|
return ip.IsPrivate() ||
|
||||||
|
ip.IsLoopback() ||
|
||||||
|
ip.IsUnspecified() ||
|
||||||
|
ip.IsLinkLocalUnicast() ||
|
||||||
|
ip.IsMulticast() ||
|
||||||
|
cgnatNetwork.Contains(ip)
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,11 +7,10 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/henrygd/beszel/internal/alerts"
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||||
|
|
||||||
@@ -30,6 +29,43 @@ func jsonReader(v any) io.Reader {
|
|||||||
return bytes.NewReader(data)
|
return bytes.NewReader(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsInternalURL(t *testing.T) {
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
internal bool
|
||||||
|
}{
|
||||||
|
{name: "loopback ipv4", url: "generic://127.0.0.1", internal: true},
|
||||||
|
{name: "private ipv4", url: "generic://10.0.0.1", internal: true},
|
||||||
|
{name: "localhost hostname", url: "generic://localhost", internal: true},
|
||||||
|
{name: "localhost with path", url: "generic+http://localhost/api/v1/postStuff", internal: true},
|
||||||
|
{name: "loopback with port and path", url: "generic+http://127.0.0.1:8080/api/v1/postStuff", internal: true},
|
||||||
|
{name: "public hostname", url: "generic+https://beszel.dev/api/v1/postStuff", internal: false},
|
||||||
|
{name: "cloud metadata ipv4", url: "generic://169.254.169.254", internal: true},
|
||||||
|
{name: "link-local ipv4", url: "generic://169.254.1.1", internal: true},
|
||||||
|
{name: "link-local ipv6", url: "generic://[fe80::1]", internal: true},
|
||||||
|
{name: "mapped link-local ipv4", url: "generic://[::ffff:169.254.169.254]", internal: true},
|
||||||
|
{name: "cgnat lower boundary", url: "generic://100.64.0.0", internal: true},
|
||||||
|
{name: "cgnat upper boundary", url: "generic://100.127.255.255", internal: true},
|
||||||
|
{name: "below cgnat", url: "generic://100.63.255.255", internal: false},
|
||||||
|
{name: "above cgnat", url: "generic://100.128.0.0", internal: false},
|
||||||
|
{name: "multicast ipv4", url: "generic://224.0.0.1", internal: true},
|
||||||
|
{name: "multicast ipv6", url: "generic://[ff02::1]", internal: true},
|
||||||
|
{name: "public ipv4", url: "generic://8.8.8.8", internal: false},
|
||||||
|
{name: "public ipv6", url: "generic://[2001:4860:4860::8888]", internal: false},
|
||||||
|
{name: "token style service url", url: "discord://abc123@123456789", internal: false},
|
||||||
|
{name: "single label service url", url: "slack://token@team/channel", internal: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run(testCase.name, func(t *testing.T) {
|
||||||
|
internal, err := alerts.IsInternalURL(testCase.url)
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Equal(t, testCase.internal, internal)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestUserAlertsApi(t *testing.T) {
|
func TestUserAlertsApi(t *testing.T) {
|
||||||
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
@@ -421,17 +457,6 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
hub, user := beszelTests.GetHubWithUser(t)
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
|
|
||||||
var delivered atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
delivered.Add(1)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
localURL := "generic+" + server.URL
|
|
||||||
|
|
||||||
readonlyUser, err := beszelTests.CreateUserWithRole(hub, "readonly@example.com", "password123", "readonly")
|
|
||||||
assert.NoError(t, err)
|
|
||||||
readonlyToken, err := readonlyUser.NewAuthToken()
|
|
||||||
assert.NoError(t, err)
|
|
||||||
userToken, err := user.NewAuthToken()
|
userToken, err := user.NewAuthToken()
|
||||||
|
|
||||||
adminUser, err := beszelTests.CreateUserWithRole(hub, "admin@example.com", "password123", "admin")
|
adminUser, err := beszelTests.CreateUserWithRole(hub, "admin@example.com", "password123", "admin")
|
||||||
@@ -456,11 +481,11 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
ExpectedContent: []string{"requires valid"},
|
ExpectedContent: []string{"requires valid"},
|
||||||
TestAppFactory: testAppFactory,
|
TestAppFactory: testAppFactory,
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "POST /test-notification - invalid service reports error",
|
Name: "POST /test-notification - with external auth should succeed",
|
||||||
Method: http.MethodPost,
|
Method: http.MethodPost,
|
||||||
URL: "/api/beszel/test-notification",
|
URL: "/api/beszel/test-notification",
|
||||||
TestAppFactory: testAppFactory,
|
TestAppFactory: testAppFactory,
|
||||||
@@ -468,7 +493,7 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": userToken,
|
"Authorization": userToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": "unknown://example.com",
|
"url": "generic://8.8.8.8",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
@@ -510,10 +535,10 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": adminUserToken,
|
"Authorization": adminUserToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":false"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "POST /test-notification - internal url with superuser auth should succeed",
|
Name: "POST /test-notification - internal url with superuser auth should succeed",
|
||||||
@@ -524,28 +549,14 @@ func TestSendTestNotification(t *testing.T) {
|
|||||||
"Authorization": superuserToken,
|
"Authorization": superuserToken,
|
||||||
},
|
},
|
||||||
Body: jsonReader(map[string]any{
|
Body: jsonReader(map[string]any{
|
||||||
"url": localURL,
|
"url": "generic://127.0.0.1",
|
||||||
}),
|
}),
|
||||||
ExpectedStatus: 200,
|
ExpectedStatus: 200,
|
||||||
ExpectedContent: []string{"\"err\":"},
|
ExpectedContent: []string{"\"err\":"},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, url := range []string{localURL, "smtp://user:pass@127.0.0.1/?fromAddress=sender@example.com&toAddresses=recipient@example.com", "mqtt://127.0.0.1/topic"} {
|
|
||||||
scenarios = append(scenarios, beszelTests.ApiScenario{
|
|
||||||
Name: "readonly cannot send to " + url,
|
|
||||||
Method: http.MethodPost,
|
|
||||||
URL: "/api/beszel/test-notification",
|
|
||||||
TestAppFactory: testAppFactory,
|
|
||||||
Headers: map[string]string{"Authorization": readonlyToken},
|
|
||||||
Body: jsonReader(map[string]any{"url": url}),
|
|
||||||
ExpectedStatus: 403,
|
|
||||||
ExpectedContent: []string{"Only admins"},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, scenario := range scenarios {
|
for _, scenario := range scenarios {
|
||||||
scenario.Test(t)
|
scenario.Test(t)
|
||||||
}
|
}
|
||||||
assert.EqualValues(t, 2, delivered.Load(), "only admin and superuser requests should reach the server")
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
package alerts
|
package alerts
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
@@ -19,9 +18,6 @@ type CachedAlertData struct {
|
|||||||
Triggered bool
|
Triggered bool
|
||||||
Min uint8
|
Min uint8
|
||||||
PendingSince time.Time
|
PendingSince time.Time
|
||||||
// Immutable after publication; decoded only when the alert record changes.
|
|
||||||
MonitorStates map[string]string
|
|
||||||
MonitorStatesValid bool
|
|
||||||
// Created types.DateTime
|
// Created types.DateTime
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -34,18 +30,11 @@ func (a *CachedAlertData) PopulateFromRecord(record *core.Record) {
|
|||||||
a.Triggered = record.GetBool("triggered")
|
a.Triggered = record.GetBool("triggered")
|
||||||
a.Min = uint8(record.GetInt("min"))
|
a.Min = uint8(record.GetInt("min"))
|
||||||
a.PendingSince = record.GetDateTime("pending_since").Time()
|
a.PendingSince = record.GetDateTime("pending_since").Time()
|
||||||
if a.Name == alertNameNetworkMonitorLoss {
|
|
||||||
var state networkMonitorAlertState
|
|
||||||
a.MonitorStatesValid = record.UnmarshalJSONField("state", &state) == nil
|
|
||||||
a.MonitorStates = state.Monitors
|
|
||||||
}
|
|
||||||
// a.Created = record.GetDateTime("created")
|
// a.Created = record.GetDateTime("created")
|
||||||
}
|
}
|
||||||
|
|
||||||
// AlertsCache provides an in-memory cache for system alerts.
|
// AlertsCache provides an in-memory cache for system alerts.
|
||||||
type AlertsCache struct {
|
type AlertsCache struct {
|
||||||
// Serialize lazy loads with updates so a late load cannot replace newer state.
|
|
||||||
loadMu sync.Mutex
|
|
||||||
app core.App
|
app core.App
|
||||||
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
store *store.Store[string, *store.Store[string, CachedAlertData]]
|
||||||
populated bool
|
populated bool
|
||||||
@@ -80,8 +69,6 @@ func (c *AlertsCache) bindEvents() *AlertsCache {
|
|||||||
|
|
||||||
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
// PopulateFromDB clears current entries and loads all alerts from the database into the cache.
|
||||||
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
func (c *AlertsCache) PopulateFromDB(force bool) error {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
if !force && c.populated {
|
if !force && c.populated {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -91,7 +78,7 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
}
|
}
|
||||||
c.store.RemoveAll()
|
c.store.RemoveAll()
|
||||||
for _, record := range records {
|
for _, record := range records {
|
||||||
c.update(record)
|
c.Update(record)
|
||||||
}
|
}
|
||||||
c.populated = true
|
c.populated = true
|
||||||
return nil
|
return nil
|
||||||
@@ -99,12 +86,6 @@ func (c *AlertsCache) PopulateFromDB(force bool) error {
|
|||||||
|
|
||||||
// Update adds or updates an alert record in the cache.
|
// Update adds or updates an alert record in the cache.
|
||||||
func (c *AlertsCache) Update(record *core.Record) {
|
func (c *AlertsCache) Update(record *core.Record) {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
c.update(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *AlertsCache) update(record *core.Record) {
|
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -121,8 +102,6 @@ func (c *AlertsCache) update(record *core.Record) {
|
|||||||
|
|
||||||
// Delete removes an alert record from the cache.
|
// Delete removes an alert record from the cache.
|
||||||
func (c *AlertsCache) Delete(record *core.Record) {
|
func (c *AlertsCache) Delete(record *core.Record) {
|
||||||
c.loadMu.Lock()
|
|
||||||
defer c.loadMu.Unlock()
|
|
||||||
systemID := record.GetString("system")
|
systemID := record.GetString("system")
|
||||||
if systemID == "" {
|
if systemID == "" {
|
||||||
return
|
return
|
||||||
@@ -136,23 +115,18 @@ func (c *AlertsCache) Delete(record *core.Record) {
|
|||||||
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
func (c *AlertsCache) GetSystemAlerts(systemID string) []CachedAlertData {
|
||||||
systemStore, ok := c.store.GetOk(systemID)
|
systemStore, ok := c.store.GetOk(systemID)
|
||||||
if !ok {
|
if !ok {
|
||||||
c.loadMu.Lock()
|
// Populate cache for this system
|
||||||
defer c.loadMu.Unlock()
|
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
||||||
systemStore, ok = c.store.GetOk(systemID)
|
if err != nil {
|
||||||
if !ok {
|
return nil
|
||||||
// Populate cache for this system
|
|
||||||
records, err := c.app.FindAllRecords("alerts", dbx.NewExp("system={:system}", dbx.Params{"system": systemID}))
|
|
||||||
if err != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
systemStore = store.New(map[string]CachedAlertData{})
|
|
||||||
for _, record := range records {
|
|
||||||
var ca CachedAlertData
|
|
||||||
ca.PopulateFromRecord(record)
|
|
||||||
systemStore.Set(record.Id, ca)
|
|
||||||
}
|
|
||||||
c.store.Set(systemID, systemStore)
|
|
||||||
}
|
}
|
||||||
|
systemStore = store.New(map[string]CachedAlertData{})
|
||||||
|
for _, record := range records {
|
||||||
|
var ca CachedAlertData
|
||||||
|
ca.PopulateFromRecord(record)
|
||||||
|
systemStore.Set(record.Id, ca)
|
||||||
|
}
|
||||||
|
c.store.Set(systemID, systemStore)
|
||||||
}
|
}
|
||||||
all := systemStore.GetAll()
|
all := systemStore.GetAll()
|
||||||
alerts := make([]CachedAlertData, 0, len(all))
|
alerts := make([]CachedAlertData, 0, len(all))
|
||||||
|
|||||||
@@ -9,12 +9,6 @@ import (
|
|||||||
|
|
||||||
// On triggered alert record delete, set matching alert history record to resolved
|
// On triggered alert record delete, set matching alert history record to resolved
|
||||||
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
if err := resolveNetworkMonitorHistory(e.App, e.Record.Id); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
if !e.Record.GetBool("triggered") {
|
if !e.Record.GetBool("triggered") {
|
||||||
return e.Next()
|
return e.Next()
|
||||||
}
|
}
|
||||||
@@ -24,10 +18,6 @@ func resolveHistoryOnAlertDelete(e *core.RecordEvent) error {
|
|||||||
|
|
||||||
// On alert record update, update alert history record
|
// On alert record update, update alert history record
|
||||||
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
func updateHistoryOnAlertUpdate(e *core.RecordEvent) error {
|
||||||
// Network monitor incidents have separate history entries per monitor.
|
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
original := e.Record.Original()
|
original := e.Record.Original()
|
||||||
new := e.Record
|
new := e.Record
|
||||||
|
|
||||||
|
|||||||
@@ -1,269 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"math"
|
|
||||||
"net"
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
const alertNameNetworkMonitorLoss = "NetworkMonitorLoss"
|
|
||||||
|
|
||||||
// networkMonitorAlertState is this alert type's persisted runtime state.
|
|
||||||
// Monitor IDs map to their open history entries independently of history retention.
|
|
||||||
type networkMonitorAlertState struct {
|
|
||||||
Monitors map[string]string `json:"monitors"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) bindNetworkMonitorAlertEvents() {
|
|
||||||
// Hidden fields are still writable through the record API unless protected.
|
|
||||||
protectState := func(e *core.RecordRequestEvent) error {
|
|
||||||
e.Record.Set("state", e.Record.Original().Get("state"))
|
|
||||||
oldName, newName := e.Record.Original().GetString("name"), e.Record.GetString("name")
|
|
||||||
if oldName != "" && (oldName == alertNameNetworkMonitorLoss || newName == alertNameNetworkMonitorLoss) &&
|
|
||||||
(oldName != newName || e.Record.GetString("system") != e.Record.Original().GetString("system")) {
|
|
||||||
return e.BadRequestError("Delete and recreate the alert to change its type or system", nil)
|
|
||||||
}
|
|
||||||
if e.Record.GetString("name") == alertNameNetworkMonitorLoss {
|
|
||||||
if !e.HasSuperuserAuth() && (e.Auth == nil || !userHasSystem(e.App, e.Auth.Id, e.Record.GetString("system"))) {
|
|
||||||
return e.ForbiddenError("You do not have access to this system", nil)
|
|
||||||
}
|
|
||||||
e.Record.Set("triggered", e.Record.Original().GetBool("triggered"))
|
|
||||||
value := e.Record.GetFloat("value")
|
|
||||||
if math.IsNaN(value) || math.IsInf(value, 0) || value < 0 || value >= 100 {
|
|
||||||
return e.BadRequestError("Monitor loss threshold must be at least 0 and below 100", nil)
|
|
||||||
}
|
|
||||||
e.Record.Set("min", 0)
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
am.hub.OnRecordCreateRequest("alerts").BindFunc(protectState)
|
|
||||||
am.hub.OnRecordUpdateRequest("alerts").BindFunc(protectState)
|
|
||||||
cleanup := func(e *core.RecordEvent) error {
|
|
||||||
if err := e.Next(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return am.evaluateNetworkMonitorAlerts(e.App, e.Record.GetString("system"), nil)
|
|
||||||
}
|
|
||||||
am.hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(cleanup)
|
|
||||||
am.hub.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
if e.Record.GetBool("enabled") || !e.Record.Original().GetBool("enabled") {
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
return cleanup(e)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// HandleNetworkMonitorAlerts runs after the full monitoring transaction commits,
|
|
||||||
// using its exact payload (dashboard requests can replace the cached payload).
|
|
||||||
// Omitted results and disconnected systems never imply recovery.
|
|
||||||
func (am *AlertManager) HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error {
|
|
||||||
if systemRecord.GetString("status") != "up" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
alerts := am.alertsCache.GetAlertsByName(systemRecord.Id, alertNameNetworkMonitorLoss)
|
|
||||||
if len(alerts) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
monitors, err := am.networkMonitors.get(systemRecord.Id)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !networkMonitorTransitionPending(alerts, monitors, results, time.Now()) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// The cache only predicts a transition. Reload and recheck under the DB
|
|
||||||
// transaction before persisting, including current system/monitor status.
|
|
||||||
return am.evaluateNetworkMonitorAlerts(am.hub, systemRecord.Id, results)
|
|
||||||
}
|
|
||||||
|
|
||||||
// networkMonitorTransitionPending does no IO and never mutates cached maps.
|
|
||||||
func networkMonitorTransitionPending(alerts []CachedAlertData, monitors map[string]int, results map[string]monitor.Result, now time.Time) bool {
|
|
||||||
for _, alert := range alerts {
|
|
||||||
if !alert.MonitorStatesValid || alert.Triggered != (len(alert.MonitorStates) > 0) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for id := range alert.MonitorStates {
|
|
||||||
if _, enabled := monitors[id]; !enabled {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for id, result := range results {
|
|
||||||
interval, enabled := monitors[id]
|
|
||||||
if !enabled || !monitorResultReady(result, interval, now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_, active := alert.MonitorStates[id]
|
|
||||||
if (result.PacketLoss1h > alert.Value) != active {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (am *AlertManager) evaluateNetworkMonitorAlerts(app core.App, systemID string, results map[string]monitor.Result) error {
|
|
||||||
var messages []AlertMessageData
|
|
||||||
err := app.RunInTransaction(func(tx core.App) error {
|
|
||||||
// Read configuration inside the transaction so concurrent threshold changes,
|
|
||||||
// disabling, and evaluations cannot overwrite each other's incident state.
|
|
||||||
alerts, err := tx.FindAllRecords("alerts", dbx.HashExp{"system": systemID, "name": alertNameNetworkMonitorLoss})
|
|
||||||
if err != nil || len(alerts) == 0 {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
system, err := tx.FindRecordById("systems", systemID)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
// System deletion cascades to its alerts.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
monitors, err := tx.FindAllRecords("network_monitors", dbx.HashExp{"system": systemID, "enabled": true})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
enabled := make(map[string]*core.Record, len(monitors))
|
|
||||||
for _, m := range monitors {
|
|
||||||
enabled[m.Id] = m
|
|
||||||
}
|
|
||||||
now := time.Now()
|
|
||||||
for _, alert := range alerts {
|
|
||||||
var state networkMonitorAlertState
|
|
||||||
if err := alert.UnmarshalJSONField("state", &state); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
states := state.Monitors
|
|
||||||
if states == nil {
|
|
||||||
states = map[string]string{}
|
|
||||||
}
|
|
||||||
changed := false
|
|
||||||
// Removing or disabling a monitor closes its incident silently.
|
|
||||||
for id, historyID := range states {
|
|
||||||
if _, ok := enabled[id]; !ok {
|
|
||||||
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
delete(states, id)
|
|
||||||
changed = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if system.GetString("status") == "up" {
|
|
||||||
for _, m := range monitors {
|
|
||||||
result, ok := results[m.Id]
|
|
||||||
if !ok || !monitorResultReady(result, m.GetInt("interval"), now) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
historyID, active := states[m.Id]
|
|
||||||
triggered := result.PacketLoss1h > alert.GetFloat("value")
|
|
||||||
if triggered == active {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
label := m.GetString("target")
|
|
||||||
if m.GetString("protocol") == "tcp" {
|
|
||||||
label = net.JoinHostPort(label, strconv.Itoa(m.GetInt("port")))
|
|
||||||
}
|
|
||||||
if triggered {
|
|
||||||
collection, err := tx.FindCachedCollectionByNameOrId("alerts_history")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
history := core.NewRecord(collection)
|
|
||||||
history.Load(map[string]any{
|
|
||||||
"alert_id": alert.Id, "user": alert.GetString("user"), "system": systemID,
|
|
||||||
"name": alertNameNetworkMonitorLoss, "monitor_name": label, "value": result.PacketLoss1h,
|
|
||||||
})
|
|
||||||
if err := tx.Save(history); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
states[m.Id] = history.Id
|
|
||||||
} else {
|
|
||||||
if err := resolveMonitorIncident(tx, historyID, now); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
delete(states, m.Id)
|
|
||||||
}
|
|
||||||
changed = true
|
|
||||||
state, comparison := "loss", "exceeds"
|
|
||||||
if !triggered {
|
|
||||||
state, comparison = "recovered", "is at or below"
|
|
||||||
}
|
|
||||||
messages = append(messages, AlertMessageData{
|
|
||||||
UserID: alert.GetString("user"), SystemID: systemID,
|
|
||||||
Title: fmt.Sprintf("Network monitor %s on %s: %s", state, system.GetString("name"), label),
|
|
||||||
Message: fmt.Sprintf("%s on %s: loss over the past hour is %.2f%%, which %s the %.2f%% threshold.", label, system.GetString("name"), result.PacketLoss1h, comparison, alert.GetFloat("value")),
|
|
||||||
Link: am.hub.MakeLink("system", systemID), LinkText: "View " + system.GetString("name"),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if changed || alert.GetBool("triggered") != (len(states) > 0) {
|
|
||||||
alert.Set("state", networkMonitorAlertState{Monitors: states})
|
|
||||||
alert.Set("triggered", len(states) > 0)
|
|
||||||
if err := tx.Save(alert); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Match other alert types: persist transitions before delivery, and respect
|
|
||||||
// the user's existing notification destinations and quiet hours.
|
|
||||||
for _, message := range messages {
|
|
||||||
if err := am.SendAlert(message); err != nil {
|
|
||||||
app.Logger().Error("Failed to send network monitor alert", "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorResultReady(result monitor.Result, interval int, now time.Time) bool {
|
|
||||||
// Three completed attempts provide a short warm-up, including after an agent
|
|
||||||
// restart.
|
|
||||||
if result.SampleCount < 3 || result.LastProbeAt <= 0 || math.IsNaN(result.PacketLoss1h) || math.IsInf(result.PacketLoss1h, 0) || result.PacketLoss1h < 0 || result.PacketLoss1h > 100 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// Never interpret an empty one-hour window as zero loss.
|
|
||||||
maxAge := min(time.Hour, max(3*time.Duration(interval)*time.Second, 3*time.Minute))
|
|
||||||
age := now.Sub(time.UnixMilli(result.LastProbeAt))
|
|
||||||
return age >= -time.Minute && age <= maxAge
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveMonitorIncident(app core.App, id string, now time.Time) error {
|
|
||||||
record, err := app.FindRecordById("alerts_history", id)
|
|
||||||
if errors.Is(err, sql.ErrNoRows) {
|
|
||||||
// History can be purged independently.
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !record.GetDateTime("resolved").IsZero() {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
record.Set("resolved", now.UTC())
|
|
||||||
return app.Save(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
func resolveNetworkMonitorHistory(app core.App, alertID string) error {
|
|
||||||
records, err := app.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alertID, "resolved": ""})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, record := range records {
|
|
||||||
record.Set("resolved", time.Now().UTC())
|
|
||||||
if err := app.Save(record); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@@ -1,513 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package alerts_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func networkAlertSetup(t *testing.T) (*beszelTests.TestHub, *core.Record, *core.Record, []*core.Record) {
|
|
||||||
t.Helper()
|
|
||||||
hub, system, alert := systemdTestSetup(t, false)
|
|
||||||
t.Cleanup(hub.Cleanup)
|
|
||||||
alert.Set("name", "NetworkMonitorLoss")
|
|
||||||
alert.Set("value", 5)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
var monitors []*core.Record
|
|
||||||
for _, name := range []string{"gateway", "website"} {
|
|
||||||
record, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
|
||||||
"system": system.Id, "target": name + ".example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
monitors = append(monitors, record)
|
|
||||||
}
|
|
||||||
// Avoid starting a system update worker in tests.
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
system.Set("status", "up")
|
|
||||||
return hub, system, alert, monitors
|
|
||||||
}
|
|
||||||
|
|
||||||
func monitorResult(loss float64) monitor.Result {
|
|
||||||
return monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 60, PacketLoss1h: loss}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertIndependentIncidents(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(0)}
|
|
||||||
check := func(active bool, open, sent int) {
|
|
||||||
t.Helper()
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, active, record.GetBool("triggered"))
|
|
||||||
total, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.EqualValues(t, open, total)
|
|
||||||
assert.Equal(t, count+sent, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 1)
|
|
||||||
message := hub.TestMailer.Messages()[count]
|
|
||||||
assert.Contains(t, message.Text, "gateway.example.com")
|
|
||||||
assert.Contains(t, message.Text, "10.00%")
|
|
||||||
assert.Contains(t, message.Text, "5.00%")
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 1)
|
|
||||||
// Persisted monitor state prevents duplicate notifications after a hub restart.
|
|
||||||
am = alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results[monitors[1].Id] = monitorResult(20)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 2, 2)
|
|
||||||
results[monitors[0].Id] = monitorResult(5) // Equality is a recovery.
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(true, 1, 3)
|
|
||||||
results[monitors[1].Id] = monitorResult(0)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
check(false, 0, 4)
|
|
||||||
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, histories, 2)
|
|
||||||
for _, history := range histories {
|
|
||||||
assert.NotEmpty(t, history.GetString("monitor_name"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertTargetLabel(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
protocol, target, label string
|
|
||||||
port int
|
|
||||||
}{
|
|
||||||
{"icmp", "gateway.example.com", "gateway.example.com", 0},
|
|
||||||
{"http", "https://example.com/health", "https://example.com/health", 0},
|
|
||||||
{"tcp", "example.com", "example.com:8443", 8443},
|
|
||||||
{"tcp", "2001:db8::1", "[2001:db8::1]:443", 443},
|
|
||||||
} {
|
|
||||||
t.Run(tc.label, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
m := monitors[0]
|
|
||||||
m.Set("protocol", tc.protocol)
|
|
||||||
m.Set("target", tc.target)
|
|
||||||
m.Set("port", tc.port)
|
|
||||||
require.NoError(t, hub.Save(m))
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{m.Id: monitorResult(10)}))
|
|
||||||
histories, err := hub.FindAllRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, histories, 1)
|
|
||||||
assert.Equal(t, tc.label, histories[0].GetString("monitor_name"))
|
|
||||||
assert.Contains(t, hub.TestMailer.Messages()[hub.TestMailer.TotalSend()-1].Text, tc.label)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertIgnoresUnknownResults(t *testing.T) {
|
|
||||||
for _, scenario := range []string{"missing", "stale", "warmup", "no probes", "down", "paused", "future", "expired hourly window"} {
|
|
||||||
t.Run(scenario, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
apply := func(loss float64) {
|
|
||||||
result := monitorResult(loss)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: result}
|
|
||||||
switch scenario {
|
|
||||||
case "missing":
|
|
||||||
results = nil
|
|
||||||
case "stale":
|
|
||||||
result.LastProbeAt = time.Now().Add(-10 * time.Minute).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "expired hourly window":
|
|
||||||
monitors[0].Set("interval", 3600)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
result.LastProbeAt = time.Now().Add(-2 * time.Hour).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "future":
|
|
||||||
result.LastProbeAt = time.Now().Add(time.Hour).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "warmup":
|
|
||||||
result.SampleCount = 2
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "no probes":
|
|
||||||
result.SampleCount = 0
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
case "down":
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": scenario}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
case "paused":
|
|
||||||
record, err := hub.FindRecordById("systems", system.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
record.Set("status", "paused")
|
|
||||||
require.NoError(t, hub.Save(record))
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
}
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
apply(100)
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "up"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, map[string]monitor.Result{monitors[0].Id: monitorResult(10)}))
|
|
||||||
apply(0)
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend(), "unknown data must not recover an incident")
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, record.GetBool("triggered"))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertCleanup(t *testing.T) {
|
|
||||||
for _, scenario := range []string{"disable monitor", "delete monitor", "disable alert", "purge history", "delete system"} {
|
|
||||||
t.Run(scenario, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10), monitors[1].Id: monitorResult(20)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
switch scenario {
|
|
||||||
case "disable monitor":
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
case "delete monitor":
|
|
||||||
require.NoError(t, hub.Delete(monitors[0]))
|
|
||||||
case "disable alert":
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
case "delete system":
|
|
||||||
require.NoError(t, hub.Delete(system))
|
|
||||||
case "purge history":
|
|
||||||
history, err := hub.FindAllRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, record := range history {
|
|
||||||
require.NoError(t, hub.Delete(record))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
open, err := hub.CountRecords("alerts_history", dbx.HashExp{"alert_id": alert.Id, "resolved": ""})
|
|
||||||
require.NoError(t, err)
|
|
||||||
if scenario == "disable monitor" || scenario == "delete monitor" {
|
|
||||||
assert.EqualValues(t, 1, open)
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, record.GetBool("triggered"))
|
|
||||||
require.NoError(t, hub.Delete(monitors[1]))
|
|
||||||
record, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, record.GetBool("triggered"))
|
|
||||||
} else {
|
|
||||||
assert.Zero(t, open)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertPerUserThresholds(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
user, err := beszelTests.CreateUser(hub, "monitor2@example.com", "password")
|
|
||||||
require.NoError(t, err)
|
|
||||||
other, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": system.Id, "user": user.Id, "value": 20})
|
|
||||||
require.NoError(t, err)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
other, err = hub.FindRecordById("alerts", other.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, other.GetBool("triggered"))
|
|
||||||
// Editing the threshold re-evaluates on the next batch, without losing state.
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
alert.Set("value", 15)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, alert.GetBool("triggered"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertAPI(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
value float64
|
|
||||||
direct, denied, patch bool
|
|
||||||
status int
|
|
||||||
}{
|
|
||||||
{name: "zero threshold", value: 0, status: 200},
|
|
||||||
{name: "fractional threshold", value: 5.5, status: 200},
|
|
||||||
{name: "negative threshold", value: -1, status: 400},
|
|
||||||
{name: "unreachable threshold", value: 100, status: 400},
|
|
||||||
{name: "bulk inaccessible system", value: 5, denied: true, status: 200},
|
|
||||||
{name: "direct inaccessible system", value: 5, direct: true, denied: true, status: 403},
|
|
||||||
{name: "direct invalid threshold", value: -1, direct: true, status: 400},
|
|
||||||
{name: "direct private state", value: 5, direct: true, status: 200},
|
|
||||||
{name: "patch preserves state", value: 10, direct: true, patch: true, status: 200},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
owner := user.Id
|
|
||||||
if tc.denied {
|
|
||||||
other, err := beszelTests.CreateUser(hub, "other@example.com", "password")
|
|
||||||
require.NoError(t, err)
|
|
||||||
owner = other.Id
|
|
||||||
}
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, owner, "paused")
|
|
||||||
require.NoError(t, err)
|
|
||||||
token, err := user.NewAuthToken()
|
|
||||||
require.NoError(t, err)
|
|
||||||
body := map[string]any{"name": "NetworkMonitorLoss", "value": tc.value, "min": 60, "systems": []string{systems[0].Id}, "overwrite": true}
|
|
||||||
url, method := "/api/beszel/user-alerts", "POST"
|
|
||||||
if tc.direct {
|
|
||||||
url = "/api/collections/alerts/records"
|
|
||||||
body["system"], body["user"] = systems[0].Id, user.Id
|
|
||||||
body["state"], body["triggered"] = map[string]any{"monitors": map[string]string{"fake": "fake"}}, true
|
|
||||||
}
|
|
||||||
if tc.patch {
|
|
||||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "NetworkMonitorLoss", "system": systems[0].Id, "user": user.Id, "value": 5, "triggered": true, "state": map[string]any{"monitors": map[string]string{"real": "history"}}})
|
|
||||||
require.NoError(t, err)
|
|
||||||
url += "/" + alert.Id
|
|
||||||
method = "PATCH"
|
|
||||||
body["triggered"] = false
|
|
||||||
}
|
|
||||||
content := `"success":true`
|
|
||||||
if tc.direct {
|
|
||||||
content = `"name":"NetworkMonitorLoss"`
|
|
||||||
}
|
|
||||||
if tc.status == 400 {
|
|
||||||
content = `"status":400`
|
|
||||||
}
|
|
||||||
if tc.status == 403 {
|
|
||||||
content = `"status":403`
|
|
||||||
}
|
|
||||||
scenario := beszelTests.ApiScenario{
|
|
||||||
Name: tc.name, Method: method, URL: url, Body: jsonReader(body),
|
|
||||||
Headers: map[string]string{"Authorization": token}, ExpectedStatus: tc.status, ExpectedContent: []string{content},
|
|
||||||
TestAppFactory: func(testing.TB) *pbTests.TestApp { return hub.TestApp },
|
|
||||||
}
|
|
||||||
scenario.Test(t)
|
|
||||||
records, err := hub.FindAllRecords("alerts")
|
|
||||||
require.NoError(t, err)
|
|
||||||
if tc.status != 200 || tc.denied {
|
|
||||||
assert.Empty(t, records)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
require.Len(t, records, 1)
|
|
||||||
assert.Equal(t, tc.value, records[0].GetFloat("value"))
|
|
||||||
assert.Zero(t, records[0].GetInt("min"))
|
|
||||||
state := struct {
|
|
||||||
Monitors map[string]string `json:"monitors"`
|
|
||||||
}{}
|
|
||||||
require.NoError(t, records[0].UnmarshalJSONField("state", &state))
|
|
||||||
states := state.Monitors
|
|
||||||
if tc.patch {
|
|
||||||
assert.Equal(t, map[string]string{"real": "history"}, states)
|
|
||||||
assert.True(t, records[0].GetBool("triggered"))
|
|
||||||
} else {
|
|
||||||
assert.Empty(t, states)
|
|
||||||
assert.False(t, records[0].GetBool("triggered"))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type monitorCountingHub struct {
|
|
||||||
*beszelTests.TestHub
|
|
||||||
transactions atomic.Int64
|
|
||||||
beforeTransaction func()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h *monitorCountingHub) RunInTransaction(fn func(core.App) error) error {
|
|
||||||
h.transactions.Add(1)
|
|
||||||
if h.beforeTransaction != nil {
|
|
||||||
h.beforeTransaction()
|
|
||||||
}
|
|
||||||
return h.App.RunInTransaction(fn)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Count actual SQL on both DB connections, including queries through record APIs.
|
|
||||||
func monitorSQLCounter(t *testing.T, app core.App) *atomic.Int64 {
|
|
||||||
t.Helper()
|
|
||||||
count := &atomic.Int64{}
|
|
||||||
for _, builder := range []dbx.Builder{app.ConcurrentDB(), app.NonconcurrentDB()} {
|
|
||||||
db := builder.(*dbx.DB)
|
|
||||||
old := db.LogFunc
|
|
||||||
db.LogFunc = func(string, ...any) { count.Add(1) }
|
|
||||||
t.Cleanup(func() { db.LogFunc = old })
|
|
||||||
}
|
|
||||||
return count
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertSteadyStateNoDatabaseWork(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
counted := &monitorCountingHub{TestHub: hub}
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
|
||||||
sql := monitorSQLCounter(t, hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
evaluate := func() { t.Helper(); require.NoError(t, am.HandleNetworkMonitorAlerts(system, results)) }
|
|
||||||
noWork := func() {
|
|
||||||
t.Helper()
|
|
||||||
sql.Store(0)
|
|
||||||
counted.transactions.Store(0)
|
|
||||||
for range 100 {
|
|
||||||
evaluate()
|
|
||||||
}
|
|
||||||
assert.Zero(t, sql.Load(), "steady state must not issue SQL")
|
|
||||||
assert.Zero(t, counted.transactions.Load(), "steady state must not open transactions")
|
|
||||||
}
|
|
||||||
// One-time lazy loads are permitted, including on hub restart.
|
|
||||||
evaluate()
|
|
||||||
assert.Positive(t, sql.Load())
|
|
||||||
noWork()
|
|
||||||
// Realtime metric saves invoke record hooks but must not invalidate config.
|
|
||||||
fresh, err := hub.FindRecordById("network_monitors", monitors[0].Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
monitors[0] = fresh
|
|
||||||
monitors[0].Set("loss1h", 0)
|
|
||||||
monitors[0].Set("res", 100)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
noWork()
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
evaluate()
|
|
||||||
assert.Positive(t, sql.Load(), "transitions must still be persisted")
|
|
||||||
assert.EqualValues(t, 1, counted.transactions.Load())
|
|
||||||
noWork()
|
|
||||||
// Missing and stale observations must not enter the transaction either.
|
|
||||||
results = nil
|
|
||||||
noWork()
|
|
||||||
results = map[string]monitor.Result{monitors[0].Id: {SampleCount: 60, LastProbeAt: time.Now().Add(-10 * time.Minute).UnixMilli()}}
|
|
||||||
noWork()
|
|
||||||
results[monitors[0].Id] = monitorResult(0)
|
|
||||||
evaluate()
|
|
||||||
noWork()
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
noWork()
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertConfigCacheInvalidation(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
// Widening the interval makes this observation fresh. A stale interval cache
|
|
||||||
// would miss the failure indefinitely, even though results keep arriving.
|
|
||||||
result := monitorResult(10)
|
|
||||||
result.LastProbeAt = time.Now().Add(-4 * time.Minute).UnixMilli()
|
|
||||||
results[monitors[0].Id] = result
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
monitors[0].Set("interval", 120)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
// Disable, then re-enable the same ID: its new failure must be detected.
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
monitors[0].Set("enabled", true)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+2, hub.TestMailer.TotalSend())
|
|
||||||
// A new monitor must also become eligible without restarting the hub.
|
|
||||||
created, err := beszelTests.CreateRecord(hub, "network_monitors", map[string]any{
|
|
||||||
"system": system.Id, "name": "new", "target": "new.example.com", "protocol": "icmp", "interval": 60, "enabled": true,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
results[created.Id] = monitorResult(10)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+3, hub.TestMailer.TotalSend())
|
|
||||||
// Threshold changes refresh cached config and preserve the active incidents.
|
|
||||||
alert, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
alert.Set("value", 15)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+5, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertRevalidatesCandidate(t *testing.T) {
|
|
||||||
for _, change := range []string{"threshold", "disable alert", "disable monitor", "down"} {
|
|
||||||
t.Run(change, func(t *testing.T) {
|
|
||||||
hub, system, alert, monitors := networkAlertSetup(t)
|
|
||||||
counted := &monitorCountingHub{TestHub: hub}
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(counted)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
// Change the DB after the cache predicts a transition, before its transaction.
|
|
||||||
counted.beforeTransaction = func() {
|
|
||||||
counted.beforeTransaction = nil
|
|
||||||
switch change {
|
|
||||||
case "threshold":
|
|
||||||
alert.Set("value", 20)
|
|
||||||
require.NoError(t, hub.Save(alert))
|
|
||||||
case "disable alert":
|
|
||||||
require.NoError(t, hub.Delete(alert))
|
|
||||||
case "disable monitor":
|
|
||||||
monitors[0].Set("enabled", false)
|
|
||||||
require.NoError(t, hub.Save(monitors[0]))
|
|
||||||
case "down":
|
|
||||||
_, err := hub.DB().Update("systems", dbx.Params{"status": "down"}, dbx.HashExp{"id": system.Id}).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.EqualValues(t, 1, counted.transactions.Load())
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
histories, err := hub.CountRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, histories)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertConcurrentEvaluations(t *testing.T) {
|
|
||||||
hub, system, _, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(10)}
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
errs := make(chan error, 8)
|
|
||||||
for range 8 {
|
|
||||||
wg.Go(func() { errs <- am.HandleNetworkMonitorAlerts(system, results) })
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(errs)
|
|
||||||
for err := range errs {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertCacheAfterRollback(t *testing.T) {
|
|
||||||
hub, system, _, monitors := networkAlertSetup(t)
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
results := map[string]monitor.Result{monitors[0].Id: monitorResult(0)}
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
_, err := hub.DB().NewQuery(`CREATE TRIGGER fail_alert BEFORE UPDATE ON alerts BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
count := hub.TestMailer.TotalSend()
|
|
||||||
results[monitors[0].Id] = monitorResult(10)
|
|
||||||
require.Error(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count, hub.TestMailer.TotalSend())
|
|
||||||
histories, err := hub.CountRecords("alerts_history")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, histories)
|
|
||||||
_, err = hub.DB().NewQuery("DROP TRIGGER fail_alert").Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
// A failed transition must not be published to the cache and mask the retry.
|
|
||||||
require.NoError(t, am.HandleNetworkMonitorAlerts(system, results))
|
|
||||||
assert.Equal(t, count+1, hub.TestMailer.TotalSend())
|
|
||||||
}
|
|
||||||
@@ -47,7 +47,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName, alertNameNetworkMonitorLoss)
|
alerts := am.alertsCache.GetAlertsExcludingNames(systemRecord.Id, "Status", alertNameSystemdFailed, containerAlertName)
|
||||||
if len(alerts) == 0 {
|
if len(alerts) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -78,7 +78,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, pool := range data.Stats.ZfsPools {
|
for _, pool := range data.Stats.ZfsPools {
|
||||||
if pool != nil && !pool.Raw && pool.Total > 0 {
|
if pool != nil && pool.Total > 0 {
|
||||||
usedPct := pool.Used / pool.Total * 100
|
usedPct := pool.Used / pool.Total * 100
|
||||||
if usedPct > maxUsedPct {
|
if usedPct > maxUsedPct {
|
||||||
maxUsedPct = usedPct
|
maxUsedPct = usedPct
|
||||||
@@ -256,7 +256,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
}
|
}
|
||||||
// add zfs pool usage from historical record
|
// add zfs pool usage from historical record
|
||||||
for key, pool := range stats.ZfsPools {
|
for key, pool := range stats.ZfsPools {
|
||||||
if !pool.Raw && pool.Total > 0 {
|
if pool.Total > 0 {
|
||||||
zfsKey := zfsDiskAlertKey(key)
|
zfsKey := zfsDiskAlertKey(key)
|
||||||
if _, ok := alert.mapSums[zfsKey]; !ok {
|
if _, ok := alert.mapSums[zfsKey]; !ok {
|
||||||
alert.mapSums[zfsKey] = 0.0
|
alert.mapSums[zfsKey] = 0.0
|
||||||
@@ -319,11 +319,6 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
|||||||
if sumPct > maxPct {
|
if sumPct > maxPct {
|
||||||
maxPct = sumPct
|
maxPct = sumPct
|
||||||
alert.descriptor = diskAlertDescriptor(key)
|
alert.descriptor = diskAlertDescriptor(key)
|
||||||
if poolKey, ok := strings.CutPrefix(key, "zfs:"); ok {
|
|
||||||
if pool := data.Stats.ZfsPools[poolKey]; pool != nil && pool.DisplayName != "" {
|
|
||||||
alert.descriptor = diskAlertDescriptor(zfsDiskAlertKey(pool.DisplayName))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
alert.val = float64(maxPct / float32(alert.count))
|
alert.val = float64(maxPct / float32(alert.count))
|
||||||
@@ -375,7 +370,7 @@ func zfsDiskAlertKey(poolName string) string {
|
|||||||
|
|
||||||
func diskAlertDescriptor(key string) string {
|
func diskAlertDescriptor(key string) string {
|
||||||
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
||||||
return fmt.Sprintf("Usage of storage pool %s", poolName)
|
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
|
||||||
}
|
}
|
||||||
return fmt.Sprintf("Usage of %s", key)
|
return fmt.Sprintf("Usage of %s", key)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,9 +11,8 @@ import (
|
|||||||
|
|
||||||
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
func NewTestAlertManagerWithoutWorker(app hubLike) *AlertManager {
|
||||||
return &AlertManager{
|
return &AlertManager{
|
||||||
hub: app,
|
hub: app,
|
||||||
alertsCache: NewAlertsCache(app),
|
alertsCache: NewAlertsCache(app),
|
||||||
networkMonitors: newNetworkMonitorCache(app),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,6 +100,10 @@ func (am *AlertManager) SetAlertTriggered(alert CachedAlertData, triggered bool)
|
|||||||
return am.setAlertTriggered(alert, triggered)
|
return am.setAlertTriggered(alert, triggered)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func IsInternalURL(rawURL string) (bool, error) {
|
||||||
|
return isInternalURL(rawURL)
|
||||||
|
}
|
||||||
|
|
||||||
// BuildContainerLogExcerpt exposes buildContainerLogExcerpt for testing.
|
// BuildContainerLogExcerpt exposes buildContainerLogExcerpt for testing.
|
||||||
func BuildContainerLogExcerpt(raw string) string {
|
func BuildContainerLogExcerpt(raw string) string {
|
||||||
return buildContainerLogExcerpt(raw)
|
return buildContainerLogExcerpt(raw)
|
||||||
|
|||||||
@@ -1,66 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package alerts_test
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestPersistedWebhooksUseCurrentOwnerRole(t *testing.T) {
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
am := alerts.NewTestAlertManagerWithoutWorker(hub)
|
|
||||||
|
|
||||||
var delivered atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
delivered.Add(1)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
|
|
||||||
settings, err := hub.FindFirstRecordByFilter("user_settings", "user={:user}", dbx.Params{"user": user.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
settings.Set("settings", alerts.UserNotificationSettings{Webhooks: []string{"generic+" + server.URL}})
|
|
||||||
require.NoError(t, hub.Save(settings))
|
|
||||||
message := alerts.AlertMessageData{UserID: user.Id, Title: "Test", Message: "Persisted webhook"}
|
|
||||||
|
|
||||||
// Keep the same URL and manager while changing roles, so cached privileges
|
|
||||||
// or treating previously saved URLs as trusted would fail this test.
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
role string
|
|
||||||
want int32
|
|
||||||
}{
|
|
||||||
{"regular user", "user", 0},
|
|
||||||
{"readonly user", "readonly", 0},
|
|
||||||
{"promoted admin", "admin", 1},
|
|
||||||
{"demoted admin", "user", 1},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
user.Set("role", tc.role)
|
|
||||||
require.NoError(t, hub.Save(user))
|
|
||||||
// Webhook errors are logged; SendAlert continues to email delivery.
|
|
||||||
require.NoError(t, am.SendAlert(message))
|
|
||||||
require.Equal(t, tc.want, delivered.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("missing owner fails closed", func(t *testing.T) {
|
|
||||||
// Model an orphaned settings record without deleting it through the
|
|
||||||
// normal user deletion cascade.
|
|
||||||
const missingOwner = "missingowner123"
|
|
||||||
settings.Set("user", missingOwner)
|
|
||||||
require.NoError(t, hub.SaveNoValidate(settings))
|
|
||||||
message.UserID = missingOwner
|
|
||||||
err := am.SendAlert(message)
|
|
||||||
require.ErrorContains(t, err, "load notification owner")
|
|
||||||
require.EqualValues(t, 1, delivered.Load())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -46,15 +46,12 @@ func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth
|
|||||||
}
|
}
|
||||||
|
|
||||||
systemName := systemRecord.GetString("name")
|
systemName := systemRecord.GetString("name")
|
||||||
poolName := e.Record.GetString("display_name")
|
poolName := e.Record.GetString("name")
|
||||||
if poolName == "" {
|
|
||||||
poolName = e.Record.GetString("name")
|
|
||||||
}
|
|
||||||
|
|
||||||
title := fmt.Sprintf("Storage pool %s on %s: %s", newHealth, systemName, poolName)
|
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
|
||||||
message := fmt.Sprintf("Storage pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
|
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
|
||||||
if oldSeverity > 0 {
|
if oldSeverity > 0 {
|
||||||
message = fmt.Sprintf("Storage pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
|
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
|
||||||
}
|
}
|
||||||
|
|
||||||
userIDs := systemRecord.GetStringSlice("users")
|
userIDs := systemRecord.GetStringSlice("users")
|
||||||
@@ -119,7 +116,7 @@ func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolNam
|
|||||||
record.Set("user", userID)
|
record.Set("user", userID)
|
||||||
record.Set("system", systemID)
|
record.Set("system", systemID)
|
||||||
record.Set("alert_id", alertID)
|
record.Set("alert_id", alertID)
|
||||||
record.Set("name", "Storage Pool: "+poolName)
|
record.Set("name", "ZFS Pool: "+poolName)
|
||||||
return app.Save(record)
|
return app.Save(record)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -143,37 +143,3 @@ func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
|
|||||||
assert.False(t, diskAlert.GetBool("triggered"),
|
assert.False(t, diskAlert.GetBool("triggered"),
|
||||||
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
|
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDiskAlertIgnoresRawPool(t *testing.T) {
|
|
||||||
for _, minutes := range []int{0, 2} {
|
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
|
||||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
|
||||||
require.NoError(t, err)
|
|
||||||
alert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{"name": "Disk", "system": systems[0].Id, "user": user.Id, "value": 80, "min": minutes})
|
|
||||||
require.NoError(t, err)
|
|
||||||
pools := map[string]*system.ZfsPool{"btrfs": {Total: 100, Used: 99, Raw: true}}
|
|
||||||
for _, offset := range []time.Duration{-180, -90, -60, -30} {
|
|
||||||
data, err := json.Marshal(system.Stats{ZfsPools: pools})
|
|
||||||
require.NoError(t, err)
|
|
||||||
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{"system": systems[0].Id, "type": "1m", "stats": string(data)})
|
|
||||||
require.NoError(t, err)
|
|
||||||
record.SetRaw("created", time.Now().UTC().Add(offset*time.Second).Format(types.DefaultDateLayout))
|
|
||||||
require.NoError(t, hub.SaveNoValidate(record))
|
|
||||||
}
|
|
||||||
require.NoError(t, hub.GetAlertManager().HandleSystemAlerts(systems[0], &system.CombinedData{Stats: system.Stats{ZfsPools: pools}}))
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
record, err := hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, record.GetBool("triggered"))
|
|
||||||
if minutes > 0 {
|
|
||||||
// A current usable sample must not make raw historical values eligible.
|
|
||||||
pools["btrfs"].Raw = false
|
|
||||||
require.NoError(t, hub.GetAlertManager().HandleSystemAlerts(systems[0], &system.CombinedData{Stats: system.Stats{ZfsPools: pools}}))
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
record, err = hub.FindRecordById("alerts", alert.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, record.GetBool("triggered"))
|
|
||||||
}
|
|
||||||
hub.Cleanup()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -10,6 +10,6 @@ import (
|
|||||||
|
|
||||||
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
|
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
|
||||||
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
|
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
|
||||||
assert.Equal(t, "Usage of storage pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
|
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
|
||||||
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
|
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
|
|||||||
|
|
||||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
|
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
|
||||||
lastMessage := hub.TestMailer.LastMessage()
|
lastMessage := hub.TestMailer.LastMessage()
|
||||||
assert.Contains(t, lastMessage.Subject, "Storage pool DEGRADED on test-system")
|
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
|
||||||
assert.Contains(t, lastMessage.Subject, "tank")
|
assert.Contains(t, lastMessage.Subject, "tank")
|
||||||
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
|
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
|
||||||
}
|
}
|
||||||
@@ -76,7 +76,7 @@ func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
|
|||||||
|
|
||||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
|
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
|
||||||
lastMessage := hub.TestMailer.LastMessage()
|
lastMessage := hub.TestMailer.LastMessage()
|
||||||
assert.Contains(t, lastMessage.Subject, "Storage pool FAULTED on test-system")
|
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
|
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
|
||||||
@@ -239,7 +239,7 @@ func TestZfsPoolAlertWritesHistory(t *testing.T) {
|
|||||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
require.Len(t, history, 1, "expected one history entry per user")
|
require.Len(t, history, 1, "expected one history entry per user")
|
||||||
assert.Equal(t, "Storage Pool: tank", history[0].GetString("name"))
|
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
|
||||||
assert.Equal(t, system.Id, history[0].GetString("system"))
|
assert.Equal(t, system.Id, history[0].GetString("system"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,79 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
// networkMonitorCache keeps just the enabled monitor IDs and probe intervals
|
|
||||||
// needed for the alert fast path. Names and targets are read only on transitions.
|
|
||||||
// Returned maps are immutable; configuration changes invalidate the whole entry.
|
|
||||||
type networkMonitorCache struct {
|
|
||||||
app core.App
|
|
||||||
mu sync.RWMutex
|
|
||||||
systems map[string]map[string]int
|
|
||||||
}
|
|
||||||
|
|
||||||
func newNetworkMonitorCache(app core.App) *networkMonitorCache {
|
|
||||||
c := &networkMonitorCache{app: app, systems: make(map[string]map[string]int)}
|
|
||||||
invalidate := func(e *core.RecordEvent) error {
|
|
||||||
c.invalidate(e.Record.GetString("system"))
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
app.OnRecordAfterCreateSuccess("network_monitors").BindFunc(invalidate)
|
|
||||||
app.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(invalidate)
|
|
||||||
app.OnRecordAfterUpdateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
old := e.Record.Original()
|
|
||||||
// Realtime metric saves also invoke this hook. They must not evict config.
|
|
||||||
if old.GetString("system") != e.Record.GetString("system") ||
|
|
||||||
old.GetBool("enabled") != e.Record.GetBool("enabled") ||
|
|
||||||
old.GetInt("interval") != e.Record.GetInt("interval") {
|
|
||||||
c.invalidate(old.GetString("system"))
|
|
||||||
c.invalidate(e.Record.GetString("system"))
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
})
|
|
||||||
app.OnRecordAfterDeleteSuccess("systems").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
c.invalidate(e.Record.Id)
|
|
||||||
return e.Next()
|
|
||||||
})
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *networkMonitorCache) invalidate(systemID string) {
|
|
||||||
c.mu.Lock()
|
|
||||||
delete(c.systems, systemID)
|
|
||||||
c.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *networkMonitorCache) get(systemID string) (map[string]int, error) {
|
|
||||||
c.mu.RLock()
|
|
||||||
monitors, ok := c.systems[systemID]
|
|
||||||
c.mu.RUnlock()
|
|
||||||
if ok {
|
|
||||||
return monitors, nil
|
|
||||||
}
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
if monitors, ok := c.systems[systemID]; ok {
|
|
||||||
return monitors, nil
|
|
||||||
}
|
|
||||||
// Keep the lock through the load so a concurrent config change cannot be
|
|
||||||
// invalidated first and then overwritten by the older query result.
|
|
||||||
var rows []struct {
|
|
||||||
ID string `db:"id"`
|
|
||||||
Interval int `db:"interval"`
|
|
||||||
}
|
|
||||||
if err := c.app.DB().Select("id", "interval").From("network_monitors").
|
|
||||||
Where(dbx.HashExp{"system": systemID, "enabled": true}).All(&rows); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
monitors = make(map[string]int, len(rows))
|
|
||||||
for _, row := range rows {
|
|
||||||
monitors[row.ID] = row.Interval
|
|
||||||
}
|
|
||||||
c.systems[systemID] = monitors
|
|
||||||
return monitors, nil
|
|
||||||
}
|
|
||||||
@@ -1,150 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/netip"
|
|
||||||
"sync/atomic"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nicholas-fedor/shoutrrr/pkg/router"
|
|
||||||
"github.com/nicholas-fedor/shoutrrr/pkg/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
errInternalDestination = errors.New("Only admins can send to internal destinations")
|
|
||||||
errUnrestrictedService = errors.New("Only admins can use this notification service") // Restrict services w/o custom connection support
|
|
||||||
publicNotificationDialer = &net.Dialer{
|
|
||||||
Timeout: 10 * time.Second,
|
|
||||||
// Control checks each resolved address immediately before connecting.
|
|
||||||
Control: func(_, address string, _ syscall.RawConn) error { return checkNotificationAddress(address) },
|
|
||||||
}
|
|
||||||
publicNotificationClient = newPublicNotificationClient()
|
|
||||||
)
|
|
||||||
|
|
||||||
func newPublicNotificationClient() *http.Client {
|
|
||||||
return &http.Client{
|
|
||||||
Timeout: 15 * time.Second,
|
|
||||||
Transport: &http.Transport{
|
|
||||||
// Do not use proxies: they can resolve the target themselves and
|
|
||||||
// bypass the destination check on our socket.
|
|
||||||
DialContext: publicNotificationDialer.DialContext,
|
|
||||||
TLSHandshakeTimeout: 10 * time.Second,
|
|
||||||
IdleConnTimeout: 90 * time.Second,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func checkNotificationAddress(address string) error {
|
|
||||||
addr, err := netip.ParseAddrPort(address)
|
|
||||||
if err != nil || addr.Addr().Zone() != "" {
|
|
||||||
return errInternalDestination
|
|
||||||
}
|
|
||||||
ip := net.IP(addr.Addr().AsSlice())
|
|
||||||
if !ip.IsGlobalUnicast() || isInternalIP(ip) {
|
|
||||||
return errInternalDestination
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func sendPublicNotification(rawURL, message string) error {
|
|
||||||
client := ¬ificationClient{Client: publicNotificationClient}
|
|
||||||
service, err := newPublicNotificationService(rawURL, types.SenderOptions{HTTPClient: client, DialContext: client.dialContext})
|
|
||||||
if err == nil {
|
|
||||||
if closer, ok := service.(io.Closer); ok {
|
|
||||||
defer closer.Close()
|
|
||||||
}
|
|
||||||
err = service.Send(message, &types.Params{})
|
|
||||||
}
|
|
||||||
// Some services format errors without preserving their error chain.
|
|
||||||
if client.blocked.Load() {
|
|
||||||
return errInternalDestination
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
type notificationClient struct {
|
|
||||||
*http.Client
|
|
||||||
blocked atomic.Bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *notificationClient) Do(req *http.Request) (*http.Response, error) {
|
|
||||||
response, err := c.Client.Do(req)
|
|
||||||
if errors.Is(err, errInternalDestination) {
|
|
||||||
c.blocked.Store(true)
|
|
||||||
}
|
|
||||||
return response, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *notificationClient) dialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
conn, err := publicNotificationDialer.DialContext(ctx, network, address)
|
|
||||||
if errors.Is(err, errInternalDestination) {
|
|
||||||
c.blocked.Store(true)
|
|
||||||
}
|
|
||||||
return conn, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func newPublicNotificationService(rawURL string, opts types.SenderOptions) (types.Service, error) {
|
|
||||||
r := &router.ServiceRouter{}
|
|
||||||
scheme, serviceURL, err := r.ExtractServiceName(rawURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
service, err := r.NewService(scheme)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
httpSetter, httpOK := service.(types.HTTPClientSetter)
|
|
||||||
dialSetter, dialOK := service.(types.DialContextSetter)
|
|
||||||
if (!httpOK || opts.HTTPClient == nil) && (!dialOK || opts.DialContext == nil) {
|
|
||||||
return nil, errUnrestrictedService
|
|
||||||
}
|
|
||||||
if serviceURL.Scheme != scheme {
|
|
||||||
custom, ok := service.(types.CustomURLService)
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: %s", router.ErrCustomURLsNotSupported, scheme)
|
|
||||||
}
|
|
||||||
serviceURL, err = custom.GetServiceURLFromCustom(serviceURL)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Shoutrrr v0.20.0 CreateSenderWithOptions injects only AFTER Initialize.
|
|
||||||
// Matrix can log in during Initialize, so inject before it as well.
|
|
||||||
if httpOK {
|
|
||||||
httpSetter.SetHTTPClient(opts.HTTPClient)
|
|
||||||
}
|
|
||||||
if dialOK {
|
|
||||||
dialSetter.SetDialContext(opts.DialContext)
|
|
||||||
}
|
|
||||||
if err := service.Initialize(serviceURL, nil); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Some initializers replace their HTTP client with a default client.
|
|
||||||
if httpOK {
|
|
||||||
httpSetter.SetHTTPClient(opts.HTTPClient)
|
|
||||||
}
|
|
||||||
if dialOK {
|
|
||||||
dialSetter.SetDialContext(opts.DialContext)
|
|
||||||
}
|
|
||||||
return service, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
var cgnatNetwork = &net.IPNet{
|
|
||||||
IP: net.IPv4(100, 64, 0, 0),
|
|
||||||
Mask: net.CIDRMask(10, 32),
|
|
||||||
}
|
|
||||||
|
|
||||||
func isInternalIP(ip net.IP) bool {
|
|
||||||
return ip.IsPrivate() ||
|
|
||||||
ip.IsLoopback() ||
|
|
||||||
ip.IsUnspecified() ||
|
|
||||||
ip.IsLinkLocalUnicast() ||
|
|
||||||
ip.IsMulticast() ||
|
|
||||||
cgnatNetwork.Contains(ip)
|
|
||||||
}
|
|
||||||
@@ -1,217 +0,0 @@
|
|||||||
package alerts
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/binary"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"sync/atomic"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/nicholas-fedor/shoutrrr/pkg/types"
|
|
||||||
"golang.org/x/net/dns/dnsmessage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCheckNotificationAddress(t *testing.T) {
|
|
||||||
for _, host := range []string{"127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.0.1", "169.254.169.254", "100.64.0.0", "100.127.255.255", "0.0.0.0", "224.0.0.1", "255.255.255.255", "::1", "::", "fc00::1", "fe80::1", "ff02::1", "::ffff:127.0.0.1", "::ffff:169.254.169.254", "fe80::1%lo", "localhost", "consul"} {
|
|
||||||
t.Run(host, func(t *testing.T) {
|
|
||||||
if err := checkNotificationAddress(net.JoinHostPort(host, "80")); !errors.Is(err, errInternalDestination) {
|
|
||||||
t.Fatalf("expected blocked address, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
for _, host := range []string{"8.8.8.8", "100.63.255.255", "100.128.0.0", "2001:4860:4860::8888"} {
|
|
||||||
if err := checkNotificationAddress(net.JoinHostPort(host, "443")); err != nil {
|
|
||||||
t.Errorf("public address %s: %v", host, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPublicNotificationBlocksInternalRequests(t *testing.T) {
|
|
||||||
var hits atomic.Int32
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
hits.Add(1)
|
|
||||||
}))
|
|
||||||
defer server.Close()
|
|
||||||
host := strings.TrimPrefix(server.URL, "http://")
|
|
||||||
for _, rawURL := range []string{
|
|
||||||
"generic+http://" + host,
|
|
||||||
"generic+https://" + host,
|
|
||||||
"generic+http://localhost:" + strings.Split(host, ":")[1],
|
|
||||||
"matrix://user:password@" + host + "/room?disabletls=yes",
|
|
||||||
"mattermost://" + host + "/token?disabletls=yes",
|
|
||||||
} {
|
|
||||||
t.Run(rawURL, func(t *testing.T) {
|
|
||||||
if err := sendPublicNotification(rawURL, "test"); !errors.Is(err, errInternalDestination) {
|
|
||||||
t.Fatalf("expected internal destination error, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
if hits.Load() != 0 {
|
|
||||||
t.Fatal("internal server received a request")
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
type notificationRoundTripper func(*http.Request) (*http.Response, error)
|
|
||||||
|
|
||||||
func (f notificationRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
|
||||||
|
|
||||||
func TestPublicNotificationRedirect(t *testing.T) {
|
|
||||||
client := newPublicNotificationClient()
|
|
||||||
defer client.CloseIdleConnections()
|
|
||||||
transport := client.Transport
|
|
||||||
client.Transport = notificationRoundTripper(func(r *http.Request) (*http.Response, error) {
|
|
||||||
if r.URL.Host == "public.example" {
|
|
||||||
return &http.Response{StatusCode: 307, Header: http.Header{"Location": {"http://127.0.0.1/"}}, Body: io.NopCloser(strings.NewReader("")), Request: r}, nil
|
|
||||||
}
|
|
||||||
return transport.RoundTrip(r)
|
|
||||||
})
|
|
||||||
_, err := client.Get("http://public.example/")
|
|
||||||
if !errors.Is(err, errInternalDestination) {
|
|
||||||
t.Fatalf("expected redirect to be blocked, got %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPublicNotificationServiceClient(t *testing.T) {
|
|
||||||
for _, rawURL := range []string{"generic+http://public.example/path", "discord://token@123456789", "slack://hook:AAAAAAAAA-BBBBBBBBB-123456789123456789123456@webhook"} {
|
|
||||||
t.Run(rawURL, func(t *testing.T) {
|
|
||||||
var hits int
|
|
||||||
client := &http.Client{Transport: notificationRoundTripper(func(r *http.Request) (*http.Response, error) {
|
|
||||||
hits++
|
|
||||||
body := `{"ok":true}`
|
|
||||||
if strings.HasPrefix(rawURL, "slack:") {
|
|
||||||
body = "ok"
|
|
||||||
}
|
|
||||||
return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: r}, nil
|
|
||||||
})}
|
|
||||||
service, err := newPublicNotificationService(rawURL, types.SenderOptions{HTTPClient: client})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := service.Send("test", &types.Params{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if hits == 0 {
|
|
||||||
t.Fatal("injected client was not used")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPublicNotificationDNS(t *testing.T) {
|
|
||||||
// Supply deterministic DNS responses over an in-memory TCP connection.
|
|
||||||
// The first lookup sees a public IP; subsequent lookups see loopback.
|
|
||||||
var rebound atomic.Bool
|
|
||||||
resolver := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
client, server := net.Pipe()
|
|
||||||
go func() {
|
|
||||||
defer server.Close()
|
|
||||||
var size [2]byte
|
|
||||||
if _, err := io.ReadFull(server, size[:]); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
buf := make([]byte, binary.BigEndian.Uint16(size[:]))
|
|
||||||
if _, err := io.ReadFull(server, buf); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var msg dnsmessage.Message
|
|
||||||
if err := msg.Unpack(buf); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
msg.Header.Response = true
|
|
||||||
msg.Header.RecursionAvailable = true
|
|
||||||
q := msg.Questions[0]
|
|
||||||
if q.Type == dnsmessage.TypeA {
|
|
||||||
ip := [4]byte{8, 8, 8, 8}
|
|
||||||
if rebound.Load() {
|
|
||||||
ip = [4]byte{127, 0, 0, 1}
|
|
||||||
}
|
|
||||||
msg.Answers = []dnsmessage.Resource{{Header: dnsmessage.ResourceHeader{Name: q.Name, Type: q.Type, Class: dnsmessage.ClassINET}, Body: &dnsmessage.AResource{A: ip}}}
|
|
||||||
}
|
|
||||||
buf, err := msg.Pack()
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
binary.BigEndian.PutUint16(size[:], uint16(len(buf)))
|
|
||||||
server.Write(append(size[:], buf...))
|
|
||||||
}()
|
|
||||||
return client, nil
|
|
||||||
}}
|
|
||||||
// These tests do not run in parallel; restore the process resolver afterward.
|
|
||||||
previous := net.DefaultResolver
|
|
||||||
net.DefaultResolver = resolver
|
|
||||||
t.Cleanup(func() { net.DefaultResolver = previous })
|
|
||||||
ips, err := resolver.LookupIP(context.Background(), "ip4", "rebind.example")
|
|
||||||
if err != nil || len(ips) != 1 || !ips[0].Equal(net.IPv4(8, 8, 8, 8)) {
|
|
||||||
t.Fatalf("initial DNS lookup: %v, %v", ips, err)
|
|
||||||
}
|
|
||||||
rebound.Store(true)
|
|
||||||
client := newPublicNotificationClient()
|
|
||||||
defer client.CloseIdleConnections()
|
|
||||||
for _, host := range []string{"rebind.example", "consul"} {
|
|
||||||
guarded := ¬ificationClient{Client: client}
|
|
||||||
conn, dialErr := guarded.dialContext(context.Background(), "tcp", net.JoinHostPort(host, "25"))
|
|
||||||
if conn != nil {
|
|
||||||
conn.Close()
|
|
||||||
}
|
|
||||||
if !errors.Is(dialErr, errInternalDestination) || !guarded.blocked.Load() {
|
|
||||||
t.Errorf("expected TCP dial-time rejection for %s, got %v", host, dialErr)
|
|
||||||
}
|
|
||||||
_, err := client.Get("http://" + host + "/")
|
|
||||||
if !errors.Is(err, errInternalDestination) {
|
|
||||||
t.Errorf("expected dial-time rejection for %s, got %v", host, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestPublicNotificationTCP(t *testing.T) {
|
|
||||||
for _, rawURL := range []string{
|
|
||||||
"smtp://user:pass@HOST:25/?fromAddress=sender@example.com&toAddresses=recipient@example.com",
|
|
||||||
"smtp://user:pass@HOST:465/?fromAddress=sender@example.com&toAddresses=recipient@example.com",
|
|
||||||
"mqtt://HOST:1883/topic",
|
|
||||||
"mqtts://HOST:8883/topic",
|
|
||||||
} {
|
|
||||||
t.Run(rawURL, func(t *testing.T) {
|
|
||||||
t.Parallel()
|
|
||||||
t.Run("internal destination", func(t *testing.T) {
|
|
||||||
err := sendPublicNotification(strings.ReplaceAll(rawURL, "HOST", "127.0.0.1"), "test")
|
|
||||||
if !errors.Is(err, errInternalDestination) {
|
|
||||||
t.Fatalf("expected blocked destination, got %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
t.Run("public destination uses injected dialer", func(t *testing.T) {
|
|
||||||
var calls atomic.Int32
|
|
||||||
stopped := errors.New("test dial stopped")
|
|
||||||
service, err := newPublicNotificationService(strings.ReplaceAll(rawURL, "HOST", "8.8.8.8"), types.SenderOptions{
|
|
||||||
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
||||||
calls.Add(1)
|
|
||||||
if network != "tcp" || !strings.HasPrefix(address, "8.8.8.8:") {
|
|
||||||
t.Errorf("unexpected dial: %s %s", network, address)
|
|
||||||
}
|
|
||||||
if err := checkNotificationAddress(address); err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
}
|
|
||||||
return nil, stopped
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if closer, ok := service.(io.Closer); ok {
|
|
||||||
defer closer.Close()
|
|
||||||
}
|
|
||||||
if err := service.Send("test", &types.Params{}); err == nil {
|
|
||||||
t.Fatal("expected dial failure")
|
|
||||||
}
|
|
||||||
if calls.Load() == 0 {
|
|
||||||
t.Fatal("custom dialer was not used")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -24,8 +24,6 @@ const (
|
|||||||
GetSystemdInfo
|
GetSystemdInfo
|
||||||
// Request ZFS detail data from agent
|
// Request ZFS detail data from agent
|
||||||
GetZfsData
|
GetZfsData
|
||||||
// Sync network monitor configuration to agent
|
|
||||||
SyncNetworkMonitors
|
|
||||||
// Add new actions here...
|
// Add new actions here...
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ RUN go mod download
|
|||||||
# Copy source files
|
# Copy source files
|
||||||
COPY . ./
|
COPY . ./
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates && update-ca-certificates
|
|
||||||
|
|
||||||
# Build
|
# Build
|
||||||
ARG TARGETOS TARGETARCH
|
ARG TARGETOS TARGETARCH
|
||||||
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
RUN CGO_ENABLED=0 GOGC=75 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -ldflags "-w -s" -o /agent ./internal/cmd/agent
|
||||||
@@ -21,7 +19,6 @@ RUN rm -rf /tmp/*
|
|||||||
# --------------------------
|
# --------------------------
|
||||||
FROM scratch
|
FROM scratch
|
||||||
COPY --from=builder /agent /agent
|
COPY --from=builder /agent /agent
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
|
||||||
|
|
||||||
# this is so we don't need to create the /tmp directory in the scratch container
|
# this is so we don't need to create the /tmp directory in the scratch container
|
||||||
COPY --from=builder /tmp /tmp
|
COPY --from=builder /tmp /tmp
|
||||||
@@ -32,4 +29,4 @@ COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
|||||||
# Ensure data persistence across container recreations
|
# Ensure data persistence across container recreations
|
||||||
VOLUME ["/var/lib/beszel-agent"]
|
VOLUME ["/var/lib/beszel-agent"]
|
||||||
|
|
||||||
ENTRYPOINT ["/agent"]
|
ENTRYPOINT ["/agent"]
|
||||||
@@ -20,9 +20,9 @@ FROM alpine:3.23
|
|||||||
|
|
||||||
COPY --from=builder /agent /agent
|
COPY --from=builder /agent /agent
|
||||||
|
|
||||||
RUN apk add --no-cache -X https://dl-cdn.alpinelinux.org/alpine/edge/testing igt-gpu-tools nvtop smartmontools zfs
|
RUN apk add --no-cache -X https://dl-cdn.alpinelinux.org/alpine/edge/testing igt-gpu-tools nvtop smartmontools
|
||||||
|
|
||||||
# Ensure data persistence across container recreations
|
# Ensure data persistence across container recreations
|
||||||
VOLUME ["/var/lib/beszel-agent"]
|
VOLUME ["/var/lib/beszel-agent"]
|
||||||
|
|
||||||
ENTRYPOINT ["/agent"]
|
ENTRYPOINT ["/agent"]
|
||||||
@@ -43,11 +43,6 @@ COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
|||||||
# Copy smartmontools binaries and config files
|
# Copy smartmontools binaries and config files
|
||||||
COPY --from=smartmontools-builder /usr/sbin/smartctl /usr/sbin/smartctl
|
COPY --from=smartmontools-builder /usr/sbin/smartctl /usr/sbin/smartctl
|
||||||
|
|
||||||
# Install ZFS userspace utilities (zpool, zfs) for pool/dataset monitoring
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
zfsutils-linux \
|
|
||||||
&& apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Ensure data persistence across container recreations
|
# Ensure data persistence across container recreations
|
||||||
VOLUME ["/var/lib/beszel-agent"]
|
VOLUME ["/var/lib/beszel-agent"]
|
||||||
|
|
||||||
|
|||||||
@@ -65,32 +65,6 @@ RUN set -eux; \
|
|||||||
cp -v "$interp" "/out/rootfs$interp"; \
|
cp -v "$interp" "/out/rootfs$interp"; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------------
|
|
||||||
# ZFS utilities builder stage
|
|
||||||
# --------------------------
|
|
||||||
FROM --platform=$TARGETPLATFORM debian:bookworm-slim AS zfsutils-builder
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
zfsutils-linux \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
# Copy the zpool/zfs binaries and their required runtime libraries
|
|
||||||
RUN set -eux; \
|
|
||||||
mkdir -p /out/rootfs/lib /out/rootfs/lib64 /out/rootfs/usr/lib; \
|
|
||||||
for bin in /usr/sbin/zpool /usr/sbin/zfs; do \
|
|
||||||
mkdir -p "/out/rootfs$(dirname "$bin")"; \
|
|
||||||
cp -v "$bin" "/out/rootfs$bin"; \
|
|
||||||
ldd "$bin" \
|
|
||||||
| awk '{print $3}' \
|
|
||||||
| grep '^/' \
|
|
||||||
| xargs -r -I '{}' sh -c 'mkdir -p "/out/rootfs$(dirname "{}")"; cp -v "{}" "/out/rootfs{}"'; \
|
|
||||||
interp="$(ldd "$bin" | awk "/ld-linux/ {print \$1}")"; \
|
|
||||||
if [ -n "$interp" ] && [ -e "$interp" ]; then \
|
|
||||||
mkdir -p "/out/rootfs$(dirname "$interp")"; \
|
|
||||||
cp -v "$interp" "/out/rootfs$interp"; \
|
|
||||||
fi; \
|
|
||||||
done
|
|
||||||
|
|
||||||
# --------------------------
|
# --------------------------
|
||||||
# Final image: lightweight multi-arch NVIDIA agent (slim)
|
# Final image: lightweight multi-arch NVIDIA agent (slim)
|
||||||
# --------------------------
|
# --------------------------
|
||||||
@@ -104,9 +78,6 @@ COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
|||||||
COPY --from=smartmontools-builder /usr/sbin/smartctl /usr/sbin/smartctl
|
COPY --from=smartmontools-builder /usr/sbin/smartctl /usr/sbin/smartctl
|
||||||
COPY --from=smartmontools-builder /out/rootfs/ /
|
COPY --from=smartmontools-builder /out/rootfs/ /
|
||||||
|
|
||||||
# Copy ZFS utilities (zpool, zfs) binaries and required runtime libraries
|
|
||||||
COPY --from=zfsutils-builder /out/rootfs/ /
|
|
||||||
|
|
||||||
# nvidia-smi is intentionally not bundled.
|
# nvidia-smi is intentionally not bundled.
|
||||||
# Mount the host binary instead, for example:
|
# Mount the host binary instead, for example:
|
||||||
# - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi:ro
|
# - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi:ro
|
||||||
|
|||||||
@@ -186,12 +186,11 @@ type Stats struct {
|
|||||||
NetworkRecv float64 `json:"nr,omitzero" cbor:"4,keyasint,omitzero"` // deprecated 0.18.3 (MB) - keep field for old agents/records
|
NetworkRecv float64 `json:"nr,omitzero" cbor:"4,keyasint,omitzero"` // deprecated 0.18.3 (MB) - keep field for old agents/records
|
||||||
Bandwidth [2]uint64 `json:"b,omitzero" cbor:"9,keyasint,omitzero"` // [sent bytes, recv bytes]
|
Bandwidth [2]uint64 `json:"b,omitzero" cbor:"9,keyasint,omitzero"` // [sent bytes, recv bytes]
|
||||||
|
|
||||||
Health DockerHealth `json:"-" cbor:"5,keyasint"`
|
Health DockerHealth `json:"-" cbor:"5,keyasint"`
|
||||||
Status string `json:"-" cbor:"6,keyasint"`
|
Status string `json:"-" cbor:"6,keyasint"`
|
||||||
Id string `json:"-" cbor:"7,keyasint"`
|
Id string `json:"-" cbor:"7,keyasint"`
|
||||||
Image string `json:"-" cbor:"8,keyasint"`
|
Image string `json:"-" cbor:"8,keyasint"`
|
||||||
Ports string `json:"-" cbor:"10,keyasint"`
|
Ports string `json:"-" cbor:"10,keyasint"`
|
||||||
UpdateAvailable bool `json:"u,omitzero" cbor:"11,keyasint,omitzero"`
|
|
||||||
// PrevCpu [2]uint64 `json:"-"`
|
// PrevCpu [2]uint64 `json:"-"`
|
||||||
CpuSystem uint64 `json:"-"`
|
CpuSystem uint64 `json:"-"`
|
||||||
CpuContainer uint64 `json:"-"`
|
CpuContainer uint64 `json:"-"`
|
||||||
|
|||||||
@@ -1,102 +0,0 @@
|
|||||||
package monitor
|
|
||||||
|
|
||||||
import "time"
|
|
||||||
|
|
||||||
// MaxProbeTimeout is the longest agent probe timeout (currently HTTP).
|
|
||||||
// Hub requests that run a probe must allow this time in addition to transport overhead.
|
|
||||||
const MaxProbeTimeout = 10 * time.Second
|
|
||||||
|
|
||||||
type SyncAction uint8
|
|
||||||
|
|
||||||
const (
|
|
||||||
// SyncActionReplace indicates a full sync where the provided configs should replace all existing monitors for the system.
|
|
||||||
SyncActionReplace SyncAction = iota
|
|
||||||
// SyncActionUpsert indicates an incremental sync where the provided config should be added or updated.
|
|
||||||
SyncActionUpsert
|
|
||||||
// SyncActionDelete indicates an incremental sync where the provided config should be removed.
|
|
||||||
SyncActionDelete
|
|
||||||
)
|
|
||||||
|
|
||||||
// Config defines a network monitor task sent from hub to agent.
|
|
||||||
type Config struct {
|
|
||||||
// ID is the stable network_monitors record ID generated by the hub.
|
|
||||||
ID string `cbor:"0,keyasint"`
|
|
||||||
Target string `cbor:"1,keyasint"`
|
|
||||||
Protocol string `cbor:"2,keyasint"` // "icmp", "tcp", "http", or "dns"
|
|
||||||
Port uint16 `cbor:"3,keyasint,omitempty"`
|
|
||||||
Interval uint16 `cbor:"4,keyasint"` // seconds
|
|
||||||
}
|
|
||||||
|
|
||||||
// SyncRequest defines an incremental or full monitor sync request sent to the agent.
|
|
||||||
type SyncRequest struct {
|
|
||||||
Action SyncAction `cbor:"0,keyasint"`
|
|
||||||
Config Config `cbor:"1,keyasint,omitempty"`
|
|
||||||
Configs []Config `cbor:"2,keyasint,omitempty"`
|
|
||||||
RunNow bool `cbor:"3,keyasint,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// SyncResponse returns the immediate result for an upsert when requested.
|
|
||||||
type SyncResponse struct {
|
|
||||||
Result Result `cbor:"0,keyasint,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Result holds aggregated monitor results for a single target.
|
|
||||||
//
|
|
||||||
// 0: avg response in microseconds
|
|
||||||
//
|
|
||||||
// 1: 1h average response in microseconds
|
|
||||||
//
|
|
||||||
// 2: min response in microseconds
|
|
||||||
//
|
|
||||||
// 3: 1h min response in microseconds
|
|
||||||
//
|
|
||||||
// 4: max response in microseconds
|
|
||||||
//
|
|
||||||
// 5: 1h max response in microseconds
|
|
||||||
//
|
|
||||||
// 6: packet loss percentage (0-100)
|
|
||||||
//
|
|
||||||
// 7: 1h packet loss percentage (0-100)
|
|
||||||
type Result struct {
|
|
||||||
AvgResponse int64 `cbor:"0,keyasint,omitempty"`
|
|
||||||
AvgResponse1h int64 `cbor:"1,keyasint,omitempty"`
|
|
||||||
MinResponse int64 `cbor:"2,keyasint,omitempty"`
|
|
||||||
MinResponse1h int64 `cbor:"3,keyasint,omitempty"`
|
|
||||||
MaxResponse int64 `cbor:"4,keyasint,omitempty"`
|
|
||||||
MaxResponse1h int64 `cbor:"5,keyasint,omitempty"`
|
|
||||||
PacketLoss float64 `cbor:"6,keyasint,omitempty"`
|
|
||||||
PacketLoss1h float64 `cbor:"7,keyasint,omitempty"`
|
|
||||||
// LastProbeAt is the latest completed probe's Unix timestamp in milliseconds.
|
|
||||||
LastProbeAt int64 `cbor:"8,keyasint"`
|
|
||||||
// SampleCount includes all completed probes since this monitor started.
|
|
||||||
// Used for alert warm-up even when the interval is longer than 20 minutes.
|
|
||||||
SampleCount int64 `cbor:"9,keyasint,omitempty"`
|
|
||||||
// Counts and sum cover the current response window (or latest-sample
|
|
||||||
// fallback), not the hourly window or lifetime SampleCount.
|
|
||||||
TotalCount int64 `cbor:"10,keyasint"`
|
|
||||||
SuccessCount int64 `cbor:"11,keyasint"`
|
|
||||||
ResponseSum int64 `cbor:"12,keyasint"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Stats holds response times in microseconds and packet loss percentage (0-100).
|
|
||||||
type Stats struct {
|
|
||||||
ResAvg float64 `json:"res_avg" db:"-"` // Derived for display; not stored.
|
|
||||||
ResMin float64 `json:"res_min" db:"res_min"`
|
|
||||||
ResMax float64 `json:"res_max" db:"res_max"`
|
|
||||||
Loss float64 `json:"loss" db:"-"` // Derived for display; not stored.
|
|
||||||
TotalCount int64 `json:"-" db:"total_count"`
|
|
||||||
SuccessCount int64 `json:"-" db:"success_count"`
|
|
||||||
ResponseSum int64 `json:"-" db:"res_sum"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s Stats) FromResult(result Result) Stats {
|
|
||||||
return Stats{
|
|
||||||
ResAvg: float64(result.AvgResponse),
|
|
||||||
ResMin: float64(result.MinResponse),
|
|
||||||
ResMax: float64(result.MaxResponse),
|
|
||||||
Loss: result.PacketLoss,
|
|
||||||
TotalCount: result.TotalCount,
|
|
||||||
SuccessCount: result.SuccessCount,
|
|
||||||
ResponseSum: result.ResponseSum,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -60,15 +59,11 @@ type Stats struct {
|
|||||||
|
|
||||||
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
|
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
|
||||||
type ZfsPool struct {
|
type ZfsPool struct {
|
||||||
DisplayName string `json:"n,omitempty" cbor:"8,keyasint,omitempty"`
|
Total float64 `json:"d" cbor:"0,keyasint"` // total capacity in GiB
|
||||||
HideUsage bool `json:"hu,omitempty" cbor:"6,keyasint,omitempty"` // equivalent filesystem usage chart exists
|
Used float64 `json:"du" cbor:"1,keyasint"` // allocated in GiB
|
||||||
HideIO bool `json:"hi,omitempty" cbor:"7,keyasint,omitempty"` // equivalent filesystem I/O chart exists
|
ReadBytes uint64 `json:"rb,omitzero" cbor:"2,keyasint,omitzero"` // read throughput in bytes/s
|
||||||
Raw bool `json:"raw,omitempty" cbor:"5,keyasint,omitempty"`
|
WriteBytes uint64 `json:"wb,omitzero" cbor:"3,keyasint,omitzero"` // write throughput in bytes/s
|
||||||
Total float64 `json:"d" cbor:"0,keyasint"` // total capacity in GiB
|
Health string `json:"h,omitempty" cbor:"4,keyasint,omitempty"` // ONLINE, DEGRADED, FAULTED, ...
|
||||||
Used float64 `json:"du" cbor:"1,keyasint"` // allocated in GiB
|
|
||||||
ReadBytes uint64 `json:"rb,omitzero" cbor:"2,keyasint,omitzero"` // read throughput in bytes/s
|
|
||||||
WriteBytes uint64 `json:"wb,omitzero" cbor:"3,keyasint,omitzero"` // write throughput in bytes/s
|
|
||||||
Health string `json:"h,omitempty" cbor:"4,keyasint,omitempty"` // ONLINE, DEGRADED, FAULTED, ...
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
|
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
|
||||||
@@ -211,6 +206,5 @@ type CombinedData struct {
|
|||||||
Details *Details `cbor:"4,keyasint,omitempty"`
|
Details *Details `cbor:"4,keyasint,omitempty"`
|
||||||
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
|
// SystemdServicesUpdated distinguishes a fresh empty snapshot from a response
|
||||||
// that omitted systemd data (for example, a short-cache dashboard request).
|
// that omitted systemd data (for example, a short-cache dashboard request).
|
||||||
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
|
SystemdServicesUpdated bool `json:"systemdUpdated,omitempty" cbor:"5,keyasint,omitempty"`
|
||||||
Monitors map[string]monitor.Result `cbor:"6,keyasint"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,47 +1,23 @@
|
|||||||
// Package zfs defines the ZFS detail data exchanged between agent and hub.
|
// Package zfs defines the ZFS detail data exchanged between agent and hub.
|
||||||
package zfs
|
package zfs
|
||||||
|
|
||||||
import "strings"
|
|
||||||
|
|
||||||
// ZfsData is the detail payload returned by the agent for the GetZfsData action.
|
// ZfsData is the detail payload returned by the agent for the GetZfsData action.
|
||||||
type ZfsData struct {
|
type ZfsData struct {
|
||||||
Pools []*PoolDetail `json:"pools,omitempty"`
|
Pools []*PoolDetail `json:"pools,omitempty"`
|
||||||
Complete bool `json:"complete,omitempty"`
|
Complete bool `json:"complete,omitempty"`
|
||||||
// Backends whose inventories are complete, even when another backend failed.
|
|
||||||
CompleteBackends []string `json:"completeBackends,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// CanRefreshPool also governs deletion: missing pools may only be removed
|
|
||||||
// after a successful inventory of their backend. Complete supports old agents.
|
|
||||||
func (data *ZfsData) CanRefreshPool(name string) bool {
|
|
||||||
if data.Complete {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
backend := "zfs"
|
|
||||||
if strings.HasPrefix(name, "b:") {
|
|
||||||
backend = "btrfs"
|
|
||||||
}
|
|
||||||
for _, complete := range data.CompleteBackends {
|
|
||||||
if complete == backend {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// PoolDetail holds the verbose state of a single pool: capacity, health,
|
// PoolDetail holds the verbose state of a single pool: capacity, health,
|
||||||
// scrub, vdev, and dataset information.
|
// scrub, vdev, and dataset information.
|
||||||
type PoolDetail struct {
|
type PoolDetail struct {
|
||||||
DisplayName string `json:"displayName,omitempty"`
|
Name string `json:"name"`
|
||||||
Raw bool `json:"raw,omitempty"`
|
Health string `json:"health,omitempty"`
|
||||||
Name string `json:"name"`
|
Size uint64 `json:"size,omitempty"` // bytes
|
||||||
Health string `json:"health,omitempty"`
|
Alloc uint64 `json:"alloc,omitempty"` // bytes
|
||||||
Size uint64 `json:"size,omitempty"` // bytes
|
Free uint64 `json:"free,omitempty"` // bytes
|
||||||
Alloc uint64 `json:"alloc,omitempty"` // bytes
|
Scrub *Scrub `json:"scrub,omitempty"`
|
||||||
Free uint64 `json:"free,omitempty"` // bytes
|
Vdevs []*Vdev `json:"vdevs,omitempty"`
|
||||||
Scrub *Scrub `json:"scrub,omitempty"`
|
Datasets []*Dataset `json:"datasets,omitempty"`
|
||||||
Vdevs []*Vdev `json:"vdevs,omitempty"`
|
|
||||||
Datasets []*Dataset `json:"datasets,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Scrub holds the scrub (or resilver) status of a pool.
|
// Scrub holds the scrub (or resilver) status of a pool.
|
||||||
|
|||||||
+1
-74
@@ -2,17 +2,13 @@ package hub
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
"log/slog"
|
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"uuid"
|
|
||||||
|
|
||||||
"github.com/blang/semver"
|
"github.com/blang/semver"
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
"github.com/henrygd/beszel/internal/alerts"
|
"github.com/henrygd/beszel/internal/alerts"
|
||||||
"github.com/henrygd/beszel/internal/ghupdate"
|
"github.com/henrygd/beszel/internal/ghupdate"
|
||||||
@@ -82,81 +78,12 @@ func (h *Hub) registerMiddlewares(se *core.ServeEvent) {
|
|||||||
}
|
}
|
||||||
// authenticate with trusted header
|
// authenticate with trusted header
|
||||||
if trustedHeader, _ := utils.GetEnv("TRUSTED_AUTH_HEADER"); trustedHeader != "" {
|
if trustedHeader, _ := utils.GetEnv("TRUSTED_AUTH_HEADER"); trustedHeader != "" {
|
||||||
// only honor the header from these peers, if set
|
|
||||||
trustedProxies, restricted := parseTrustedProxies()
|
|
||||||
se.Router.BindFunc(func(e *core.RequestEvent) error {
|
se.Router.BindFunc(func(e *core.RequestEvent) error {
|
||||||
if restricted && !isTrustedProxy(trustedProxies, e.Request.RemoteAddr) {
|
|
||||||
return e.Next()
|
|
||||||
}
|
|
||||||
return authorizeRequestWithEmail(e, e.Request.Header.Get(trustedHeader))
|
return authorizeRequestWithEmail(e, e.Request.Header.Get(trustedHeader))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// parseTrustedProxies reads TRUSTED_PROXY_IPS (comma-separated IPs or CIDRs).
|
|
||||||
// restricted is false when the variable is unset or empty, meaning the trusted
|
|
||||||
// header is accepted from any peer. Invalid entries are skipped with a warning,
|
|
||||||
// so a typo narrows the allowlist rather than widening it.
|
|
||||||
func parseTrustedProxies() (prefixes []netip.Prefix, restricted bool) {
|
|
||||||
value, _ := utils.GetEnv("TRUSTED_PROXY_IPS")
|
|
||||||
if value == "" {
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
for entry := range strings.SplitSeq(value, ",") {
|
|
||||||
entry = strings.TrimSpace(entry)
|
|
||||||
if entry == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if prefix, err := parseProxyPrefix(entry); err == nil {
|
|
||||||
prefixes = append(prefixes, prefix)
|
|
||||||
} else {
|
|
||||||
slog.Warn("Ignoring invalid TRUSTED_PROXY_IPS entry", "entry", entry)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return prefixes, true
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseProxyPrefix parses an IP or CIDR into a masked prefix. IPv4-mapped IPv6
|
|
||||||
// entries are converted to IPv4 so they match IPv4 peers.
|
|
||||||
func parseProxyPrefix(entry string) (netip.Prefix, error) {
|
|
||||||
prefix, err := netip.ParsePrefix(entry)
|
|
||||||
if err != nil {
|
|
||||||
addr, err := netip.ParseAddr(entry)
|
|
||||||
if err != nil {
|
|
||||||
return netip.Prefix{}, err
|
|
||||||
}
|
|
||||||
addr = addr.Unmap()
|
|
||||||
return netip.PrefixFrom(addr, addr.BitLen()), nil
|
|
||||||
}
|
|
||||||
if prefix.Addr().Is4In6() {
|
|
||||||
if prefix.Bits() < 96 {
|
|
||||||
return netip.Prefix{}, fmt.Errorf("%s covers more than the IPv4-mapped range", entry)
|
|
||||||
}
|
|
||||||
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
|
|
||||||
}
|
|
||||||
return prefix.Masked(), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// isTrustedProxy reports whether the peer address of a request (host:port) is
|
|
||||||
// within one of the prefixes.
|
|
||||||
func isTrustedProxy(prefixes []netip.Prefix, remoteAddr string) bool {
|
|
||||||
host, _, err := net.SplitHostPort(remoteAddr)
|
|
||||||
if err != nil {
|
|
||||||
host = remoteAddr
|
|
||||||
}
|
|
||||||
addr, err := netip.ParseAddr(host)
|
|
||||||
if err != nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
addr = addr.Unmap().WithZone("")
|
|
||||||
for _, prefix := range prefixes {
|
|
||||||
if prefix.Contains(addr) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// registerApiRoutes registers custom API routes
|
// registerApiRoutes registers custom API routes
|
||||||
func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
|
func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
|
||||||
// auth protected routes
|
// auth protected routes
|
||||||
|
|||||||
@@ -6,16 +6,12 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
|
||||||
"sort"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/migrations"
|
"github.com/henrygd/beszel/internal/migrations"
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/apis"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
pbTests "github.com/pocketbase/pocketbase/tests"
|
pbTests "github.com/pocketbase/pocketbase/tests"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -30,59 +26,6 @@ func jsonReader(v any) io.Reader {
|
|||||||
return bytes.NewReader(data)
|
return bytes.NewReader(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
type gatedReader struct {
|
|
||||||
data []byte
|
|
||||||
started chan struct{}
|
|
||||||
release chan struct{}
|
|
||||||
offset int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *gatedReader) Read(p []byte) (int, error) {
|
|
||||||
if r.offset == 0 {
|
|
||||||
close(r.started)
|
|
||||||
<-r.release
|
|
||||||
}
|
|
||||||
if r.offset >= len(r.data) {
|
|
||||||
return 0, io.EOF
|
|
||||||
}
|
|
||||||
n := copy(p, r.data[r.offset:])
|
|
||||||
r.offset += n
|
|
||||||
return n, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func firstUserTestMux(t *testing.T) (*beszelTests.TestHub, http.Handler) {
|
|
||||||
t.Helper()
|
|
||||||
hub, err := beszelTests.NewTestHub(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
|
||||||
_ = hub.StartHub()
|
|
||||||
|
|
||||||
router, err := apis.NewRouter(hub.TestApp)
|
|
||||||
require.NoError(t, err)
|
|
||||||
serveEvent := &core.ServeEvent{App: hub.TestApp, Router: router}
|
|
||||||
|
|
||||||
var handler http.Handler
|
|
||||||
err = hub.TestApp.OnServe().Trigger(serveEvent, func(e *core.ServeEvent) error {
|
|
||||||
var buildErr error
|
|
||||||
handler, buildErr = e.Router.BuildMux()
|
|
||||||
return buildErr
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NotNil(t, handler)
|
|
||||||
return hub, handler
|
|
||||||
}
|
|
||||||
|
|
||||||
func postFirstUser(handler http.Handler, email string) *httptest.ResponseRecorder {
|
|
||||||
body, _ := json.Marshal(map[string]string{
|
|
||||||
"email": email,
|
|
||||||
"password": "password123",
|
|
||||||
})
|
|
||||||
req := httptest.NewRequest(http.MethodPost, "/api/beszel/create-user", bytes.NewReader(body))
|
|
||||||
req.Header.Set("Content-Type", "application/json")
|
|
||||||
recorder := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(recorder, req)
|
|
||||||
return recorder
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestApiRoutesAuthentication(t *testing.T) {
|
func TestApiRoutesAuthentication(t *testing.T) {
|
||||||
hub, user := beszelTests.GetHubWithUser(t)
|
hub, user := beszelTests.GetHubWithUser(t)
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
@@ -846,87 +789,6 @@ func TestFirstUserCreation(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestFirstUserBootstrapAtomicity(t *testing.T) {
|
|
||||||
t.Run("concurrent complete requests produce exactly one winner", func(t *testing.T) {
|
|
||||||
hub, handler := firstUserTestMux(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
start := make(chan struct{})
|
|
||||||
statuses := make(chan int, 2)
|
|
||||||
for _, email := range []string{"first@example.com", "second@example.com"} {
|
|
||||||
go func(email string) {
|
|
||||||
<-start
|
|
||||||
statuses <- postFirstUser(handler, email).Code
|
|
||||||
}(email)
|
|
||||||
}
|
|
||||||
close(start)
|
|
||||||
|
|
||||||
got := []int{<-statuses, <-statuses}
|
|
||||||
sort.Ints(got)
|
|
||||||
require.Equal(t, []int{http.StatusOK, http.StatusForbidden}, got)
|
|
||||||
|
|
||||||
users, err := hub.FindAllRecords("users")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, users, 1)
|
|
||||||
superusers, err := hub.FindAllRecords(core.CollectionNameSuperusers)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, superusers, 1)
|
|
||||||
require.NotEqual(t, migrations.TempAdminEmail, superusers[0].Email())
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("partial body cannot retain stale bootstrap authorization", func(t *testing.T) {
|
|
||||||
hub, handler := firstUserTestMux(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
|
|
||||||
body, err := json.Marshal(map[string]string{
|
|
||||||
"email": "parked@example.com",
|
|
||||||
"password": "password123",
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
gated := &gatedReader{
|
|
||||||
data: body,
|
|
||||||
started: make(chan struct{}),
|
|
||||||
release: make(chan struct{}),
|
|
||||||
}
|
|
||||||
parkedRequest := httptest.NewRequest(http.MethodPost, "/api/beszel/create-user", gated)
|
|
||||||
parkedRequest.Header.Set("Content-Type", "application/json")
|
|
||||||
parkedRecorder := httptest.NewRecorder()
|
|
||||||
parkedDone := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
handler.ServeHTTP(parkedRecorder, parkedRequest)
|
|
||||||
close(parkedDone)
|
|
||||||
}()
|
|
||||||
|
|
||||||
select {
|
|
||||||
case <-gated.started:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("parked request did not begin reading its body")
|
|
||||||
}
|
|
||||||
|
|
||||||
operatorRecorder := postFirstUser(handler, "operator@example.com")
|
|
||||||
require.Equal(t, http.StatusOK, operatorRecorder.Code)
|
|
||||||
lateRecorder := postFirstUser(handler, "late@example.com")
|
|
||||||
require.Equal(t, http.StatusForbidden, lateRecorder.Code)
|
|
||||||
|
|
||||||
close(gated.release)
|
|
||||||
select {
|
|
||||||
case <-parkedDone:
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("parked request did not finish")
|
|
||||||
}
|
|
||||||
require.Equal(t, http.StatusForbidden, parkedRecorder.Code)
|
|
||||||
|
|
||||||
users, err := hub.FindAllRecords("users")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, users, 1)
|
|
||||||
require.Equal(t, "operator@example.com", users[0].Email())
|
|
||||||
superusers, err := hub.FindAllRecords(core.CollectionNameSuperusers)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, superusers, 1)
|
|
||||||
require.Equal(t, "operator@example.com", superusers[0].Email())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCreateUserEndpointAvailability(t *testing.T) {
|
func TestCreateUserEndpointAvailability(t *testing.T) {
|
||||||
t.Run("CreateUserEndpoint available when no users exist", func(t *testing.T) {
|
t.Run("CreateUserEndpoint available when no users exist", func(t *testing.T) {
|
||||||
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
||||||
@@ -1107,79 +969,6 @@ func TestTrustedHeaderMiddleware(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTrustedHeaderProxyAllowlist(t *testing.T) {
|
|
||||||
var hubs []*beszelTests.TestHub
|
|
||||||
|
|
||||||
defer func() {
|
|
||||||
for _, hub := range hubs {
|
|
||||||
hub.Cleanup()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
testAppFactory := func(t testing.TB) *pbTests.TestApp {
|
|
||||||
hub, _ := beszelTests.NewTestHub(t.TempDir())
|
|
||||||
hubs = append(hubs, hub)
|
|
||||||
hub.StartHub()
|
|
||||||
return hub.TestApp
|
|
||||||
}
|
|
||||||
|
|
||||||
// httptest requests arrive from 192.0.2.1:1234
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
proxies string
|
|
||||||
expectedStatus int
|
|
||||||
expectedContent []string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "peer inside an allowed range",
|
|
||||||
proxies: "10.0.0.0/8, 192.0.2.0/24",
|
|
||||||
expectedStatus: 200,
|
|
||||||
expectedContent: []string{"\"key\":", "\"v\":"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "peer is the listed address",
|
|
||||||
proxies: "192.0.2.1",
|
|
||||||
expectedStatus: 200,
|
|
||||||
expectedContent: []string{"\"key\":", "\"v\":"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "peer outside the allowlist",
|
|
||||||
proxies: "10.0.0.0/8",
|
|
||||||
expectedStatus: 401,
|
|
||||||
expectedContent: []string{"requires valid"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "allowlist with no valid entry",
|
|
||||||
proxies: "proxy.internal",
|
|
||||||
expectedStatus: 401,
|
|
||||||
expectedContent: []string{"requires valid"},
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_AUTH_HEADER", "X-Beszel-Trusted")
|
|
||||||
t.Setenv("TRUSTED_PROXY_IPS", tc.proxies)
|
|
||||||
|
|
||||||
scenario := beszelTests.ApiScenario{
|
|
||||||
Name: "GET /getkey - with trusted header",
|
|
||||||
Method: http.MethodGet,
|
|
||||||
URL: "/api/beszel/getkey",
|
|
||||||
Headers: map[string]string{
|
|
||||||
"X-Beszel-Trusted": "user@test.com",
|
|
||||||
},
|
|
||||||
ExpectedStatus: tc.expectedStatus,
|
|
||||||
ExpectedContent: tc.expectedContent,
|
|
||||||
TestAppFactory: testAppFactory,
|
|
||||||
BeforeTestFunc: func(t testing.TB, app *pbTests.TestApp, e *core.ServeEvent) {
|
|
||||||
beszelTests.CreateUser(app, "user@test.com", "password123")
|
|
||||||
},
|
|
||||||
}
|
|
||||||
scenario.Test(t)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestUpdateEndpoint(t *testing.T) {
|
func TestUpdateEndpoint(t *testing.T) {
|
||||||
t.Setenv("CHECK_UPDATES", "true")
|
t.Setenv("CHECK_UPDATES", "true")
|
||||||
|
|
||||||
|
|||||||
@@ -78,7 +78,7 @@ func setCollectionAuthSettings(app core.App) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services", "network_monitor_stats"}, collectionRules{
|
if err := applyCollectionRules(app, []string{"containers", "container_stats", "system_stats", "systemd_services"}, collectionRules{
|
||||||
list: &systemScopedReadRule,
|
list: &systemScopedReadRule,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -99,16 +99,6 @@ func setCollectionAuthSettings(app core.App) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
|
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
|
||||||
list: &systemScopedWriteRule,
|
|
||||||
view: &systemScopedWriteRule,
|
|
||||||
create: &systemScopedWriteRule,
|
|
||||||
update: &systemScopedWriteRule,
|
|
||||||
delete: &systemScopedWriteRule,
|
|
||||||
}); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := applyCollectionRules(app, []string{"network_monitors"}, collectionRules{
|
|
||||||
list: &systemScopedReadRule,
|
list: &systemScopedReadRule,
|
||||||
view: &systemScopedReadRule,
|
view: &systemScopedReadRule,
|
||||||
create: &systemScopedWriteRule,
|
create: &systemScopedWriteRule,
|
||||||
|
|||||||
@@ -88,8 +88,8 @@ func TestCollectionRulesDefault(t *testing.T) {
|
|||||||
// fingerprints collection
|
// fingerprints collection
|
||||||
fingerprintsCollection, err := hub.FindCollectionByNameOrId("fingerprints")
|
fingerprintsCollection, err := hub.FindCollectionByNameOrId("fingerprints")
|
||||||
require.NoError(t, err, "Failed to find fingerprints collection")
|
require.NoError(t, err, "Failed to find fingerprints collection")
|
||||||
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.ListRule)
|
assert.Equal(t, isUserInSystemUsers, *fingerprintsCollection.ListRule)
|
||||||
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.ViewRule)
|
assert.Equal(t, isUserInSystemUsers, *fingerprintsCollection.ViewRule)
|
||||||
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.CreateRule)
|
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.CreateRule)
|
||||||
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.UpdateRule)
|
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.UpdateRule)
|
||||||
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.DeleteRule)
|
assert.Equal(t, isUserInSystemUsersNotReadonly, *fingerprintsCollection.DeleteRule)
|
||||||
@@ -216,8 +216,8 @@ func TestCollectionRulesShareAllSystems(t *testing.T) {
|
|||||||
// fingerprints collection
|
// fingerprints collection
|
||||||
fingerprintsCollection, err := hub.FindCollectionByNameOrId("fingerprints")
|
fingerprintsCollection, err := hub.FindCollectionByNameOrId("fingerprints")
|
||||||
require.NoError(t, err, "Failed to find fingerprints collection")
|
require.NoError(t, err, "Failed to find fingerprints collection")
|
||||||
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.ListRule)
|
assert.Equal(t, isUser, *fingerprintsCollection.ListRule)
|
||||||
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.ViewRule)
|
assert.Equal(t, isUser, *fingerprintsCollection.ViewRule)
|
||||||
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.CreateRule)
|
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.CreateRule)
|
||||||
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.UpdateRule)
|
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.UpdateRule)
|
||||||
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.DeleteRule)
|
assert.Equal(t, isUserNotReadonly, *fingerprintsCollection.DeleteRule)
|
||||||
|
|||||||
@@ -6,8 +6,8 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"uuid"
|
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
|
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
|
|||||||
@@ -106,12 +106,9 @@ func (h *Hub) StartHub() error {
|
|||||||
|
|
||||||
// TODO: move to users package
|
// TODO: move to users package
|
||||||
// handle default values for user / user_settings creation
|
// handle default values for user / user_settings creation
|
||||||
h.App.OnRecordAuthWithOAuth2Request("users").BindFunc(h.um.InitializeOAuthUserRole)
|
|
||||||
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
|
h.App.OnRecordCreate("users").BindFunc(h.um.InitializeUserRole)
|
||||||
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
|
h.App.OnRecordCreate("user_settings").BindFunc(h.um.InitializeUserSettings)
|
||||||
|
|
||||||
bindNetworkMonitorsEvents(h)
|
|
||||||
|
|
||||||
pb, ok := h.App.(*pocketbase.PocketBase)
|
pb, ok := h.App.(*pocketbase.PocketBase)
|
||||||
if !ok {
|
if !ok {
|
||||||
return errors.New("not a pocketbase app")
|
return errors.New("not a pocketbase app")
|
||||||
@@ -125,8 +122,6 @@ func (h *Hub) initialize(app core.App) error {
|
|||||||
settings := app.Settings()
|
settings := app.Settings()
|
||||||
// batch requests (for alerts)
|
// batch requests (for alerts)
|
||||||
settings.Batch.Enabled = true
|
settings.Batch.Enabled = true
|
||||||
settings.Batch.MaxRequests = 100
|
|
||||||
settings.Batch.MaxBodySize = 1 << 20 // 1 MiB
|
|
||||||
// set URL if APP_URL env is set
|
// set URL if APP_URL env is set
|
||||||
if appURL, isSet := utils.GetEnv("APP_URL"); isSet {
|
if appURL, isSet := utils.GetEnv("APP_URL"); isSet {
|
||||||
h.appURL = appURL
|
h.appURL = appURL
|
||||||
|
|||||||
@@ -1,158 +0,0 @@
|
|||||||
package hub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strconv"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/hub/systems"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
// generateMonitorID creates a stable hash ID for a monitor based on its configuration and the system it belongs to.
|
|
||||||
func generateMonitorID(systemId string, config monitor.Config) string {
|
|
||||||
args := []string{systemId, config.Target, config.Protocol}
|
|
||||||
// only use port for TCP monitors, since for other protocols it's not relevant as standalone value
|
|
||||||
if config.Protocol == "tcp" {
|
|
||||||
args = append(args, strconv.FormatUint(uint64(config.Port), 10))
|
|
||||||
}
|
|
||||||
return systems.MakeStableHashId(args...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// bindNetworkMonitorsEvents keeps monitor records and agent monitor state in sync.
|
|
||||||
func bindNetworkMonitorsEvents(hub *Hub) {
|
|
||||||
// on create, make sure the id is set to a stable hash
|
|
||||||
hub.OnRecordCreate("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
systemID := e.Record.GetString("system")
|
|
||||||
config := monitorConfigFromRecord(e.Record)
|
|
||||||
id := generateMonitorID(systemID, *config)
|
|
||||||
e.Record.Set("id", id)
|
|
||||||
return e.Next()
|
|
||||||
})
|
|
||||||
|
|
||||||
// sync monitor to agent on creation and persist the first result immediately when available
|
|
||||||
hub.OnRecordAfterCreateSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
err := e.Next()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if !e.Record.GetBool("enabled") {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
// If connected, run the monitor immediately. Paused systems may be absent
|
|
||||||
// from the manager; their monitors will sync when they reconnect.
|
|
||||||
system, err := hub.sm.GetSystem(e.Record.GetString("system"))
|
|
||||||
if err == nil && system.Status == "up" {
|
|
||||||
go hub.upsertNetworkMonitor(e.Record, true)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
// On API update requests, if the monitor config changed in a way that requires a new ID, create a new
|
|
||||||
// record with the new ID and delete the old one. Otherwise, just update the existing monitor on the agent.
|
|
||||||
hub.OnRecordUpdateRequest("network_monitors").BindFunc(func(e *core.RecordRequestEvent) error {
|
|
||||||
systemID := e.Record.GetString("system")
|
|
||||||
// only tcp uses port - set other protocols port to zero
|
|
||||||
if e.Record.GetString("protocol") != "tcp" {
|
|
||||||
e.Record.Set("port", 0)
|
|
||||||
}
|
|
||||||
ID := generateMonitorID(systemID, *monitorConfigFromRecord(e.Record))
|
|
||||||
if ID != e.Record.Id {
|
|
||||||
newRecord := copyMonitorToNewRecord(e.Record, ID)
|
|
||||||
if err := e.App.Save(newRecord); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if err := e.App.Delete(e.Record); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
err := e.Next()
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if e.Record.GetBool("enabled") {
|
|
||||||
// if the monitor is enabled, sync the updated config to the agent now
|
|
||||||
runNow := !e.Record.Original().GetBool("enabled")
|
|
||||||
err = hub.upsertNetworkMonitor(e.Record, runNow)
|
|
||||||
} else {
|
|
||||||
// if the monitor is paused, remove it from the agent
|
|
||||||
err = hub.deleteNetworkMonitor(e.Record)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
hub.Logger().Warn("failed to sync updated monitor", "system", systemID, "monitor", e.Record.Id, "err", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
// sync monitor to agent on delete
|
|
||||||
hub.OnRecordAfterDeleteSuccess("network_monitors").BindFunc(func(e *core.RecordEvent) error {
|
|
||||||
if err := hub.deleteNetworkMonitor(e.Record); err != nil {
|
|
||||||
hub.Logger().Warn("failed to delete monitor on agent", "system", e.Record.GetString("system"), "monitor", e.Record.Id, "err", err)
|
|
||||||
}
|
|
||||||
return e.Next()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// monitorConfigFromRecord builds a monitor config from a network_monitors record.
|
|
||||||
func monitorConfigFromRecord(record *core.Record) *monitor.Config {
|
|
||||||
return &monitor.Config{
|
|
||||||
ID: record.Id,
|
|
||||||
Target: record.GetString("target"),
|
|
||||||
Protocol: record.GetString("protocol"),
|
|
||||||
Port: uint16(record.GetInt("port")),
|
|
||||||
Interval: uint16(record.GetInt("interval")),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setMonitorResultFields stores the latest monitor result values on the record.
|
|
||||||
func setMonitorResultFields(record *core.Record, result monitor.Result) {
|
|
||||||
nowString := time.Now().UTC().Format(types.DefaultDateLayout)
|
|
||||||
record.Set("res", result.AvgResponse)
|
|
||||||
record.Set("resAvg1h", result.AvgResponse1h)
|
|
||||||
record.Set("resMin1h", result.MinResponse1h)
|
|
||||||
record.Set("resMax1h", result.MaxResponse1h)
|
|
||||||
record.Set("loss1h", result.PacketLoss1h)
|
|
||||||
record.Set("updated", nowString)
|
|
||||||
}
|
|
||||||
|
|
||||||
// copyMonitorToNewRecord creates a new record with the same field values as the old one.
|
|
||||||
// This is used when the monitor config changes in a way that requires a new ID, so we need
|
|
||||||
// to create a new record with the new ID and delete the old one.
|
|
||||||
func copyMonitorToNewRecord(oldRecord *core.Record, newID string) *core.Record {
|
|
||||||
collection := oldRecord.Collection()
|
|
||||||
newRecord := core.NewRecord(collection)
|
|
||||||
newRecord.Id = newID
|
|
||||||
fields := []string{"system", "target", "protocol", "port", "interval", "enabled"}
|
|
||||||
for _, field := range fields {
|
|
||||||
newRecord.Set(field, oldRecord.Get(field))
|
|
||||||
}
|
|
||||||
return newRecord
|
|
||||||
}
|
|
||||||
|
|
||||||
// upsertNetworkMonitor creates or updates the record's monitor on the target system. If runNow
|
|
||||||
// is true, it will also trigger an immediate monitor run and update the record with the result.
|
|
||||||
func (h *Hub) upsertNetworkMonitor(record *core.Record, runNow bool) error {
|
|
||||||
systemID := record.GetString("system")
|
|
||||||
system, err := h.sm.GetSystem(systemID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
result, err := system.UpsertNetworkMonitor(*monitorConfigFromRecord(record), runNow)
|
|
||||||
if err != nil || result == nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
setMonitorResultFields(record, *result)
|
|
||||||
return h.App.SaveNoValidate(record)
|
|
||||||
}
|
|
||||||
|
|
||||||
// deleteNetworkMonitor removes the record's monitor from the target system.
|
|
||||||
func (h *Hub) deleteNetworkMonitor(record *core.Record) error {
|
|
||||||
systemID := record.GetString("system")
|
|
||||||
system, err := h.sm.GetSystem(systemID)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return system.DeleteNetworkMonitor(record.Id)
|
|
||||||
}
|
|
||||||
@@ -1,225 +0,0 @@
|
|||||||
package hub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"encoding/json"
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/pocketbase/pocketbase/apis"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCreateNetworkMonitorsOnPausedSystem(t *testing.T) {
|
|
||||||
for _, batch := range []bool{false, true} {
|
|
||||||
name := "single"
|
|
||||||
if batch {
|
|
||||||
name = "batch"
|
|
||||||
}
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
hub, testApp, err := createTestHub(t)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer cleanupTestHub(hub, testApp)
|
|
||||||
bindNetworkMonitorsEvents(hub)
|
|
||||||
|
|
||||||
user, err := createTestUser(hub)
|
|
||||||
require.NoError(t, err)
|
|
||||||
system, err := createTestRecord(hub, "systems", map[string]any{
|
|
||||||
"name": "Paused", "host": "localhost", "port": "45876",
|
|
||||||
"status": "paused", "users": []string{user.Id},
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
// Paused systems are not loaded into the manager at startup.
|
|
||||||
_, err = hub.sm.GetSystem(system.Id)
|
|
||||||
require.Error(t, err)
|
|
||||||
|
|
||||||
payload := func(target string) map[string]any {
|
|
||||||
return map[string]any{
|
|
||||||
"system": system.Id, "target": target, "protocol": "icmp",
|
|
||||||
"interval": 60, "enabled": true,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
url := "/api/collections/network_monitors/records"
|
|
||||||
var body any = payload("1.1.1.1")
|
|
||||||
count := 1
|
|
||||||
if batch {
|
|
||||||
body = map[string]any{"requests": []map[string]any{
|
|
||||||
{"method": "POST", "url": url, "body": payload("1.1.1.1")},
|
|
||||||
{"method": "POST", "url": url, "body": payload("8.8.8.8")},
|
|
||||||
}}
|
|
||||||
url = "/api/batch"
|
|
||||||
count = 2
|
|
||||||
}
|
|
||||||
data, err := json.Marshal(body)
|
|
||||||
require.NoError(t, err)
|
|
||||||
token, err := user.NewAuthToken()
|
|
||||||
require.NoError(t, err)
|
|
||||||
router, err := apis.NewRouter(hub)
|
|
||||||
require.NoError(t, err)
|
|
||||||
handler, err := router.BuildMux()
|
|
||||||
require.NoError(t, err)
|
|
||||||
request := httptest.NewRequest(http.MethodPost, url, bytes.NewReader(data))
|
|
||||||
request.Header.Set("Content-Type", "application/json")
|
|
||||||
request.Header.Set("Authorization", token)
|
|
||||||
response := httptest.NewRecorder()
|
|
||||||
handler.ServeHTTP(response, request)
|
|
||||||
assert.Equal(t, http.StatusOK, response.Code, response.Body.String())
|
|
||||||
|
|
||||||
records, err := hub.FindAllRecords("network_monitors")
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, records, count)
|
|
||||||
for _, record := range records {
|
|
||||||
assert.Equal(t, system.Id, record.GetString("system"))
|
|
||||||
assert.True(t, record.GetBool("enabled"))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGenerateMonitorID(t *testing.T) {
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
systemID string
|
|
||||||
config monitor.Config
|
|
||||||
expected string
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "HTTP monitor on example.com",
|
|
||||||
systemID: "sys123",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "http",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 0,
|
|
||||||
Interval: 60,
|
|
||||||
},
|
|
||||||
expected: "a20a5827",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP monitor on example.com with different port",
|
|
||||||
systemID: "sys123",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "http",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 8080,
|
|
||||||
Interval: 60,
|
|
||||||
},
|
|
||||||
expected: "a20a5827",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "HTTP monitor on example.com with different system ID",
|
|
||||||
systemID: "sys1234",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "http",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 80,
|
|
||||||
Interval: 60,
|
|
||||||
},
|
|
||||||
expected: "ab602ae7",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "Same monitor, different interval",
|
|
||||||
systemID: "sys1234",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "http",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 80,
|
|
||||||
Interval: 120,
|
|
||||||
},
|
|
||||||
expected: "ab602ae7",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ICMP monitor on 1.1.1.1",
|
|
||||||
systemID: "sys456",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "icmp",
|
|
||||||
Target: "1.1.1.1",
|
|
||||||
Port: 0,
|
|
||||||
Interval: 10,
|
|
||||||
},
|
|
||||||
expected: "6d13a4a4",
|
|
||||||
}, {
|
|
||||||
name: "ICMP monitor on 1.1.1.1 with different system ID",
|
|
||||||
systemID: "sys4567",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "icmp",
|
|
||||||
Target: "1.1.1.1",
|
|
||||||
Port: 0,
|
|
||||||
Interval: 10,
|
|
||||||
},
|
|
||||||
expected: "ddd6c81",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "TCP monitor on example.com with port 443",
|
|
||||||
systemID: "sys789",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "tcp",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 443,
|
|
||||||
Interval: 30,
|
|
||||||
},
|
|
||||||
expected: "677b991",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "TCP monitor on example.com with port 8443",
|
|
||||||
systemID: "sys789",
|
|
||||||
config: monitor.Config{
|
|
||||||
Protocol: "tcp",
|
|
||||||
Target: "example.com",
|
|
||||||
Port: 8443,
|
|
||||||
Interval: 30,
|
|
||||||
},
|
|
||||||
expected: "84167969",
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tt := range tests {
|
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
|
||||||
got := generateMonitorID(tt.systemID, tt.config)
|
|
||||||
assert.Equal(t, tt.expected, got, "generateMonitorID() = %v, want %v", got, tt.expected)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCopyMonitorToNewRecordDropsResultFields(t *testing.T) {
|
|
||||||
hub, testApp, err := createTestHub(t)
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer cleanupTestHub(hub, testApp)
|
|
||||||
|
|
||||||
collection, err := hub.FindCachedCollectionByNameOrId("network_monitors")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Nil(t, collection.Fields.GetByName("name"))
|
|
||||||
|
|
||||||
oldRecord := core.NewRecord(collection)
|
|
||||||
oldRecord.Load(map[string]any{
|
|
||||||
"system": "sys123",
|
|
||||||
"target": "https://example.com",
|
|
||||||
"protocol": "http",
|
|
||||||
"port": 443,
|
|
||||||
"interval": 60,
|
|
||||||
"enabled": true,
|
|
||||||
"res": 1200,
|
|
||||||
"resAvg1h": 1300,
|
|
||||||
"resMin1h": 900,
|
|
||||||
"resMax1h": 1600,
|
|
||||||
"loss1h": 5,
|
|
||||||
"updated": "2026-04-29 12:00:00.000Z",
|
|
||||||
})
|
|
||||||
|
|
||||||
newRecord := copyMonitorToNewRecord(oldRecord, "next12345")
|
|
||||||
|
|
||||||
assert.Equal(t, "next12345", newRecord.Id)
|
|
||||||
assert.Equal(t, "https://example.com", newRecord.GetString("target"))
|
|
||||||
assert.Equal(t, "http", newRecord.GetString("protocol"))
|
|
||||||
assert.Equal(t, 443, newRecord.GetInt("port"))
|
|
||||||
assert.True(t, newRecord.GetBool("enabled"))
|
|
||||||
assert.Zero(t, newRecord.GetFloat("res"))
|
|
||||||
assert.Zero(t, newRecord.GetFloat("resAvg1h"))
|
|
||||||
assert.Zero(t, newRecord.GetFloat("resMin1h"))
|
|
||||||
assert.Zero(t, newRecord.GetFloat("resMax1h"))
|
|
||||||
assert.Zero(t, newRecord.GetFloat("loss1h"))
|
|
||||||
assert.Equal(t, "", newRecord.GetString("updated"))
|
|
||||||
}
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCreateContainerRecordsPersistsImageUpdateAvailability(t *testing.T) {
|
|
||||||
_, app := newTestSystemWithHub(t)
|
|
||||||
|
|
||||||
const (
|
|
||||||
systemID = "system123"
|
|
||||||
containerID = "abcdef123456"
|
|
||||||
image = "nginx:latest"
|
|
||||||
)
|
|
||||||
|
|
||||||
data := &container.Stats{
|
|
||||||
Id: containerID,
|
|
||||||
Name: "web",
|
|
||||||
Image: image,
|
|
||||||
UpdateAvailable: true,
|
|
||||||
}
|
|
||||||
require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
|
|
||||||
|
|
||||||
var record struct {
|
|
||||||
Image string `db:"image"`
|
|
||||||
UpdateAvailable bool `db:"updatable"`
|
|
||||||
}
|
|
||||||
require.NoError(t, app.DB().Select("image", "updatable").From("containers").
|
|
||||||
Where(dbx.HashExp{"id": containerID}).One(&record))
|
|
||||||
assert.Equal(t, image, record.Image)
|
|
||||||
assert.True(t, record.UpdateAvailable)
|
|
||||||
|
|
||||||
data.UpdateAvailable = false
|
|
||||||
require.NoError(t, createContainerRecords(app, []*container.Stats{data}, systemID))
|
|
||||||
require.NoError(t, app.DB().Select("image", "updatable").From("containers").
|
|
||||||
Where(dbx.HashExp{"id": containerID}).One(&record))
|
|
||||||
assert.Equal(t, image, record.Image)
|
|
||||||
assert.False(t, record.UpdateAvailable)
|
|
||||||
}
|
|
||||||
@@ -1,224 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/subscriptions"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestNetworkMonitorProbePruning(t *testing.T) {
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
monitors map[string]monitor.Result
|
|
||||||
fail bool
|
|
||||||
want map[string]int64
|
|
||||||
}{
|
|
||||||
{"nil report", nil, false, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
|
|
||||||
{"empty report", map[string]monitor.Result{}, false, map[string]int64{}},
|
|
||||||
{"removed monitor", map[string]monitor.Result{"monitor1": {LastProbeAt: 1000}}, false, map[string]int64{"monitor1": 1000}},
|
|
||||||
{"rolled back report", map[string]monitor.Result{}, true, map[string]int64{"monitor1": 1000, "monitor2": 1000}},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
sys.lastSavedMonitorProbe = map[string]int64{"monitor1": 1000, "monitor2": 1000}
|
|
||||||
// Preserve the distinction between nil and empty across the agent transport.
|
|
||||||
encoded, err := cbor.Marshal(system.CombinedData{Monitors: tc.monitors})
|
|
||||||
require.NoError(t, err)
|
|
||||||
var data system.CombinedData
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &data))
|
|
||||||
if tc.fail {
|
|
||||||
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
_, err = sys.createRecords(&data)
|
|
||||||
if tc.fail {
|
|
||||||
require.Error(t, err)
|
|
||||||
} else {
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.want, sys.lastSavedMonitorProbe)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorStatsFreshness(t *testing.T) {
|
|
||||||
for _, realtime := range []bool{false, true} {
|
|
||||||
name := "sql"
|
|
||||||
if realtime {
|
|
||||||
name = "realtime"
|
|
||||||
}
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
if realtime {
|
|
||||||
client := subscriptions.NewDefaultClient()
|
|
||||||
client.Subscribe("network_monitors/*")
|
|
||||||
app.SubscriptionsBroker().Register(client)
|
|
||||||
t.Cleanup(func() { app.SubscriptionsBroker().Unregister(client.Id()) })
|
|
||||||
}
|
|
||||||
col, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, id := range []string{"monitor1", "monitor2"} {
|
|
||||||
record := core.NewRecord(col)
|
|
||||||
record.Id = id
|
|
||||||
record.Set("system", sys.Id)
|
|
||||||
require.NoError(t, app.SaveNoValidate(record))
|
|
||||||
}
|
|
||||||
data := &system.CombinedData{Monitors: map[string]monitor.Result{
|
|
||||||
"monitor1": {LastProbeAt: 1000, AvgResponse: 20, TotalCount: 6, SuccessCount: 6, ResponseSum: 123},
|
|
||||||
"monitor2": {LastProbeAt: 1000, PacketLoss: 100, TotalCount: 1},
|
|
||||||
}}
|
|
||||||
count := func(want int64) {
|
|
||||||
t.Helper()
|
|
||||||
got, err := app.CountRecords("network_monitor_stats")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, want, got)
|
|
||||||
}
|
|
||||||
save := func() {
|
|
||||||
t.Helper()
|
|
||||||
_, err := sys.createRecords(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
save()
|
|
||||||
count(2)
|
|
||||||
stored, err := app.FindAllRecords("network_monitor_stats")
|
|
||||||
require.NoError(t, err)
|
|
||||||
for _, record := range stored {
|
|
||||||
result := data.Monitors[record.GetString("monitor")]
|
|
||||||
assert.EqualValues(t, result.TotalCount, record.GetInt("total_count"))
|
|
||||||
assert.EqualValues(t, result.SuccessCount, record.GetInt("success_count"))
|
|
||||||
assert.EqualValues(t, result.ResponseSum, record.GetInt("res_sum"))
|
|
||||||
}
|
|
||||||
// A resume can overlap the scheduled update with the same probe.
|
|
||||||
errs := make(chan error, 4)
|
|
||||||
for range 4 {
|
|
||||||
go func() {
|
|
||||||
_, err := sys.createRecords(data)
|
|
||||||
errs <- err
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
for range 4 {
|
|
||||||
require.NoError(t, <-errs)
|
|
||||||
}
|
|
||||||
count(2)
|
|
||||||
|
|
||||||
// A rolling hourly value can change without a new probe.
|
|
||||||
result := data.Monitors["monitor1"]
|
|
||||||
result.AvgResponse1h = 42
|
|
||||||
data.Monitors["monitor1"] = result
|
|
||||||
save()
|
|
||||||
count(2)
|
|
||||||
record, err := app.FindRecordById("network_monitors", "monitor1")
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 42, record.GetInt("resAvg1h"))
|
|
||||||
|
|
||||||
// Identical response values and failed probes still count as new measurements.
|
|
||||||
for id, result := range data.Monitors {
|
|
||||||
result.LastProbeAt = 301000
|
|
||||||
data.Monitors[id] = result
|
|
||||||
}
|
|
||||||
save()
|
|
||||||
count(4)
|
|
||||||
|
|
||||||
// A failed individual insert must remain retryable, even if others commit.
|
|
||||||
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_monitor_insert BEFORE INSERT ON network_monitor_stats WHEN NEW.monitor = 'monitor1' BEGIN SELECT RAISE(ABORT, 'test insert failure'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
for id, result := range data.Monitors {
|
|
||||||
result.LastProbeAt = 601000
|
|
||||||
data.Monitors[id] = result
|
|
||||||
}
|
|
||||||
save()
|
|
||||||
count(5)
|
|
||||||
assert.Equal(t, int64(301000), sys.lastSavedMonitorProbe["monitor1"])
|
|
||||||
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor2"])
|
|
||||||
_, err = app.DB().NewQuery("DROP TRIGGER fail_monitor_insert").Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
save()
|
|
||||||
count(6)
|
|
||||||
|
|
||||||
// Failure after inserting stats rolls back the whole transaction and its markers.
|
|
||||||
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system_update BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
result = data.Monitors["monitor1"]
|
|
||||||
result.LastProbeAt = 901000
|
|
||||||
data.Monitors["monitor1"] = result
|
|
||||||
_, err = sys.createRecords(data)
|
|
||||||
require.Error(t, err)
|
|
||||||
count(6)
|
|
||||||
assert.Equal(t, int64(601000), sys.lastSavedMonitorProbe["monitor1"])
|
|
||||||
_, err = app.DB().NewQuery("DROP TRIGGER fail_system_update").Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
save()
|
|
||||||
count(7)
|
|
||||||
|
|
||||||
// Clock rollback is a new probe identity, not a reason to stall writes.
|
|
||||||
result.LastProbeAt = 500
|
|
||||||
data.Monitors["monitor1"] = result
|
|
||||||
save()
|
|
||||||
count(8)
|
|
||||||
|
|
||||||
// Recreated systems intentionally accept the first result without restoring state.
|
|
||||||
sys = &System{Id: sys.Id, manager: sys.manager}
|
|
||||||
save()
|
|
||||||
count(10)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Observes the committed DB through the hub, not the transaction's app.
|
|
||||||
type monitorAlertHub struct {
|
|
||||||
stubHub
|
|
||||||
handle func(*core.Record, map[string]monitor.Result) error
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h monitorAlertHub) HandleNetworkMonitorAlerts(record *core.Record, results map[string]monitor.Result) error {
|
|
||||||
return h.handle(record, results)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorAlertsAfterCommit(t *testing.T) {
|
|
||||||
for _, realtime := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprint(realtime), func(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
if realtime {
|
|
||||||
client := subscriptions.NewDefaultClient()
|
|
||||||
client.Subscribe("network_monitors/*")
|
|
||||||
app.SubscriptionsBroker().Register(client)
|
|
||||||
}
|
|
||||||
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
|
||||||
require.NoError(t, err)
|
|
||||||
record := core.NewRecord(collection)
|
|
||||||
record.Set("system", sys.Id)
|
|
||||||
require.NoError(t, app.SaveNoValidate(record))
|
|
||||||
called := 0
|
|
||||||
result := monitor.Result{LastProbeAt: time.Now().UnixMilli(), SampleCount: 3, PacketLoss1h: 10}
|
|
||||||
sys.manager.hub = monitorAlertHub{stubHub: stubHub{app}, handle: func(systemRecord *core.Record, results map[string]monitor.Result) error {
|
|
||||||
called++
|
|
||||||
assert.Equal(t, sys.Id, systemRecord.Id)
|
|
||||||
assert.Equal(t, result, results[record.Id])
|
|
||||||
saved, err := app.FindRecordById("network_monitors", record.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 10.0, saved.GetFloat("loss1h"))
|
|
||||||
return nil
|
|
||||||
}}
|
|
||||||
data := &system.CombinedData{Monitors: map[string]monitor.Result{record.Id: result}}
|
|
||||||
_, err = sys.createRecords(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, 1, called)
|
|
||||||
// A transaction that fails after writing monitor stats must not notify.
|
|
||||||
_, err = app.DB().NewQuery(`CREATE TRIGGER fail_system BEFORE UPDATE ON systems BEGIN SELECT RAISE(ABORT, 'test rollback'); END`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = sys.createRecords(data)
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.Equal(t, 1, called)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,144 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/http"
|
|
||||||
"net/http/httptest"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/blang/semver"
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/hub/ws"
|
|
||||||
"github.com/lxzan/gws"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
type monitorSyncClient struct {
|
|
||||||
gws.BuiltinEventHandler
|
|
||||||
requests chan common.HubRequest[monitor.SyncRequest]
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *monitorSyncClient) OnMessage(conn *gws.Conn, message *gws.Message) {
|
|
||||||
defer message.Close()
|
|
||||||
var req common.HubRequest[monitor.SyncRequest]
|
|
||||||
if err := cbor.Unmarshal(message.Bytes(), &req); err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c.requests <- req
|
|
||||||
data, _ := cbor.Marshal(monitor.SyncResponse{})
|
|
||||||
response, _ := cbor.Marshal(common.AgentResponse{Id: req.Id, Data: data})
|
|
||||||
_ = conn.WriteMessage(gws.OpcodeBinary, response)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Avoid the production delayed disconnect notification; these tests explicitly
|
|
||||||
// remove each connection from the manager before reconnecting.
|
|
||||||
type monitorSyncServer struct{ ws.Handler }
|
|
||||||
|
|
||||||
func (*monitorSyncServer) OnClose(*gws.Conn, error) {}
|
|
||||||
|
|
||||||
func TestNetworkMonitorSyncSkipsOlderAgents(t *testing.T) {
|
|
||||||
for _, version := range []string{"0.0.0", "0.18.0", "0.19.0"} {
|
|
||||||
t.Run(version, func(t *testing.T) {
|
|
||||||
// No transport: attempting to send any request would fail.
|
|
||||||
sys := &System{agentVersion: semver.MustParse(version)}
|
|
||||||
require.NoError(t, sys.SyncNetworkMonitors(nil))
|
|
||||||
result, err := sys.UpsertNetworkMonitor(monitor.Config{ID: "test"}, true)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Nil(t, result)
|
|
||||||
require.NoError(t, sys.DeleteNetworkMonitor("test"))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestNetworkMonitorReconnectSync(t *testing.T) {
|
|
||||||
for _, change := range []string{"delete", "disable"} {
|
|
||||||
t.Run(change, func(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
record, err := app.FindRecordById("systems", sys.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
// Suppress unrelated system-stat requests while exercising reconnects.
|
|
||||||
record.Set("status", paused)
|
|
||||||
require.NoError(t, app.SaveNoValidate(record))
|
|
||||||
collection, err := app.FindCachedCollectionByNameOrId("network_monitors")
|
|
||||||
require.NoError(t, err)
|
|
||||||
probe := core.NewRecord(collection)
|
|
||||||
probe.Load(map[string]any{
|
|
||||||
"system": sys.Id, "target": "localhost", "protocol": "tcp",
|
|
||||||
"port": 80, "interval": 60, "enabled": true,
|
|
||||||
})
|
|
||||||
require.NoError(t, app.SaveNoValidate(probe))
|
|
||||||
|
|
||||||
sm := NewSystemManager(stubHub{app})
|
|
||||||
t.Cleanup(func() {
|
|
||||||
sm.cancel()
|
|
||||||
_ = sm.RemoveSystem(sys.Id)
|
|
||||||
sm.smartFetchMap.StopCleaner()
|
|
||||||
sm.zfsFetchMap.StopCleaner()
|
|
||||||
})
|
|
||||||
version := semver.MustParse("0.20.0")
|
|
||||||
connections := make(chan *ws.WsConn, 1)
|
|
||||||
upgrader := gws.NewUpgrader(&monitorSyncServer{}, nil)
|
|
||||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
conn, err := upgrader.Upgrade(w, r)
|
|
||||||
if err != nil {
|
|
||||||
t.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wsConn := ws.NewWsConnection(conn, version)
|
|
||||||
conn.Session().Store("wsConn", wsConn)
|
|
||||||
connections <- wsConn
|
|
||||||
conn.ReadLoop()
|
|
||||||
}))
|
|
||||||
t.Cleanup(server.Close)
|
|
||||||
client := &monitorSyncClient{requests: make(chan common.HubRequest[monitor.SyncRequest], 2)}
|
|
||||||
connect := func() monitor.SyncRequest {
|
|
||||||
t.Helper()
|
|
||||||
conn, _, err := gws.NewClient(client, &gws.ClientOption{Addr: "ws" + strings.TrimPrefix(server.URL, "http")})
|
|
||||||
require.NoError(t, err)
|
|
||||||
t.Cleanup(func() { _ = conn.NetConn().Close() })
|
|
||||||
go conn.ReadLoop()
|
|
||||||
select {
|
|
||||||
case wsConn := <-connections:
|
|
||||||
require.NoError(t, sm.AddWebSocketSystem(sys.Id, version, wsConn))
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("websocket connection was not established")
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case req := <-client.requests:
|
|
||||||
require.Equal(t, common.SyncNetworkMonitors, req.Action)
|
|
||||||
require.Equal(t, monitor.SyncActionReplace, req.Data.Action)
|
|
||||||
return req.Data
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("reconnected agent did not receive a monitor replacement")
|
|
||||||
return monitor.SyncRequest{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
initial := connect()
|
|
||||||
require.Len(t, initial.Configs, 1)
|
|
||||||
require.Equal(t, probe.Id, initial.Configs[0].ID)
|
|
||||||
require.NoError(t, sm.RemoveSystem(sys.Id))
|
|
||||||
if change == "delete" {
|
|
||||||
require.NoError(t, app.Delete(probe))
|
|
||||||
} else {
|
|
||||||
probe.Set("enabled", false)
|
|
||||||
require.NoError(t, app.SaveNoValidate(probe))
|
|
||||||
}
|
|
||||||
require.Empty(t, connect().Configs, "reconnect must clear the agent's previous probe")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestGetMonitorConfigsForSystemQueryError(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
_, err := app.DB().NewQuery("DROP TABLE network_monitors").Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = sys.manager.GetMonitorConfigsForSystem(sys.Id)
|
|
||||||
require.Error(t, err, "a failed query must not be treated as an empty monitor set")
|
|
||||||
}
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel"
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SyncNetworkMonitors sends monitor configurations to the agent.
|
|
||||||
func (sys *System) SyncNetworkMonitors(configs []monitor.Config) error {
|
|
||||||
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{Action: monitor.SyncActionReplace, Configs: configs})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// UpsertNetworkMonitor sends a single monitor configuration change to the agent.
|
|
||||||
func (sys *System) UpsertNetworkMonitor(config monitor.Config, runNow bool) (*monitor.Result, error) {
|
|
||||||
resp, err := sys.syncNetworkMonitors(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionUpsert,
|
|
||||||
Config: config,
|
|
||||||
RunNow: runNow,
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if resp.Result == (monitor.Result{}) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
result := resp.Result
|
|
||||||
return &result, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// DeleteNetworkMonitor removes a single monitor task from the agent.
|
|
||||||
func (sys *System) DeleteNetworkMonitor(id string) error {
|
|
||||||
_, err := sys.syncNetworkMonitors(monitor.SyncRequest{
|
|
||||||
Action: monitor.SyncActionDelete,
|
|
||||||
Config: monitor.Config{ID: id},
|
|
||||||
})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sys *System) syncNetworkMonitors(req monitor.SyncRequest) (monitor.SyncResponse, error) {
|
|
||||||
if sys.agentVersion.LT(beszel.MinVersionNetworkMonitors) {
|
|
||||||
return monitor.SyncResponse{}, nil
|
|
||||||
}
|
|
||||||
timeout := 5 * time.Second
|
|
||||||
if req.Action == monitor.SyncActionUpsert && req.RunNow {
|
|
||||||
// Allow the probe to finish, including a timeout result, while preserving
|
|
||||||
// the normal request budget for transport and response handling.
|
|
||||||
timeout += monitor.MaxProbeTimeout
|
|
||||||
}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
|
||||||
defer cancel()
|
|
||||||
var result monitor.SyncResponse
|
|
||||||
return result, sys.request(ctx, common.SyncNetworkMonitors, req, &result)
|
|
||||||
}
|
|
||||||
@@ -9,7 +9,6 @@ import (
|
|||||||
"math/rand"
|
"math/rand"
|
||||||
"net"
|
"net"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -19,7 +18,6 @@ import (
|
|||||||
"github.com/henrygd/beszel/internal/hub/ws"
|
"github.com/henrygd/beszel/internal/hub/ws"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/container"
|
"github.com/henrygd/beszel/internal/entities/container"
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
@@ -32,8 +30,6 @@ import (
|
|||||||
"github.com/lxzan/gws"
|
"github.com/lxzan/gws"
|
||||||
"github.com/pocketbase/dbx"
|
"github.com/pocketbase/dbx"
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
"github.com/pocketbase/pocketbase/tools/security"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/types"
|
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,10 +52,6 @@ type System struct {
|
|||||||
smartInterval time.Duration // Interval for periodic SMART data updates
|
smartInterval time.Duration // Interval for periodic SMART data updates
|
||||||
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
||||||
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
||||||
// Serialize persistence from scheduled updates and resumes through commit.
|
|
||||||
recordsMu sync.Mutex
|
|
||||||
// Protected by recordsMu; realtime reads don't consume probes.
|
|
||||||
lastSavedMonitorProbe map[string]int64
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sm *SystemManager) NewSystem(systemId string) *System {
|
func (sm *SystemManager) NewSystem(systemId string) *System {
|
||||||
@@ -219,15 +211,11 @@ func (sys *System) handlePaused() {
|
|||||||
|
|
||||||
// createRecords updates the system record and adds system_stats and container_stats records
|
// createRecords updates the system record and adds system_stats and container_stats records
|
||||||
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
|
func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error) {
|
||||||
sys.recordsMu.Lock()
|
|
||||||
defer sys.recordsMu.Unlock()
|
|
||||||
|
|
||||||
systemRecord, err := sys.getRecord(sys.manager.hub)
|
systemRecord, err := sys.getRecord(sys.manager.hub)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
hub := sys.manager.hub
|
hub := sys.manager.hub
|
||||||
savedMonitorProbes := make(map[string]int64)
|
|
||||||
err = hub.RunInTransaction(func(txApp core.App) error {
|
err = hub.RunInTransaction(func(txApp core.App) error {
|
||||||
// add system_stats record
|
// add system_stats record
|
||||||
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
|
systemStatsCollection, err := txApp.FindCachedCollectionByNameOrId("system_stats")
|
||||||
@@ -278,56 +266,19 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if data.Monitors != nil {
|
|
||||||
if err := sys.updateNetworkMonitorsRecords(txApp, data.Monitors, savedMonitorProbes); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
|
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
|
||||||
systemRecord.Set("status", up)
|
systemRecord.Set("status", up)
|
||||||
// Distinguish an idle GPU from a system without GPU data (#2312)
|
systemRecord.Set("info", data.Info)
|
||||||
info := struct {
|
|
||||||
system.Info
|
|
||||||
GpuPct *float64 `json:"g,omitempty"`
|
|
||||||
}{Info: data.Info}
|
|
||||||
if len(data.Stats.GPUData) > 0 {
|
|
||||||
info.GpuPct = &data.Info.GpuPct
|
|
||||||
}
|
|
||||||
systemRecord.Set("info", info)
|
|
||||||
if err := txApp.SaveNoValidate(systemRecord); err != nil {
|
if err := txApp.SaveNoValidate(systemRecord); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
// Publish only successful inserts after the entire transaction commits.
|
|
||||||
if err == nil && len(savedMonitorProbes) > 0 {
|
|
||||||
if sys.lastSavedMonitorProbe == nil {
|
|
||||||
sys.lastSavedMonitorProbe = savedMonitorProbes
|
|
||||||
} else {
|
|
||||||
for id, timestamp := range savedMonitorProbes {
|
|
||||||
sys.lastSavedMonitorProbe[id] = timestamp
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A non-nil report includes cached results for all remaining monitors.
|
|
||||||
if err == nil && data.Monitors != nil {
|
|
||||||
for id := range sys.lastSavedMonitorProbe {
|
|
||||||
if _, exists := data.Monitors[id]; !exists {
|
|
||||||
delete(sys.lastSavedMonitorProbe, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == nil {
|
|
||||||
if alertErr := hub.HandleNetworkMonitorAlerts(systemRecord, data.Monitors); alertErr != nil {
|
|
||||||
hub.Logger().Error("Error handling network monitor alerts", "err", alertErr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return systemRecord, err
|
return systemRecord, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,14 +322,9 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
|
|||||||
|
|
||||||
valueStrings := make([]string, 0, len(data))
|
valueStrings := make([]string, 0, len(data))
|
||||||
for i, service := range data {
|
for i, service := range data {
|
||||||
// Agent payloads can contain null entries. Reject the snapshot before
|
|
||||||
// executing any queries so existing service records remain intact.
|
|
||||||
if service == nil {
|
|
||||||
return fmt.Errorf("null systemd service at index %d", i)
|
|
||||||
}
|
|
||||||
suffix := fmt.Sprintf("%d", i)
|
suffix := fmt.Sprintf("%d", i)
|
||||||
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
|
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:state%[1]s}, {:sub%[1]s}, {:cpu%[1]s}, {:cpuPeak%[1]s}, {:memory%[1]s}, {:memPeak%[1]s}, {:updated})", suffix))
|
||||||
params["id"+suffix] = MakeStableHashId(systemId, service.Name)
|
params["id"+suffix] = makeStableHashId(systemId, service.Name)
|
||||||
params["name"+suffix] = service.Name
|
params["name"+suffix] = service.Name
|
||||||
params["state"+suffix] = service.State
|
params["state"+suffix] = service.State
|
||||||
params["sub"+suffix] = service.Sub
|
params["sub"+suffix] = service.Sub
|
||||||
@@ -404,106 +350,6 @@ func createSystemdStatsRecords(app core.App, data []*systemd.Service, systemId s
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sys *System) updateNetworkMonitorsRecords(app core.App, monitorResults map[string]monitor.Result, savedProbes map[string]int64) error {
|
|
||||||
if len(monitorResults) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var err error
|
|
||||||
systemId := sys.Id
|
|
||||||
const monitorCollectionName = "network_monitors"
|
|
||||||
|
|
||||||
// If realtime updates are active, we save via PocketBase records to trigger realtime events.
|
|
||||||
// Otherwise we can do a more efficient direct update via SQL
|
|
||||||
realtimeActive := utils.RealtimeActiveForCollection(app, monitorCollectionName, func(filterQuery string) bool {
|
|
||||||
return !strings.Contains(filterQuery, "system") || strings.Contains(filterQuery, systemId)
|
|
||||||
})
|
|
||||||
|
|
||||||
now := time.Now().UTC()
|
|
||||||
nowMilli := now.UnixMilli()
|
|
||||||
nowString := now.Format(types.DefaultDateLayout)
|
|
||||||
var db dbx.Builder
|
|
||||||
var updateQuery *dbx.Query
|
|
||||||
if !realtimeActive {
|
|
||||||
db = app.DB()
|
|
||||||
monitorFields := []string{"res", "resMin1h", "resMax1h", "resAvg1h", "loss1h", "updated"}
|
|
||||||
setClauses := make([]string, len(monitorFields))
|
|
||||||
for i, f := range monitorFields {
|
|
||||||
setClauses[i] = fmt.Sprintf("%s={:%s}", f, f)
|
|
||||||
}
|
|
||||||
queryString := fmt.Sprintf("UPDATE %s SET %s WHERE id={:id}", monitorCollectionName, strings.Join(setClauses, ", "))
|
|
||||||
updateQuery = db.NewQuery(queryString)
|
|
||||||
}
|
|
||||||
|
|
||||||
// update network_monitors records
|
|
||||||
for id, result := range monitorResults {
|
|
||||||
monitorData := map[string]any{
|
|
||||||
"id": id,
|
|
||||||
"res": result.AvgResponse,
|
|
||||||
"resAvg1h": result.AvgResponse1h,
|
|
||||||
"resMin1h": result.MinResponse1h,
|
|
||||||
"resMax1h": result.MaxResponse1h,
|
|
||||||
"loss1h": result.PacketLoss1h,
|
|
||||||
"updated": nowString,
|
|
||||||
}
|
|
||||||
switch realtimeActive {
|
|
||||||
case true:
|
|
||||||
var record *core.Record
|
|
||||||
record, err = app.FindRecordById(monitorCollectionName, id)
|
|
||||||
if err == nil {
|
|
||||||
record.Load(monitorData)
|
|
||||||
err = app.SaveNoValidate(record)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
_, err = updateQuery.Bind(dbx.Params(monitorData)).Execute()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
app.Logger().Warn("Failed to update monitor", "system", systemId, "monitor", id, "err", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// handle stats collection — one record per monitor
|
|
||||||
const statsCollectionName = "network_monitor_stats"
|
|
||||||
|
|
||||||
var statsCollection *core.Collection
|
|
||||||
if realtimeActive {
|
|
||||||
statsCollection, _ = app.FindCachedCollectionByNameOrId(statsCollectionName)
|
|
||||||
}
|
|
||||||
|
|
||||||
for monitorId, result := range monitorResults {
|
|
||||||
// Compare identity, not ordering, so agent clock changes don't stall writes.
|
|
||||||
if result.LastProbeAt == sys.lastSavedMonitorProbe[monitorId] {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
statsRecordData := map[string]any{
|
|
||||||
"system": systemId,
|
|
||||||
"monitor": monitorId,
|
|
||||||
"type": "1m",
|
|
||||||
"created": nowMilli,
|
|
||||||
"res_min": result.MinResponse,
|
|
||||||
"res_max": result.MaxResponse,
|
|
||||||
"total_count": result.TotalCount,
|
|
||||||
"success_count": result.SuccessCount,
|
|
||||||
"res_sum": result.ResponseSum,
|
|
||||||
}
|
|
||||||
switch realtimeActive {
|
|
||||||
case true:
|
|
||||||
record := core.NewRecord(statsCollection)
|
|
||||||
record.Load(statsRecordData)
|
|
||||||
err = app.SaveNoValidate(record)
|
|
||||||
default:
|
|
||||||
statsRecordData["id"] = security.PseudorandomStringWithAlphabet(10, core.DefaultIdAlphabet)
|
|
||||||
_, err = db.Insert(statsCollectionName, dbx.Params(statsRecordData)).Execute()
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
app.Logger().Error("Failed to update monitor stats", "system", systemId, "monitor", monitorId, "err", err)
|
|
||||||
} else {
|
|
||||||
savedProbes[monitorId] = result.LastProbeAt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// createContainerRecords creates container records
|
// createContainerRecords creates container records
|
||||||
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
|
func createContainerRecords(app core.App, data []*container.Stats, systemId string) error {
|
||||||
if len(data) == 0 {
|
if len(data) == 0 {
|
||||||
@@ -517,7 +363,7 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
|
|||||||
valueStrings := make([]string, 0, len(data))
|
valueStrings := make([]string, 0, len(data))
|
||||||
for i, container := range data {
|
for i, container := range data {
|
||||||
suffix := fmt.Sprintf("%d", i)
|
suffix := fmt.Sprintf("%d", i)
|
||||||
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updateAvailable%[1]s}, {:updated})", suffix))
|
valueStrings = append(valueStrings, fmt.Sprintf("({:id%[1]s}, {:system}, {:name%[1]s}, {:image%[1]s}, {:ports%[1]s}, {:status%[1]s}, {:health%[1]s}, {:cpu%[1]s}, {:memory%[1]s}, {:net%[1]s}, {:updated})", suffix))
|
||||||
params["id"+suffix] = container.Id
|
params["id"+suffix] = container.Id
|
||||||
params["name"+suffix] = container.Name
|
params["name"+suffix] = container.Name
|
||||||
params["image"+suffix] = container.Image
|
params["image"+suffix] = container.Image
|
||||||
@@ -531,10 +377,9 @@ func createContainerRecords(app core.App, data []*container.Stats, systemId stri
|
|||||||
netBytes = uint64((container.NetworkSent + container.NetworkRecv) * 1024 * 1024)
|
netBytes = uint64((container.NetworkSent + container.NetworkRecv) * 1024 * 1024)
|
||||||
}
|
}
|
||||||
params["net"+suffix] = netBytes
|
params["net"+suffix] = netBytes
|
||||||
params["updateAvailable"+suffix] = container.UpdateAvailable
|
|
||||||
}
|
}
|
||||||
queryString := fmt.Sprintf(
|
queryString := fmt.Sprintf(
|
||||||
"INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updatable, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updatable = excluded.updatable, updated = excluded.updated",
|
"INSERT INTO containers (id, system, name, image, ports, status, health, cpu, memory, net, updated) VALUES %s ON CONFLICT(id) DO UPDATE SET system = excluded.system, name = excluded.name, image = excluded.image, ports = excluded.ports, status = excluded.status, health = excluded.health, cpu = excluded.cpu, memory = excluded.memory, net = excluded.net, updated = excluded.updated",
|
||||||
strings.Join(valueStrings, ","),
|
strings.Join(valueStrings, ","),
|
||||||
)
|
)
|
||||||
_, err := app.DB().NewQuery(queryString).Bind(params).Execute()
|
_, err := app.DB().NewQuery(queryString).Bind(params).Execute()
|
||||||
@@ -763,7 +608,7 @@ func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
|
|||||||
return &result, err
|
return &result, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func MakeStableHashId(strings ...string) string {
|
func makeStableHashId(strings ...string) string {
|
||||||
hash := fnv.New32a()
|
hash := fnv.New32a()
|
||||||
for _, str := range strings {
|
for _, str := range strings {
|
||||||
hash.Write([]byte(str))
|
hash.Write([]byte(str))
|
||||||
|
|||||||
@@ -1,44 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestCreateRecordsGPUUtilization(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
gpu bool
|
|
||||||
usage float64
|
|
||||||
}{
|
|
||||||
{"no GPU", false, 0},
|
|
||||||
{"active GPU", true, 42.5},
|
|
||||||
{"idle GPU", true, 0},
|
|
||||||
{"GPU removed", false, 0},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
data := &system.CombinedData{Info: system.Info{GpuPct: tc.usage, Cpu: 12.5}}
|
|
||||||
if tc.gpu {
|
|
||||||
data.Stats.GPUData = map[string]system.GPUData{"0": {Name: "GPU", Usage: tc.usage}}
|
|
||||||
}
|
|
||||||
_, err := sys.createRecords(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
record, err := app.FindRecordById("systems", sys.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
var info map[string]any
|
|
||||||
require.NoError(t, record.UnmarshalJSONField("info", &info))
|
|
||||||
assert.Equal(t, 12.5, info["cpu"])
|
|
||||||
if tc.gpu {
|
|
||||||
assert.Equal(t, tc.usage, info["g"])
|
|
||||||
} else {
|
|
||||||
assert.NotContains(t, info, "g")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -4,12 +4,10 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/hub/ws"
|
"github.com/henrygd/beszel/internal/hub/ws"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/hub/expirymap"
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
|
|
||||||
@@ -18,7 +16,6 @@ import (
|
|||||||
"github.com/henrygd/beszel"
|
"github.com/henrygd/beszel"
|
||||||
|
|
||||||
"github.com/blang/semver"
|
"github.com/blang/semver"
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
"github.com/pocketbase/pocketbase/tools/store"
|
"github.com/pocketbase/pocketbase/tools/store"
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
@@ -45,17 +42,13 @@ var errSystemExists = errors.New("system exists")
|
|||||||
// SystemManager manages a collection of monitored systems and their connections.
|
// SystemManager manages a collection of monitored systems and their connections.
|
||||||
// It handles system lifecycle, status updates, and maintains both SSH and WebSocket connections.
|
// It handles system lifecycle, status updates, and maintains both SSH and WebSocket connections.
|
||||||
type SystemManager struct {
|
type SystemManager struct {
|
||||||
hub hubLike // Hub interface for database and alert operations
|
hub hubLike // Hub interface for database and alert operations
|
||||||
systems *store.Store[string, *System] // Thread-safe store of active systems
|
systems *store.Store[string, *System] // Thread-safe store of active systems
|
||||||
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
|
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
|
||||||
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
|
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
|
||||||
zfsFetchMap *expirymap.ExpiryMap[zfsFetchState] // Stores last ZFS fetch time/result; TTL is only for cleanup
|
zfsFetchMap *expirymap.ExpiryMap[zfsFetchState] // Stores last ZFS fetch time/result; TTL is only for cleanup
|
||||||
realtimeMutex sync.Mutex // Protects all realtime worker and subscription state
|
ctx context.Context // Cancelled when the app terminates
|
||||||
activeSubscriptions map[string]*subscriptionInfo // Realtime subscriptions keyed by system ID
|
cancel context.CancelFunc // Cancels ctx and all child system contexts
|
||||||
realtimeWorkerStop chan struct{} // Stops the current realtime worker generation
|
|
||||||
realtimeWorkerRun bool // Whether a realtime worker has been started
|
|
||||||
ctx context.Context // Cancelled when the app terminates
|
|
||||||
cancel context.CancelFunc // Cancels ctx and all child system contexts
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// hubLike defines the interface requirements for the hub dependency.
|
// hubLike defines the interface requirements for the hub dependency.
|
||||||
@@ -64,7 +57,6 @@ type hubLike interface {
|
|||||||
core.App
|
core.App
|
||||||
GetSSHKey(dataDir string) (ssh.Signer, error)
|
GetSSHKey(dataDir string) (ssh.Signer, error)
|
||||||
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
|
HandleSystemAlerts(systemRecord *core.Record, data *system.CombinedData) error
|
||||||
HandleNetworkMonitorAlerts(systemRecord *core.Record, results map[string]monitor.Result) error
|
|
||||||
HandleStatusAlerts(status string, systemRecord *core.Record) error
|
HandleStatusAlerts(status string, systemRecord *core.Record) error
|
||||||
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
|
HandleContainerAlerts(systemRecord *core.Record, data *system.CombinedData, fetchLogs func(containerID string) (string, error)) error
|
||||||
CancelPendingStatusAlerts(systemID string)
|
CancelPendingStatusAlerts(systemID string)
|
||||||
@@ -75,11 +67,10 @@ type hubLike interface {
|
|||||||
// The hub must implement the hubLike interface to provide database and alert functionality.
|
// The hub must implement the hubLike interface to provide database and alert functionality.
|
||||||
func NewSystemManager(hub hubLike) *SystemManager {
|
func NewSystemManager(hub hubLike) *SystemManager {
|
||||||
sm := &SystemManager{
|
sm := &SystemManager{
|
||||||
systems: store.New(map[string]*System{}),
|
systems: store.New(map[string]*System{}),
|
||||||
hub: hub,
|
hub: hub,
|
||||||
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
|
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
|
||||||
zfsFetchMap: expirymap.New[zfsFetchState](time.Hour),
|
zfsFetchMap: expirymap.New[zfsFetchState](time.Hour),
|
||||||
activeSubscriptions: make(map[string]*subscriptionInfo),
|
|
||||||
}
|
}
|
||||||
sm.ctx, sm.cancel = context.WithCancel(context.Background())
|
sm.ctx, sm.cancel = context.WithCancel(context.Background())
|
||||||
return sm
|
return sm
|
||||||
@@ -147,7 +138,6 @@ func (sm *SystemManager) bindEventHooks() {
|
|||||||
// onTerminate cancels SystemManager context on app shutdown
|
// onTerminate cancels SystemManager context on app shutdown
|
||||||
func (sm *SystemManager) onTerminate(e *core.TerminateEvent) error {
|
func (sm *SystemManager) onTerminate(e *core.TerminateEvent) error {
|
||||||
sm.cancel()
|
sm.cancel()
|
||||||
sm.stopRealtimeWorker()
|
|
||||||
return e.Next()
|
return e.Next()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -353,20 +343,6 @@ func (sm *SystemManager) AddWebSocketSystem(systemId string, agentVersion semver
|
|||||||
if err := sm.AddRecord(systemRecord, system); err != nil {
|
if err := sm.AddRecord(systemRecord, system); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sync network monitors to the newly connected agent
|
|
||||||
go func() {
|
|
||||||
configs, err := sm.GetMonitorConfigsForSystem(systemId)
|
|
||||||
if err != nil {
|
|
||||||
sm.hub.Logger().Warn("failed to load monitors for agent", "system", systemId, "err", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// An empty set must also replace any probes retained across a disconnect.
|
|
||||||
if err := system.SyncNetworkMonitors(configs); err != nil {
|
|
||||||
sm.hub.Logger().Warn("failed to sync monitors to agent", "system", systemId, "err", err)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -379,16 +355,6 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetMonitorConfigsForSystem returns all enabled monitor configs for a system.
|
|
||||||
func (sm *SystemManager) GetMonitorConfigsForSystem(systemID string) ([]monitor.Config, error) {
|
|
||||||
var configs []monitor.Config
|
|
||||||
err := sm.hub.DB().
|
|
||||||
NewQuery("SELECT id, target, protocol, port, interval FROM network_monitors WHERE system = {:system} AND enabled = true").
|
|
||||||
Bind(dbx.Params{"system": systemID}).
|
|
||||||
All(&configs)
|
|
||||||
return configs, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
||||||
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
||||||
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
||||||
@@ -424,12 +390,11 @@ func (sm *SystemManager) createSSHClientConfig() error {
|
|||||||
|
|
||||||
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
|
// deactivateAlerts finds all triggered alerts for a system and sets them to inactive.
|
||||||
// This is called when a system is paused or goes offline to prevent continued alerts.
|
// This is called when a system is paused or goes offline to prevent continued alerts.
|
||||||
// Monitor incidents remain open: a missing observation does not establish recovery.
|
|
||||||
func deactivateAlerts(app core.App, systemID string) error {
|
func deactivateAlerts(app core.App, systemID string) error {
|
||||||
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
|
// Note: Direct SQL updates don't trigger SSE, so we use the PocketBase API
|
||||||
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
|
// _, err := app.DB().NewQuery(fmt.Sprintf("UPDATE alerts SET triggered = false WHERE system = '%s'", systemID)).Execute()
|
||||||
|
|
||||||
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1 && name != 'NetworkMonitorLoss'", systemID), "", -1, 0)
|
alerts, err := app.FindRecordsByFilter("alerts", fmt.Sprintf("system = '%s' && triggered = 1", systemID), "", -1, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,29 +3,25 @@ package systems
|
|||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/common"
|
"github.com/henrygd/beszel/internal/common"
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
|
||||||
"github.com/henrygd/beszel/internal/hub/utils"
|
|
||||||
"github.com/pocketbase/dbx"
|
|
||||||
"github.com/pocketbase/pocketbase/apis"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
"github.com/pocketbase/pocketbase/core"
|
||||||
"github.com/pocketbase/pocketbase/tools/subscriptions"
|
"github.com/pocketbase/pocketbase/tools/subscriptions"
|
||||||
)
|
)
|
||||||
|
|
||||||
type subscriptionInfo struct {
|
type subscriptionInfo struct {
|
||||||
subscription string
|
subscription string
|
||||||
connectedClients int
|
connectedClients uint8
|
||||||
fetching bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type realtimeFetch struct {
|
var (
|
||||||
systemID string
|
activeSubscriptions = make(map[string]*subscriptionInfo)
|
||||||
subscription string
|
workerRunning bool
|
||||||
info *subscriptionInfo
|
tickerStopChan chan struct{}
|
||||||
}
|
realtimeMutex sync.Mutex
|
||||||
|
)
|
||||||
|
|
||||||
// onRealtimeConnectRequest handles client connection events for realtime subscriptions.
|
// onRealtimeConnectRequest handles client connection events for realtime subscriptions.
|
||||||
// It cleans up existing subscriptions when a client connects.
|
// It cleans up existing subscriptions when a client connects.
|
||||||
@@ -42,19 +38,6 @@ func (sm *SystemManager) onRealtimeConnectRequest(e *core.RealtimeConnectRequest
|
|||||||
// onRealtimeSubscribeRequest handles client subscription events for realtime metrics.
|
// onRealtimeSubscribeRequest handles client subscription events for realtime metrics.
|
||||||
// It tracks new subscriptions and unsubscriptions to manage the realtime worker lifecycle.
|
// It tracks new subscriptions and unsubscriptions to manage the realtime worker lifecycle.
|
||||||
func (sm *SystemManager) onRealtimeSubscribeRequest(e *core.RealtimeSubscribeRequestEvent) error {
|
func (sm *SystemManager) onRealtimeSubscribeRequest(e *core.RealtimeSubscribeRequestEvent) error {
|
||||||
// Parse with PocketBase's own subscription parser before changing the real
|
|
||||||
// client. Reject the entire request if any metrics target is inaccessible.
|
|
||||||
requested := subscriptions.NewDefaultClient()
|
|
||||||
requested.Subscribe(e.Subscriptions...)
|
|
||||||
for topic, options := range requested.Subscriptions() {
|
|
||||||
if !strings.HasPrefix(topic, "rt_metrics") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
system, err := sm.GetSystem(options.Query["system"])
|
|
||||||
if err != nil || !system.HasUser(e.App, e.Auth) {
|
|
||||||
return e.NotFoundError("", nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
oldSubs := e.Client.Subscriptions()
|
oldSubs := e.Client.Subscriptions()
|
||||||
// after e.Next() is the result of the subscribe request
|
// after e.Next() is the result of the subscribe request
|
||||||
err := e.Next()
|
err := e.Next()
|
||||||
@@ -64,7 +47,14 @@ func (sm *SystemManager) onRealtimeSubscribeRequest(e *core.RealtimeSubscribeReq
|
|||||||
for k, options := range newSubs {
|
for k, options := range newSubs {
|
||||||
if _, ok := oldSubs[k]; !ok {
|
if _, ok := oldSubs[k]; !ok {
|
||||||
if strings.HasPrefix(k, "rt_metrics") {
|
if strings.HasPrefix(k, "rt_metrics") {
|
||||||
sm.addRealtimeSubscription(options.Query["system"], k)
|
systemId := options.Query["system"]
|
||||||
|
if _, ok := activeSubscriptions[systemId]; !ok {
|
||||||
|
activeSubscriptions[systemId] = &subscriptionInfo{
|
||||||
|
subscription: k,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
activeSubscriptions[systemId].connectedClients += 1
|
||||||
|
sm.onRealtimeSubscriptionAdded()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -78,76 +68,72 @@ func (sm *SystemManager) onRealtimeSubscribeRequest(e *core.RealtimeSubscribeReq
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// addRealtimeSubscription tracks a subscriber and starts a worker if necessary.
|
// onRealtimeSubscriptionAdded initializes or starts the realtime worker when the first subscription is added.
|
||||||
func (sm *SystemManager) addRealtimeSubscription(systemID, subscription string) {
|
// It ensures only one worker runs at a time.
|
||||||
sm.realtimeMutex.Lock()
|
func (sm *SystemManager) onRealtimeSubscriptionAdded() {
|
||||||
defer sm.realtimeMutex.Unlock()
|
realtimeMutex.Lock()
|
||||||
|
defer realtimeMutex.Unlock()
|
||||||
|
|
||||||
if sm.activeSubscriptions == nil {
|
// Start the worker if it's not already running
|
||||||
sm.activeSubscriptions = make(map[string]*subscriptionInfo)
|
if !workerRunning {
|
||||||
}
|
workerRunning = true
|
||||||
info, ok := sm.activeSubscriptions[systemID]
|
// Create a new stop channel for this worker instance
|
||||||
if !ok {
|
tickerStopChan = make(chan struct{})
|
||||||
info = &subscriptionInfo{subscription: subscription}
|
go sm.startRealtimeWorker()
|
||||||
sm.activeSubscriptions[systemID] = info
|
|
||||||
}
|
|
||||||
info.connectedClients++
|
|
||||||
|
|
||||||
if !sm.realtimeWorkerRun {
|
|
||||||
sm.realtimeWorkerRun = true
|
|
||||||
stop := make(chan struct{})
|
|
||||||
sm.realtimeWorkerStop = stop
|
|
||||||
go sm.startRealtimeWorker(stop)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stopRealtimeWorker stops the current worker generation, if any.
|
// checkSubscriptions stops the realtime worker when there are no active subscriptions.
|
||||||
func (sm *SystemManager) stopRealtimeWorker() {
|
// This prevents unnecessary resource usage when no clients are listening for realtime data.
|
||||||
sm.realtimeMutex.Lock()
|
func (sm *SystemManager) checkSubscriptions() {
|
||||||
defer sm.realtimeMutex.Unlock()
|
if !workerRunning || len(activeSubscriptions) > 0 {
|
||||||
sm.stopRealtimeWorkerLocked()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SystemManager) stopRealtimeWorkerLocked() {
|
|
||||||
if !sm.realtimeWorkerRun {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
close(sm.realtimeWorkerStop)
|
|
||||||
sm.realtimeWorkerStop = nil
|
realtimeMutex.Lock()
|
||||||
sm.realtimeWorkerRun = false
|
defer realtimeMutex.Unlock()
|
||||||
|
|
||||||
|
// Signal the worker to stop
|
||||||
|
if tickerStopChan != nil {
|
||||||
|
select {
|
||||||
|
case tickerStopChan <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark worker as stopped (will be reset when next subscription comes in)
|
||||||
|
workerRunning = false
|
||||||
}
|
}
|
||||||
|
|
||||||
// removeRealtimeSubscription removes a realtime subscription and checks if the worker should be stopped.
|
// removeRealtimeSubscription removes a realtime subscription and checks if the worker should be stopped.
|
||||||
// It only processes subscriptions with the "rt_metrics" prefix and triggers cleanup when subscriptions are removed.
|
// It only processes subscriptions with the "rt_metrics" prefix and triggers cleanup when subscriptions are removed.
|
||||||
func (sm *SystemManager) removeRealtimeSubscription(subscription string, options subscriptions.SubscriptionOptions) {
|
func (sm *SystemManager) removeRealtimeSubscription(subscription string, options subscriptions.SubscriptionOptions) {
|
||||||
if strings.HasPrefix(subscription, "rt_metrics") {
|
if strings.HasPrefix(subscription, "rt_metrics") {
|
||||||
systemID := options.Query["system"]
|
systemId := options.Query["system"]
|
||||||
sm.realtimeMutex.Lock()
|
if info, ok := activeSubscriptions[systemId]; ok {
|
||||||
if info, ok := sm.activeSubscriptions[systemID]; ok {
|
info.connectedClients -= 1
|
||||||
info.connectedClients--
|
|
||||||
if info.connectedClients <= 0 {
|
if info.connectedClients <= 0 {
|
||||||
delete(sm.activeSubscriptions, systemID)
|
delete(activeSubscriptions, systemId)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(sm.activeSubscriptions) == 0 {
|
sm.checkSubscriptions()
|
||||||
sm.stopRealtimeWorkerLocked()
|
|
||||||
}
|
|
||||||
sm.realtimeMutex.Unlock()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startRealtimeWorker runs the main loop for fetching realtime data from agents.
|
// startRealtimeWorker runs the main loop for fetching realtime data from agents.
|
||||||
// It continuously fetches system data and broadcasts it to subscribed clients via WebSocket.
|
// It continuously fetches system data and broadcasts it to subscribed clients via WebSocket.
|
||||||
func (sm *SystemManager) startRealtimeWorker(stop <-chan struct{}) {
|
func (sm *SystemManager) startRealtimeWorker() {
|
||||||
sm.fetchRealtimeDataAndNotify()
|
sm.fetchRealtimeDataAndNotify()
|
||||||
ticker := time.NewTicker(time.Second)
|
tick := time.Tick(1 * time.Second)
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-stop:
|
case <-tickerStopChan:
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-tick:
|
||||||
|
if len(activeSubscriptions) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
sm.fetchRealtimeDataAndNotify()
|
sm.fetchRealtimeDataAndNotify()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -155,95 +141,27 @@ func (sm *SystemManager) startRealtimeWorker(stop <-chan struct{}) {
|
|||||||
|
|
||||||
// fetchRealtimeDataAndNotify fetches realtime data for all active subscriptions and notifies the clients.
|
// fetchRealtimeDataAndNotify fetches realtime data for all active subscriptions and notifies the clients.
|
||||||
func (sm *SystemManager) fetchRealtimeDataAndNotify() {
|
func (sm *SystemManager) fetchRealtimeDataAndNotify() {
|
||||||
for _, fetch := range sm.claimRealtimeFetches() {
|
for systemId, info := range activeSubscriptions {
|
||||||
system, err := sm.GetSystem(fetch.systemID)
|
system, err := sm.GetSystem(systemId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sm.finishRealtimeFetch(fetch)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
go func(fetch realtimeFetch) {
|
go func() {
|
||||||
defer sm.finishRealtimeFetch(fetch)
|
|
||||||
data, err := system.fetchDataFromAgent(common.DataRequestOptions{CacheTimeMs: 1000})
|
data, err := system.fetchDataFromAgent(common.DataRequestOptions{CacheTimeMs: 1000})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
bytes, err := marshalRealtimeData(data)
|
bytes, err := json.Marshal(data)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
notify(sm.hub, system, fetch.subscription, bytes)
|
notify(sm.hub, info.subscription, bytes)
|
||||||
}
|
}
|
||||||
}(fetch)
|
}()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// claimRealtimeFetches takes a stable snapshot and marks each selected system as
|
|
||||||
// in flight. Slow agents are skipped on later ticks until their fetch completes.
|
|
||||||
func (sm *SystemManager) claimRealtimeFetches() []realtimeFetch {
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
defer sm.realtimeMutex.Unlock()
|
|
||||||
|
|
||||||
fetches := make([]realtimeFetch, 0, len(sm.activeSubscriptions))
|
|
||||||
for systemID, info := range sm.activeSubscriptions {
|
|
||||||
if info.fetching {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
info.fetching = true
|
|
||||||
fetches = append(fetches, realtimeFetch{
|
|
||||||
systemID: systemID,
|
|
||||||
subscription: info.subscription,
|
|
||||||
info: info,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
return fetches
|
|
||||||
}
|
|
||||||
|
|
||||||
func (sm *SystemManager) finishRealtimeFetch(fetch realtimeFetch) {
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
defer sm.realtimeMutex.Unlock()
|
|
||||||
// A subscription may have been removed and recreated while the old request
|
|
||||||
// was running. Only release the exact entry claimed by this request.
|
|
||||||
if info := sm.activeSubscriptions[fetch.systemID]; info == fetch.info {
|
|
||||||
info.fetching = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// marshalRealtimeData marshals combined agent data for a realtime broadcast, converting
|
|
||||||
// the per-monitor results into the derived metric fields the frontend charts expect.
|
|
||||||
func marshalRealtimeData(data *system.CombinedData) ([]byte, error) {
|
|
||||||
if len(data.Monitors) == 0 {
|
|
||||||
return json.Marshal(data)
|
|
||||||
}
|
|
||||||
monitorStats := make(map[string]monitor.Stats, len(data.Monitors))
|
|
||||||
for id, result := range data.Monitors {
|
|
||||||
monitorStats[id] = monitor.Stats{}.FromResult(result)
|
|
||||||
}
|
|
||||||
return json.Marshal(struct {
|
|
||||||
*system.CombinedData
|
|
||||||
Monitors map[string]monitor.Stats `json:"Monitors"`
|
|
||||||
}{data, monitorStats})
|
|
||||||
}
|
|
||||||
|
|
||||||
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
|
// notify broadcasts realtime data to all clients subscribed to a specific subscription.
|
||||||
// Custom topics bypass collection rules, so check current access for every
|
// It iterates through all connected clients and sends the data only to those with matching subscriptions.
|
||||||
// recipient, including clients whose authentication or membership was revoked.
|
func notify(app core.App, subscription string, data []byte) error {
|
||||||
func notify(app core.App, system *System, subscription string, data []byte) error {
|
|
||||||
shareAll, _ := utils.GetEnv("SHARE_ALL_SYSTEMS")
|
|
||||||
members := make(map[string]struct{})
|
|
||||||
if shareAll != "true" {
|
|
||||||
// Refresh once per broadcast so membership changes take effect on the
|
|
||||||
// next update without querying the database for every recipient.
|
|
||||||
var recordData struct{ Users string }
|
|
||||||
if err := app.DB().NewQuery("SELECT users FROM systems WHERE id={:id}").
|
|
||||||
Bind(dbx.Params{"id": system.Id}).One(&recordData); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var userIDs []string
|
|
||||||
if err := json.Unmarshal([]byte(recordData.Users), &userIDs); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for _, id := range userIDs {
|
|
||||||
members[id] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
message := subscriptions.Message{
|
message := subscriptions.Message{
|
||||||
Name: subscription,
|
Name: subscription,
|
||||||
Data: data,
|
Data: data,
|
||||||
@@ -252,13 +170,6 @@ func notify(app core.App, system *System, subscription string, data []byte) erro
|
|||||||
if !client.HasSubscription(subscription) {
|
if !client.HasSubscription(subscription) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
auth, _ := client.Get(apis.RealtimeClientAuthKey).(*core.Record)
|
|
||||||
if auth == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, member := members[auth.Id]; shareAll != "true" && !member {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
client.Send(message)
|
client.Send(message)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1,229 +0,0 @@
|
|||||||
package systems
|
|
||||||
|
|
||||||
import (
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/pocketbase/pocketbase/apis"
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
pbtests "github.com/pocketbase/pocketbase/tests"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/hook"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/store"
|
|
||||||
"github.com/pocketbase/pocketbase/tools/subscriptions"
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestRealtimeAuthorization(t *testing.T) {
|
|
||||||
t.Setenv("SHARE_ALL_SYSTEMS", "false")
|
|
||||||
t.Setenv("BESZEL_HUB_SHARE_ALL_SYSTEMS", "")
|
|
||||||
app, err := pbtests.NewTestApp(t.TempDir())
|
|
||||||
require.NoError(t, err)
|
|
||||||
t.Cleanup(app.Cleanup)
|
|
||||||
_, err = app.DB().NewQuery(`CREATE TABLE IF NOT EXISTS systems (id TEXT PRIMARY KEY, users TEXT)`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
_, err = app.DB().NewQuery(`INSERT INTO systems (id, users) VALUES ('target', '["member"]')`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
member := core.NewRecord(core.NewAuthCollection("users"))
|
|
||||||
member.Id = "member"
|
|
||||||
outsider := core.NewRecord(member.Collection())
|
|
||||||
outsider.Id = "outsider"
|
|
||||||
system := &System{Id: "target"}
|
|
||||||
sm := newRealtimeTestManager()
|
|
||||||
sm.systems.Set(system.Id, system)
|
|
||||||
// Keep the lifecycle bookkeeping active without starting an agent worker.
|
|
||||||
sm.realtimeWorkerRun = true
|
|
||||||
sm.realtimeWorkerStop = make(chan struct{})
|
|
||||||
t.Cleanup(sm.stopRealtimeWorker)
|
|
||||||
topic := `rt_metrics?options={"query":{"system":"target"}}`
|
|
||||||
|
|
||||||
for _, tc := range []struct {
|
|
||||||
name string
|
|
||||||
auth *core.Record
|
|
||||||
topic string
|
|
||||||
share bool
|
|
||||||
allowed bool
|
|
||||||
}{
|
|
||||||
{"guest", nil, topic, false, false},
|
|
||||||
{"outsider", outsider, topic, false, false},
|
|
||||||
{"member", member, topic, false, true},
|
|
||||||
{"missing system", member, `rt_metrics`, false, false},
|
|
||||||
{"unknown system", member, `rt_metrics?options={"query":{"system":"missing"}}`, false, false},
|
|
||||||
{"malformed options", member, `rt_metrics?options=invalid`, false, false},
|
|
||||||
{"prefix variant", outsider, `rt_metrics_extra?options={"query":{"system":"target"}}`, false, false},
|
|
||||||
{"shared outsider", outsider, topic, true, true},
|
|
||||||
{"shared guest", nil, topic, true, false},
|
|
||||||
{"other topic", nil, "systems/*", false, true},
|
|
||||||
} {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
if tc.share {
|
|
||||||
t.Setenv("BESZEL_HUB_SHARE_ALL_SYSTEMS", "true")
|
|
||||||
}
|
|
||||||
client := subscriptions.NewDefaultClient()
|
|
||||||
client.Subscribe("existing")
|
|
||||||
e := &core.RealtimeSubscribeRequestEvent{
|
|
||||||
RequestEvent: &core.RequestEvent{App: app, Auth: tc.auth},
|
|
||||||
Client: client, Subscriptions: []string{tc.topic},
|
|
||||||
}
|
|
||||||
called := false
|
|
||||||
h := &hook.Hook[*core.RealtimeSubscribeRequestEvent]{}
|
|
||||||
h.BindFunc(sm.onRealtimeSubscribeRequest)
|
|
||||||
err := h.Trigger(e, func(e *core.RealtimeSubscribeRequestEvent) error {
|
|
||||||
called = true
|
|
||||||
client.Unsubscribe()
|
|
||||||
client.Subscribe(e.Subscriptions...)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if tc.allowed {
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.True(t, called)
|
|
||||||
} else {
|
|
||||||
require.Error(t, err)
|
|
||||||
assert.False(t, called)
|
|
||||||
assert.True(t, client.HasSubscription("existing"))
|
|
||||||
assert.False(t, client.HasSubscription(tc.topic))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("broadcast checks current access", func(t *testing.T) {
|
|
||||||
client := subscriptions.NewDefaultClient()
|
|
||||||
client.Subscribe(topic)
|
|
||||||
app.SubscriptionsBroker().Register(client)
|
|
||||||
defer app.SubscriptionsBroker().Unregister(client.Id())
|
|
||||||
secondClient := subscriptions.NewDefaultClient()
|
|
||||||
secondClient.Subscribe(topic)
|
|
||||||
app.SubscriptionsBroker().Register(secondClient)
|
|
||||||
defer app.SubscriptionsBroker().Unregister(secondClient.Id())
|
|
||||||
check := func(auth *core.Record, allowed bool) {
|
|
||||||
t.Helper()
|
|
||||||
client.Set(apis.RealtimeClientAuthKey, auth)
|
|
||||||
secondClient.Set(apis.RealtimeClientAuthKey, auth)
|
|
||||||
done := make(chan struct{})
|
|
||||||
go func() {
|
|
||||||
notify(app, system, topic, []byte(`{"cpu":42}`))
|
|
||||||
close(done)
|
|
||||||
}()
|
|
||||||
// Even on failure, drain pending sends and join the broadcaster before
|
|
||||||
// unregistering clients, which closes their channels.
|
|
||||||
defer func() {
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-client.Channel():
|
|
||||||
case <-secondClient.Channel():
|
|
||||||
case <-done:
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
var received [2]int
|
|
||||||
timer := time.NewTimer(time.Second)
|
|
||||||
defer timer.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case msg := <-client.Channel():
|
|
||||||
received[0]++
|
|
||||||
assert.Equal(t, topic, msg.Name)
|
|
||||||
case msg := <-secondClient.Channel():
|
|
||||||
received[1]++
|
|
||||||
assert.Equal(t, topic, msg.Name)
|
|
||||||
case <-done:
|
|
||||||
want := [2]int{}
|
|
||||||
if allowed {
|
|
||||||
want = [2]int{1, 1}
|
|
||||||
}
|
|
||||||
assert.Equal(t, want, received)
|
|
||||||
return
|
|
||||||
case <-timer.C:
|
|
||||||
t.Fatal("broadcast did not finish")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
check(nil, false)
|
|
||||||
check(outsider, false)
|
|
||||||
check(member, true)
|
|
||||||
_, err := app.DB().NewQuery(`UPDATE systems SET users = '[]'`).Execute()
|
|
||||||
require.NoError(t, err)
|
|
||||||
check(member, false)
|
|
||||||
t.Setenv("BESZEL_HUB_SHARE_ALL_SYSTEMS", "true")
|
|
||||||
check(outsider, true)
|
|
||||||
check(nil, false)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func newRealtimeTestManager() *SystemManager {
|
|
||||||
return &SystemManager{
|
|
||||||
systems: store.New(map[string]*System{}),
|
|
||||||
activeSubscriptions: make(map[string]*subscriptionInfo),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRealtimeFetchesDoNotOverlapPerSystem(t *testing.T) {
|
|
||||||
sm := newRealtimeTestManager()
|
|
||||||
sm.activeSubscriptions["one"] = &subscriptionInfo{subscription: "rt_metrics_one"}
|
|
||||||
sm.activeSubscriptions["two"] = &subscriptionInfo{subscription: "rt_metrics_two"}
|
|
||||||
|
|
||||||
first := sm.claimRealtimeFetches()
|
|
||||||
require.Len(t, first, 2)
|
|
||||||
assert.Empty(t, sm.claimRealtimeFetches())
|
|
||||||
|
|
||||||
sm.finishRealtimeFetch(first[0])
|
|
||||||
next := sm.claimRealtimeFetches()
|
|
||||||
require.Len(t, next, 1)
|
|
||||||
assert.Equal(t, first[0].systemID, next[0].systemID)
|
|
||||||
|
|
||||||
sm.finishRealtimeFetch(first[1])
|
|
||||||
sm.finishRealtimeFetch(next[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestFinishingOldRealtimeFetchDoesNotReleaseReplacement(t *testing.T) {
|
|
||||||
sm := newRealtimeTestManager()
|
|
||||||
oldInfo := &subscriptionInfo{subscription: "old"}
|
|
||||||
sm.activeSubscriptions["system"] = oldInfo
|
|
||||||
|
|
||||||
fetch := sm.claimRealtimeFetches()[0]
|
|
||||||
newInfo := &subscriptionInfo{subscription: "new", fetching: true}
|
|
||||||
sm.activeSubscriptions["system"] = newInfo
|
|
||||||
|
|
||||||
sm.finishRealtimeFetch(fetch)
|
|
||||||
assert.True(t, newInfo.fetching)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRealtimeSubscriptionLifecycle(t *testing.T) {
|
|
||||||
sm := newRealtimeTestManager()
|
|
||||||
options := subscriptions.SubscriptionOptions{Query: map[string]string{"system": "system"}}
|
|
||||||
|
|
||||||
sm.addRealtimeSubscription("system", "rt_metrics")
|
|
||||||
sm.addRealtimeSubscription("system", "rt_metrics")
|
|
||||||
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
firstStop := sm.realtimeWorkerStop
|
|
||||||
assert.True(t, sm.realtimeWorkerRun)
|
|
||||||
assert.Equal(t, 2, sm.activeSubscriptions["system"].connectedClients)
|
|
||||||
sm.realtimeMutex.Unlock()
|
|
||||||
|
|
||||||
sm.removeRealtimeSubscription("rt_metrics", options)
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
assert.True(t, sm.realtimeWorkerRun)
|
|
||||||
assert.Equal(t, 1, sm.activeSubscriptions["system"].connectedClients)
|
|
||||||
sm.realtimeMutex.Unlock()
|
|
||||||
|
|
||||||
sm.removeRealtimeSubscription("rt_metrics", options)
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
assert.False(t, sm.realtimeWorkerRun)
|
|
||||||
assert.Empty(t, sm.activeSubscriptions)
|
|
||||||
sm.realtimeMutex.Unlock()
|
|
||||||
select {
|
|
||||||
case <-firstStop:
|
|
||||||
default:
|
|
||||||
t.Fatal("worker stop channel was not closed")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later subscription must get a new stop channel owned by its worker.
|
|
||||||
sm.addRealtimeSubscription("system", "rt_metrics")
|
|
||||||
sm.realtimeMutex.Lock()
|
|
||||||
secondStop := sm.realtimeWorkerStop
|
|
||||||
assert.NotEqual(t, firstStop, secondStop)
|
|
||||||
sm.realtimeMutex.Unlock()
|
|
||||||
sm.stopRealtimeWorker()
|
|
||||||
}
|
|
||||||
@@ -77,7 +77,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
|
|||||||
|
|
||||||
currentIDs := make(map[string]struct{}, len(smartData))
|
currentIDs := make(map[string]struct{}, len(smartData))
|
||||||
for deviceKey := range smartData {
|
for deviceKey := range smartData {
|
||||||
currentIDs[MakeStableHashId(sys.Id, deviceKey)] = struct{}{}
|
currentIDs[makeStableHashId(sys.Id, deviceKey)] = struct{}{}
|
||||||
}
|
}
|
||||||
|
|
||||||
err = hub.RunInTransaction(func(txApp core.App) error {
|
err = hub.RunInTransaction(func(txApp core.App) error {
|
||||||
@@ -115,7 +115,7 @@ func (sys *System) saveSmartDevices(smartData map[string]smart.SmartData, comple
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
|
func (sys *System) upsertSmartDeviceRecord(app core.App, collection *core.Collection, deviceKey string, device smart.SmartData) error {
|
||||||
recordID := MakeStableHashId(sys.Id, deviceKey)
|
recordID := makeStableHashId(sys.Id, deviceKey)
|
||||||
|
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/henrygd/beszel/internal/entities/monitor"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/smart"
|
"github.com/henrygd/beszel/internal/entities/smart"
|
||||||
esystem "github.com/henrygd/beszel/internal/entities/system"
|
esystem "github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/hub/expirymap"
|
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||||
@@ -29,8 +28,7 @@ func (stubHub) GetSSHKey(dataDir string) (ssh.Signer, error) { return nil, nil }
|
|||||||
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
|
func (stubHub) HandleSystemAlerts(systemRecord *core.Record, data *esystem.CombinedData) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
func (stubHub) HandleNetworkMonitorAlerts(*core.Record, map[string]monitor.Result) error { return nil }
|
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
|
||||||
func (stubHub) HandleStatusAlerts(status string, systemRecord *core.Record) error { return nil }
|
|
||||||
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
|
func (stubHub) HandleContainerAlerts(systemRecord *core.Record, data *esystem.CombinedData, fetchLogs func(containerID string) (string, error)) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -214,7 +212,7 @@ func TestSaveSmartDevices_IncompleteDataDoesNotRemoveDevices(t *testing.T) {
|
|||||||
}, false))
|
}, false))
|
||||||
|
|
||||||
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
|
assert.Len(t, countSmartDeviceRecords(t, testApp, sys.Id), 2)
|
||||||
recordA, err := testApp.FindRecordById("smart_devices", MakeStableHashId(sys.Id, "AAA"))
|
recordA, err := testApp.FindRecordById("smart_devices", makeStableHashId(sys.Id, "AAA"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.EqualValues(t, 42, recordA.GetInt("temp"))
|
assert.EqualValues(t, 42, recordA.GetInt("temp"))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,9 +14,9 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
serviceName := "nginx.service"
|
serviceName := "nginx.service"
|
||||||
|
|
||||||
// Call multiple times and ensure same result
|
// Call multiple times and ensure same result
|
||||||
id1 := MakeStableHashId(systemId, serviceName)
|
id1 := makeStableHashId(systemId, serviceName)
|
||||||
id2 := MakeStableHashId(systemId, serviceName)
|
id2 := makeStableHashId(systemId, serviceName)
|
||||||
id3 := MakeStableHashId(systemId, serviceName)
|
id3 := makeStableHashId(systemId, serviceName)
|
||||||
|
|
||||||
assert.Equal(t, id1, id2)
|
assert.Equal(t, id1, id2)
|
||||||
assert.Equal(t, id2, id3)
|
assert.Equal(t, id2, id3)
|
||||||
@@ -29,10 +29,10 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
serviceName1 := "nginx.service"
|
serviceName1 := "nginx.service"
|
||||||
serviceName2 := "apache.service"
|
serviceName2 := "apache.service"
|
||||||
|
|
||||||
id1 := MakeStableHashId(systemId1, serviceName1)
|
id1 := makeStableHashId(systemId1, serviceName1)
|
||||||
id2 := MakeStableHashId(systemId2, serviceName1)
|
id2 := makeStableHashId(systemId2, serviceName1)
|
||||||
id3 := MakeStableHashId(systemId1, serviceName2)
|
id3 := makeStableHashId(systemId1, serviceName2)
|
||||||
id4 := MakeStableHashId(systemId2, serviceName2)
|
id4 := makeStableHashId(systemId2, serviceName2)
|
||||||
|
|
||||||
// All IDs should be different
|
// All IDs should be different
|
||||||
assert.NotEqual(t, id1, id2)
|
assert.NotEqual(t, id1, id2)
|
||||||
@@ -56,14 +56,14 @@ func TestGetSystemdServiceId(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range testCases {
|
for _, tc := range testCases {
|
||||||
id := MakeStableHashId(tc.systemId, tc.serviceName)
|
id := makeStableHashId(tc.systemId, tc.serviceName)
|
||||||
// FNV-32 produces 8 hex characters
|
// FNV-32 produces 8 hex characters
|
||||||
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
|
assert.Len(t, id, 8, "ID should be 8 characters for systemId='%s', serviceName='%s'", tc.systemId, tc.serviceName)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
t.Run("hexadecimal output", func(t *testing.T) {
|
t.Run("hexadecimal output", func(t *testing.T) {
|
||||||
id := MakeStableHashId("test-system", "test-service")
|
id := makeStableHashId("test-system", "test-service")
|
||||||
assert.NotEmpty(t, id)
|
assert.NotEmpty(t, id)
|
||||||
|
|
||||||
// Should only contain hexadecimal characters
|
// Should only contain hexadecimal characters
|
||||||
|
|||||||
@@ -32,12 +32,13 @@ func (sys *System) FetchAndSaveZfsPools(force bool) error {
|
|||||||
sys.recordZfsFetchResult(err, 0)
|
sys.recordZfsFetchResult(err, 0)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
err = sys.saveZfsPools(zfsData)
|
if zfsData == nil || !zfsData.Complete {
|
||||||
poolCount := 0
|
err = errIncompleteZfsData
|
||||||
if zfsData != nil {
|
sys.recordZfsFetchResult(err, 0)
|
||||||
poolCount = len(zfsData.Pools)
|
return err
|
||||||
}
|
}
|
||||||
sys.recordZfsFetchResult(err, poolCount)
|
err = sys.saveZfsPools(zfsData)
|
||||||
|
sys.recordZfsFetchResult(err, len(zfsData.Pools))
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -78,7 +79,7 @@ func (sys *System) zfsFetchInterval() time.Duration {
|
|||||||
// saveZfsPools saves ZFS pool detail data to the zfs_pools collection and
|
// saveZfsPools saves ZFS pool detail data to the zfs_pools collection and
|
||||||
// removes records for pools no longer reported by a complete agent inventory.
|
// removes records for pools no longer reported by a complete agent inventory.
|
||||||
func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
||||||
if zfsData == nil || (!zfsData.CanRefreshPool("zfs") && !zfsData.CanRefreshPool("b:")) {
|
if zfsData == nil || !zfsData.Complete {
|
||||||
return errIncompleteZfsData
|
return errIncompleteZfsData
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,10 +89,10 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = hub.RunInTransaction(func(txApp core.App) error {
|
return hub.RunInTransaction(func(txApp core.App) error {
|
||||||
alive := make(map[string]bool, len(zfsData.Pools))
|
alive := make(map[string]bool, len(zfsData.Pools))
|
||||||
for _, pool := range zfsData.Pools {
|
for _, pool := range zfsData.Pools {
|
||||||
if pool == nil || !zfsData.CanRefreshPool(pool.Name) {
|
if pool == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
alive[pool.Name] = true
|
alive[pool.Name] = true
|
||||||
@@ -110,7 +111,7 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, record := range existing {
|
for _, record := range existing {
|
||||||
if name := record.GetString("name"); zfsData.CanRefreshPool(name) && !alive[name] {
|
if !alive[record.GetString("name")] {
|
||||||
if err := txApp.Delete(record); err != nil {
|
if err := txApp.Delete(record); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -118,18 +119,10 @@ func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Report partial failure only after committing healthy backend updates.
|
|
||||||
if !zfsData.Complete {
|
|
||||||
return errIncompleteZfsData
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
||||||
recordID := MakeStableHashId(sys.Id, pool.Name)
|
recordID := makeStableHashId(sys.Id, pool.Name)
|
||||||
|
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -142,12 +135,10 @@ func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection
|
|||||||
|
|
||||||
record.Set("system", sys.Id)
|
record.Set("system", sys.Id)
|
||||||
record.Set("name", pool.Name)
|
record.Set("name", pool.Name)
|
||||||
record.Set("display_name", pool.DisplayName)
|
|
||||||
record.Set("health", pool.Health)
|
record.Set("health", pool.Health)
|
||||||
record.Set("size", pool.Size)
|
record.Set("size", pool.Size)
|
||||||
record.Set("alloc", pool.Alloc)
|
record.Set("alloc", pool.Alloc)
|
||||||
record.Set("free", pool.Free)
|
record.Set("free", pool.Free)
|
||||||
record.Set("raw", pool.Raw)
|
|
||||||
record.Set("scrub", pool.Scrub)
|
record.Set("scrub", pool.Scrub)
|
||||||
record.Set("vdevs", pool.Vdevs)
|
record.Set("vdevs", pool.Vdevs)
|
||||||
record.Set("datasets", pool.Datasets)
|
record.Set("datasets", pool.Datasets)
|
||||||
@@ -171,7 +162,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
|
|||||||
if pool == nil {
|
if pool == nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
recordID := MakeStableHashId(sys.Id, name)
|
recordID := makeStableHashId(sys.Id, name)
|
||||||
record, err := app.FindRecordById(collection, recordID)
|
record, err := app.FindRecordById(collection, recordID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if !errors.Is(err, sql.ErrNoRows) {
|
if !errors.Is(err, sql.ErrNoRows) {
|
||||||
@@ -181,9 +172,7 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
|
|||||||
record.Set("id", recordID)
|
record.Set("id", recordID)
|
||||||
record.Set("system", sys.Id)
|
record.Set("system", sys.Id)
|
||||||
record.Set("name", name)
|
record.Set("name", name)
|
||||||
record.Set("display_name", pool.DisplayName)
|
|
||||||
record.Set("health", pool.Health)
|
record.Set("health", pool.Health)
|
||||||
record.Set("raw", pool.Raw)
|
|
||||||
record.Set("size", uint64(pool.Total*gib))
|
record.Set("size", uint64(pool.Total*gib))
|
||||||
record.Set("alloc", uint64(pool.Used*gib))
|
record.Set("alloc", uint64(pool.Used*gib))
|
||||||
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
|
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
|
||||||
@@ -192,15 +181,10 @@ func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsP
|
|||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if record.GetString("health") == pool.Health && record.GetBool("raw") == pool.Raw && record.GetString("display_name") == pool.DisplayName {
|
if record.GetString("health") == pool.Health {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
record.Set("display_name", pool.DisplayName)
|
|
||||||
record.Set("health", pool.Health)
|
record.Set("health", pool.Health)
|
||||||
record.Set("raw", pool.Raw)
|
|
||||||
record.Set("size", uint64(pool.Total*gib))
|
|
||||||
record.Set("alloc", uint64(pool.Used*gib))
|
|
||||||
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
|
|
||||||
if err := app.SaveNoValidate(record); err != nil {
|
if err := app.SaveNoValidate(record); err != nil {
|
||||||
return fmt.Errorf("updating ZFS pool health %q: %w", name, err)
|
return fmt.Errorf("updating ZFS pool health %q: %w", name, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -123,44 +123,6 @@ func TestSaveZfsPoolsIncompletePreservesRecords(t *testing.T) {
|
|||||||
assert.Len(t, records, 1)
|
assert.Len(t, records, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSavePartialBackendInventory(t *testing.T) {
|
|
||||||
for _, healthy := range []string{"zfs", "btrfs"} {
|
|
||||||
t.Run(healthy, func(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
healthyKey, failedKey := "tank", "b:uuid"
|
|
||||||
if healthy == "btrfs" {
|
|
||||||
healthyKey, failedKey = failedKey, healthyKey
|
|
||||||
}
|
|
||||||
initial := &zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{
|
|
||||||
{Name: healthyKey, Alloc: 10}, {Name: failedKey, Alloc: 10},
|
|
||||||
}}
|
|
||||||
require.NoError(t, sys.saveZfsPools(initial))
|
|
||||||
failedID := MakeStableHashId(sys.Id, failedKey)
|
|
||||||
before, err := app.FindRecordById("zfs_pools", failedID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
partial := &zfs.ZfsData{CompleteBackends: []string{healthy}, Pools: []*zfs.PoolDetail{
|
|
||||||
{Name: healthyKey, Alloc: 20}, {Name: failedKey, Alloc: 99},
|
|
||||||
}}
|
|
||||||
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
|
|
||||||
fresh, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, healthyKey))
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.EqualValues(t, 20, fresh.GetInt("alloc"))
|
|
||||||
cached, err := app.FindRecordById("zfs_pools", failedID)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.EqualValues(t, 10, cached.GetInt("alloc"))
|
|
||||||
assert.Equal(t, before.GetDateTime("details_updated"), cached.GetDateTime("details_updated"))
|
|
||||||
// An empty successful backend can prune, even while the other fails.
|
|
||||||
partial.Pools = nil
|
|
||||||
assert.ErrorIs(t, sys.saveZfsPools(partial), errIncompleteZfsData)
|
|
||||||
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Len(t, records, 1)
|
|
||||||
assert.Equal(t, failedKey, records[0].GetString("name"))
|
|
||||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true}))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
||||||
sys, app := newTestSystemWithHub(t)
|
sys, app := newTestSystemWithHub(t)
|
||||||
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
||||||
@@ -169,7 +131,7 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
|||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||||
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
||||||
}))
|
}))
|
||||||
record, err := app.FindRecordById(collection, MakeStableHashId(sys.Id, "tank"))
|
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
firstUpdated := record.GetDateTime("updated")
|
firstUpdated := record.GetDateTime("updated")
|
||||||
assert.Equal(t, "ONLINE", record.GetString("health"))
|
assert.Equal(t, "ONLINE", record.GetString("health"))
|
||||||
@@ -189,42 +151,3 @@ func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
|||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
assert.Equal(t, "DEGRADED", record.GetString("health"))
|
assert.Equal(t, "DEGRADED", record.GetString("health"))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestZfsRawCapacityPersistence(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{{Name: "btrfs", Size: 200, Alloc: 10, Raw: true}}}))
|
|
||||||
record, err := app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "btrfs"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.True(t, record.GetBool("raw"))
|
|
||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{"btrfs": {Total: 1, Used: 0.25}}))
|
|
||||||
record, err = app.FindRecordById("zfs_pools", record.Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.False(t, record.GetBool("raw"))
|
|
||||||
assert.EqualValues(t, 1024*1024*1024, record.GetInt("size"))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestBtrfsDisplayNameKeepsRecordIdentity(t *testing.T) {
|
|
||||||
sys, app := newTestSystemWithHub(t)
|
|
||||||
key := "b:11111111-1111-4111-8111-111111111111"
|
|
||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
|
||||||
key: {DisplayName: "tank", Health: "ONLINE"},
|
|
||||||
"tank": {Health: "ONLINE"},
|
|
||||||
}))
|
|
||||||
id := MakeStableHashId(sys.Id, key)
|
|
||||||
record, err := app.FindRecordById("zfs_pools", id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "tank", record.GetString("display_name"))
|
|
||||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{key: {DisplayName: "renamed", Health: "ONLINE"}}))
|
|
||||||
record, err = app.FindRecordById("zfs_pools", id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, key, record.GetString("name"))
|
|
||||||
assert.Equal(t, "renamed", record.GetString("display_name"))
|
|
||||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true, Pools: []*zfs.PoolDetail{
|
|
||||||
{Name: key, DisplayName: "detail name", Health: "ONLINE"}, {Name: "tank", Health: "ONLINE"},
|
|
||||||
}}))
|
|
||||||
record, err = app.FindRecordById("zfs_pools", id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Equal(t, "detail name", record.GetString("display_name"))
|
|
||||||
_, err = app.FindRecordById("zfs_pools", MakeStableHashId(sys.Id, "tank"))
|
|
||||||
require.NoError(t, err)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -3,11 +3,9 @@
|
|||||||
package systems_test
|
package systems_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/fxamacker/cbor/v2"
|
|
||||||
"github.com/henrygd/beszel/internal/entities/system"
|
"github.com/henrygd/beszel/internal/entities/system"
|
||||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||||
"github.com/henrygd/beszel/internal/hub/systems"
|
"github.com/henrygd/beszel/internal/hub/systems"
|
||||||
@@ -17,42 +15,6 @@ import (
|
|||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestCreateRecordsRejectsNullSystemdService(t *testing.T) {
|
|
||||||
hub, user := tests.GetHubWithUser(t)
|
|
||||||
defer hub.Cleanup()
|
|
||||||
records, err := tests.CreateSystems(hub, 1, user.Id, "paused")
|
|
||||||
require.NoError(t, err)
|
|
||||||
sys, err := hub.GetSystemManager().GetSystem(records[0].Id)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.NoError(t, systems.CreateSystemdStatsRecords(hub, []*systemd.Service{
|
|
||||||
{Name: "existing.service", State: systemd.StatusFailed},
|
|
||||||
}, records[0].Id))
|
|
||||||
|
|
||||||
for _, services := range []string{`[null]`, `[{"name":"new.service"},null]`, `[null,{"name":"new.service"}]`} {
|
|
||||||
for _, encoding := range []string{"json", "cbor"} {
|
|
||||||
t.Run(encoding+"/"+services, func(t *testing.T) {
|
|
||||||
var data system.CombinedData
|
|
||||||
require.NoError(t, json.Unmarshal([]byte(`{"systemd":`+services+`}`), &data))
|
|
||||||
if encoding == "cbor" {
|
|
||||||
encoded, err := cbor.Marshal(data)
|
|
||||||
require.NoError(t, err)
|
|
||||||
data = system.CombinedData{}
|
|
||||||
require.NoError(t, cbor.Unmarshal(encoded, &data))
|
|
||||||
}
|
|
||||||
_, err := sys.CreateRecords(&data)
|
|
||||||
require.ErrorContains(t, err, "null systemd service")
|
|
||||||
var names []string
|
|
||||||
require.NoError(t, hub.DB().Select("name").From("systemd_services").
|
|
||||||
Where(dbx.HashExp{"system": records[0].Id}).Column(&names))
|
|
||||||
assert.Equal(t, []string{"existing.service"}, names)
|
|
||||||
count, err := hub.CountRecords("system_stats", dbx.HashExp{"system": records[0].Id})
|
|
||||||
require.NoError(t, err)
|
|
||||||
assert.Zero(t, count, "invalid snapshot must roll back system stats")
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestCreateRecordsHandlesSystemdAlertLifecycle(t *testing.T) {
|
func TestCreateRecordsHandlesSystemdAlertLifecycle(t *testing.T) {
|
||||||
hub, user := tests.GetHubWithUser(t)
|
hub, user := tests.GetHubWithUser(t)
|
||||||
defer hub.Cleanup()
|
defer hub.Cleanup()
|
||||||
|
|||||||
@@ -1,127 +0,0 @@
|
|||||||
//go:build testing
|
|
||||||
|
|
||||||
package hub
|
|
||||||
|
|
||||||
import (
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/stretchr/testify/assert"
|
|
||||||
"github.com/stretchr/testify/require"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestParseTrustedProxies(t *testing.T) {
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
value string
|
|
||||||
prefixes []string
|
|
||||||
restricted bool
|
|
||||||
}{
|
|
||||||
{
|
|
||||||
name: "empty",
|
|
||||||
value: "",
|
|
||||||
restricted: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "blank",
|
|
||||||
value: " , ",
|
|
||||||
prefixes: nil,
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "single addresses become host prefixes",
|
|
||||||
value: "10.0.0.5, 2001:db8::1",
|
|
||||||
prefixes: []string{"10.0.0.5/32", "2001:db8::1/128"},
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "cidrs are masked",
|
|
||||||
value: "172.16.5.9/12,fd00::1/64",
|
|
||||||
prefixes: []string{"172.16.0.0/12", "fd00::/64"},
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "ipv4-mapped entries become ipv4",
|
|
||||||
value: "::ffff:10.0.0.5, ::ffff:10.0.0.0/104",
|
|
||||||
prefixes: []string{"10.0.0.5/32", "10.0.0.0/8"},
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "invalid entries are skipped, valid ones kept",
|
|
||||||
value: "proxy.internal, 10.0.0.0/8, 300.1.1.1, ::ffff:0.0.0.0/64",
|
|
||||||
prefixes: []string{"10.0.0.0/8"},
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: "only invalid entries trust nobody",
|
|
||||||
value: "proxy.internal",
|
|
||||||
prefixes: nil,
|
|
||||||
restricted: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_PROXY_IPS", tc.value)
|
|
||||||
prefixes, restricted := parseTrustedProxies()
|
|
||||||
assert.Equal(t, tc.restricted, restricted)
|
|
||||||
var got []string
|
|
||||||
for _, p := range prefixes {
|
|
||||||
got = append(got, p.String())
|
|
||||||
}
|
|
||||||
assert.Equal(t, tc.prefixes, got)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("unset", func(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_PROXY_IPS", "")
|
|
||||||
os.Unsetenv("TRUSTED_PROXY_IPS")
|
|
||||||
prefixes, restricted := parseTrustedProxies()
|
|
||||||
assert.False(t, restricted)
|
|
||||||
assert.Nil(t, prefixes)
|
|
||||||
})
|
|
||||||
|
|
||||||
t.Run("prefixed env var takes precedence", func(t *testing.T) {
|
|
||||||
t.Setenv("TRUSTED_PROXY_IPS", "10.0.0.0/8")
|
|
||||||
t.Setenv("BESZEL_HUB_TRUSTED_PROXY_IPS", "192.168.0.0/16")
|
|
||||||
prefixes, restricted := parseTrustedProxies()
|
|
||||||
assert.True(t, restricted)
|
|
||||||
require.Len(t, prefixes, 1)
|
|
||||||
assert.Equal(t, "192.168.0.0/16", prefixes[0].String())
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsTrustedProxy(t *testing.T) {
|
|
||||||
prefixes := []netip.Prefix{
|
|
||||||
netip.MustParsePrefix("10.0.0.0/8"),
|
|
||||||
netip.MustParsePrefix("2001:db8::/32"),
|
|
||||||
netip.MustParsePrefix("fe80::/10"),
|
|
||||||
}
|
|
||||||
|
|
||||||
testCases := []struct {
|
|
||||||
name string
|
|
||||||
remoteAddr string
|
|
||||||
trusted bool
|
|
||||||
}{
|
|
||||||
{"ipv4 in prefix", "10.20.30.40:51234", true},
|
|
||||||
{"ipv4 outside prefix", "11.0.0.1:51234", false},
|
|
||||||
{"ipv6 in prefix", "[2001:db8:1::2]:443", true},
|
|
||||||
{"ipv6 outside prefix", "[2001:db9::1]:443", false},
|
|
||||||
{"ipv4-mapped ipv6 matches ipv4 prefix", "[::ffff:10.1.2.3]:80", true},
|
|
||||||
{"zone is ignored", "[fe80::1%eth0]:80", true},
|
|
||||||
{"no port", "10.1.2.3", true},
|
|
||||||
{"empty", "", false},
|
|
||||||
{"garbage", "not-an-address:80", false},
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, tc := range testCases {
|
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
|
||||||
assert.Equal(t, tc.trusted, isTrustedProxy(prefixes, tc.remoteAddr))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
t.Run("empty allowlist trusts nobody", func(t *testing.T) {
|
|
||||||
assert.False(t, isTrustedProxy(nil, "10.0.0.1:1"))
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -1,11 +1,7 @@
|
|||||||
// Package utils provides utility functions for the hub.
|
// Package utils provides utility functions for the hub.
|
||||||
package utils
|
package utils
|
||||||
|
|
||||||
import (
|
import "os"
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/pocketbase/pocketbase/core"
|
|
||||||
)
|
|
||||||
|
|
||||||
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
|
// GetEnv retrieves an environment variable with a "BESZEL_HUB_" prefix, or falls back to the unprefixed key.
|
||||||
func GetEnv(key string) (value string, exists bool) {
|
func GetEnv(key string) (value string, exists bool) {
|
||||||
@@ -14,26 +10,3 @@ func GetEnv(key string) (value string, exists bool) {
|
|||||||
}
|
}
|
||||||
return os.LookupEnv(key)
|
return os.LookupEnv(key)
|
||||||
}
|
}
|
||||||
|
|
||||||
// realtimeActiveForCollection checks if there are active WebSocket subscriptions for the given collection.
|
|
||||||
func RealtimeActiveForCollection(app core.App, collectionName string, validateFn func(filterQuery string) bool) bool {
|
|
||||||
broker := app.SubscriptionsBroker()
|
|
||||||
if broker.TotalClients() == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
for _, client := range broker.Clients() {
|
|
||||||
subs := client.Subscriptions(collectionName)
|
|
||||||
if len(subs) > 0 {
|
|
||||||
if validateFn == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for k := range subs {
|
|
||||||
filter := subs[k].Query["filter"]
|
|
||||||
if validateFn(filter) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user