feat: add ZFS monitoring (#2209)

- track pool capacity, health, I/O, scrub status, and vdev errors
- report dataset usage and correct ZFS filesystem metrics
- add pool charts, detail views, refresh controls, and health alerts
- persist pool details and include ZFS usage in disk alerts
- support configurable detail intervals and legacy agent compatibility

---------

Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
Tamás Vince
2026-09-01 12:19:36 -04:00
committed by GitHub
co-authored by hank
parent b38fb7dafa
commit 917d069ab3
46 changed files with 3768 additions and 15 deletions
+14
View File
@@ -48,6 +48,7 @@ type Agent struct {
keys []gossh.PublicKey // SSH public keys
smartManager *SmartManager // Manages SMART data
systemdManager *systemdManager // Manages systemd services
zfsManager *ZfsManager // Manages ZFS pool and dataset data
}
// NewAgent creates a new agent with the given data directory for persisting data.
@@ -121,6 +122,19 @@ func NewAgent(dataDir ...string) (agent *Agent, err error) {
// initialize handler registry
agent.handlerRegistry = NewHandlerRegistry()
agent.zfsManager = newZfsManager()
// ZFS_INTERVAL env var to update ZFS detail data at this interval
if zfsIntervalEnv, exists := utils.GetEnv("ZFS_INTERVAL"); exists {
if duration, err := time.ParseDuration(zfsIntervalEnv); err == nil && duration > 0 {
agent.zfsManager.detailInterval = duration
agent.systemDetails.ZfsInterval = duration
slog.Info("ZFS_INTERVAL", "duration", duration)
} else {
slog.Warn("Invalid ZFS_INTERVAL", "err", err)
}
}
// initialize disk info
agent.initializeDiskInfo()
+35 -7
View File
@@ -537,7 +537,16 @@ func normalizeDeviceName(value string) string {
func (a *Agent) initializeDiskIoStats(diskIoCounters map[string]disk.IOCountersStat) {
a.fsNames = a.fsNames[:0]
now := time.Now()
// ZFS datasets have no /proc/diskstats entry, so they are excluded from
// I/O tracking instead of warning about a missing device (#1541).
var zfsMountpoints map[string]bool
if a.zfsManager != nil {
zfsMountpoints = a.zfsManager.ZfsMountpoints()
}
for device, stats := range a.fsStats {
if zfsMountpoints[stats.Mountpoint] {
continue
}
// skip if not in diskIoCounters
d, exists := diskIoCounters[device]
if !exists {
@@ -562,20 +571,31 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
!a.lastDiskUsageUpdate.IsZero() &&
time.Since(a.lastDiskUsageUpdate) < a.diskUsageCacheDuration
// ZFS dataset mountpoints use `zfs list` values because statfs(2) reports
// dataset-level usage that excludes child datasets (#1541).
var zfsUsage map[string]zfsDatasetUsage
if a.zfsManager != nil {
zfsUsage = a.zfsManager.DatasetUsage()
}
// disk usage
for _, stats := range a.fsStats {
// Skip non-root filesystems if caching is active
if cacheExtraFs && !stats.Root {
continue
}
if d, err := disk.Usage(stats.Mountpoint); err == nil {
stats.DiskTotal = utils.BytesToGigabytes(d.Total)
stats.DiskUsed = utils.BytesToGigabytes(d.Used)
if stats.Root {
systemStats.DiskTotal = utils.BytesToGigabytes(d.Total)
systemStats.DiskUsed = utils.BytesToGigabytes(d.Used)
systemStats.DiskPct = utils.TwoDecimals(d.UsedPercent)
var total, used uint64
var usedPct float64
if u, ok := zfsUsage[stats.Mountpoint]; ok {
total = u.used + u.avail
used = u.used
if total > 0 {
usedPct = float64(used) / float64(total) * 100
}
} else if d, err := disk.Usage(stats.Mountpoint); err == nil {
total = d.Total
used = d.Used
usedPct = d.UsedPercent
} else {
// reset stats if error (likely unmounted)
slog.Error("Error getting disk stats", "name", stats.Mountpoint, "err", err)
@@ -583,6 +603,14 @@ func (a *Agent) updateDiskUsage(systemStats *system.Stats) {
stats.DiskUsed = 0
stats.TotalRead = 0
stats.TotalWrite = 0
continue
}
stats.DiskTotal = utils.BytesToGigabytes(total)
stats.DiskUsed = utils.BytesToGigabytes(used)
if stats.Root {
systemStats.DiskTotal = stats.DiskTotal
systemStats.DiskUsed = stats.DiskUsed
systemStats.DiskPct = utils.TwoDecimals(usedPct)
}
}
+109
View File
@@ -0,0 +1,109 @@
//go:build testing
package agent
import (
"testing"
"github.com/henrygd/beszel/agent/zfs"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/shirou/gopsutil/v4/disk"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestUpdateDiskUsageZfsMountpoint verifies that a filesystem whose mountpoint
// is a ZFS dataset reports `zfs list` usage (which includes child datasets)
// instead of the dataset-scoped statfs values (#1541).
func TestUpdateDiskUsageZfsMountpoint(t *testing.T) {
zm := &ZfsManager{}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
}, nil
}
agent := &Agent{
fsStats: map[string]*system.FsStats{
"tank": {Root: false, Mountpoint: "/tank"},
},
zfsManager: zm,
}
var stats system.Stats
agent.updateDiskUsage(&stats)
fs := agent.fsStats["tank"]
require.NotNil(t, fs)
assert.Equal(t, 22350.81, fs.DiskTotal) // (used + avail) in GiB
assert.Equal(t, 11175.87, fs.DiskUsed)
// Non-root filesystems do not populate system-level stats.
assert.Equal(t, float64(0), stats.DiskTotal)
}
// TestUpdateDiskUsageZfsRootPopulatesSystemStats verifies the root disk values
// are derived from ZFS usage when the root mountpoint is a ZFS dataset.
func TestUpdateDiskUsageZfsRootPopulatesSystemStats(t *testing.T) {
zm := &ZfsManager{}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{
{Name: "rpool/ROOT/pve-1", Used: 900000000000, Avail: 300000000000, Mountpoint: "/"},
}, nil
}
agent := &Agent{
fsStats: map[string]*system.FsStats{
"rpool/ROOT/pve-1": {Root: true, Mountpoint: "/"},
},
zfsManager: zm,
}
var stats system.Stats
agent.updateDiskUsage(&stats)
assert.Equal(t, 1117.59, agent.fsStats["rpool/ROOT/pve-1"].DiskTotal)
assert.Equal(t, 838.19, agent.fsStats["rpool/ROOT/pve-1"].DiskUsed)
assert.Equal(t, 75.0, stats.DiskPct)
assert.Equal(t, 1117.59, stats.DiskTotal)
assert.Equal(t, 838.19, stats.DiskUsed)
}
// TestUpdateDiskUsageWithoutZfsManager falls back to statfs when no manager is
// present (e.g. tests constructing bare Agent values).
func TestUpdateDiskUsageWithoutZfsManager(t *testing.T) {
agent := &Agent{
fsStats: map[string]*system.FsStats{
"root": {Root: true, Mountpoint: "/"},
},
}
var stats system.Stats
agent.updateDiskUsage(&stats)
assert.True(t, agent.fsStats["root"].DiskTotal > 0, "root usage should come from statfs")
assert.True(t, stats.DiskTotal > 0)
}
// TestInitializeDiskIoStatsSkipsZfsMountpoints verifies ZFS filesystems are
// excluded from diskstats I/O tracking instead of warning about a missing device.
func TestInitializeDiskIoStatsSkipsZfsMountpoints(t *testing.T) {
zm := &ZfsManager{}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{{Name: "tank", Mountpoint: "/tank"}}, nil
}
agent := &Agent{
fsStats: map[string]*system.FsStats{
"tank": {Root: false, Mountpoint: "/tank"},
"sda1": {Root: false, Mountpoint: "/mnt/data"},
},
zfsManager: zm,
diskPrev: make(map[uint16]map[string]prevDisk),
}
agent.initializeDiskIoStats(map[string]disk.IOCountersStat{
"sda1": {Name: "sda1", ReadBytes: 100, WriteBytes: 100},
})
assert.Equal(t, []string{"sda1"}, agent.fsNames)
assert.Equal(t, uint64(100), agent.fsStats["sda1"].TotalRead)
// ZFS entry is present but untouched by diskstats initialization.
assert.Equal(t, uint64(0), agent.fsStats["tank"].TotalRead)
}
+18
View File
@@ -51,6 +51,7 @@ func NewHandlerRegistry() *HandlerRegistry {
registry.Register(common.GetContainerInfo, &GetContainerInfoHandler{})
registry.Register(common.GetSmartData, &GetSmartDataHandler{})
registry.Register(common.GetSystemdInfo, &GetSystemdInfoHandler{})
registry.Register(common.GetZfsData, &GetZfsDataHandler{})
return registry
}
@@ -178,6 +179,23 @@ func (h *GetSmartDataHandler) Handle(hctx *HandlerContext) error {
}, hctx.RequestID)
}
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
// GetZfsDataHandler handles ZFS detail data requests
type GetZfsDataHandler struct{}
func (h *GetZfsDataHandler) Handle(hctx *HandlerContext) error {
if hctx.Agent.zfsManager == nil {
return hctx.SendResponse(nil, hctx.RequestID)
}
var req common.ZfsDataRequest
if err := cbor.Unmarshal(hctx.Request.Data, &req); err != nil {
return err
}
return hctx.SendResponse(hctx.Agent.zfsManager.GetDetail(req.Force), hctx.RequestID)
}
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////
+28
View File
@@ -4,8 +4,10 @@ package agent
import (
"testing"
"time"
"github.com/fxamacker/cbor/v2"
"github.com/henrygd/beszel/agent/zfs"
"github.com/henrygd/beszel/internal/common"
"github.com/henrygd/beszel/internal/entities/smart"
"github.com/stretchr/testify/assert"
@@ -30,6 +32,32 @@ func TestNewAgentResponseSmartData(t *testing.T) {
assert.True(t, response.SmartComplete)
}
func TestGetZfsDataHandlerForceRefresh(t *testing.T) {
poolCalls := 0
zm := &ZfsManager{detailInterval: time.Hour}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
poolCalls++
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
}
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
zm.GetDetail(false)
requestData, err := cbor.Marshal(common.ZfsDataRequest{Force: true})
assert.NoError(t, err)
ctx := &HandlerContext{
Agent: &Agent{zfsManager: zm},
Request: &common.HubRequest[cbor.RawMessage]{
Action: common.GetZfsData,
Data: requestData,
},
SendResponse: func(any, *uint32) error { return nil },
}
assert.NoError(t, (&GetZfsDataHandler{}).Handle(ctx))
assert.Equal(t, 2, poolCalls)
}
func (m *MockHandler) Handle(ctx *HandlerContext) error {
if m.handleFunc != nil {
return m.handleFunc(ctx)
+3
View File
@@ -219,6 +219,9 @@ func (a *Agent) getSystemStats(cacheTimeMs uint16) system.Stats {
// disk i/o (cache-aware per interval)
a.updateDiskIo(cacheTimeMs, &systemStats)
// zfs pool stats
a.zfsManager.Update(&systemStats)
// network stats (per cache interval)
a.updateNetworkStats(cacheTimeMs, &systemStats)
+9
View File
@@ -0,0 +1,9 @@
tank 12000000000000 11999000000000 /tank
tank/apps 1000000000000 11999000000000 /tank/apps
tank/backup 2000000000000 11999000000000 /tank/backup
tank/media 1000000000000 11999000000000 /tank/my media
rpool 900000000000 300000000000 -
rpool/ROOT 1000000000 300000000000 -
rpool/ROOT/pve-1 890000000000 300000000000 /
rpool/data 9000000000 300000000000 -
rpool/data/subvol-100-disk-0 400000000000 300000000000 /subvol-100-disk-0
+2
View File
@@ -0,0 +1,2 @@
tank 23999000000000 12000000000000 11999000000000 ONLINE
rpool 1200000000000 900000000000 300000000000 DEGRADED
+29
View File
@@ -0,0 +1,29 @@
pool: tank
state: ONLINE
scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025
config:
NAME STATE READ WRITE CKSUM
tank ONLINE 0 0 0
mirror-0 ONLINE 0 0 0
sda ONLINE 0 0 0
sdb ONLINE 0 0 0
errors: No known data errors
pool: rpool
state: DEGRADED
status: One or more devices could not be used because the label is missing or
invalid. Sufficient replicas exist for the pool to continue functioning in a
degraded state.
scan: scrub in progress since Sun Jun 8 01:00:00 2025
10.00% done, 01:30:00 to go, 0.00/s
config:
NAME STATE READ WRITE CKSUM
rpool DEGRADED 0 0 0
mirror-0 DEGRADED 0 0 0
sda ONLINE 0 0 0
sdb FAULTED 1 2 3
errors: 1 data errors, use '-v' for a list
+160
View File
@@ -0,0 +1,160 @@
// Package zfs provides functions to read ZFS statistics.
package zfs
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"os"
"os/exec"
"strconv"
"strings"
"time"
)
var commandTimeout = 10 * time.Second
var commandOutput = func(name string, args ...string) ([]byte, error) {
ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, name, args...)
cmd.Env = append(os.Environ(), "LC_ALL=C", "LANG=C")
out, err := cmd.Output()
if ctx.Err() != nil {
return nil, fmt.Errorf("%s timed out after %s: %w", name, commandTimeout, ctx.Err())
}
return out, err
}
// ErrNoZfs is returned when the ZFS utilities or kernel interfaces are unavailable.
var ErrNoZfs = errors.New("zfs utilities unavailable")
// PoolStat is a snapshot of a ZFS pool's capacity and health.
type PoolStat struct {
Name string
Size uint64 // total capacity in bytes
Alloc uint64 // allocated bytes
Free uint64 // free bytes
Health string // ONLINE, DEGRADED, FAULTED, ...
}
// PoolKernelStat is the inexpensive pool telemetry exposed by the ZFS kernel.
// NRead and NWrite are cumulative byte counters since the pool was imported.
type PoolKernelStat struct {
Name string
Health string
NRead uint64
NWrite uint64
}
// PoolIoStats holds calculated per-second I/O rates for a pool.
type PoolIoStats struct {
NRead uint64
NWrite uint64
}
// Dataset is a single ZFS dataset with usage information.
type Dataset struct {
Name string
Used uint64
Avail uint64
Mountpoint string
}
// PoolStats returns capacity and health for all pools on the system using
// `zpool list`. Frequent health and I/O sampling uses PoolKernelStats instead.
func PoolStats() ([]PoolStat, error) {
out, err := commandOutput("zpool", "list", "-Hp", "-o", "name,size,alloc,free,health")
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && strings.Contains(string(exitErr.Stderr), "no pools available") {
return nil, nil
}
return nil, fmt.Errorf("zpool list: %w", err)
}
return parseZpoolListOutput(out)
}
// Datasets returns all datasets on the system with usage and mountpoint
// information using `zfs list` (recursive by default).
func Datasets() ([]Dataset, error) {
out, err := commandOutput("zfs", "list", "-Hp", "-o", "name,used,avail,mountpoint")
if err != nil {
return nil, fmt.Errorf("zfs list: %w", err)
}
return parseZfsListOutput(out)
}
// parseZpoolListOutput parses `zpool list -Hp -o name,size,alloc,free,health` output.
// Columns are tab-separated; numeric columns are raw bytes.
func parseZpoolListOutput(out []byte) ([]PoolStat, error) {
var pools []PoolStat
scanner := bufio.NewScanner(bytes.NewReader(out))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
if line == "no pools available" && len(pools) == 0 {
return nil, nil
}
fields := strings.Split(line, "\t")
if len(fields) < 5 {
return nil, fmt.Errorf("unexpected zpool list line: %q", line)
}
size, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing size for pool %q: %w", fields[0], err)
}
alloc, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing alloc for pool %q: %w", fields[0], err)
}
free, err := strconv.ParseUint(fields[3], 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing free for pool %q: %w", fields[0], err)
}
pools = append(pools, PoolStat{
Name: fields[0],
Size: size,
Alloc: alloc,
Free: free,
Health: fields[4],
})
}
return pools, scanner.Err()
}
// parseZfsListOutput parses `zfs list -Hp -o name,used,avail,mountpoint` output.
// The mountpoint column may contain spaces, so it is split on tabs only.
func parseZfsListOutput(out []byte) ([]Dataset, error) {
var datasets []Dataset
scanner := bufio.NewScanner(bytes.NewReader(out))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
fields := strings.SplitN(line, "\t", 4)
if len(fields) < 4 {
return nil, fmt.Errorf("unexpected zfs list line: %q", line)
}
used, err := strconv.ParseUint(fields[1], 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing used for dataset %q: %w", fields[0], err)
}
avail, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return nil, fmt.Errorf("parsing avail for dataset %q: %w", fields[0], err)
}
datasets = append(datasets, Dataset{
Name: fields[0],
Used: used,
Avail: avail,
Mountpoint: fields[3],
})
}
return datasets, scanner.Err()
}
+8
View File
@@ -3,9 +3,17 @@
package zfs
import (
"errors"
"golang.org/x/sys/unix"
)
func ARCSize() (uint64, error) {
return unix.SysctlUint64("kstat.zfs.misc.arcstats.size")
}
// FreeBSD does not expose Linux's per-pool procfs kstats. Capacity, health,
// and detail collection still work through the cached utilities.
func PoolKernelStats() ([]PoolKernelStat, error) {
return nil, errors.ErrUnsupported
}
+166 -1
View File
@@ -5,14 +5,18 @@ package zfs
import (
"bufio"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
)
var procZfsPath = "/proc/spl/kstat/zfs"
func ARCSize() (uint64, error) {
file, err := os.Open("/proc/spl/kstat/zfs/arcstats")
file, err := os.Open(filepath.Join(procZfsPath, "arcstats"))
if err != nil {
return 0, err
}
@@ -29,6 +33,167 @@ func ARCSize() (uint64, error) {
return strconv.ParseUint(fields[2], 10, 64)
}
}
if err := scanner.Err(); err != nil {
return 0, err
}
return 0, fmt.Errorf("size field not found in arcstats")
}
// PoolKernelStats reads pool state and cumulative I/O counters directly from
// procfs. These kstats are the same interfaces used by node_exporter's Linux
// ZFS collector and avoid keeping a `zpool iostat` subprocess alive.
func PoolKernelStats() ([]PoolKernelStat, error) {
poolDirs := make(map[string]struct{})
for _, filename := range []string{"state", "io", "objset-*"} {
paths, err := filepath.Glob(filepath.Join(procZfsPath, "*", filename))
if err != nil {
return nil, err
}
for _, path := range paths {
poolDirs[filepath.Dir(path)] = struct{}{}
}
}
if len(poolDirs) == 0 {
return nil, ErrNoZfs
}
pools := make([]PoolKernelStat, 0, len(poolDirs))
for poolDir := range poolDirs {
nread, nwrite, err := readPoolCounters(poolDir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
continue // pool may have been exported after the glob
}
return nil, err
}
state, err := os.ReadFile(filepath.Join(poolDir, "state"))
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
pools = append(pools, PoolKernelStat{
Name: filepath.Base(poolDir), Health: strings.ToUpper(strings.TrimSpace(string(state))),
NRead: nread, NWrite: nwrite,
})
}
if len(pools) == 0 {
return nil, ErrNoZfs
}
return pools, nil
}
// readPoolCounters supports both ZFS kernel interfaces. OpenZFS through 2.3
// exposes aggregate vdev counters in "io". When that file is unavailable, sum
// the logical I/O counters exposed for each dataset in the pool.
func readPoolCounters(poolDir string) (uint64, uint64, error) {
nread, nwrite, err := readPoolIO(filepath.Join(poolDir, "io"))
if err == nil || !errors.Is(err, os.ErrNotExist) {
return nread, nwrite, err
}
return readPoolObjsets(poolDir)
}
func readPoolIO(path string) (uint64, uint64, error) {
file, err := os.Open(path)
if err != nil {
return 0, 0, err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 2 || fields[0] != "nread" {
continue
}
if !scanner.Scan() {
break
}
values := strings.Fields(scanner.Text())
if len(values) < 2 {
break
}
nread, err := strconv.ParseUint(values[0], 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("parsing nread in %s: %w", path, err)
}
nwrite, err := strconv.ParseUint(values[1], 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("parsing nwritten in %s: %w", path, err)
}
return nread, nwrite, nil
}
if err := scanner.Err(); err != nil {
return 0, 0, err
}
return 0, 0, fmt.Errorf("I/O counters not found in %s", path)
}
func readPoolObjsets(poolDir string) (uint64, uint64, error) {
paths, err := filepath.Glob(filepath.Join(poolDir, "objset-*"))
if err != nil {
return 0, 0, err
}
if len(paths) == 0 {
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
}
var totalRead, totalWrite uint64
objsetsRead := 0
for _, path := range paths {
nread, nwrite, err := readObjsetIO(path)
if errors.Is(err, os.ErrNotExist) {
continue // dataset may have been destroyed after the glob
}
if err != nil {
return 0, 0, err
}
totalRead += nread
totalWrite += nwrite
objsetsRead++
}
if objsetsRead == 0 {
return 0, 0, fmt.Errorf("dataset I/O counters not found in %s", poolDir)
}
return totalRead, totalWrite, nil
}
func readObjsetIO(path string) (uint64, uint64, error) {
file, err := os.Open(path)
if err != nil {
return 0, 0, err
}
defer file.Close()
var nread, nwrite uint64
var foundRead, foundWrite bool
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
if len(fields) < 3 {
continue
}
var target *uint64
switch fields[0] {
case "nread":
target = &nread
foundRead = true
case "nwritten":
target = &nwrite
foundWrite = true
default:
continue
}
value, err := strconv.ParseUint(fields[2], 10, 64)
if err != nil {
return 0, 0, fmt.Errorf("parsing %s in %s: %w", fields[0], path, err)
}
*target = value
}
if err := scanner.Err(); err != nil {
return 0, 0, err
}
if !foundRead || !foundWrite {
return 0, 0, fmt.Errorf("incomplete I/O counters in %s", path)
}
return nread, nwrite, nil
}
+90
View File
@@ -0,0 +1,90 @@
//go:build testing && linux
package zfs
import (
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPoolKernelStats(t *testing.T) {
root := t.TempDir()
oldPath := procZfsPath
procZfsPath = root
t.Cleanup(func() { procZfsPath = oldPath })
poolDir := filepath.Join(root, "tank")
require.NoError(t, os.MkdirAll(poolDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "io"), []byte(
"11 3 0x00 1 80 0 0\n"+
"nread nwritten reads writes wtime wlentime wupdate rtime rlentime rupdate wcnt rcnt\n"+
"1884160 6450688 22 978 0 0 0 0 0 0 0 0\n",
), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("DEGRADED\n"), 0o644))
stats, err := PoolKernelStats()
require.NoError(t, err)
require.Len(t, stats, 1)
assert.Equal(t, PoolKernelStat{
Name: "tank", Health: "DEGRADED", NRead: 1884160, NWrite: 6450688,
}, stats[0])
}
func TestPoolKernelStatsOpenZfs24(t *testing.T) {
root := t.TempDir()
oldPath := procZfsPath
procZfsPath = root
t.Cleanup(func() { procZfsPath = oldPath })
poolDir := filepath.Join(root, "tank")
require.NoError(t, os.MkdirAll(poolDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "state"), []byte("ONLINE\n"), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x1"), []byte(
"34 1 0x01 28 7872 0 0\n"+
"name type data\n"+
"dataset_name 7 tank\n"+
"nwritten 4 2000\n"+
"nread 4 1000\n",
), 0o644))
require.NoError(t, os.WriteFile(filepath.Join(poolDir, "objset-0x2"), []byte(
"34 1 0x01 28 7872 0 0\n"+
"name type data\n"+
"dataset_name 7 tank/videos\n"+
"nwritten 4 400\n"+
"nread 4 300\n",
), 0o644))
stats, err := PoolKernelStats()
require.NoError(t, err)
require.Len(t, stats, 1)
assert.Equal(t, PoolKernelStat{
Name: "tank", Health: "ONLINE", NRead: 1300, NWrite: 2400,
}, stats[0])
}
func TestPoolKernelStatsNoZfs(t *testing.T) {
oldPath := procZfsPath
procZfsPath = t.TempDir()
t.Cleanup(func() { procZfsPath = oldPath })
_, err := PoolKernelStats()
assert.ErrorIs(t, err, ErrNoZfs)
}
func TestReadPoolIORejectsMalformedCounters(t *testing.T) {
path := filepath.Join(t.TempDir(), "io")
require.NoError(t, os.WriteFile(path, []byte("nread nwritten\nnope 10\n"), 0o644))
_, _, err := readPoolIO(path)
require.Error(t, err)
}
func TestReadObjsetIORequiresAllCounters(t *testing.T) {
path := filepath.Join(t.TempDir(), "objset-0x1")
require.NoError(t, os.WriteFile(path, []byte("nread 4 10\n"), 0o644))
_, _, err := readObjsetIO(path)
require.Error(t, err)
}
+150
View File
@@ -0,0 +1,150 @@
package zfs
import (
"bufio"
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
)
// PoolStatus holds parsed `zpool status` information for one pool.
type PoolStatus struct {
Name string
State string // ONLINE, DEGRADED, FAULTED, ...
Scrub ScrubStatus
Vdevs []VdevStatus
}
// ScrubStatus holds the scrub (or resilver) status parsed from the scan line.
type ScrubStatus struct {
State string // NONE, SCANNING, FINISHED, CANCELED
Progress string // e.g. "10.00%" while scanning
Errors uint64
}
// VdevStatus is a single vdev row (mirror, raidz, or leaf disk).
type VdevStatus struct {
Name string
State string
ReadErrs uint64
WriteErrs uint64
ChecksumErrs uint64
}
var (
progressRe = regexp.MustCompile(`(\d+\.\d+)%\s+done`)
errorsRe = regexp.MustCompile(`with\s+(\d+)\s+errors`)
)
// PoolStatuses runs `zpool status` and parses per-pool state, scrub, and vdev
// information. The human-readable format has been stable across OpenZFS
// releases; rows are matched by their tabular shape rather than position.
func PoolStatuses() ([]PoolStatus, error) {
out, err := commandOutput("zpool", "status")
if err != nil {
return nil, fmt.Errorf("zpool status: %w", err)
}
return parseZpoolStatusOutput(out)
}
// parseZpoolStatusOutput parses the output of `zpool status`.
func parseZpoolStatusOutput(out []byte) ([]PoolStatus, error) {
var pools []PoolStatus
var current *PoolStatus
inConfig := false
scanContinuation := false // next non-blank line continues the scan line (progress)
scanner := bufio.NewScanner(bytes.NewReader(out))
for scanner.Scan() {
line := scanner.Text()
trimmed := strings.TrimSpace(line)
switch {
case strings.HasPrefix(trimmed, "pool:"):
pools = append(pools, PoolStatus{Name: strings.TrimSpace(strings.TrimPrefix(trimmed, "pool:"))})
current = &pools[len(pools)-1]
inConfig = false
scanContinuation = false
case current == nil:
continue
case strings.HasPrefix(trimmed, "state:"):
current.State = strings.TrimSpace(strings.TrimPrefix(trimmed, "state:"))
case strings.HasPrefix(trimmed, "scan:"):
current.Scrub = parseScanLine(trimmed)
// zpool status prints the progress percentage on the line after scan.
scanContinuation = true
case trimmed == "config:":
inConfig = true
case scanContinuation:
// The line after scan: may be an indented progress continuation.
if m := progressRe.FindStringSubmatch(trimmed); m != nil {
current.Scrub.Progress = m[1] + "%"
}
scanContinuation = false
case inConfig && (line == "" || strings.HasPrefix(line, " ") || strings.HasPrefix(line, "\t")):
// Table rows are indented; blank lines separate sections. The
// column header and the pool's own row are skipped.
if trimmed != "" && !strings.HasPrefix(trimmed, "NAME") {
if vdev, ok := parseVdevLine(trimmed, current.Name); ok {
current.Vdevs = append(current.Vdevs, vdev)
}
}
case inConfig:
// unindented line (errors:, status:, next pool:) ends the table
inConfig = false
}
}
return pools, scanner.Err()
}
// parseScanLine maps a `scan:` line to a ScrubStatus.
func parseScanLine(line string) ScrubStatus {
var scrub ScrubStatus
switch {
case strings.Contains(line, "in progress"):
scrub.State = "SCANNING"
case strings.Contains(line, "canceled"):
scrub.State = "CANCELED"
case strings.Contains(line, "repaired"), strings.Contains(line, "resilvered"):
scrub.State = "FINISHED"
default:
scrub.State = "NONE"
}
if m := progressRe.FindStringSubmatch(line); m != nil {
scrub.Progress = m[1] + "%"
}
if m := errorsRe.FindStringSubmatch(line); m != nil {
if n, err := strconv.ParseUint(m[1], 10, 64); err == nil {
scrub.Errors = n
}
}
return scrub
}
// parseVdevLine parses one row of the config table. Rows have the shape
// "NAME STATE READ WRITE CKSUM [extra...]". The first data row is the pool
// itself and is skipped since it duplicates pool-level info.
func parseVdevLine(line, poolName string) (VdevStatus, bool) {
fields := strings.Fields(line)
if len(fields) < 5 {
return VdevStatus{}, false
}
if fields[0] == poolName {
return VdevStatus{}, false
}
read, err1 := strconv.ParseUint(fields[2], 10, 64)
write, err2 := strconv.ParseUint(fields[3], 10, 64)
cksum, err3 := strconv.ParseUint(fields[4], 10, 64)
if err1 != nil || err2 != nil || err3 != nil {
return VdevStatus{}, false
}
return VdevStatus{
Name: fields[0],
State: fields[1],
ReadErrs: read,
WriteErrs: write,
ChecksumErrs: cksum,
}, true
}
+143
View File
@@ -0,0 +1,143 @@
//go:build testing
package zfs
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func fixturePath(name string) string {
return filepath.Join("..", "test-data", "zfs", name)
}
func TestParseZpoolListOutput(t *testing.T) {
data, err := os.ReadFile(fixturePath("zpool_list.txt"))
require.NoError(t, err)
pools, err := parseZpoolListOutput(data)
require.NoError(t, err)
require.Len(t, pools, 2)
assert.Equal(t, PoolStat{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "ONLINE"}, pools[0])
assert.Equal(t, PoolStat{Name: "rpool", Size: 1200000000000, Alloc: 900000000000, Free: 300000000000, Health: "DEGRADED"}, pools[1])
}
func TestParseZpoolListOutputIgnoresEmptyLines(t *testing.T) {
pools, err := parseZpoolListOutput([]byte("tank\t100\t50\t50\tONLINE\n\n"))
require.NoError(t, err)
require.Len(t, pools, 1)
assert.Equal(t, "tank", pools[0].Name)
}
func TestParseZpoolListOutputNoPools(t *testing.T) {
pools, err := parseZpoolListOutput([]byte("no pools available\n"))
require.NoError(t, err)
assert.Empty(t, pools)
}
func TestParseZpoolListOutputRejectsMalformedLine(t *testing.T) {
_, err := parseZpoolListOutput([]byte("tank\t100\t50\n"))
require.Error(t, err)
_, err = parseZpoolListOutput([]byte("tank\tnotanumber\t50\t50\tONLINE\n"))
require.Error(t, err)
}
func TestParseZfsListOutput(t *testing.T) {
data, err := os.ReadFile(fixturePath("zfs_list.txt"))
require.NoError(t, err)
datasets, err := parseZfsListOutput(data)
require.NoError(t, err)
require.Len(t, datasets, 9)
// Mountpoint with a space must be kept intact (tab-split only).
assert.Equal(t, "/tank/my media", datasets[3].Mountpoint)
// Unmounted datasets/zvols report "-".
assert.Equal(t, "-", datasets[4].Mountpoint)
assert.Equal(t, uint64(12000000000000), datasets[0].Used)
assert.Equal(t, uint64(11999000000000), datasets[0].Avail)
}
func TestParseZpoolStatusOutput(t *testing.T) {
data, err := os.ReadFile(fixturePath("zpool_status.txt"))
require.NoError(t, err)
pools, err := parseZpoolStatusOutput(data)
require.NoError(t, err)
require.Len(t, pools, 2)
tank := pools[0]
assert.Equal(t, "tank", tank.Name)
assert.Equal(t, "ONLINE", tank.State)
assert.Equal(t, "FINISHED", tank.Scrub.State)
assert.Equal(t, "", tank.Scrub.Progress)
assert.Equal(t, uint64(0), tank.Scrub.Errors)
// Pool row itself is skipped; mirror + 2 disks remain.
require.Len(t, tank.Vdevs, 3)
assert.Equal(t, "mirror-0", tank.Vdevs[0].Name)
assert.Equal(t, "sda", tank.Vdevs[1].Name)
assert.Equal(t, "sdb", tank.Vdevs[2].Name)
rpool := pools[1]
assert.Equal(t, "rpool", rpool.Name)
assert.Equal(t, "DEGRADED", rpool.State)
assert.Equal(t, "SCANNING", rpool.Scrub.State)
assert.Equal(t, "10.00%", rpool.Scrub.Progress)
require.Len(t, rpool.Vdevs, 3)
assert.Equal(t, "FAULTED", rpool.Vdevs[2].State)
assert.Equal(t, uint64(1), rpool.Vdevs[2].ReadErrs)
assert.Equal(t, uint64(2), rpool.Vdevs[2].WriteErrs)
assert.Equal(t, uint64(3), rpool.Vdevs[2].ChecksumErrs)
}
func TestParseScanLine(t *testing.T) {
assert.Equal(t, "FINISHED", parseScanLine("scan: scrub repaired 0B in 00:05:12 with 0 errors on Sun Jun 1 02:00:12 2025").State)
assert.Equal(t, uint64(3), parseScanLine("scan: scrub repaired 10G in 01:00:00 with 3 errors on Sun Jun 1 02:00:12 2025").Errors)
assert.Equal(t, "SCANNING", parseScanLine("scan: scrub in progress since Sun Jun 8 01:00:00 2025").State)
assert.Equal(t, "CANCELED", parseScanLine("scan: scrub canceled on Sun Jun 1 02:00:12 2025").State)
assert.Equal(t, "FINISHED", parseScanLine("scan: resilvered 1.23G in 00:01:00 with 0 errors on Sun Jun 1 02:00:12 2025").State)
assert.Equal(t, "NONE", parseScanLine("scan: none requested").State)
}
func TestCommandOutputForcesLocaleAndTimesOut(t *testing.T) {
t.Setenv("BESZEL_ZFS_COMMAND_HELPER", "1")
out, err := commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "locale")
require.NoError(t, err)
assert.Equal(t, "C/C", string(out))
oldTimeout := commandTimeout
commandTimeout = 20 * time.Millisecond
t.Cleanup(func() { commandTimeout = oldTimeout })
_, err = commandOutput(os.Args[0], "-test.run=TestZfsCommandHelperProcess", "--", "sleep")
require.Error(t, err)
assert.Contains(t, err.Error(), "timed out")
}
func TestZfsCommandHelperProcess(t *testing.T) {
if os.Getenv("BESZEL_ZFS_COMMAND_HELPER") != "1" {
return
}
mode := ""
for i, arg := range os.Args {
if arg == "--" && i+1 < len(os.Args) {
mode = os.Args[i+1]
break
}
}
switch strings.TrimSpace(mode) {
case "locale":
_, _ = fmt.Printf("%s/%s", os.Getenv("LC_ALL"), os.Getenv("LANG"))
case "sleep":
time.Sleep(time.Second)
}
os.Exit(0)
}
+4
View File
@@ -7,3 +7,7 @@ import "errors"
func ARCSize() (uint64, error) {
return 0, errors.ErrUnsupported
}
func PoolKernelStats() ([]PoolKernelStat, error) {
return nil, errors.ErrUnsupported
}
+320
View File
@@ -0,0 +1,320 @@
package agent
import (
"log/slog"
"strings"
"sync"
"time"
"github.com/henrygd/beszel/agent/zfs"
"github.com/henrygd/beszel/internal/entities/system"
zfsentity "github.com/henrygd/beszel/internal/entities/zfs"
)
// zfsDatasetUsage holds usage values for a ZFS dataset mountpoint.
type zfsDatasetUsage struct {
used uint64
avail uint64
}
// datasetUsageRefreshInterval controls how often `zfs list` is re-run for the
// mountpoint usage map. Dataset inventory changes rarely.
const datasetUsageRefreshInterval = 5 * time.Minute
// poolStatsRefreshInterval controls how often `zpool list` is re-run for pool
// capacity. Health and I/O are read from procfs on Linux, so the utility only
// needs to refresh slow-moving space accounting.
const poolStatsRefreshInterval = time.Minute
type poolKernelSample struct {
nread uint64
nwrite uint64
at time.Time
}
// ZfsManager collects ZFS pool and dataset statistics. Collection functions
// are fields so unit tests can substitute them (same pattern as
// diskDiscovery.usageFn). It is safe for concurrent use by a single goroutine
// only; callers must hold the agent lock like updateDiskUsage does.
type ZfsManager struct {
poolStatsFn func() ([]zfs.PoolStat, error) // capacity/health source
datasetsFn func() ([]zfs.Dataset, error) // dataset inventory source
kernelStatsFn func() ([]zfs.PoolKernelStat, error) // procfs pool state/I/O source
poolStatusesFn func() ([]zfs.PoolStatus, error) // scrub/vdev detail source
poolData []zfs.PoolStat // cached pool inventory (TTL below)
lastPoolStats time.Time
kernelSamples map[string]poolKernelSample
datasetUsage map[string]zfsDatasetUsage // mountpoint -> usage
lastUsageRefresh time.Time
// Detail data (pools, vdevs, scrub, datasets) is cached and refreshed on
// an interval. Accessed from handler goroutines, so it is mutex-protected.
detailMu sync.Mutex
detail *zfsentity.ZfsData
lastDetailRefresh time.Time
detailInterval time.Duration
}
// newZfsManager creates a ZfsManager wired to the system's ZFS utilities.
func newZfsManager() *ZfsManager {
return &ZfsManager{
poolStatsFn: zfs.PoolStats,
datasetsFn: zfs.Datasets,
kernelStatsFn: zfs.PoolKernelStats,
poolStatusesFn: zfs.PoolStatuses,
detailInterval: time.Hour,
}
}
// Update refreshes systemStats.ZfsPools with the latest pool data. I/O
// throughput and health come from inexpensive kernel kstats on Linux. Pool
// capacity and dataset usage come from separately cached utility calls. It is
// a no-op when ZFS is absent.
func (zm *ZfsManager) Update(systemStats *system.Stats) {
pools := zm.poolStats()
if len(pools) == 0 {
return
}
kernelStats, ioRates := zm.kernelStats()
if systemStats.ZfsPools == nil {
systemStats.ZfsPools = make(map[string]*system.ZfsPool, len(pools))
}
for i := range pools {
pool := &pools[i]
// Full precision, matching the dataset values below; the frontend
// formats any magnitude.
stats := &system.ZfsPool{
Total: float64(pool.Size) / (1024 * 1024 * 1024),
Used: float64(pool.Alloc) / (1024 * 1024 * 1024),
Health: pool.Health,
}
if kernel, exists := kernelStats[pool.Name]; exists && kernel.Health != "" {
stats.Health = kernel.Health
}
if io, exists := ioRates[pool.Name]; exists {
stats.ReadBytes = io.NRead
stats.WriteBytes = io.NWrite
}
slog.Debug("ZFS pool sample", "pool", pool.Name, "health", stats.Health, "used_gb", stats.Used, "read_bps", stats.ReadBytes, "write_bps", stats.WriteBytes)
systemStats.ZfsPools[pool.Name] = stats
}
}
// poolStats returns the cached pool inventory, re-running `zpool list` at most
// every poolStatsRefreshInterval. On failure the previous inventory is
// retained and the refresh is retried on the next cadence.
func (zm *ZfsManager) poolStats() []zfs.PoolStat {
if zm.lastPoolStats.IsZero() || time.Since(zm.lastPoolStats) >= poolStatsRefreshInterval {
pools, err := zm.poolStatsFn()
if err != nil {
slog.Debug("ZFS pool stats unavailable", "err", err)
} else {
zm.poolData = pools
}
zm.lastPoolStats = time.Now()
}
return zm.poolData
}
// kernelStats reads cumulative pool counters and converts them to per-second
// rates. Counter decreases indicate a pool export/import and reset the
// baseline instead of producing an underflow spike.
func (zm *ZfsManager) kernelStats() (map[string]zfs.PoolKernelStat, map[string]zfs.PoolIoStats) {
if zm.kernelStatsFn == nil {
return nil, nil
}
stats, err := zm.kernelStatsFn()
if err != nil {
slog.Debug("ZFS kernel stats unavailable", "err", err)
return nil, nil
}
now := time.Now()
byName := make(map[string]zfs.PoolKernelStat, len(stats))
rates := make(map[string]zfs.PoolIoStats, len(stats))
nextSamples := make(map[string]poolKernelSample, len(stats))
for _, stat := range stats {
byName[stat.Name] = stat
if previous, ok := zm.kernelSamples[stat.Name]; ok && now.After(previous.at) &&
stat.NRead >= previous.nread && stat.NWrite >= previous.nwrite {
seconds := now.Sub(previous.at).Seconds()
rates[stat.Name] = zfs.PoolIoStats{
NRead: uint64(float64(stat.NRead-previous.nread) / seconds),
NWrite: uint64(float64(stat.NWrite-previous.nwrite) / seconds),
}
}
nextSamples[stat.Name] = poolKernelSample{nread: stat.NRead, nwrite: stat.NWrite, at: now}
}
zm.kernelSamples = nextSamples
return byName, rates
}
// refreshDatasetUsage re-runs `zfs list` when the refresh window has elapsed
// and rebuilds the mountpoint-keyed usage map.
func (zm *ZfsManager) refreshDatasetUsage() {
if !zm.lastUsageRefresh.IsZero() && time.Since(zm.lastUsageRefresh) < datasetUsageRefreshInterval {
return
}
datasets, err := zm.datasetsFn()
if err != nil {
slog.Debug("ZFS dataset usage unavailable", "err", err)
} else {
usage := make(map[string]zfsDatasetUsage, len(datasets))
for _, ds := range datasets {
if ds.Mountpoint != "" && ds.Mountpoint != "-" {
usage[ds.Mountpoint] = zfsDatasetUsage{used: ds.Used, avail: ds.Avail}
}
}
zm.datasetUsage = usage
}
zm.lastUsageRefresh = time.Now()
}
// DatasetUsage returns ZFS dataset usage keyed by mountpoint, refreshed at
// most every datasetUsageRefreshInterval. On failure the previous map is
// retained and a debug log is emitted.
func (zm *ZfsManager) DatasetUsage() map[string]zfsDatasetUsage {
zm.refreshDatasetUsage()
return zm.datasetUsage
}
// GetDetail returns ZFS detail data (pool health, scrub, vdevs, datasets).
// Scheduled requests use the cached snapshot until stale; manual requests can
// force collection. On failure the previous snapshot is retained.
func (zm *ZfsManager) GetDetail(force bool) *zfsentity.ZfsData {
zm.detailMu.Lock()
defer zm.detailMu.Unlock()
if force || zm.detail == nil || time.Since(zm.lastDetailRefresh) >= zm.detailInterval {
if data, err := zm.collectDetail(zm.detail); err != nil {
slog.Debug("ZFS detail collection failed", "err", err)
if zm.detail == nil {
return &zfsentity.ZfsData{}
}
return &zfsentity.ZfsData{Pools: zm.detail.Pools}
} else {
zm.detail = data
zm.lastDetailRefresh = time.Now()
}
}
if zm.detail == nil {
return &zfsentity.ZfsData{}
}
return zm.detail
}
// collectDetail builds a ZfsData payload from the current system state.
func (zm *ZfsManager) collectDetail(previous *zfsentity.ZfsData) (*zfsentity.ZfsData, error) {
pools, err := zm.poolStatsFn()
if err != nil {
return nil, err
}
if len(pools) == 0 {
return &zfsentity.ZfsData{Pools: []*zfsentity.PoolDetail{}, Complete: true}, nil
}
statuses, statusErr := zm.poolStatusesFn()
if statusErr != nil {
slog.Debug("ZFS pool status unavailable", "err", statusErr)
}
datasets, datasetsErr := zm.datasetsFn()
if datasetsErr != nil {
slog.Debug("ZFS datasets unavailable", "err", datasetsErr)
}
statusByPool := make(map[string]zfs.PoolStatus, len(statuses))
for _, st := range statuses {
statusByPool[st.Name] = st
}
previousByPool := make(map[string]*zfsentity.PoolDetail)
if previous != nil {
for _, pool := range previous.Pools {
if pool != nil {
previousByPool[pool.Name] = pool
}
}
}
data := &zfsentity.ZfsData{Pools: make([]*zfsentity.PoolDetail, 0, len(pools)), Complete: true}
for i := range pools {
p := &pools[i]
detail := &zfsentity.PoolDetail{
Name: p.Name,
Health: p.Health,
Size: p.Size,
Alloc: p.Alloc,
Free: p.Free,
}
if st, ok := statusByPool[p.Name]; statusErr == nil && ok {
if st.Scrub.State != "" && st.Scrub.State != "NONE" {
detail.Scrub = &zfsentity.Scrub{
State: st.Scrub.State,
Progress: st.Scrub.Progress,
Errors: st.Scrub.Errors,
}
}
for _, v := range st.Vdevs {
detail.Vdevs = append(detail.Vdevs, &zfsentity.Vdev{
Name: v.Name,
State: v.State,
ReadErrs: v.ReadErrs,
WriteErrs: v.WriteErrs,
ChecksumErrs: v.ChecksumErrs,
})
}
} else {
if cached := previousByPool[p.Name]; cached != nil {
detail.Scrub = cached.Scrub
detail.Vdevs = cached.Vdevs
}
}
if datasetsErr == nil {
foundDataset := false
for _, ds := range datasets {
if poolOfDataset(ds.Name) == p.Name {
foundDataset = true
detail.Datasets = append(detail.Datasets, &zfsentity.Dataset{
Name: ds.Name,
Used: ds.Used,
Avail: ds.Avail,
Mountpoint: ds.Mountpoint,
})
}
}
if !foundDataset {
if cached := previousByPool[p.Name]; cached != nil {
detail.Datasets = cached.Datasets
}
}
} else if cached := previousByPool[p.Name]; cached != nil {
detail.Datasets = cached.Datasets
}
data.Pools = append(data.Pools, detail)
}
return data, nil
}
// poolOfDataset returns the pool name for a dataset name (everything before
// the first '/'). Datasets without a separator belong to a pool of the same
// name.
func poolOfDataset(name string) string {
if idx := strings.IndexByte(name, '/'); idx >= 0 {
return name[:idx]
}
return name
}
// ZfsMountpoints returns the set of mountpoints backed by ZFS datasets.
func (zm *ZfsManager) ZfsMountpoints() map[string]bool {
usage := zm.DatasetUsage()
mountpoints := make(map[string]bool, len(usage))
for mountpoint := range usage {
mountpoints[mountpoint] = true
}
return mountpoints
}
+245
View File
@@ -0,0 +1,245 @@
//go:build testing
package agent
import (
"testing"
"time"
"github.com/henrygd/beszel/agent/zfs"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestUpdatePopulatesZfsPools(t *testing.T) {
zm := &ZfsManager{}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
return []zfs.PoolStat{{Name: "tank", Size: 23999000000000, Alloc: 12000000000000, Free: 11999000000000, Health: "DEGRADED"}}, nil
}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{
{Name: "tank/apps", Used: 5000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
{Name: "tank/backup", Used: 6000000000000, Avail: 11999000000000, Mountpoint: "/tank/backup"},
// Small zvol (Proxmox VM EFI disk): must not round to zero.
{Name: "rpool/vm-100-disk-2", Used: 4194304, Avail: 0, Mountpoint: "-"},
}, nil
}
var kernelCalls int
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
kernelCalls++
return []zfs.PoolKernelStat{{
Name: "tank", Health: "ONLINE",
NRead: uint64(kernelCalls-1) * 1250, NWrite: uint64(kernelCalls-1) * 5120,
}}, nil
}
var stats system.Stats
// The first kernel sample establishes the cumulative-counter baseline.
zm.Update(&stats)
zm.kernelSamples["tank"] = poolKernelSample{at: time.Now().Add(-time.Second)}
zm.Update(&stats)
require.NotNil(t, stats.ZfsPools)
require.Contains(t, stats.ZfsPools, "tank")
assert.InDelta(t, 22350.8105, stats.ZfsPools["tank"].Total, 0.0001) // Size in GiB
assert.InDelta(t, 11175.8709, stats.ZfsPools["tank"].Used, 0.0001) // Alloc in GiB
assert.Equal(t, "ONLINE", stats.ZfsPools["tank"].Health)
assert.InDelta(t, 1250, stats.ZfsPools["tank"].ReadBytes, 5)
assert.InDelta(t, 5120, stats.ZfsPools["tank"].WriteBytes, 5)
}
// TestUpdateKernelStatsMissing verifies pools without a kernel sample report zero
// I/O instead of erroring.
func TestUpdateKernelStatsMissing(t *testing.T) {
zm := &ZfsManager{}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
return []zfs.PoolStat{{Name: "tank", Size: 1, Alloc: 1, Health: "ONLINE"}}, nil
}
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
return nil, zfs.ErrNoZfs
}
var stats system.Stats
zm.Update(&stats)
require.NotNil(t, stats.ZfsPools)
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
}
func TestUpdateKernelCounterReset(t *testing.T) {
zm := &ZfsManager{}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
return []zfs.PoolStat{{Name: "tank", Health: "ONLINE"}}, nil
}
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
zm.kernelSamples = map[string]poolKernelSample{
"tank": {nread: 100, nwrite: 200, at: time.Now().Add(-time.Second)},
}
zm.kernelStatsFn = func() ([]zfs.PoolKernelStat, error) {
return []zfs.PoolKernelStat{{Name: "tank", Health: "ONLINE", NRead: 10, NWrite: 20}}, nil
}
var stats system.Stats
zm.Update(&stats)
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].ReadBytes)
assert.Equal(t, uint64(0), stats.ZfsPools["tank"].WriteBytes)
}
func TestUpdateNoZfs(t *testing.T) {
zm := &ZfsManager{}
calls := 0
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
calls++
return nil, zfs.ErrNoZfs
}
var stats system.Stats
zm.Update(&stats)
zm.Update(&stats)
assert.Nil(t, stats.ZfsPools)
assert.Equal(t, 1, calls, "failed pool discovery should be cached until the next refresh interval")
}
func TestUpdateEmptyPools(t *testing.T) {
zm := &ZfsManager{}
calls := 0
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
calls++
return nil, nil
}
var stats system.Stats
zm.Update(&stats)
zm.Update(&stats)
assert.Nil(t, stats.ZfsPools)
assert.Equal(t, 1, calls, "an empty pool inventory should be cached until the next refresh interval")
}
func TestDatasetUsage(t *testing.T) {
zm := &ZfsManager{}
calls := 0
zm.datasetsFn = func() ([]zfs.Dataset, error) {
calls++
return []zfs.Dataset{
{Name: "tank", Used: 12000000000000, Avail: 11999000000000, Mountpoint: "/tank"},
{Name: "tank/apps", Used: 1000000000000, Avail: 11999000000000, Mountpoint: "/tank/apps"},
{Name: "rpool", Used: 900000000000, Avail: 300000000000, Mountpoint: "-"}, // zvol/unmounted: excluded
}, nil
}
usage := zm.DatasetUsage()
require.Len(t, usage, 2)
assert.Equal(t, zfsDatasetUsage{used: 12000000000000, avail: 11999000000000}, usage["/tank"])
assert.Equal(t, zfsDatasetUsage{used: 1000000000000, avail: 11999000000000}, usage["/tank/apps"])
assert.Equal(t, 1, calls)
// Second call within the refresh window must not re-run the collector.
zm.DatasetUsage()
assert.Equal(t, 1, calls)
}
func TestDatasetUsageRefreshOnErrorKeepsPrevious(t *testing.T) {
zm := &ZfsManager{}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{{Name: "tank", Used: 1, Avail: 1, Mountpoint: "/tank"}}, nil
}
assert.Len(t, zm.DatasetUsage(), 1)
// Force refresh window expiry, then a failing collector.
zm.lastUsageRefresh = time.Now().Add(-10 * time.Minute)
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return nil, zfs.ErrNoZfs
}
usage := zm.DatasetUsage()
assert.Len(t, usage, 1, "previous usage should be retained on error")
}
func TestGetDetailForceRefresh(t *testing.T) {
zm := &ZfsManager{detailInterval: time.Hour}
poolCalls := 0
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
poolCalls++
return []zfs.PoolStat{{Name: "tank", Alloc: uint64(poolCalls)}}, nil
}
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
first := zm.GetDetail(false)
assert.True(t, first.Complete)
require.Len(t, first.Pools, 1)
assert.Equal(t, uint64(1), first.Pools[0].Alloc)
cached := zm.GetDetail(false)
require.Len(t, cached.Pools, 1)
assert.Equal(t, uint64(1), cached.Pools[0].Alloc)
assert.Equal(t, 1, poolCalls)
refreshed := zm.GetDetail(true)
assert.True(t, refreshed.Complete)
require.Len(t, refreshed.Pools, 1)
assert.Equal(t, uint64(2), refreshed.Pools[0].Alloc)
assert.Equal(t, 2, poolCalls)
}
func TestGetDetailSuccessfulEmptyInventoryClearsCache(t *testing.T) {
zm := &ZfsManager{detailInterval: time.Hour}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
return []zfs.PoolStat{{Name: "tank"}}, nil
}
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, nil }
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, nil }
require.Len(t, zm.GetDetail(false).Pools, 1)
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, nil }
empty := zm.GetDetail(true)
assert.True(t, empty.Complete)
assert.Empty(t, empty.Pools)
}
func TestGetDetailFailureReturnsIncompleteCachedInventory(t *testing.T) {
zm := &ZfsManager{detailInterval: time.Hour}
zm.poolStatsFn = func() ([]zfs.PoolStat, error) {
return []zfs.PoolStat{{Name: "tank"}}, nil
}
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) {
return []zfs.PoolStatus{{Name: "tank", Vdevs: []zfs.VdevStatus{{Name: "mirror-0"}}}}, nil
}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{{Name: "tank/data"}}, nil
}
first := zm.GetDetail(false)
require.True(t, first.Complete)
require.Len(t, first.Pools[0].Vdevs, 1)
require.Len(t, first.Pools[0].Datasets, 1)
zm.poolStatusesFn = func() ([]zfs.PoolStatus, error) { return nil, zfs.ErrNoZfs }
zm.datasetsFn = func() ([]zfs.Dataset, error) { return nil, zfs.ErrNoZfs }
partial := zm.GetDetail(true)
require.True(t, partial.Complete)
require.Len(t, partial.Pools[0].Vdevs, 1)
require.Len(t, partial.Pools[0].Datasets, 1)
zm.poolStatsFn = func() ([]zfs.PoolStat, error) { return nil, zfs.ErrNoZfs }
lastSuccessfulRefresh := zm.lastDetailRefresh
failed := zm.GetDetail(true)
assert.False(t, failed.Complete)
require.Len(t, failed.Pools, 1)
assert.Equal(t, "tank", failed.Pools[0].Name)
assert.Equal(t, lastSuccessfulRefresh, zm.lastDetailRefresh)
}
func TestZfsMountpoints(t *testing.T) {
zm := &ZfsManager{}
zm.datasetsFn = func() ([]zfs.Dataset, error) {
return []zfs.Dataset{
{Name: "tank", Mountpoint: "/tank"},
{Name: "rpool/ROOT/pve-1", Mountpoint: "/"},
}, nil
}
mountpoints := zm.ZfsMountpoints()
assert.Len(t, mountpoints, 2)
assert.True(t, mountpoints["/tank"])
assert.True(t, mountpoints["/"])
}
+4 -1
View File
@@ -6,7 +6,7 @@ import "github.com/blang/semver"
const (
// Version is the current version of the application.
Version = "0.18.8"
Version = "0.18.9"
// AppName is the name of the application.
AppName = "beszel"
)
@@ -16,3 +16,6 @@ var MinVersionCbor = semver.MustParse("0.12.0")
// MinVersionAgentResponse is the minimum supported version for AgentResponse compatibility.
var MinVersionAgentResponse = semver.MustParse("0.13.0")
// MinVersionZfsData is the minimum agent version that supports ZFS detail requests.
var MinVersionZfsData = semver.MustParse("0.18.9")
+9
View File
@@ -57,12 +57,18 @@ type SystemAlertStats struct {
Battery [2]uint8 `json:"bat"`
Batteries map[string]uint8 `json:"bats"`
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
}
type SystemAlertGPUData struct {
Usage float64 `json:"u"`
}
type SystemAlertZfsPool struct {
Total float64 `json:"d"`
Used float64 `json:"du"`
}
type SystemAlertData struct {
systemRecord *core.Record
alertData CachedAlertData
@@ -111,6 +117,9 @@ func (am *AlertManager) bindEvents() {
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
// Populate all alerts into cache on startup
+30 -1
View File
@@ -44,6 +44,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
maxUsedPct = usedPct
}
}
for _, pool := range data.Stats.ZfsPools {
if pool != nil && pool.Total > 0 {
usedPct := pool.Used / pool.Total * 100
if usedPct > maxUsedPct {
maxUsedPct = usedPct
}
}
}
val = maxUsedPct
case "Temperature":
if data.Info.DashboardTemp < 1 {
@@ -208,6 +216,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
}
}
// add zfs pool usage from historical record
for key, pool := range stats.ZfsPools {
if pool.Total > 0 {
zfsKey := zfsDiskAlertKey(key)
if _, ok := alert.mapSums[zfsKey]; !ok {
alert.mapSums[zfsKey] = 0.0
}
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
}
}
case "Temperature":
if alert.mapSums == nil {
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
@@ -255,7 +273,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
sumPct := float32(value)
if sumPct > maxPct {
maxPct = sumPct
alert.descriptor = fmt.Sprintf("Usage of %s", key)
alert.descriptor = diskAlertDescriptor(key)
}
}
alert.val = float64(maxPct / float32(alert.count))
@@ -301,6 +319,17 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
return nil
}
func zfsDiskAlertKey(poolName string) string {
return "zfs:" + poolName
}
func diskAlertDescriptor(key string) string {
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
}
return fmt.Sprintf("Usage of %s", key)
}
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
return legacy != [2]uint8{} || len(batteries) > 0
}
+142
View File
@@ -0,0 +1,142 @@
package alerts
import (
"fmt"
"time"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
// handleZfsPoolAlert sends alerts when a ZFS pool health state worsens and
// resolves the alert history entry when the pool recovers. Like the SMART
// hook, this is automatic and does not require user opt-in.
func (am *AlertManager) handleZfsPoolAlert(e *core.RecordEvent) error {
return am.handleZfsPoolHealthAlert(e, e.Record.Original().GetString("health"))
}
func (am *AlertManager) handleZfsPoolCreateAlert(e *core.RecordEvent) error {
return am.handleZfsPoolHealthAlert(e, "")
}
func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth string) error {
newHealth := e.Record.GetString("health")
oldSeverity := zfsPoolSeverity(oldHealth)
newSeverity := zfsPoolSeverity(newHealth)
systemID := e.Record.GetString("system")
if systemID == "" {
return e.Next()
}
systemRecord, err := e.App.FindRecordById("systems", systemID)
if err != nil {
e.App.Logger().Error("Failed to find system for ZFS alert", "err", err, "systemID", systemID)
return e.Next()
}
// Pool recovered to a healthy state: resolve any open history entries.
if newSeverity == 1 && oldSeverity > 1 {
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
return e.Next()
}
if !shouldSendZfsPoolAlert(oldSeverity, newSeverity) {
return e.Next()
}
systemName := systemRecord.GetString("name")
poolName := e.Record.GetString("name")
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
if oldSeverity > 0 {
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
}
userIDs := systemRecord.GetStringSlice("users")
if len(userIDs) == 0 {
return e.Next()
}
for _, userID := range userIDs {
if err := am.SendAlert(AlertMessageData{
UserID: userID,
SystemID: systemID,
Title: title,
Message: message,
Link: am.hub.MakeLink("system", systemID),
LinkText: "View " + systemName,
}); err != nil {
e.App.Logger().Error("Failed to send ZFS alert", "err", err, "userID", userID)
}
_ = createZfsPoolHistoryRecord(e.App, userID, systemID, e.Record.Id, poolName)
}
return e.Next()
}
// resolveZfsPoolHistoryOnDelete resolves open alert history entries when a
// pool record is deleted (manually or because the pool disappeared), so the
// UI does not keep showing an ongoing alert for a pool that no longer exists.
func resolveZfsPoolHistoryOnDelete(e *core.RecordEvent) error {
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
return e.Next()
}
// shouldSendZfsPoolAlert reports whether a health transition warrants an alert.
// First observations of unhealthy pools and worsening transitions are reported.
func shouldSendZfsPoolAlert(oldSeverity, newSeverity int) bool {
return newSeverity > 1 && (oldSeverity == 0 || newSeverity > oldSeverity)
}
// zfsPoolSeverity ranks pool health states: healthy (1), degraded (2),
// failed/unavailable (3), unknown (0).
func zfsPoolSeverity(health string) int {
switch health {
case "ONLINE":
return 1
case "DEGRADED":
return 2
case "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED":
return 3
default:
return 0
}
}
// createZfsPoolHistoryRecord logs a pool health alert in the alerts history so
// it is visible in the UI without creating an editable alert configuration.
func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolName string) error {
collection, err := app.FindCachedCollectionByNameOrId("alerts_history")
if err != nil {
return err
}
record := core.NewRecord(collection)
record.Set("user", userID)
record.Set("system", systemID)
record.Set("alert_id", alertID)
record.Set("name", "ZFS Pool: "+poolName)
return app.Save(record)
}
// resolveAllAlertHistoryRecords resolves every open history entry for an alert
// record id (one per system user).
func resolveAllAlertHistoryRecords(app core.App, alertID string) {
records, err := app.FindRecordsByFilter(
"alerts_history",
"alert_id={:alert_id} && resolved=null",
"", 0, 0,
dbx.Params{"alert_id": alertID},
)
if err != nil || len(records) == 0 {
return
}
now := time.Now().UTC()
for _, record := range records {
record.Set("resolved", now)
if err := app.Save(record); err != nil {
app.Logger().Error("Failed to resolve ZFS alert history", "err", err, "recordId", record.Id)
}
}
}
+145
View File
@@ -0,0 +1,145 @@
//go:build testing
package alerts_test
import (
"encoding/json"
"testing"
"time"
"github.com/henrygd/beszel/internal/entities/system"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/tools/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestDiskAlertZfsPoolMultiMinute verifies that ZFS pool usage participates in
// the Disk threshold alert using historical per-minute values, mirroring the
// extra-filesystem behavior.
func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
require.NoError(t, err)
systemRecord := systems[0]
diskAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
"name": "Disk",
"system": systemRecord.Id,
"user": user.Id,
"value": 80, // threshold: 80%
"min": 2, // requires historical averaging
})
require.NoError(t, err)
am := hub.GetAlertManager()
now := time.Now().UTC()
poolHigh := map[string]*system.ZfsPool{
"tank": {Total: 1000, Used: 920}, // 92% - above threshold
}
recordTimes := []time.Duration{
-180 * time.Second,
-90 * time.Second,
-60 * time.Second,
-30 * time.Second,
}
for _, offset := range recordTimes {
stats := system.Stats{
DiskPct: 30, // root disk at 30% - below threshold
ZfsPools: poolHigh,
}
statsJSON, _ := json.Marshal(stats)
recordTime := now.Add(offset)
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
"system": systemRecord.Id,
"type": "1m",
"stats": string(statsJSON),
})
require.NoError(t, err)
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
err = hub.SaveNoValidate(record)
require.NoError(t, err)
}
combinedDataHigh := &system.CombinedData{
Stats: system.Stats{
DiskPct: 30,
ZfsPools: poolHigh,
},
Info: system.Info{
DiskPct: 30,
},
}
systemRecord.Set("updated", now)
err = hub.SaveNoValidate(systemRecord)
require.NoError(t, err)
err = am.HandleSystemAlerts(systemRecord, combinedDataHigh)
require.NoError(t, err)
time.Sleep(20 * time.Millisecond)
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
require.NoError(t, err)
assert.True(t, diskAlert.GetBool("triggered"),
"Alert should be triggered when ZFS pool average (92%%) exceeds threshold (80%%)")
// --- Resolution: pool drops to 50%, alert should resolve ---
poolLow := map[string]*system.ZfsPool{
"tank": {Total: 1000, Used: 500}, // 50% - below threshold
}
newNow := now.Add(2 * time.Minute)
for _, offset := range recordTimes {
stats := system.Stats{
DiskPct: 30,
ZfsPools: poolLow,
}
statsJSON, _ := json.Marshal(stats)
recordTime := newNow.Add(offset)
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
"system": systemRecord.Id,
"type": "1m",
"stats": string(statsJSON),
})
require.NoError(t, err)
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
err = hub.SaveNoValidate(record)
require.NoError(t, err)
}
combinedDataLow := &system.CombinedData{
Stats: system.Stats{
DiskPct: 30,
ZfsPools: poolLow,
},
Info: system.Info{
DiskPct: 30,
},
}
systemRecord.Set("updated", newNow)
err = hub.SaveNoValidate(systemRecord)
require.NoError(t, err)
err = am.HandleSystemAlerts(systemRecord, combinedDataLow)
require.NoError(t, err)
time.Sleep(20 * time.Millisecond)
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
require.NoError(t, err)
assert.False(t, diskAlert.GetBool("triggered"),
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
}
+15
View File
@@ -0,0 +1,15 @@
//go:build testing
package alerts
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
}
+292
View File
@@ -0,0 +1,292 @@
//go:build testing
package alerts_test
import (
"testing"
"time"
beszelTests "github.com/henrygd/beszel/internal/tests"
"github.com/pocketbase/pocketbase/core"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
// Re-fetch so PocketBase tracks original values
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "DEGRADED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
lastMessage := hub.TestMailer.LastMessage()
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
assert.Contains(t, lastMessage.Subject, "tank")
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
}
func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "rpool",
"health": "DEGRADED",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
lastMessage := hub.TestMailer.LastMessage()
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
}
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
assert.NoError(t, err)
// Trigger a worsening alert first
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "expected alerts for initial DEGRADED state and DEGRADED -> FAULTED")
// Recovery back to ONLINE must not send a new alert
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "ONLINE")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "recovery should not send a new alert")
// And the open history entry should have been resolved
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
requireHistoryResolved(t, history)
}
func TestZfsPoolAlertUnknownHealthDoesNotResolve(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
require.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
require.NoError(t, err)
pool.Set("health", "")
require.NoError(t, hub.Save(pool))
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
require.NoError(t, err)
require.Len(t, history, 1, "unknown health must not resolve an active alert")
}
func TestZfsPoolAlertUnknownToFaulted(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should alert when a previously unknown pool becomes FAULTED")
}
func TestZfsPoolAlertOnInitialUnhealthyState(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
require.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "DEGRADED",
})
require.NoError(t, err)
time.Sleep(50 * time.Millisecond)
require.EqualValues(t, 1, hub.TestMailer.TotalSend())
assert.Contains(t, hub.TestMailer.LastMessage().Text, "first observed as DEGRADED")
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
require.NoError(t, err)
require.NoError(t, hub.Save(pool))
time.Sleep(50 * time.Millisecond)
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "unchanged unhealthy health must not duplicate alerts")
}
func TestZfsPoolAlertWritesHistory(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1, "expected one history entry per user")
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
assert.Equal(t, system.Id, history[0].GetString("system"))
}
func TestZfsPoolAlertResolvedOnRecordDelete(t *testing.T) {
hub, user := beszelTests.GetHubWithUser(t)
defer hub.Cleanup()
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
"name": "test-system",
"users": []string{user.Id},
"host": "127.0.0.1",
})
assert.NoError(t, err)
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
"system": system.Id,
"name": "tank",
"health": "ONLINE",
})
assert.NoError(t, err)
// Trigger an alert so an open history entry exists.
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
assert.NoError(t, err)
pool.Set("health", "FAULTED")
err = hub.Save(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1, "expected one open history entry")
// Deleting the pool record must resolve the open entry.
err = hub.Delete(pool)
assert.NoError(t, err)
time.Sleep(50 * time.Millisecond)
history, err = hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
assert.NoError(t, err)
require.Len(t, history, 1)
requireHistoryResolved(t, history)
}
func requireHistoryResolved(t *testing.T, history []*core.Record) {
t.Helper()
for _, record := range history {
assert.False(t, record.GetDateTime("resolved").Time().IsZero(), "expected history entry to be resolved")
}
}
+6
View File
@@ -22,6 +22,8 @@ const (
GetSmartData
// Request detailed systemd service info from agent
GetSystemdInfo
// Request ZFS detail data from agent
GetZfsData
// Add new actions here...
)
@@ -64,6 +66,10 @@ type DataRequestOptions struct {
IncludeDetails bool `cbor:"1,keyasint"`
}
type ZfsDataRequest struct {
Force bool `cbor:"0,keyasint,omitempty"`
}
type ContainerLogsRequest struct {
ContainerID string `cbor:"0,keyasint"`
}
+1 -1
View File
@@ -23,7 +23,7 @@ COPY --from=builder /agent /agent
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
RUN apk add --no-cache smartmontools
RUN apk add --no-cache smartmontools zfs
# Ensure data persistence across container recreations
VOLUME ["/var/lib/beszel-agent"]
+15 -3
View File
@@ -33,8 +33,6 @@ type Stats struct {
MaxNetworkSent float64 `json:"nsm,omitempty" cbor:"-"`
MaxNetworkRecv float64 `json:"nrm,omitempty" cbor:"-"`
Temperatures map[string]float64 `json:"t,omitempty" cbor:"20,keyasint,omitempty"`
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
ExtraFs map[string]*FsStats `json:"efs,omitempty" cbor:"21,keyasint,omitempty"`
GPUData map[string]GPUData `json:"g,omitempty" cbor:"22,keyasint,omitempty"`
// LoadAvg1 float64 `json:"l1,omitempty" cbor:"23,keyasint,omitempty"`
@@ -52,7 +50,20 @@ type Stats struct {
CpuCoresUsage Uint8Slice `json:"cpus,omitempty" cbor:"34,keyasint,omitempty"` // per-core busy usage [CPU0..]
DiskIoStats [6]float64 `json:"dios,omitzero" cbor:"35,keyasint,omitzero"` // [read time %, write time %, io utilization %, r_await ms, w_await ms, weighted io %]
MaxDiskIoStats [6]float64 `json:"diosm,omitzero" cbor:"-"` // max values for DiskIoStats
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
ZfsPools map[string]*ZfsPool `json:"z,omitempty" cbor:"39,keyasint,omitempty"` // ZFS pool metrics, keyed by pool name
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
}
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
type ZfsPool struct {
Total float64 `json:"d" cbor:"0,keyasint"` // total capacity in GiB
Used float64 `json:"du" cbor:"1,keyasint"` // allocated in GiB
ReadBytes uint64 `json:"rb,omitzero" cbor:"2,keyasint,omitzero"` // read throughput in bytes/s
WriteBytes uint64 `json:"wb,omitzero" cbor:"3,keyasint,omitzero"` // write throughput in bytes/s
Health string `json:"h,omitempty" cbor:"4,keyasint,omitempty"` // ONLINE, DEGRADED, FAULTED, ...
}
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
@@ -183,6 +194,7 @@ type Details struct {
Podman bool `cbor:"8,keyasint,omitempty"`
MemoryTotal uint64 `cbor:"9,keyasint"`
SmartInterval time.Duration `cbor:"10,keyasint,omitempty"`
ZfsInterval time.Duration `cbor:"11,keyasint,omitempty"` // interval for ZFS detail refresh
}
// Final data structure to return to the hub
+45
View File
@@ -0,0 +1,45 @@
// Package zfs defines the ZFS detail data exchanged between agent and hub.
package zfs
// ZfsData is the detail payload returned by the agent for the GetZfsData action.
type ZfsData struct {
Pools []*PoolDetail `json:"pools,omitempty"`
Complete bool `json:"complete,omitempty"`
}
// PoolDetail holds the verbose state of a single pool: capacity, health,
// scrub, vdev, and dataset information.
type PoolDetail struct {
Name string `json:"name"`
Health string `json:"health,omitempty"`
Size uint64 `json:"size,omitempty"` // bytes
Alloc uint64 `json:"alloc,omitempty"` // bytes
Free uint64 `json:"free,omitempty"` // bytes
Scrub *Scrub `json:"scrub,omitempty"`
Vdevs []*Vdev `json:"vdevs,omitempty"`
Datasets []*Dataset `json:"datasets,omitempty"`
}
// Scrub holds the scrub (or resilver) status of a pool.
type Scrub struct {
State string `json:"state,omitempty"` // NONE, SCANNING, FINISHED, CANCELED
Progress string `json:"progress,omitempty"`
Errors uint64 `json:"errors,omitempty"`
}
// Vdev is a single vdev (mirror, raidz, or leaf disk) with error counters.
type Vdev struct {
Name string `json:"name"`
State string `json:"state,omitempty"`
ReadErrs uint64 `json:"readErrs,omitempty"`
WriteErrs uint64 `json:"writeErrs,omitempty"`
ChecksumErrs uint64 `json:"checksumErrs,omitempty"`
}
// Dataset is a single ZFS dataset with usage information.
type Dataset struct {
Name string `json:"name"`
Used uint64 `json:"used,omitempty"`
Avail uint64 `json:"avail,omitempty"`
Mountpoint string `json:"mount,omitempty"`
}
+22
View File
@@ -125,6 +125,8 @@ func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts)
// refresh SMART devices for a system
apiAuth.POST("/smart/refresh", h.refreshSmartData).BindFunc(excludeReadOnlyRole)
// refresh ZFS pool details for a system
apiAuth.POST("/zfs/refresh", h.refreshZfsData).BindFunc(excludeReadOnlyRole)
// get systemd service details
apiAuth.GET("/systemd/info", h.getSystemdInfo)
// /containers routes
@@ -389,3 +391,23 @@ func (h *Hub) refreshSmartData(e *core.RequestEvent) error {
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
}
// refreshZfsData handles POST /api/beszel/zfs/refresh requests
// Fetches fresh ZFS detail data from the agent and updates the collection
func (h *Hub) refreshZfsData(e *core.RequestEvent) error {
systemID := e.Request.URL.Query().Get("system")
if systemID == "" {
return e.BadRequestError("Invalid system parameter", nil)
}
system, err := h.sm.GetSystem(systemID)
if err != nil || !system.HasUser(e.App, e.Auth) {
return e.NotFoundError("", nil)
}
if err := system.FetchAndSaveZfsPools(true); err != nil {
return e.InternalServerError("", err)
}
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
}
+6
View File
@@ -91,6 +91,12 @@ func setCollectionAuthSettings(app core.App) error {
}); err != nil {
return err
}
if err := applyCollectionRules(app, []string{"zfs_pools"}, collectionRules{
list: &systemScopedReadRule,
view: &systemScopedReadRule,
}); err != nil {
return err
}
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
list: &systemScopedReadRule,
+37
View File
@@ -21,6 +21,7 @@ import (
"github.com/henrygd/beszel/internal/entities/smart"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/systemd"
"github.com/henrygd/beszel/internal/entities/zfs"
"github.com/henrygd/beszel"
@@ -49,6 +50,8 @@ type System struct {
detailsFetched atomic.Bool // True if static system details have been fetched and saved
smartFetching atomic.Bool // True if SMART devices are currently being fetched
smartInterval time.Duration // Interval for periodic SMART data updates
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
}
func (sm *SystemManager) NewSystem(systemId string) *System {
@@ -154,6 +157,12 @@ func (sys *System) update() error {
// to prevent premature expiration leading to new fetch if interval is different.
sys.manager.smartFetchMap.UpdateExpiration(sys.Id, sys.smartInterval+time.Minute)
}
// update zfs interval if it's set on the agent side
if data.Details.ZfsInterval > 0 {
sys.zfsInterval = data.Details.ZfsInterval
sys.manager.hub.Logger().Info("ZFS interval updated from agent details", "system", sys.Id, "interval", sys.zfsInterval.String())
sys.manager.zfsFetchMap.UpdateExpiration(sys.Id, sys.zfsInterval+time.Minute)
}
}
// Fetch and save SMART devices when system first comes online or at intervals
@@ -170,6 +179,20 @@ func (sys *System) update() error {
}
}
// Fetch and save ZFS pool details when system first comes online or at intervals
if backgroundZfsFetchEnabled() && sys.detailsFetched.Load() && sys.supportsZfsData() {
if sys.zfsInterval <= 0 {
sys.zfsInterval = time.Hour
}
if sys.shouldFetchZfs() && sys.zfsFetching.CompareAndSwap(false, true) {
sys.manager.hub.Logger().Info("ZFS fetch", "system", sys.Id, "interval", sys.zfsInterval.String())
go func() {
defer sys.zfsFetching.Store(false)
_ = sys.FetchAndSaveZfsPools(false)
}()
}
}
return err
}
@@ -241,6 +264,10 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
}
}
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
return err
}
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
systemRecord.Set("status", up)
systemRecord.Set("info", data.Info)
@@ -558,6 +585,15 @@ func (sys *System) FetchSmartDataFromAgent() (smart.SmartDataResponse, error) {
return result, err
}
// FetchZfsDataFromAgent fetches ZFS detail data from the agent.
func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
var result zfs.ZfsData
err := sys.request(ctx, common.GetZfsData, common.ZfsDataRequest{Force: force}, &result)
return &result, err
}
func makeStableHashId(strings ...string) string {
hash := fnv.New32a()
for _, str := range strings {
@@ -728,6 +764,7 @@ func (s *System) createSSHClient() error {
}
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
s.manager.resetFailedSmartFetchState(s.Id)
s.manager.resetFailedZfsFetchState(s.Id)
return nil
}
+11
View File
@@ -46,6 +46,7 @@ type SystemManager struct {
systems *store.Store[string, *System] // Thread-safe store of active systems
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
zfsFetchMap *expirymap.ExpiryMap[zfsFetchState] // Stores last ZFS fetch time/result; TTL is only for cleanup
ctx context.Context // Cancelled when the app terminates
cancel context.CancelFunc // Cancels ctx and all child system contexts
}
@@ -67,6 +68,7 @@ func NewSystemManager(hub hubLike) *SystemManager {
systems: store.New(map[string]*System{}),
hub: hub,
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
zfsFetchMap: expirymap.New[zfsFetchState](time.Hour),
}
sm.ctx, sm.cancel = context.WithCancel(context.Background())
return sm
@@ -343,6 +345,15 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
}
}
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
// agent reconnect retries ZFS discovery immediately after configuration changes.
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
state, ok := sm.zfsFetchMap.GetOk(systemID)
if ok && !state.Successful {
sm.zfsFetchMap.Remove(systemID)
}
}
// createSSHClientConfig initializes the SSH client configuration for connecting to an agent's server
func (sm *SystemManager) createSSHClientConfig() error {
privateKey, err := sm.hub.GetSSHKey("")
+193
View File
@@ -0,0 +1,193 @@
package systems
import (
"database/sql"
"errors"
"fmt"
"time"
"github.com/henrygd/beszel"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/zfs"
"github.com/pocketbase/dbx"
"github.com/pocketbase/pocketbase/core"
)
var errIncompleteZfsData = errors.New("incomplete ZFS pool inventory")
type zfsFetchState struct {
LastAttempt int64
Successful bool
}
func (sys *System) supportsZfsData() bool {
return sys.agentVersion.GTE(beszel.MinVersionZfsData)
}
// FetchAndSaveZfsPools fetches ZFS detail data from the agent and saves it to
// the database. force bypasses the agent's detail cache for manual refreshes.
func (sys *System) FetchAndSaveZfsPools(force bool) error {
zfsData, err := sys.FetchZfsDataFromAgent(force)
if err != nil {
sys.recordZfsFetchResult(err, 0)
return err
}
if zfsData == nil || !zfsData.Complete {
err = errIncompleteZfsData
sys.recordZfsFetchResult(err, 0)
return err
}
err = sys.saveZfsPools(zfsData)
sys.recordZfsFetchResult(err, len(zfsData.Pools))
return err
}
// recordZfsFetchResult stores a cooldown entry for the ZFS interval and marks
// whether the last fetch produced any pools, so failed setup can retry on reconnect.
func (sys *System) recordZfsFetchResult(err error, poolCount int) {
if sys.manager == nil {
return
}
interval := sys.zfsFetchInterval()
success := err == nil && poolCount > 0
if sys.manager.hub != nil {
sys.manager.hub.Logger().Info("ZFS fetch result", "system", sys.Id, "success", success, "pools", poolCount, "interval", interval.String(), "err", err)
}
sys.manager.zfsFetchMap.Set(sys.Id, zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: success}, interval+time.Minute)
}
// shouldFetchZfs returns true when there is no active ZFS cooldown entry for this system.
func (sys *System) shouldFetchZfs() bool {
if sys.manager == nil {
return true
}
state, ok := sys.manager.zfsFetchMap.GetOk(sys.Id)
if !ok {
return true
}
return !time.UnixMilli(state.LastAttempt).Add(sys.zfsFetchInterval()).After(time.Now())
}
// zfsFetchInterval returns the agent-provided ZFS interval or the default when unset.
func (sys *System) zfsFetchInterval() time.Duration {
if sys.zfsInterval > 0 {
return sys.zfsInterval
}
return time.Hour
}
// saveZfsPools saves ZFS pool detail data to the zfs_pools collection and
// removes records for pools no longer reported by a complete agent inventory.
func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
if zfsData == nil || !zfsData.Complete {
return errIncompleteZfsData
}
hub := sys.manager.hub
collection, err := hub.FindCachedCollectionByNameOrId("zfs_pools")
if err != nil {
return err
}
return hub.RunInTransaction(func(txApp core.App) error {
alive := make(map[string]bool, len(zfsData.Pools))
for _, pool := range zfsData.Pools {
if pool == nil {
continue
}
alive[pool.Name] = true
if err := sys.upsertZfsPoolRecord(txApp, collection, pool); err != nil {
return err
}
}
existing, err := txApp.FindRecordsByFilter(
collection,
"system={:system}",
"", 0, 0,
dbx.Params{"system": sys.Id},
)
if err != nil {
return err
}
for _, record := range existing {
if !alive[record.GetString("name")] {
if err := txApp.Delete(record); err != nil {
return err
}
}
}
return nil
})
}
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
recordID := makeStableHashId(sys.Id, pool.Name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
if !errors.Is(err, sql.ErrNoRows) {
return err
}
record = core.NewRecord(collection)
record.Set("id", recordID)
}
record.Set("system", sys.Id)
record.Set("name", pool.Name)
record.Set("health", pool.Health)
record.Set("size", pool.Size)
record.Set("alloc", pool.Alloc)
record.Set("free", pool.Free)
record.Set("scrub", pool.Scrub)
record.Set("vdevs", pool.Vdevs)
record.Set("datasets", pool.Datasets)
record.Set("details_updated", time.Now().UTC())
return app.SaveNoValidate(record)
}
// syncZfsPoolHealth persists newly discovered pools and health transitions from
// regular system samples. Detailed fields remain owned by the hourly refresh.
func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsPool) error {
if len(pools) == 0 {
return nil
}
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
if err != nil {
return err
}
const gib = 1024 * 1024 * 1024
for name, pool := range pools {
if pool == nil {
continue
}
recordID := makeStableHashId(sys.Id, name)
record, err := app.FindRecordById(collection, recordID)
if err != nil {
if !errors.Is(err, sql.ErrNoRows) {
return err
}
record = core.NewRecord(collection)
record.Set("id", recordID)
record.Set("system", sys.Id)
record.Set("name", name)
record.Set("health", pool.Health)
record.Set("size", uint64(pool.Total*gib))
record.Set("alloc", uint64(pool.Used*gib))
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
if err := app.SaveNoValidate(record); err != nil {
return fmt.Errorf("creating ZFS pool summary %q: %w", name, err)
}
continue
}
if record.GetString("health") == pool.Health {
continue
}
record.Set("health", pool.Health)
if err := app.SaveNoValidate(record); err != nil {
return fmt.Errorf("updating ZFS pool health %q: %w", name, err)
}
}
return nil
}
+153
View File
@@ -0,0 +1,153 @@
//go:build testing
package systems
import (
"errors"
"testing"
"time"
"github.com/blang/semver"
"github.com/henrygd/beszel/internal/entities/system"
"github.com/henrygd/beszel/internal/entities/zfs"
"github.com/henrygd/beszel/internal/hub/expirymap"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSupportsZfsData(t *testing.T) {
sys := &System{agentVersion: semver.MustParse("0.18.8")}
assert.False(t, sys.supportsZfsData())
sys.agentVersion = semver.MustParse("0.18.9")
assert.True(t, sys.supportsZfsData())
}
func TestRecordZfsFetchResult(t *testing.T) {
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
t.Cleanup(sm.zfsFetchMap.StopCleaner)
sys := &System{
Id: "system-1",
manager: sm,
zfsInterval: time.Hour,
}
// Successful fetch with pools
sys.recordZfsFetchResult(nil, 2)
state, ok := sm.zfsFetchMap.GetOk(sys.Id)
assert.True(t, ok, "expected zfs fetch result to be stored")
assert.True(t, state.Successful, "expected successful fetch state to be recorded")
// Failed fetch
sys.recordZfsFetchResult(errors.New("failed"), 0)
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
assert.True(t, ok, "expected failed zfs fetch state to be stored")
assert.False(t, state.Successful, "expected failed zfs fetch state to be marked unsuccessful")
// Successful fetch but no pools
sys.recordZfsFetchResult(nil, 0)
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
assert.True(t, ok, "expected fetch with zero pools to be stored")
assert.False(t, state.Successful, "expected fetch with zero pools to be marked unsuccessful")
}
func TestShouldFetchZfs(t *testing.T) {
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
t.Cleanup(sm.zfsFetchMap.StopCleaner)
sys := &System{
Id: "system-1",
manager: sm,
zfsInterval: time.Hour,
}
assert.True(t, sys.shouldFetchZfs(), "expected initial zfs fetch to be allowed")
sys.recordZfsFetchResult(errors.New("failed"), 0)
assert.False(t, sys.shouldFetchZfs(), "expected zfs fetch to be blocked while interval entry exists")
sm.zfsFetchMap.Remove(sys.Id)
assert.True(t, sys.shouldFetchZfs(), "expected zfs fetch to be allowed after interval entry is cleared")
}
func TestZfsFetchIntervalDefault(t *testing.T) {
sys := &System{}
assert.Equal(t, time.Hour, sys.zfsFetchInterval())
sys.zfsInterval = 5 * time.Minute
assert.Equal(t, 5*time.Minute, sys.zfsFetchInterval())
}
func TestResetFailedZfsFetchState(t *testing.T) {
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
t.Cleanup(sm.zfsFetchMap.StopCleaner)
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: false}, time.Hour)
sm.resetFailedZfsFetchState("system-1")
_, ok := sm.zfsFetchMap.GetOk("system-1")
assert.False(t, ok, "expected failed zfs fetch state to be cleared on reconnect")
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: true}, time.Hour)
sm.resetFailedZfsFetchState("system-1")
_, ok = sm.zfsFetchMap.GetOk("system-1")
assert.True(t, ok, "expected successful zfs fetch state to be preserved")
}
func TestSaveZfsPoolsCompleteEmptyPrunesFinalPool(t *testing.T) {
sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
Complete: true,
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
}))
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
require.NoError(t, err)
require.Len(t, records, 1)
assert.False(t, records[0].GetDateTime("details_updated").Time().IsZero())
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true}))
records, err = app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
require.NoError(t, err)
assert.Empty(t, records)
}
func TestSaveZfsPoolsIncompletePreservesRecords(t *testing.T) {
sys, app := newTestSystemWithHub(t)
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
Complete: true,
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
}))
assert.ErrorIs(t, sys.saveZfsPools(&zfs.ZfsData{}), errIncompleteZfsData)
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
require.NoError(t, err)
assert.Len(t, records, 1)
}
func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
sys, app := newTestSystemWithHub(t)
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
require.NoError(t, err)
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
}))
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
require.NoError(t, err)
firstUpdated := record.GetDateTime("updated")
assert.Equal(t, "ONLINE", record.GetString("health"))
assert.EqualValues(t, 100*1024*1024*1024, record.GetInt("size"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 30, Health: "ONLINE"},
}))
record, err = app.FindRecordById(collection, record.Id)
require.NoError(t, err)
assert.Equal(t, firstUpdated, record.GetDateTime("updated"))
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 30, Health: "DEGRADED"},
}))
record, err = app.FindRecordById(collection, record.Id)
require.NoError(t, err)
assert.Equal(t, "DEGRADED", record.GetString("health"))
}
@@ -7,3 +7,6 @@ package systems
// The hub integration tests create/replace systems and clean up the test apps quickly.
// Background SMART fetching can outlive teardown and crash in PocketBase internals (nil DB).
func backgroundSmartFetchEnabled() bool { return true }
// Background ZFS fetching follows the same policy as SMART fetching.
func backgroundZfsFetchEnabled() bool { return true }
@@ -17,6 +17,9 @@ import (
// the automatic background fetch during tests.
func backgroundSmartFetchEnabled() bool { return false }
// Background ZFS fetching follows the same policy as SMART fetching.
func backgroundZfsFetchEnabled() bool { return false }
// TESTING ONLY: GetSystemCount returns the number of systems in the store
func (sm *SystemManager) GetSystemCount() int {
return sm.systems.Length()
@@ -115,6 +118,7 @@ func (sm *SystemManager) RemoveAllSystems() {
sm.RemoveSystem(system.Id)
}
sm.smartFetchMap.StopCleaner()
sm.zfsFetchMap.StopCleaner()
}
// ResetContextForTesting replaces the manager context for a new synctest bubble.
+184
View File
@@ -0,0 +1,184 @@
package migrations
import (
"github.com/pocketbase/pocketbase/core"
m "github.com/pocketbase/pocketbase/migrations"
)
// Creates the zfs_pools collection for per-system ZFS pool detail data
// (pool health, capacity, scrub, vdevs, datasets). Upserts rather than
// deletes missing collections, so it is safe on fresh and existing installs.
func init() {
m.Register(func(app core.App) error {
// update collections
jsonData := `[
{
"createRule": null,
"deleteRule": null,
"fields": [
{
"autogeneratePattern": "[a-z0-9]{15}",
"hidden": false,
"id": "text3208210256",
"max": 15,
"min": 15,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"cascadeDelete": true,
"collectionId": "2hz5ncl8tizk5nx",
"hidden": false,
"id": "relation1204987316",
"maxSelect": 1,
"minSelect": 0,
"name": "system",
"presentable": false,
"required": true,
"system": false,
"type": "relation"
},
{
"autogeneratePattern": "",
"hidden": false,
"id": "text7739291048",
"max": 0,
"min": 0,
"name": "name",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{
"autogeneratePattern": "",
"hidden": false,
"id": "text5528164482",
"max": 0,
"min": 0,
"name": "health",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": false,
"system": false,
"type": "text"
},
{
"hidden": false,
"id": "number8862034195",
"max": null,
"min": null,
"name": "size",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"hidden": false,
"id": "number4418907321",
"max": null,
"min": null,
"name": "alloc",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"hidden": false,
"id": "number2904183765",
"max": null,
"min": null,
"name": "free",
"onlyInt": true,
"presentable": false,
"required": false,
"system": false,
"type": "number"
},
{
"hidden": false,
"id": "json4466109723",
"maxSize": 0,
"name": "scrub",
"presentable": false,
"required": false,
"system": false,
"type": "json"
},
{
"hidden": false,
"id": "json9012873456",
"maxSize": 0,
"name": "vdevs",
"presentable": false,
"required": false,
"system": false,
"type": "json"
},
{
"hidden": false,
"id": "json7182045639",
"maxSize": 0,
"name": "datasets",
"presentable": false,
"required": false,
"system": false,
"type": "json"
},
{
"hidden": false,
"id": "date9274163058",
"max": "",
"min": "",
"name": "details_updated",
"presentable": false,
"required": false,
"system": false,
"type": "date"
},
{
"hidden": false,
"id": "autodate3332085495",
"name": "updated",
"onCreate": true,
"onUpdate": true,
"presentable": false,
"system": false,
"type": "autodate"
}
],
"id": "pbc_8441057391",
"indexes": [
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
],
"listRule": null,
"name": "zfs_pools",
"system": false,
"type": "base",
"updateRule": null,
"viewRule": null
}
]`
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
if err != nil {
return err
}
return nil
}, func(app core.App) error {
return nil
})
}
+35
View File
@@ -127,6 +127,7 @@ func (rm *RecordManager) CreateLongerRecords() {
"created": shorterRecordPeriod,
},
)).
OrderBy("created").
All(&recordIds)
// continue if not enough shorter records
@@ -196,6 +197,7 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
tempCount := float64(0)
var fanSums map[string]uint64
fanCount := uint64(0)
zfsPoolCounts := make(map[string]uint64)
// Accumulate totals
for i := range records {
@@ -336,6 +338,31 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
}
}
// Accumulate ZFS pool stats. Counts are tracked per entry so a pool
// missing from some samples is not averaged as zero.
if stats.ZfsPools != nil {
if sum.ZfsPools == nil {
sum.ZfsPools = make(map[string]*system.ZfsPool, len(stats.ZfsPools))
}
for name, value := range stats.ZfsPools {
if value == nil {
continue
}
pool := sum.ZfsPools[name]
if pool == nil {
pool = &system.ZfsPool{}
sum.ZfsPools[name] = pool
}
pool.Total += value.Total
pool.Used += value.Used
pool.ReadBytes += value.ReadBytes
pool.WriteBytes += value.WriteBytes
if value.Health != "" {
pool.Health = value.Health
}
zfsPoolCounts[name]++
}
}
// Accumulate GPU data
if stats.GPUData != nil {
if sum.GPUData == nil {
@@ -446,6 +473,14 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
}
}
// Average ZFS pool stats.
for name, pool := range sum.ZfsPools {
entryCount := zfsPoolCounts[name]
pool.Total = twoDecimals(pool.Total / float64(entryCount))
pool.Used = twoDecimals(pool.Used / float64(entryCount))
pool.ReadBytes /= entryCount
pool.WriteBytes /= entryCount
}
// Average GPU data
if sum.GPUData != nil {
for id := range sum.GPUData {
@@ -669,6 +669,33 @@ func TestAverageSystemStatsSlice_MixedOptionalFields(t *testing.T) {
assert.Equal(t, 20.0, result.GPUData["gpu0"].Usage)
}
func TestAverageSystemStatsSlice_Zfs(t *testing.T) {
input := []system.Stats{
{
ZfsPools: map[string]*system.ZfsPool{
"tank": {Total: 100, Used: 40, ReadBytes: 100, WriteBytes: 200, Health: "ONLINE"},
},
},
{},
{
ZfsPools: map[string]*system.ZfsPool{
"tank": {Total: 120, Used: 60, ReadBytes: 300, WriteBytes: 400, Health: "DEGRADED"},
"backup": {Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE"},
},
},
}
result := records.AverageSystemStatsSlice(input)
require.Len(t, result.ZfsPools, 2)
assert.Equal(t, &system.ZfsPool{
Total: 110, Used: 50, ReadBytes: 200, WriteBytes: 300, Health: "DEGRADED",
}, result.ZfsPools["tank"])
assert.Equal(t, &system.ZfsPool{
Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE",
}, result.ZfsPools["backup"])
}
// Tests with 10 records matching the common real-world case (10 x 1m -> 1 x 10m).
func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
input := make([]system.Stats, 10)
@@ -8,10 +8,11 @@ import { useSystemData } from "./system/use-system-data"
import { CpuChart, ContainerCpuChart } from "./system/charts/cpu-charts"
import { MemoryChart, ContainerMemoryChart, SwapChart } from "./system/charts/memory-charts"
import { RootDiskCharts, ExtraFsCharts } from "./system/charts/disk-charts"
import { ZfsCharts } from "./system/charts/zfs-charts"
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
import { GpuPowerChart, GpuDetailCharts } from "./system/charts/gpu-charts"
import { LazyContainersTable, LazySmartTable, LazySystemdTable } from "./system/lazy-tables"
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
import { LoadAverageChart } from "./system/charts/load-average-chart"
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
import { GpuIcon } from "../ui/icons"
@@ -63,6 +64,7 @@ export default memo(function SystemDetail({ id }: { id: string }) {
const hasContainersTable = hasContainers && compareSemVer(chartData.agentVersion, SEMVER_0_14_0) >= 0
const hasSystemd = system.info.sv
const hasGpu = hasGpuData || hasGpuPowerData
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
// keep tabsRef in sync for keyboard navigation
const tabs = ["core", "disk"]
@@ -142,6 +144,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<ExtraFsCharts systemData={systemData} />
{hasZfs && <ZfsCharts systemData={systemData} />}
{hasZfs && <LazyZfsTable systemId={system.id} />}
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
@@ -204,6 +210,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
<RootDiskCharts systemData={systemData} />
</div>
<ExtraFsCharts systemData={systemData} />
{hasZfs && <ZfsCharts systemData={systemData} />}
{hasZfs && <LazyZfsTable systemId={system.id} />}
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
</>
)}
@@ -0,0 +1,130 @@
import { t } from "@lingui/core/macro"
import AreaChartDefault from "@/components/charts/area-chart"
import { decimalString, formatBytes, toFixedFloat } from "@/lib/utils"
import type { SystemStatsRecord } from "@/types"
import { ChartCard } from "../chart-card"
import { Unit } from "@/lib/enums"
import { useStore } from "@nanostores/react"
import { $userSettings } from "@/lib/stores"
import type { SystemData } from "../use-system-data"
// Accessors for ZFS metrics
const poolUsage =
(name: string) =>
({ stats }: SystemStatsRecord) =>
stats?.z?.[name]?.du ?? 0
const poolRead =
(name: string) =>
({ stats }: SystemStatsRecord) =>
stats?.z?.[name]?.rb ?? 0
const poolWrite =
(name: string) =>
({ stats }: SystemStatsRecord) =>
stats?.z?.[name]?.wb ?? 0
export function ZfsPoolUsageChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
const { chartData, grid, dataEmpty } = systemData
const latest = chartData.systemStats.at(-1)?.stats
const pool = latest?.z?.[poolName]
if (!pool) {
return null
}
let poolTotal = pool.d
// round to nearest GB
if (poolTotal >= 100) {
poolTotal = Math.round(poolTotal)
}
return (
<ChartCard
empty={dataEmpty}
grid={grid}
title={`${poolName} ${t`Usage`}`}
description={t`Usage of ZFS pool ${poolName}`}
>
<AreaChartDefault
chartData={chartData}
domain={[0, poolTotal]}
showTotal={true}
tickFormatter={(val) => {
const { value, unit } = formatBytes(val * 1024, false, Unit.Bytes, true)
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
}}
contentFormatter={({ value }) => {
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
}}
dataPoints={[
{
label: t`Pool Usage`,
dataKey: poolUsage(poolName),
color: 4,
opacity: 0.4,
},
]}
/>
</ChartCard>
)
}
export function ZfsPoolIOChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
const { chartData, grid, dataEmpty } = systemData
const userSettings = useStore($userSettings)
if (!chartData.systemStats?.length) {
return null
}
return (
<ChartCard
empty={dataEmpty}
grid={grid}
title={`${poolName} I/O`}
description={t`Throughput of ZFS pool ${poolName}`}
>
<AreaChartDefault
chartData={chartData}
showTotal={true}
dataPoints={[
{
label: t({ message: "Write", comment: "Disk write" }),
dataKey: poolWrite(poolName),
color: 3,
opacity: 0.3,
},
{
label: t({ message: "Read", comment: "Disk read" }),
dataKey: poolRead(poolName),
color: 1,
opacity: 0.3,
},
]}
tickFormatter={(val) => {
const { value, unit } = formatBytes(val, true, userSettings.unitDisk, false)
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
}}
contentFormatter={({ value }) => {
const { value: convertedValue, unit } = formatBytes(value, true, userSettings.unitDisk, false)
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
}}
/>
</ChartCard>
)
}
/** ZFS section: one stacked usage card per pool plus per-pool I/O cards. */
export function ZfsCharts({ systemData }: { systemData: SystemData }) {
const latest = systemData.chartData.systemStats?.at(-1)?.stats
const pools = latest?.z ?? {}
if (Object.keys(pools).length === 0) {
return null
}
return (
<div className="grid xl:grid-cols-2 gap-4">
{Object.keys(pools).map((poolName) => (
<div key={poolName} className="contents">
<ZfsPoolUsageChart systemData={systemData} poolName={poolName} />
<ZfsPoolIOChart systemData={systemData} poolName={poolName} />
</div>
))}
</div>
)
}
@@ -24,6 +24,17 @@ export function LazySmartTable({ systemId }: { systemId: string }) {
)
}
const ZfsTable = lazy(() => import("./zfs-table"))
export function LazyZfsTable({ systemId }: { systemId: string }) {
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
return (
<div ref={ref} className={cn(isIntersecting && "contents")}>
{isIntersecting && <ZfsTable systemId={systemId} />}
</div>
)
}
const SystemdTable = lazy(() => import("../../systemd-table/systemd-table"))
export function LazySystemdTable({ systemId }: { systemId: string }) {
@@ -0,0 +1,652 @@
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"
import { Badge } from "@/components/ui/badge"
import { Button } from "@/components/ui/button"
import { Card, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
import { Input } from "@/components/ui/input"
import { Separator } from "@/components/ui/separator"
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
import { isReadOnlyUser, pb } from "@/lib/api"
import { cn, formatBytes, formatShortDate, hourWithSeconds, toFixedFloat } from "@/lib/utils"
import type { ZfsDataset, ZfsPoolRecord, ZfsVdev } from "@/types"
import { t } from "@lingui/core/macro"
import { Trans } from "@lingui/react/macro"
import type { Column, ColumnDef } from "@tanstack/react-table"
import {
flexRender,
getCoreRowModel,
getFilteredRowModel,
getSortedRowModel,
useReactTable,
} from "@tanstack/react-table"
import {
ActivityIcon,
BinaryIcon,
CheckCircleIcon,
CircleAlertIcon,
ClockIcon,
HardDriveDownloadIcon,
HardDriveIcon,
HardDriveUploadIcon,
LoaderCircleIcon,
MoreHorizontalIcon,
RefreshCwIcon,
RotateCwIcon,
XCircleIcon,
XIcon,
} from "lucide-react"
import { useCallback, useEffect, useMemo, useState } from "react"
const ZFS_POOL_FIELDS = "id,system,name,health,size,alloc,free,scrub,details_updated,updated"
/** Maps a zpool health string to a Badge variant. */
function healthVariant(health: string): "success" | "warning" | "danger" | "outline" {
switch (health) {
case "ONLINE":
return "success"
case "DEGRADED":
return "warning"
case "FAULTED":
case "OFFLINE":
case "UNAVAIL":
case "REMOVED":
case "SUSPENDED":
return "danger"
default:
return "outline"
}
}
function formatCapacity(bytes: number): string {
if (!bytes) return "-"
const { value, unit } = formatBytes(bytes)
return `${toFixedFloat(value, value >= 10 ? 1 : 2)} ${unit}`
}
function HeaderButton<T>({ column, name, Icon }: { column: Column<T>; name: string; Icon: React.ElementType }) {
const isSorted = column.getIsSorted()
return (
<Button
className={cn(
"h-9 px-3 flex items-center gap-2 duration-50",
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
)}
variant="ghost"
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
>
<Icon className="size-4" />
{name}
</Button>
)
}
const columns: ColumnDef<ZfsPoolRecord>[] = [
{
accessorKey: "name",
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
header: ({ column }) => <HeaderButton column={column} name={t`Pool`} Icon={HardDriveIcon} />,
cell: ({ getValue }) => <span className="font-medium ms-1.5">{getValue() as string}</span>,
},
{
accessorKey: "health",
sortingFn: (a, b) => a.original.health.localeCompare(b.original.health),
header: ({ column }) => <HeaderButton column={column} name={t`Health`} Icon={ActivityIcon} />,
cell: ({ getValue }) => {
const health = (getValue() as string) || ""
return <Badge variant={healthVariant(health)}>{health || t`Unknown`}</Badge>
},
},
{
id: "size",
accessorFn: (record) => record.size,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Size`} Icon={BinaryIcon} />,
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
},
{
id: "used",
accessorFn: (record) => record.alloc,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
},
{
id: "free",
accessorFn: (record) => record.free,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Free`} Icon={HardDriveUploadIcon} />,
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
},
{
id: "scrub",
accessorFn: (record) => record.scrub?.state ?? "",
header: ({ column }) => <HeaderButton column={column} name={t`Scrub`} Icon={RotateCwIcon} />,
cell: ({ row }) => {
const scrub = row.original.scrub
if (!scrub?.state) return <span className="ms-1.5 text-muted-foreground">{t`None`}</span>
return (
<span className="ms-1.5 tabular-nums">
{scrub.state}
{scrub.progress ? ` (${scrub.progress})` : ""}
</span>
)
},
},
{
id: "updated",
invertSorting: true,
accessorFn: (record) => record.details_updated || record.updated,
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
cell: ({ getValue }) => {
const timestamp = getValue() as string
if (!timestamp) return null
const formatter =
new Date(timestamp).toDateString() === new Date().toDateString() ? hourWithSeconds : formatShortDate
return <span className="ms-1 tabular-nums">{formatter(timestamp)}</span>
},
},
]
function VdevTable({ vdevs }: { vdevs: ZfsVdev[] }) {
if (!vdevs?.length) return null
return (
<div className="overflow-x-auto rounded-md border">
<Table>
<TableHeader>
<TableRow>
<TableHead>{t`Vdev`}</TableHead>
<TableHead>{t`State`}</TableHead>
<TableHead className="text-right">{t`Read errors`}</TableHead>
<TableHead className="text-right">{t`Write errors`}</TableHead>
<TableHead className="text-right">{t`Checksum errors`}</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{vdevs.map((vdev) => (
<TableRow key={vdev.name}>
<TableCell className="font-mono text-xs">{vdev.name}</TableCell>
<TableCell>
<Badge variant={healthVariant(vdev.state ?? "")} className="font-normal">
{vdev.state ?? "-"}
</Badge>
</TableCell>
<TableCell
className={cn("text-right tabular-nums", (vdev.readErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
>
{vdev.readErrs ?? 0}
</TableCell>
<TableCell
className={cn("text-right tabular-nums", (vdev.writeErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
>
{vdev.writeErrs ?? 0}
</TableCell>
<TableCell
className={cn(
"text-right tabular-nums",
(vdev.checksumErrs ?? 0) > 0 && "text-red-600 dark:text-red-400"
)}
>
{vdev.checksumErrs ?? 0}
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
)
}
const datasetColumns: ColumnDef<ZfsDataset>[] = [
{
accessorKey: "name",
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
header: ({ column }) => <HeaderButton column={column} name={t`Dataset`} Icon={HardDriveIcon} />,
cell: ({ getValue }) => <span className="font-mono text-xs">{getValue() as string}</span>,
},
{
id: "used",
accessorFn: (ds) => ds.used ?? 0,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
},
{
id: "avail",
accessorFn: (ds) => ds.avail ?? 0,
invertSorting: true,
header: ({ column }) => <HeaderButton column={column} name={t`Available`} Icon={HardDriveUploadIcon} />,
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
},
{
accessorKey: "mount",
sortingFn: (a, b) => (a.original.mount ?? "").localeCompare(b.original.mount ?? ""),
header: ({ column }) => <HeaderButton column={column} name={t`Mountpoint`} Icon={HardDriveIcon} />,
cell: ({ getValue }) => (
<span className="font-mono text-xs text-muted-foreground">{(getValue() as string) || "-"}</span>
),
},
]
function DatasetTable({ datasets }: { datasets: ZfsDataset[] }) {
const [filter, setFilter] = useState("")
const filtered = useMemo(() => {
if (!datasets) return []
if (!filter) return datasets
const needle = filter.toLowerCase()
return datasets.filter((ds) => ds.name.toLowerCase().includes(needle))
}, [datasets, filter])
const table = useReactTable({
data: filtered,
columns: datasetColumns,
getCoreRowModel: getCoreRowModel(),
getSortedRowModel: getSortedRowModel(),
})
if (!datasets?.length) return null
return (
<div>
<div className="mb-2 flex items-center justify-between gap-4">
<h3 className="text-base font-semibold">
<Trans>Datasets</Trans>
</h3>
<div className="relative w-64 max-w-full">
<Input
placeholder={t`Filter datasets...`}
value={filter}
onChange={(e) => setFilter(e.target.value)}
className="px-4 w-full"
/>
{filter && (
<Button
type="button"
variant="ghost"
size="icon"
aria-label={t`Clear`}
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
onClick={() => setFilter("")}
>
<XIcon className="h-4 w-4" />
</Button>
)}
</div>
</div>
<div className="max-h-80 overflow-auto rounded-md border">
<Table>
<TableHeader className="sticky top-0 z-10">
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id}>
{headerGroup.headers.map((header) => (
<TableHead key={header.id} className="px-2">
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
</TableHead>
))}
</TableRow>
))}
</TableHeader>
<TableBody>
{table.getRowModel().rows.map((row) => (
<TableRow key={row.id}>
{row.getVisibleCells().map((cell) => (
<TableCell key={cell.id} className="ps-5 whitespace-pre">
{flexRender(cell.column.columnDef.cell, cell.getContext())}
</TableCell>
))}
</TableRow>
))}
</TableBody>
</Table>
</div>
</div>
)
}
function PoolSheet({
poolId,
open,
onOpenChange,
}: {
poolId: string | null
open: boolean
onOpenChange: (open: boolean) => void
}) {
const [pool, setPool] = useState<ZfsPoolRecord | null>(null)
const [isLoading, setIsLoading] = useState(false)
useEffect(() => {
let active = true
if (!poolId) {
setPool(null)
return
}
// Only fetch when opening, not when closing (keeps data visible during close animation)
if (!open) return
setIsLoading(true)
pb.collection("zfs_pools")
.getOne(poolId)
.then((record) => active && setPool(record as ZfsPoolRecord))
.catch(() => active && setPool(null))
.finally(() => active && setIsLoading(false))
return () => {
active = false
}
}, [open, poolId])
const health = pool?.health || ""
const healthVariantValue = healthVariant(health)
const HealthIcon =
healthVariantValue === "success"
? CheckCircleIcon
: healthVariantValue === "warning"
? CircleAlertIcon
: XCircleIcon
return (
<Sheet open={open} onOpenChange={onOpenChange}>
<SheetContent className="w-full sm:max-w-220 gap-0 overflow-y-auto">
<SheetHeader className="mb-0 border-b">
<SheetTitle className="flex items-center gap-2">
{pool ? pool.name : t`ZFS Pool`}
{pool && <Badge variant={healthVariantValue}>{health}</Badge>}
</SheetTitle>
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
{pool?.size ? formatCapacity(pool.size) : null}
{pool?.alloc ? (
<>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<span>
<Trans>Used</Trans>: {formatCapacity(pool.alloc)}
</span>
</>
) : null}
{pool?.free ? (
<>
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
<span>
<Trans>Free</Trans>: {formatCapacity(pool.free)}
</span>
</>
) : null}
</SheetDescription>
</SheetHeader>
<div className="flex-1 p-4 flex flex-col gap-4">
{isLoading ? (
<div className="flex justify-center py-8">
<LoaderCircleIcon className="animate-spin size-10 opacity-60" />
</div>
) : (
<>
{pool && health && (
<Alert className="pb-3 shrink-0">
<HealthIcon className="size-4" />
<AlertTitle>
<Trans>Pool Health</Trans>: {health}
</AlertTitle>
{pool.scrub?.state && (
<AlertDescription>
<Trans>Scrub</Trans>: {pool.scrub.state}
{pool.scrub.progress ? ` (${pool.scrub.progress})` : ""}
{pool.scrub.errors ? `, ${pool.scrub.errors} errors` : ""}
</AlertDescription>
)}
</Alert>
)}
{pool?.vdevs?.length || pool?.datasets?.length ? (
<>
{pool.vdevs?.length ? <VdevTable vdevs={pool.vdevs} /> : null}
<DatasetTable datasets={pool.datasets ?? []} />
</>
) : (
!isLoading && (
<div className="py-8 text-center text-sm text-muted-foreground">
<Trans>No detail data for this pool.</Trans>
</div>
)
)}
</>
)}
</div>
</SheetContent>
</Sheet>
)
}
export default function ZfsTable({ systemId }: { systemId?: string }) {
const [zfsPools, setZfsPools] = useState<ZfsPoolRecord[]>()
const [globalFilter, setGlobalFilter] = useState("")
const [activePoolId, setActivePoolId] = useState<string | null>(null)
const [sheetOpen, setSheetOpen] = useState(false)
const [refreshingId, setRefreshingId] = useState<string | null>(null)
useEffect(() => {
let disposed = false
let unsubscribe: () => void = () => {}
// fetch initial records
pb.collection<ZfsPoolRecord>("zfs_pools")
.getFullList({
filter: systemId ? pb.filter("system={:id}", { id: systemId }) : "",
sort: "name",
fields: ZFS_POOL_FIELDS,
})
.then((records) => !disposed && setZfsPools(records))
.catch((error) => console.error("Failed to fetch ZFS pools:", error))
// subscribe to realtime updates
const pbOptions = systemId ? { filter: `system="${systemId}"` } : undefined
;(async () => {
try {
const unsubscribeNow = await pb.collection<ZfsPoolRecord>("zfs_pools").subscribe(
"*",
(event) => {
const record = event.record as ZfsPoolRecord
setZfsPools((current) => {
const pools = current ?? []
const matchesSystemScope = !systemId || record.system === systemId
if (event.action === "delete") {
return pools.filter((pool) => pool.id !== record.id)
}
if (!matchesSystemScope) {
return pools.filter((pool) => pool.id !== record.id)
}
const existingIndex = pools.findIndex((pool) => pool.id === record.id)
if (existingIndex === -1) {
return [record, ...pools]
}
const next = [...pools]
next[existingIndex] = record
return next
})
},
pbOptions
)
if (disposed) {
unsubscribeNow()
} else {
unsubscribe = unsubscribeNow
}
} catch (error) {
console.error("Failed to subscribe to ZFS pool updates:", error)
}
})()
return () => {
disposed = true
unsubscribe?.()
}
}, [systemId])
const refreshSystem = useCallback(async (systemId: string) => {
try {
await pb.send("/api/beszel/zfs/refresh", {
method: "POST",
query: { system: systemId },
})
} catch (error) {
console.error("Failed to refresh ZFS pools:", error)
}
}, [])
const handleRowRefresh = useCallback(
async (pool: ZfsPoolRecord) => {
if (!pool.system) return
setRefreshingId(pool.id)
try {
await refreshSystem(pool.system)
} finally {
setRefreshingId((id) => (id === pool.id ? null : id))
}
},
[refreshSystem]
)
const actionColumn = useMemo<ColumnDef<ZfsPoolRecord>>(
() => ({
id: "actions",
enableSorting: false,
header: () => (
<span className="sr-only">
<Trans>Actions</Trans>
</span>
),
cell: ({ row }) => {
const pool = row.original
const isRowRefreshing = refreshingId === pool.id
return (
<div className="flex justify-end">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
variant="ghost"
size="icon"
className="size-10"
onClick={(event) => event.stopPropagation()}
onMouseDown={(event) => event.stopPropagation()}
>
<span className="sr-only">
<Trans>Open menu</Trans>
</span>
<MoreHorizontalIcon className="w-5" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
<DropdownMenuItem
onClick={(event) => {
event.stopPropagation()
handleRowRefresh(pool)
}}
disabled={isRowRefreshing}
>
<RefreshCwIcon className={cn("me-2.5 size-4", isRowRefreshing && "animate-spin")} />
<Trans>Refresh</Trans>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
)
},
}),
[refreshingId, handleRowRefresh]
)
const tableColumns = useMemo(() => {
return isReadOnlyUser() ? columns : [...columns, actionColumn]
}, [actionColumn])
const table = useReactTable({
data: zfsPools || ([] as ZfsPoolRecord[]),
columns: tableColumns,
getCoreRowModel: getCoreRowModel(),
getSortedRowModel: getSortedRowModel(),
getFilteredRowModel: getFilteredRowModel(),
state: { globalFilter },
onGlobalFilterChange: setGlobalFilter,
globalFilterFn: (row, _columnId, filterValue) => {
const pool = row.original
const searchString = `${pool.name} ${pool.health ?? ""}`.toLowerCase()
return (filterValue as string)
.toLowerCase()
.split(" ")
.every((term) => searchString.includes(term))
},
})
const rows = table.getRowModel().rows
// Hide the table on system pages if there's no data
if (systemId && !zfsPools?.length && !globalFilter) {
return null
}
const openSheet = (pool: ZfsPoolRecord) => {
setActivePoolId(pool.id)
setSheetOpen(true)
}
return (
<div>
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
<CardHeader className="p-0 mb-3 sm:mb-4">
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
<div className="px-2 sm:px-1">
<CardTitle className="mb-2">ZFS</CardTitle>
<CardDescription className="flex">
<Trans>Click on a pool to view vdev and dataset details.</Trans>
</CardDescription>
</div>
<div className="relative ms-auto w-full max-w-full md:w-64">
<Input
placeholder={t`Filter...`}
value={globalFilter}
onChange={(event) => setGlobalFilter(event.target.value)}
className="px-4 w-full max-w-full md:w-64"
/>
{globalFilter && (
<Button
type="button"
variant="ghost"
size="icon"
aria-label={t`Clear`}
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
onClick={() => setGlobalFilter("")}
>
<XIcon className="h-4 w-4" />
</Button>
)}
</div>
</div>
</CardHeader>
<div className="h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto rounded-md border">
<Table>
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
{table.getHeaderGroups().map((headerGroup) => (
<TableRow key={headerGroup.id}>
{headerGroup.headers.map((header) => (
<TableHead key={header.id} className="px-2">
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
</TableHead>
))}
</TableRow>
))}
</TableHeader>
<TableBody>
{rows.map((row) => (
<TableRow
key={row.id}
data-state={row.getIsSelected() && "selected"}
className="cursor-pointer"
onClick={() => openSheet(row.original)}
>
{row.getVisibleCells().map((cell) => (
<TableCell key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</TableCell>
))}
</TableRow>
))}
</TableBody>
</Table>
</div>
</Card>
<PoolSheet poolId={activePoolId} open={sheetOpen} onOpenChange={setSheetOpen} />
</div>
)
}
+52
View File
@@ -151,6 +151,8 @@ export interface SystemStats {
f?: Record<string, number>
/** extra filesystems */
efs?: Record<string, ExtraFsStats>
/** ZFS pool metrics */
z?: Record<string, ZfsPool>
/** GPU data */
g?: Record<string, GPUData>
/** battery percent and state */
@@ -178,6 +180,56 @@ export interface GPUData {
e?: Record<string, number>
}
export interface ZfsPool {
/** total capacity (GiB) */
d: number
/** allocated (GiB) */
du: number
/** read throughput (bytes/s) */
rb?: number
/** write throughput (bytes/s) */
wb?: number
/** health: ONLINE, DEGRADED, FAULTED, ... */
h?: string
}
export interface ZfsScrub {
/** NONE, SCANNING, FINISHED, CANCELED */
state?: string
/** progress while scanning, e.g. "10.00%" */
progress?: string
errors?: number
}
export interface ZfsVdev {
name: string
state?: string
readErrs?: number
writeErrs?: number
checksumErrs?: number
}
export interface ZfsDataset {
name: string
used?: number
avail?: number
mount?: string
}
export interface ZfsPoolRecord extends RecordModel {
system: string
name: string
health: string
size: number
alloc: number
free: number
scrub: ZfsScrub | null
vdevs: ZfsVdev[] | null
datasets: ZfsDataset[] | null
details_updated: string
updated: string
}
export interface ExtraFsStats {
/** disk size (gb) */
d: number
+2
View File
@@ -16,6 +16,7 @@ It has a friendly web interface, simple configuration, and is ready to use out o
- **Lightweight**: Smaller and less resource-intensive than leading solutions.
- **Simple**: Easy setup with little manual configuration required.
- **Docker stats**: Tracks CPU, memory, and network usage history for each container.
- **ZFS**: Tracks pool capacity, health, and I/O, plus per-dataset usage.
- **Alerts**: Configurable alerts for CPU, memory, disk, bandwidth, temperature, fan speed, load average, and status.
- **Multi-user**: Users manage their own systems. Admins can share systems across users.
- **OAuth / OIDC**: Supports many OAuth2 providers. Password auth can be disabled.
@@ -53,6 +54,7 @@ The [quick start guide](https://beszel.dev/guide/getting-started) and other docu
- **Battery** - Host system battery charge.
- **Containers** - Status and metrics of all running Docker / Podman containers.
- **S.M.A.R.T.** - Host system disk health (includes eMMC wear/EOL and Linux mdraid array health via sysfs when available).
- **ZFS** - Pool capacity, usage, health, I/O throughput, scrub status, and per-dataset usage.
## Help and discussion