mirror of
https://github.com/henrygd/beszel.git
synced 2026-09-19 14:34:14 +00:00
feat: add ZFS monitoring (#2209)
- track pool capacity, health, I/O, scrub status, and vdev errors - report dataset usage and correct ZFS filesystem metrics - add pool charts, detail views, refresh controls, and health alerts - persist pool details and include ZFS usage in disk alerts - support configurable detail intervals and legacy agent compatibility --------- Co-authored-by: hank <hank@henrygd.me>
This commit is contained in:
@@ -57,12 +57,18 @@ type SystemAlertStats struct {
|
||||
Battery [2]uint8 `json:"bat"`
|
||||
Batteries map[string]uint8 `json:"bats"`
|
||||
ExtraFs map[string]SystemAlertFsStats `json:"efs"`
|
||||
ZfsPools map[string]SystemAlertZfsPool `json:"z"`
|
||||
}
|
||||
|
||||
type SystemAlertGPUData struct {
|
||||
Usage float64 `json:"u"`
|
||||
}
|
||||
|
||||
type SystemAlertZfsPool struct {
|
||||
Total float64 `json:"d"`
|
||||
Used float64 `json:"du"`
|
||||
}
|
||||
|
||||
type SystemAlertData struct {
|
||||
systemRecord *core.Record
|
||||
alertData CachedAlertData
|
||||
@@ -111,6 +117,9 @@ func (am *AlertManager) bindEvents() {
|
||||
am.hub.OnRecordAfterUpdateSuccess("alerts").BindFunc(updateHistoryOnAlertUpdate)
|
||||
am.hub.OnRecordAfterDeleteSuccess("alerts").BindFunc(resolveHistoryOnAlertDelete)
|
||||
am.hub.OnRecordAfterUpdateSuccess("smart_devices").BindFunc(am.handleSmartDeviceAlert)
|
||||
am.hub.OnRecordAfterCreateSuccess("zfs_pools").BindFunc(am.handleZfsPoolCreateAlert)
|
||||
am.hub.OnRecordAfterUpdateSuccess("zfs_pools").BindFunc(am.handleZfsPoolAlert)
|
||||
am.hub.OnRecordAfterDeleteSuccess("zfs_pools").BindFunc(resolveZfsPoolHistoryOnDelete)
|
||||
|
||||
am.hub.OnServe().BindFunc(func(e *core.ServeEvent) error {
|
||||
// Populate all alerts into cache on startup
|
||||
|
||||
@@ -44,6 +44,14 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
maxUsedPct = usedPct
|
||||
}
|
||||
}
|
||||
for _, pool := range data.Stats.ZfsPools {
|
||||
if pool != nil && pool.Total > 0 {
|
||||
usedPct := pool.Used / pool.Total * 100
|
||||
if usedPct > maxUsedPct {
|
||||
maxUsedPct = usedPct
|
||||
}
|
||||
}
|
||||
}
|
||||
val = maxUsedPct
|
||||
case "Temperature":
|
||||
if data.Info.DashboardTemp < 1 {
|
||||
@@ -208,6 +216,16 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
alert.mapSums[key] += float32(fs.DiskUsed / fs.DiskTotal * 100)
|
||||
}
|
||||
}
|
||||
// add zfs pool usage from historical record
|
||||
for key, pool := range stats.ZfsPools {
|
||||
if pool.Total > 0 {
|
||||
zfsKey := zfsDiskAlertKey(key)
|
||||
if _, ok := alert.mapSums[zfsKey]; !ok {
|
||||
alert.mapSums[zfsKey] = 0.0
|
||||
}
|
||||
alert.mapSums[zfsKey] += float32(pool.Used / pool.Total * 100)
|
||||
}
|
||||
}
|
||||
case "Temperature":
|
||||
if alert.mapSums == nil {
|
||||
alert.mapSums = make(map[string]float32, len(stats.Temperatures))
|
||||
@@ -255,7 +273,7 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
sumPct := float32(value)
|
||||
if sumPct > maxPct {
|
||||
maxPct = sumPct
|
||||
alert.descriptor = fmt.Sprintf("Usage of %s", key)
|
||||
alert.descriptor = diskAlertDescriptor(key)
|
||||
}
|
||||
}
|
||||
alert.val = float64(maxPct / float32(alert.count))
|
||||
@@ -301,6 +319,17 @@ func (am *AlertManager) HandleSystemAlerts(systemRecord *core.Record, data *syst
|
||||
return nil
|
||||
}
|
||||
|
||||
func zfsDiskAlertKey(poolName string) string {
|
||||
return "zfs:" + poolName
|
||||
}
|
||||
|
||||
func diskAlertDescriptor(key string) string {
|
||||
if poolName, ok := strings.CutPrefix(key, "zfs:"); ok {
|
||||
return fmt.Sprintf("Usage of ZFS pool %s", poolName)
|
||||
}
|
||||
return fmt.Sprintf("Usage of %s", key)
|
||||
}
|
||||
|
||||
func hasRepresentativeBattery(legacy [2]uint8, batteries map[string]uint8) bool {
|
||||
return legacy != [2]uint8{} || len(batteries) > 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
// handleZfsPoolAlert sends alerts when a ZFS pool health state worsens and
|
||||
// resolves the alert history entry when the pool recovers. Like the SMART
|
||||
// hook, this is automatic and does not require user opt-in.
|
||||
func (am *AlertManager) handleZfsPoolAlert(e *core.RecordEvent) error {
|
||||
return am.handleZfsPoolHealthAlert(e, e.Record.Original().GetString("health"))
|
||||
}
|
||||
|
||||
func (am *AlertManager) handleZfsPoolCreateAlert(e *core.RecordEvent) error {
|
||||
return am.handleZfsPoolHealthAlert(e, "")
|
||||
}
|
||||
|
||||
func (am *AlertManager) handleZfsPoolHealthAlert(e *core.RecordEvent, oldHealth string) error {
|
||||
newHealth := e.Record.GetString("health")
|
||||
oldSeverity := zfsPoolSeverity(oldHealth)
|
||||
newSeverity := zfsPoolSeverity(newHealth)
|
||||
|
||||
systemID := e.Record.GetString("system")
|
||||
if systemID == "" {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
systemRecord, err := e.App.FindRecordById("systems", systemID)
|
||||
if err != nil {
|
||||
e.App.Logger().Error("Failed to find system for ZFS alert", "err", err, "systemID", systemID)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// Pool recovered to a healthy state: resolve any open history entries.
|
||||
if newSeverity == 1 && oldSeverity > 1 {
|
||||
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
if !shouldSendZfsPoolAlert(oldSeverity, newSeverity) {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
systemName := systemRecord.GetString("name")
|
||||
poolName := e.Record.GetString("name")
|
||||
|
||||
title := fmt.Sprintf("ZFS pool %s on %s: %s", newHealth, systemName, poolName)
|
||||
message := fmt.Sprintf("ZFS pool %s (%s) was first observed as %s", poolName, systemName, newHealth)
|
||||
if oldSeverity > 0 {
|
||||
message = fmt.Sprintf("ZFS pool %s (%s) health changed from %s to %s", poolName, systemName, oldHealth, newHealth)
|
||||
}
|
||||
|
||||
userIDs := systemRecord.GetStringSlice("users")
|
||||
if len(userIDs) == 0 {
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
for _, userID := range userIDs {
|
||||
if err := am.SendAlert(AlertMessageData{
|
||||
UserID: userID,
|
||||
SystemID: systemID,
|
||||
Title: title,
|
||||
Message: message,
|
||||
Link: am.hub.MakeLink("system", systemID),
|
||||
LinkText: "View " + systemName,
|
||||
}); err != nil {
|
||||
e.App.Logger().Error("Failed to send ZFS alert", "err", err, "userID", userID)
|
||||
}
|
||||
_ = createZfsPoolHistoryRecord(e.App, userID, systemID, e.Record.Id, poolName)
|
||||
}
|
||||
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// resolveZfsPoolHistoryOnDelete resolves open alert history entries when a
|
||||
// pool record is deleted (manually or because the pool disappeared), so the
|
||||
// UI does not keep showing an ongoing alert for a pool that no longer exists.
|
||||
func resolveZfsPoolHistoryOnDelete(e *core.RecordEvent) error {
|
||||
resolveAllAlertHistoryRecords(e.App, e.Record.Id)
|
||||
return e.Next()
|
||||
}
|
||||
|
||||
// shouldSendZfsPoolAlert reports whether a health transition warrants an alert.
|
||||
// First observations of unhealthy pools and worsening transitions are reported.
|
||||
func shouldSendZfsPoolAlert(oldSeverity, newSeverity int) bool {
|
||||
return newSeverity > 1 && (oldSeverity == 0 || newSeverity > oldSeverity)
|
||||
}
|
||||
|
||||
// zfsPoolSeverity ranks pool health states: healthy (1), degraded (2),
|
||||
// failed/unavailable (3), unknown (0).
|
||||
func zfsPoolSeverity(health string) int {
|
||||
switch health {
|
||||
case "ONLINE":
|
||||
return 1
|
||||
case "DEGRADED":
|
||||
return 2
|
||||
case "FAULTED", "OFFLINE", "UNAVAIL", "REMOVED", "SUSPENDED":
|
||||
return 3
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// createZfsPoolHistoryRecord logs a pool health alert in the alerts history so
|
||||
// it is visible in the UI without creating an editable alert configuration.
|
||||
func createZfsPoolHistoryRecord(app core.App, userID, systemID, alertID, poolName string) error {
|
||||
collection, err := app.FindCachedCollectionByNameOrId("alerts_history")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
record := core.NewRecord(collection)
|
||||
record.Set("user", userID)
|
||||
record.Set("system", systemID)
|
||||
record.Set("alert_id", alertID)
|
||||
record.Set("name", "ZFS Pool: "+poolName)
|
||||
return app.Save(record)
|
||||
}
|
||||
|
||||
// resolveAllAlertHistoryRecords resolves every open history entry for an alert
|
||||
// record id (one per system user).
|
||||
func resolveAllAlertHistoryRecords(app core.App, alertID string) {
|
||||
records, err := app.FindRecordsByFilter(
|
||||
"alerts_history",
|
||||
"alert_id={:alert_id} && resolved=null",
|
||||
"", 0, 0,
|
||||
dbx.Params{"alert_id": alertID},
|
||||
)
|
||||
if err != nil || len(records) == 0 {
|
||||
return
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, record := range records {
|
||||
record.Set("resolved", now)
|
||||
if err := app.Save(record); err != nil {
|
||||
app.Logger().Error("Failed to resolve ZFS alert history", "err", err, "recordId", record.Id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/henrygd/beszel/internal/entities/system"
|
||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/tools/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestDiskAlertZfsPoolMultiMinute verifies that ZFS pool usage participates in
|
||||
// the Disk threshold alert using historical per-minute values, mirroring the
|
||||
// extra-filesystem behavior.
|
||||
func TestDiskAlertZfsPoolMultiMinute(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
systems, err := beszelTests.CreateSystems(hub, 1, user.Id, "up")
|
||||
require.NoError(t, err)
|
||||
systemRecord := systems[0]
|
||||
|
||||
diskAlert, err := beszelTests.CreateRecord(hub, "alerts", map[string]any{
|
||||
"name": "Disk",
|
||||
"system": systemRecord.Id,
|
||||
"user": user.Id,
|
||||
"value": 80, // threshold: 80%
|
||||
"min": 2, // requires historical averaging
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
am := hub.GetAlertManager()
|
||||
now := time.Now().UTC()
|
||||
|
||||
poolHigh := map[string]*system.ZfsPool{
|
||||
"tank": {Total: 1000, Used: 920}, // 92% - above threshold
|
||||
}
|
||||
|
||||
recordTimes := []time.Duration{
|
||||
-180 * time.Second,
|
||||
-90 * time.Second,
|
||||
-60 * time.Second,
|
||||
-30 * time.Second,
|
||||
}
|
||||
|
||||
for _, offset := range recordTimes {
|
||||
stats := system.Stats{
|
||||
DiskPct: 30, // root disk at 30% - below threshold
|
||||
ZfsPools: poolHigh,
|
||||
}
|
||||
statsJSON, _ := json.Marshal(stats)
|
||||
|
||||
recordTime := now.Add(offset)
|
||||
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||
"system": systemRecord.Id,
|
||||
"type": "1m",
|
||||
"stats": string(statsJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||
err = hub.SaveNoValidate(record)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
combinedDataHigh := &system.CombinedData{
|
||||
Stats: system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolHigh,
|
||||
},
|
||||
Info: system.Info{
|
||||
DiskPct: 30,
|
||||
},
|
||||
}
|
||||
|
||||
systemRecord.Set("updated", now)
|
||||
err = hub.SaveNoValidate(systemRecord)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = am.HandleSystemAlerts(systemRecord, combinedDataHigh)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||
require.NoError(t, err)
|
||||
assert.True(t, diskAlert.GetBool("triggered"),
|
||||
"Alert should be triggered when ZFS pool average (92%%) exceeds threshold (80%%)")
|
||||
|
||||
// --- Resolution: pool drops to 50%, alert should resolve ---
|
||||
|
||||
poolLow := map[string]*system.ZfsPool{
|
||||
"tank": {Total: 1000, Used: 500}, // 50% - below threshold
|
||||
}
|
||||
|
||||
newNow := now.Add(2 * time.Minute)
|
||||
for _, offset := range recordTimes {
|
||||
stats := system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolLow,
|
||||
}
|
||||
statsJSON, _ := json.Marshal(stats)
|
||||
|
||||
recordTime := newNow.Add(offset)
|
||||
record, err := beszelTests.CreateRecord(hub, "system_stats", map[string]any{
|
||||
"system": systemRecord.Id,
|
||||
"type": "1m",
|
||||
"stats": string(statsJSON),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
record.SetRaw("created", recordTime.Format(types.DefaultDateLayout))
|
||||
err = hub.SaveNoValidate(record)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
combinedDataLow := &system.CombinedData{
|
||||
Stats: system.Stats{
|
||||
DiskPct: 30,
|
||||
ZfsPools: poolLow,
|
||||
},
|
||||
Info: system.Info{
|
||||
DiskPct: 30,
|
||||
},
|
||||
}
|
||||
|
||||
systemRecord.Set("updated", newNow)
|
||||
err = hub.SaveNoValidate(systemRecord)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = am.HandleSystemAlerts(systemRecord, combinedDataLow)
|
||||
require.NoError(t, err)
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
diskAlert, err = hub.FindFirstRecordByFilter("alerts", "id={:id}", dbx.Params{"id": diskAlert.Id})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, diskAlert.GetBool("triggered"),
|
||||
"Alert should be resolved when ZFS pool average (50%%) drops below threshold (80%%)")
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestZfsDiskAlertKeyIsNamespaced(t *testing.T) {
|
||||
assert.Equal(t, "zfs:tank", zfsDiskAlertKey("tank"))
|
||||
assert.Equal(t, "Usage of ZFS pool tank", diskAlertDescriptor(zfsDiskAlertKey("tank")))
|
||||
assert.Equal(t, "Usage of tank", diskAlertDescriptor("tank"))
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
//go:build testing
|
||||
|
||||
package alerts_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
beszelTests "github.com/henrygd/beszel/internal/tests"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestZfsPoolAlertOnlineToDegraded(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Re-fetch so PocketBase tracks original values
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "DEGRADED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should have 1 email sent after pool became DEGRADED")
|
||||
lastMessage := hub.TestMailer.LastMessage()
|
||||
assert.Contains(t, lastMessage.Subject, "ZFS pool DEGRADED on test-system")
|
||||
assert.Contains(t, lastMessage.Subject, "tank")
|
||||
assert.Contains(t, lastMessage.Text, "ONLINE to DEGRADED")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertDegradedToFaulted(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "rpool",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "should alert on initial DEGRADED state and later FAULTED transition")
|
||||
lastMessage := hub.TestMailer.LastMessage()
|
||||
assert.Contains(t, lastMessage.Subject, "ZFS pool FAULTED on test-system")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertNoAlertOnRecovery(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Trigger a worsening alert first
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "expected alerts for initial DEGRADED state and DEGRADED -> FAULTED")
|
||||
|
||||
// Recovery back to ONLINE must not send a new alert
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "ONLINE")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 2, hub.TestMailer.TotalSend(), "recovery should not send a new alert")
|
||||
|
||||
// And the open history entry should have been resolved
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
requireHistoryResolved(t, history)
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertUnknownHealthDoesNotResolve(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
require.NoError(t, err)
|
||||
pool.Set("health", "")
|
||||
require.NoError(t, hub.Save(pool))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, history, 1, "unknown health must not resolve an active alert")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertUnknownToFaulted(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "should alert when a previously unknown pool becomes FAULTED")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertOnInitialUnhealthyState(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "DEGRADED",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
require.EqualValues(t, 1, hub.TestMailer.TotalSend())
|
||||
assert.Contains(t, hub.TestMailer.LastMessage().Text, "first observed as DEGRADED")
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, hub.Save(pool))
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 1, hub.TestMailer.TotalSend(), "unchanged unhealthy health must not duplicate alerts")
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertWritesHistory(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1, "expected one history entry per user")
|
||||
assert.Equal(t, "ZFS Pool: tank", history[0].GetString("name"))
|
||||
assert.Equal(t, system.Id, history[0].GetString("system"))
|
||||
}
|
||||
|
||||
func TestZfsPoolAlertResolvedOnRecordDelete(t *testing.T) {
|
||||
hub, user := beszelTests.GetHubWithUser(t)
|
||||
defer hub.Cleanup()
|
||||
|
||||
system, err := beszelTests.CreateRecord(hub, "systems", map[string]any{
|
||||
"name": "test-system",
|
||||
"users": []string{user.Id},
|
||||
"host": "127.0.0.1",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
pool, err := beszelTests.CreateRecord(hub, "zfs_pools", map[string]any{
|
||||
"system": system.Id,
|
||||
"name": "tank",
|
||||
"health": "ONLINE",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Trigger an alert so an open history entry exists.
|
||||
pool, err = hub.FindRecordById("zfs_pools", pool.Id)
|
||||
assert.NoError(t, err)
|
||||
pool.Set("health", "FAULTED")
|
||||
err = hub.Save(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err := hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id} && resolved=null", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1, "expected one open history entry")
|
||||
|
||||
// Deleting the pool record must resolve the open entry.
|
||||
err = hub.Delete(pool)
|
||||
assert.NoError(t, err)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
history, err = hub.FindRecordsByFilter("alerts_history", "alert_id={:alert_id}", "", 0, 0, map[string]any{"alert_id": pool.Id})
|
||||
assert.NoError(t, err)
|
||||
require.Len(t, history, 1)
|
||||
requireHistoryResolved(t, history)
|
||||
}
|
||||
|
||||
func requireHistoryResolved(t *testing.T, history []*core.Record) {
|
||||
t.Helper()
|
||||
for _, record := range history {
|
||||
assert.False(t, record.GetDateTime("resolved").Time().IsZero(), "expected history entry to be resolved")
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,8 @@ const (
|
||||
GetSmartData
|
||||
// Request detailed systemd service info from agent
|
||||
GetSystemdInfo
|
||||
// Request ZFS detail data from agent
|
||||
GetZfsData
|
||||
// Add new actions here...
|
||||
)
|
||||
|
||||
@@ -64,6 +66,10 @@ type DataRequestOptions struct {
|
||||
IncludeDetails bool `cbor:"1,keyasint"`
|
||||
}
|
||||
|
||||
type ZfsDataRequest struct {
|
||||
Force bool `cbor:"0,keyasint,omitempty"`
|
||||
}
|
||||
|
||||
type ContainerLogsRequest struct {
|
||||
ContainerID string `cbor:"0,keyasint"`
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ COPY --from=builder /agent /agent
|
||||
# AMD GPU name lookup (used by agent on Linux when /usr/share/libdrm/amdgpu.ids is read)
|
||||
COPY --from=builder /app/agent/test-data/amdgpu.ids /usr/share/libdrm/amdgpu.ids
|
||||
|
||||
RUN apk add --no-cache smartmontools
|
||||
RUN apk add --no-cache smartmontools zfs
|
||||
|
||||
# Ensure data persistence across container recreations
|
||||
VOLUME ["/var/lib/beszel-agent"]
|
||||
|
||||
@@ -33,8 +33,6 @@ type Stats struct {
|
||||
MaxNetworkSent float64 `json:"nsm,omitempty" cbor:"-"`
|
||||
MaxNetworkRecv float64 `json:"nrm,omitempty" cbor:"-"`
|
||||
Temperatures map[string]float64 `json:"t,omitempty" cbor:"20,keyasint,omitempty"`
|
||||
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
|
||||
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
|
||||
ExtraFs map[string]*FsStats `json:"efs,omitempty" cbor:"21,keyasint,omitempty"`
|
||||
GPUData map[string]GPUData `json:"g,omitempty" cbor:"22,keyasint,omitempty"`
|
||||
// LoadAvg1 float64 `json:"l1,omitempty" cbor:"23,keyasint,omitempty"`
|
||||
@@ -52,7 +50,20 @@ type Stats struct {
|
||||
CpuCoresUsage Uint8Slice `json:"cpus,omitempty" cbor:"34,keyasint,omitempty"` // per-core busy usage [CPU0..]
|
||||
DiskIoStats [6]float64 `json:"dios,omitzero" cbor:"35,keyasint,omitzero"` // [read time %, write time %, io utilization %, r_await ms, w_await ms, weighted io %]
|
||||
MaxDiskIoStats [6]float64 `json:"diosm,omitzero" cbor:"-"` // max values for DiskIoStats
|
||||
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
|
||||
Fans map[string]uint16 `json:"f,omitempty" cbor:"36,keyasint,omitempty"`
|
||||
Batteries map[string]uint8 `json:"bats,omitempty" cbor:"37,keyasint,omitempty"`
|
||||
ZfsPools map[string]*ZfsPool `json:"z,omitempty" cbor:"39,keyasint,omitempty"` // ZFS pool metrics, keyed by pool name
|
||||
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
|
||||
|
||||
}
|
||||
|
||||
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
|
||||
type ZfsPool struct {
|
||||
Total float64 `json:"d" cbor:"0,keyasint"` // total capacity in GiB
|
||||
Used float64 `json:"du" cbor:"1,keyasint"` // allocated in GiB
|
||||
ReadBytes uint64 `json:"rb,omitzero" cbor:"2,keyasint,omitzero"` // read throughput in bytes/s
|
||||
WriteBytes uint64 `json:"wb,omitzero" cbor:"3,keyasint,omitzero"` // write throughput in bytes/s
|
||||
Health string `json:"h,omitempty" cbor:"4,keyasint,omitempty"` // ONLINE, DEGRADED, FAULTED, ...
|
||||
}
|
||||
|
||||
// Uint8Slice wraps []uint8 to customize JSON encoding while keeping CBOR efficient.
|
||||
@@ -183,6 +194,7 @@ type Details struct {
|
||||
Podman bool `cbor:"8,keyasint,omitempty"`
|
||||
MemoryTotal uint64 `cbor:"9,keyasint"`
|
||||
SmartInterval time.Duration `cbor:"10,keyasint,omitempty"`
|
||||
ZfsInterval time.Duration `cbor:"11,keyasint,omitempty"` // interval for ZFS detail refresh
|
||||
}
|
||||
|
||||
// Final data structure to return to the hub
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// Package zfs defines the ZFS detail data exchanged between agent and hub.
|
||||
package zfs
|
||||
|
||||
// ZfsData is the detail payload returned by the agent for the GetZfsData action.
|
||||
type ZfsData struct {
|
||||
Pools []*PoolDetail `json:"pools,omitempty"`
|
||||
Complete bool `json:"complete,omitempty"`
|
||||
}
|
||||
|
||||
// PoolDetail holds the verbose state of a single pool: capacity, health,
|
||||
// scrub, vdev, and dataset information.
|
||||
type PoolDetail struct {
|
||||
Name string `json:"name"`
|
||||
Health string `json:"health,omitempty"`
|
||||
Size uint64 `json:"size,omitempty"` // bytes
|
||||
Alloc uint64 `json:"alloc,omitempty"` // bytes
|
||||
Free uint64 `json:"free,omitempty"` // bytes
|
||||
Scrub *Scrub `json:"scrub,omitempty"`
|
||||
Vdevs []*Vdev `json:"vdevs,omitempty"`
|
||||
Datasets []*Dataset `json:"datasets,omitempty"`
|
||||
}
|
||||
|
||||
// Scrub holds the scrub (or resilver) status of a pool.
|
||||
type Scrub struct {
|
||||
State string `json:"state,omitempty"` // NONE, SCANNING, FINISHED, CANCELED
|
||||
Progress string `json:"progress,omitempty"`
|
||||
Errors uint64 `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// Vdev is a single vdev (mirror, raidz, or leaf disk) with error counters.
|
||||
type Vdev struct {
|
||||
Name string `json:"name"`
|
||||
State string `json:"state,omitempty"`
|
||||
ReadErrs uint64 `json:"readErrs,omitempty"`
|
||||
WriteErrs uint64 `json:"writeErrs,omitempty"`
|
||||
ChecksumErrs uint64 `json:"checksumErrs,omitempty"`
|
||||
}
|
||||
|
||||
// Dataset is a single ZFS dataset with usage information.
|
||||
type Dataset struct {
|
||||
Name string `json:"name"`
|
||||
Used uint64 `json:"used,omitempty"`
|
||||
Avail uint64 `json:"avail,omitempty"`
|
||||
Mountpoint string `json:"mount,omitempty"`
|
||||
}
|
||||
@@ -125,6 +125,8 @@ func (h *Hub) registerApiRoutes(se *core.ServeEvent) error {
|
||||
apiAuth.DELETE("/user-alerts", alerts.DeleteUserAlerts)
|
||||
// refresh SMART devices for a system
|
||||
apiAuth.POST("/smart/refresh", h.refreshSmartData).BindFunc(excludeReadOnlyRole)
|
||||
// refresh ZFS pool details for a system
|
||||
apiAuth.POST("/zfs/refresh", h.refreshZfsData).BindFunc(excludeReadOnlyRole)
|
||||
// get systemd service details
|
||||
apiAuth.GET("/systemd/info", h.getSystemdInfo)
|
||||
// /containers routes
|
||||
@@ -389,3 +391,23 @@ func (h *Hub) refreshSmartData(e *core.RequestEvent) error {
|
||||
|
||||
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
// refreshZfsData handles POST /api/beszel/zfs/refresh requests
|
||||
// Fetches fresh ZFS detail data from the agent and updates the collection
|
||||
func (h *Hub) refreshZfsData(e *core.RequestEvent) error {
|
||||
systemID := e.Request.URL.Query().Get("system")
|
||||
if systemID == "" {
|
||||
return e.BadRequestError("Invalid system parameter", nil)
|
||||
}
|
||||
|
||||
system, err := h.sm.GetSystem(systemID)
|
||||
if err != nil || !system.HasUser(e.App, e.Auth) {
|
||||
return e.NotFoundError("", nil)
|
||||
}
|
||||
|
||||
if err := system.FetchAndSaveZfsPools(true); err != nil {
|
||||
return e.InternalServerError("", err)
|
||||
}
|
||||
|
||||
return e.JSON(http.StatusOK, map[string]string{"status": "ok"})
|
||||
}
|
||||
|
||||
@@ -91,6 +91,12 @@ func setCollectionAuthSettings(app core.App) error {
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := applyCollectionRules(app, []string{"zfs_pools"}, collectionRules{
|
||||
list: &systemScopedReadRule,
|
||||
view: &systemScopedReadRule,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := applyCollectionRules(app, []string{"fingerprints"}, collectionRules{
|
||||
list: &systemScopedReadRule,
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/henrygd/beszel/internal/entities/smart"
|
||||
"github.com/henrygd/beszel/internal/entities/system"
|
||||
"github.com/henrygd/beszel/internal/entities/systemd"
|
||||
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||
|
||||
"github.com/henrygd/beszel"
|
||||
|
||||
@@ -49,6 +50,8 @@ type System struct {
|
||||
detailsFetched atomic.Bool // True if static system details have been fetched and saved
|
||||
smartFetching atomic.Bool // True if SMART devices are currently being fetched
|
||||
smartInterval time.Duration // Interval for periodic SMART data updates
|
||||
zfsFetching atomic.Bool // True if ZFS pools are currently being fetched
|
||||
zfsInterval time.Duration // Interval for periodic ZFS detail data updates
|
||||
}
|
||||
|
||||
func (sm *SystemManager) NewSystem(systemId string) *System {
|
||||
@@ -154,6 +157,12 @@ func (sys *System) update() error {
|
||||
// to prevent premature expiration leading to new fetch if interval is different.
|
||||
sys.manager.smartFetchMap.UpdateExpiration(sys.Id, sys.smartInterval+time.Minute)
|
||||
}
|
||||
// update zfs interval if it's set on the agent side
|
||||
if data.Details.ZfsInterval > 0 {
|
||||
sys.zfsInterval = data.Details.ZfsInterval
|
||||
sys.manager.hub.Logger().Info("ZFS interval updated from agent details", "system", sys.Id, "interval", sys.zfsInterval.String())
|
||||
sys.manager.zfsFetchMap.UpdateExpiration(sys.Id, sys.zfsInterval+time.Minute)
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and save SMART devices when system first comes online or at intervals
|
||||
@@ -170,6 +179,20 @@ func (sys *System) update() error {
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch and save ZFS pool details when system first comes online or at intervals
|
||||
if backgroundZfsFetchEnabled() && sys.detailsFetched.Load() && sys.supportsZfsData() {
|
||||
if sys.zfsInterval <= 0 {
|
||||
sys.zfsInterval = time.Hour
|
||||
}
|
||||
if sys.shouldFetchZfs() && sys.zfsFetching.CompareAndSwap(false, true) {
|
||||
sys.manager.hub.Logger().Info("ZFS fetch", "system", sys.Id, "interval", sys.zfsInterval.String())
|
||||
go func() {
|
||||
defer sys.zfsFetching.Store(false)
|
||||
_ = sys.FetchAndSaveZfsPools(false)
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -241,6 +264,10 @@ func (sys *System) createRecords(data *system.CombinedData) (*core.Record, error
|
||||
}
|
||||
}
|
||||
|
||||
if err := sys.syncZfsPoolHealth(txApp, data.Stats.ZfsPools); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// update system record (do this last because it triggers alerts and we need above records to be inserted first)
|
||||
systemRecord.Set("status", up)
|
||||
systemRecord.Set("info", data.Info)
|
||||
@@ -558,6 +585,15 @@ func (sys *System) FetchSmartDataFromAgent() (smart.SmartDataResponse, error) {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// FetchZfsDataFromAgent fetches ZFS detail data from the agent.
|
||||
func (sys *System) FetchZfsDataFromAgent(force bool) (*zfs.ZfsData, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
var result zfs.ZfsData
|
||||
err := sys.request(ctx, common.GetZfsData, common.ZfsDataRequest{Force: force}, &result)
|
||||
return &result, err
|
||||
}
|
||||
|
||||
func makeStableHashId(strings ...string) string {
|
||||
hash := fnv.New32a()
|
||||
for _, str := range strings {
|
||||
@@ -728,6 +764,7 @@ func (s *System) createSSHClient() error {
|
||||
}
|
||||
s.agentVersion, _ = extractAgentVersion(string(client.Conn.ServerVersion()))
|
||||
s.manager.resetFailedSmartFetchState(s.Id)
|
||||
s.manager.resetFailedZfsFetchState(s.Id)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,7 @@ type SystemManager struct {
|
||||
systems *store.Store[string, *System] // Thread-safe store of active systems
|
||||
sshConfig *ssh.ClientConfig // SSH client configuration for system connections
|
||||
smartFetchMap *expirymap.ExpiryMap[smartFetchState] // Stores last SMART fetch time/result; TTL is only for cleanup
|
||||
zfsFetchMap *expirymap.ExpiryMap[zfsFetchState] // Stores last ZFS fetch time/result; TTL is only for cleanup
|
||||
ctx context.Context // Cancelled when the app terminates
|
||||
cancel context.CancelFunc // Cancels ctx and all child system contexts
|
||||
}
|
||||
@@ -67,6 +68,7 @@ func NewSystemManager(hub hubLike) *SystemManager {
|
||||
systems: store.New(map[string]*System{}),
|
||||
hub: hub,
|
||||
smartFetchMap: expirymap.New[smartFetchState](time.Hour),
|
||||
zfsFetchMap: expirymap.New[zfsFetchState](time.Hour),
|
||||
}
|
||||
sm.ctx, sm.cancel = context.WithCancel(context.Background())
|
||||
return sm
|
||||
@@ -343,6 +345,15 @@ func (sm *SystemManager) resetFailedSmartFetchState(systemID string) {
|
||||
}
|
||||
}
|
||||
|
||||
// resetFailedZfsFetchState clears only failed ZFS cooldown entries so a fresh
|
||||
// agent reconnect retries ZFS discovery immediately after configuration changes.
|
||||
func (sm *SystemManager) resetFailedZfsFetchState(systemID string) {
|
||||
state, ok := sm.zfsFetchMap.GetOk(systemID)
|
||||
if ok && !state.Successful {
|
||||
sm.zfsFetchMap.Remove(systemID)
|
||||
}
|
||||
}
|
||||
|
||||
// createSSHClientConfig initializes the SSH client configuration for connecting to an agent's server
|
||||
func (sm *SystemManager) createSSHClientConfig() error {
|
||||
privateKey, err := sm.hub.GetSSHKey("")
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package systems
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/henrygd/beszel"
|
||||
"github.com/henrygd/beszel/internal/entities/system"
|
||||
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||
"github.com/pocketbase/dbx"
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
)
|
||||
|
||||
var errIncompleteZfsData = errors.New("incomplete ZFS pool inventory")
|
||||
|
||||
type zfsFetchState struct {
|
||||
LastAttempt int64
|
||||
Successful bool
|
||||
}
|
||||
|
||||
func (sys *System) supportsZfsData() bool {
|
||||
return sys.agentVersion.GTE(beszel.MinVersionZfsData)
|
||||
}
|
||||
|
||||
// FetchAndSaveZfsPools fetches ZFS detail data from the agent and saves it to
|
||||
// the database. force bypasses the agent's detail cache for manual refreshes.
|
||||
func (sys *System) FetchAndSaveZfsPools(force bool) error {
|
||||
zfsData, err := sys.FetchZfsDataFromAgent(force)
|
||||
if err != nil {
|
||||
sys.recordZfsFetchResult(err, 0)
|
||||
return err
|
||||
}
|
||||
if zfsData == nil || !zfsData.Complete {
|
||||
err = errIncompleteZfsData
|
||||
sys.recordZfsFetchResult(err, 0)
|
||||
return err
|
||||
}
|
||||
err = sys.saveZfsPools(zfsData)
|
||||
sys.recordZfsFetchResult(err, len(zfsData.Pools))
|
||||
return err
|
||||
}
|
||||
|
||||
// recordZfsFetchResult stores a cooldown entry for the ZFS interval and marks
|
||||
// whether the last fetch produced any pools, so failed setup can retry on reconnect.
|
||||
func (sys *System) recordZfsFetchResult(err error, poolCount int) {
|
||||
if sys.manager == nil {
|
||||
return
|
||||
}
|
||||
interval := sys.zfsFetchInterval()
|
||||
success := err == nil && poolCount > 0
|
||||
if sys.manager.hub != nil {
|
||||
sys.manager.hub.Logger().Info("ZFS fetch result", "system", sys.Id, "success", success, "pools", poolCount, "interval", interval.String(), "err", err)
|
||||
}
|
||||
sys.manager.zfsFetchMap.Set(sys.Id, zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: success}, interval+time.Minute)
|
||||
}
|
||||
|
||||
// shouldFetchZfs returns true when there is no active ZFS cooldown entry for this system.
|
||||
func (sys *System) shouldFetchZfs() bool {
|
||||
if sys.manager == nil {
|
||||
return true
|
||||
}
|
||||
state, ok := sys.manager.zfsFetchMap.GetOk(sys.Id)
|
||||
if !ok {
|
||||
return true
|
||||
}
|
||||
return !time.UnixMilli(state.LastAttempt).Add(sys.zfsFetchInterval()).After(time.Now())
|
||||
}
|
||||
|
||||
// zfsFetchInterval returns the agent-provided ZFS interval or the default when unset.
|
||||
func (sys *System) zfsFetchInterval() time.Duration {
|
||||
if sys.zfsInterval > 0 {
|
||||
return sys.zfsInterval
|
||||
}
|
||||
return time.Hour
|
||||
}
|
||||
|
||||
// saveZfsPools saves ZFS pool detail data to the zfs_pools collection and
|
||||
// removes records for pools no longer reported by a complete agent inventory.
|
||||
func (sys *System) saveZfsPools(zfsData *zfs.ZfsData) error {
|
||||
if zfsData == nil || !zfsData.Complete {
|
||||
return errIncompleteZfsData
|
||||
}
|
||||
|
||||
hub := sys.manager.hub
|
||||
collection, err := hub.FindCachedCollectionByNameOrId("zfs_pools")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return hub.RunInTransaction(func(txApp core.App) error {
|
||||
alive := make(map[string]bool, len(zfsData.Pools))
|
||||
for _, pool := range zfsData.Pools {
|
||||
if pool == nil {
|
||||
continue
|
||||
}
|
||||
alive[pool.Name] = true
|
||||
if err := sys.upsertZfsPoolRecord(txApp, collection, pool); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
existing, err := txApp.FindRecordsByFilter(
|
||||
collection,
|
||||
"system={:system}",
|
||||
"", 0, 0,
|
||||
dbx.Params{"system": sys.Id},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, record := range existing {
|
||||
if !alive[record.GetString("name")] {
|
||||
if err := txApp.Delete(record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (sys *System) upsertZfsPoolRecord(app core.App, collection *core.Collection, pool *zfs.PoolDetail) error {
|
||||
recordID := makeStableHashId(sys.Id, pool.Name)
|
||||
|
||||
record, err := app.FindRecordById(collection, recordID)
|
||||
if err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
record = core.NewRecord(collection)
|
||||
record.Set("id", recordID)
|
||||
}
|
||||
|
||||
record.Set("system", sys.Id)
|
||||
record.Set("name", pool.Name)
|
||||
record.Set("health", pool.Health)
|
||||
record.Set("size", pool.Size)
|
||||
record.Set("alloc", pool.Alloc)
|
||||
record.Set("free", pool.Free)
|
||||
record.Set("scrub", pool.Scrub)
|
||||
record.Set("vdevs", pool.Vdevs)
|
||||
record.Set("datasets", pool.Datasets)
|
||||
record.Set("details_updated", time.Now().UTC())
|
||||
|
||||
return app.SaveNoValidate(record)
|
||||
}
|
||||
|
||||
// syncZfsPoolHealth persists newly discovered pools and health transitions from
|
||||
// regular system samples. Detailed fields remain owned by the hourly refresh.
|
||||
func (sys *System) syncZfsPoolHealth(app core.App, pools map[string]*system.ZfsPool) error {
|
||||
if len(pools) == 0 {
|
||||
return nil
|
||||
}
|
||||
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
const gib = 1024 * 1024 * 1024
|
||||
for name, pool := range pools {
|
||||
if pool == nil {
|
||||
continue
|
||||
}
|
||||
recordID := makeStableHashId(sys.Id, name)
|
||||
record, err := app.FindRecordById(collection, recordID)
|
||||
if err != nil {
|
||||
if !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
record = core.NewRecord(collection)
|
||||
record.Set("id", recordID)
|
||||
record.Set("system", sys.Id)
|
||||
record.Set("name", name)
|
||||
record.Set("health", pool.Health)
|
||||
record.Set("size", uint64(pool.Total*gib))
|
||||
record.Set("alloc", uint64(pool.Used*gib))
|
||||
record.Set("free", uint64(max(pool.Total-pool.Used, 0)*gib))
|
||||
if err := app.SaveNoValidate(record); err != nil {
|
||||
return fmt.Errorf("creating ZFS pool summary %q: %w", name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if record.GetString("health") == pool.Health {
|
||||
continue
|
||||
}
|
||||
record.Set("health", pool.Health)
|
||||
if err := app.SaveNoValidate(record); err != nil {
|
||||
return fmt.Errorf("updating ZFS pool health %q: %w", name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
//go:build testing
|
||||
|
||||
package systems
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/blang/semver"
|
||||
"github.com/henrygd/beszel/internal/entities/system"
|
||||
"github.com/henrygd/beszel/internal/entities/zfs"
|
||||
"github.com/henrygd/beszel/internal/hub/expirymap"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSupportsZfsData(t *testing.T) {
|
||||
sys := &System{agentVersion: semver.MustParse("0.18.8")}
|
||||
assert.False(t, sys.supportsZfsData())
|
||||
|
||||
sys.agentVersion = semver.MustParse("0.18.9")
|
||||
assert.True(t, sys.supportsZfsData())
|
||||
}
|
||||
|
||||
func TestRecordZfsFetchResult(t *testing.T) {
|
||||
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||
|
||||
sys := &System{
|
||||
Id: "system-1",
|
||||
manager: sm,
|
||||
zfsInterval: time.Hour,
|
||||
}
|
||||
|
||||
// Successful fetch with pools
|
||||
sys.recordZfsFetchResult(nil, 2)
|
||||
state, ok := sm.zfsFetchMap.GetOk(sys.Id)
|
||||
assert.True(t, ok, "expected zfs fetch result to be stored")
|
||||
assert.True(t, state.Successful, "expected successful fetch state to be recorded")
|
||||
|
||||
// Failed fetch
|
||||
sys.recordZfsFetchResult(errors.New("failed"), 0)
|
||||
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
|
||||
assert.True(t, ok, "expected failed zfs fetch state to be stored")
|
||||
assert.False(t, state.Successful, "expected failed zfs fetch state to be marked unsuccessful")
|
||||
|
||||
// Successful fetch but no pools
|
||||
sys.recordZfsFetchResult(nil, 0)
|
||||
state, ok = sm.zfsFetchMap.GetOk(sys.Id)
|
||||
assert.True(t, ok, "expected fetch with zero pools to be stored")
|
||||
assert.False(t, state.Successful, "expected fetch with zero pools to be marked unsuccessful")
|
||||
}
|
||||
|
||||
func TestShouldFetchZfs(t *testing.T) {
|
||||
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||
|
||||
sys := &System{
|
||||
Id: "system-1",
|
||||
manager: sm,
|
||||
zfsInterval: time.Hour,
|
||||
}
|
||||
|
||||
assert.True(t, sys.shouldFetchZfs(), "expected initial zfs fetch to be allowed")
|
||||
|
||||
sys.recordZfsFetchResult(errors.New("failed"), 0)
|
||||
assert.False(t, sys.shouldFetchZfs(), "expected zfs fetch to be blocked while interval entry exists")
|
||||
|
||||
sm.zfsFetchMap.Remove(sys.Id)
|
||||
assert.True(t, sys.shouldFetchZfs(), "expected zfs fetch to be allowed after interval entry is cleared")
|
||||
}
|
||||
|
||||
func TestZfsFetchIntervalDefault(t *testing.T) {
|
||||
sys := &System{}
|
||||
assert.Equal(t, time.Hour, sys.zfsFetchInterval())
|
||||
|
||||
sys.zfsInterval = 5 * time.Minute
|
||||
assert.Equal(t, 5*time.Minute, sys.zfsFetchInterval())
|
||||
}
|
||||
|
||||
func TestResetFailedZfsFetchState(t *testing.T) {
|
||||
sm := &SystemManager{zfsFetchMap: expirymap.New[zfsFetchState](time.Hour)}
|
||||
t.Cleanup(sm.zfsFetchMap.StopCleaner)
|
||||
|
||||
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: false}, time.Hour)
|
||||
sm.resetFailedZfsFetchState("system-1")
|
||||
_, ok := sm.zfsFetchMap.GetOk("system-1")
|
||||
assert.False(t, ok, "expected failed zfs fetch state to be cleared on reconnect")
|
||||
|
||||
sm.zfsFetchMap.Set("system-1", zfsFetchState{LastAttempt: time.Now().UnixMilli(), Successful: true}, time.Hour)
|
||||
sm.resetFailedZfsFetchState("system-1")
|
||||
_, ok = sm.zfsFetchMap.GetOk("system-1")
|
||||
assert.True(t, ok, "expected successful zfs fetch state to be preserved")
|
||||
}
|
||||
|
||||
func TestSaveZfsPoolsCompleteEmptyPrunesFinalPool(t *testing.T) {
|
||||
sys, app := newTestSystemWithHub(t)
|
||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
|
||||
Complete: true,
|
||||
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
|
||||
}))
|
||||
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, records, 1)
|
||||
assert.False(t, records[0].GetDateTime("details_updated").Time().IsZero())
|
||||
|
||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{Complete: true}))
|
||||
records, err = app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, records)
|
||||
}
|
||||
|
||||
func TestSaveZfsPoolsIncompletePreservesRecords(t *testing.T) {
|
||||
sys, app := newTestSystemWithHub(t)
|
||||
require.NoError(t, sys.saveZfsPools(&zfs.ZfsData{
|
||||
Complete: true,
|
||||
Pools: []*zfs.PoolDetail{{Name: "tank", Health: "ONLINE"}},
|
||||
}))
|
||||
assert.ErrorIs(t, sys.saveZfsPools(&zfs.ZfsData{}), errIncompleteZfsData)
|
||||
records, err := app.FindRecordsByFilter("zfs_pools", "system={:system}", "", 0, 0, map[string]any{"system": sys.Id})
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, records, 1)
|
||||
}
|
||||
|
||||
func TestSyncZfsPoolHealthWritesOnlyTransitions(t *testing.T) {
|
||||
sys, app := newTestSystemWithHub(t)
|
||||
collection, err := app.FindCachedCollectionByNameOrId("zfs_pools")
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||
"tank": {Total: 100, Used: 25, Health: "ONLINE"},
|
||||
}))
|
||||
record, err := app.FindRecordById(collection, makeStableHashId(sys.Id, "tank"))
|
||||
require.NoError(t, err)
|
||||
firstUpdated := record.GetDateTime("updated")
|
||||
assert.Equal(t, "ONLINE", record.GetString("health"))
|
||||
assert.EqualValues(t, 100*1024*1024*1024, record.GetInt("size"))
|
||||
|
||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||
"tank": {Total: 100, Used: 30, Health: "ONLINE"},
|
||||
}))
|
||||
record, err = app.FindRecordById(collection, record.Id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, firstUpdated, record.GetDateTime("updated"))
|
||||
|
||||
require.NoError(t, sys.syncZfsPoolHealth(app, map[string]*system.ZfsPool{
|
||||
"tank": {Total: 100, Used: 30, Health: "DEGRADED"},
|
||||
}))
|
||||
record, err = app.FindRecordById(collection, record.Id)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "DEGRADED", record.GetString("health"))
|
||||
}
|
||||
@@ -7,3 +7,6 @@ package systems
|
||||
// The hub integration tests create/replace systems and clean up the test apps quickly.
|
||||
// Background SMART fetching can outlive teardown and crash in PocketBase internals (nil DB).
|
||||
func backgroundSmartFetchEnabled() bool { return true }
|
||||
|
||||
// Background ZFS fetching follows the same policy as SMART fetching.
|
||||
func backgroundZfsFetchEnabled() bool { return true }
|
||||
|
||||
@@ -17,6 +17,9 @@ import (
|
||||
// the automatic background fetch during tests.
|
||||
func backgroundSmartFetchEnabled() bool { return false }
|
||||
|
||||
// Background ZFS fetching follows the same policy as SMART fetching.
|
||||
func backgroundZfsFetchEnabled() bool { return false }
|
||||
|
||||
// TESTING ONLY: GetSystemCount returns the number of systems in the store
|
||||
func (sm *SystemManager) GetSystemCount() int {
|
||||
return sm.systems.Length()
|
||||
@@ -115,6 +118,7 @@ func (sm *SystemManager) RemoveAllSystems() {
|
||||
sm.RemoveSystem(system.Id)
|
||||
}
|
||||
sm.smartFetchMap.StopCleaner()
|
||||
sm.zfsFetchMap.StopCleaner()
|
||||
}
|
||||
|
||||
// ResetContextForTesting replaces the manager context for a new synctest bubble.
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"github.com/pocketbase/pocketbase/core"
|
||||
m "github.com/pocketbase/pocketbase/migrations"
|
||||
)
|
||||
|
||||
// Creates the zfs_pools collection for per-system ZFS pool detail data
|
||||
// (pool health, capacity, scrub, vdevs, datasets). Upserts rather than
|
||||
// deletes missing collections, so it is safe on fresh and existing installs.
|
||||
func init() {
|
||||
m.Register(func(app core.App) error {
|
||||
// update collections
|
||||
jsonData := `[
|
||||
{
|
||||
"createRule": null,
|
||||
"deleteRule": null,
|
||||
"fields": [
|
||||
{
|
||||
"autogeneratePattern": "[a-z0-9]{15}",
|
||||
"hidden": false,
|
||||
"id": "text3208210256",
|
||||
"max": 15,
|
||||
"min": 15,
|
||||
"name": "id",
|
||||
"pattern": "^[a-z0-9]+$",
|
||||
"presentable": false,
|
||||
"primaryKey": true,
|
||||
"required": true,
|
||||
"system": true,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"cascadeDelete": true,
|
||||
"collectionId": "2hz5ncl8tizk5nx",
|
||||
"hidden": false,
|
||||
"id": "relation1204987316",
|
||||
"maxSelect": 1,
|
||||
"minSelect": 0,
|
||||
"name": "system",
|
||||
"presentable": false,
|
||||
"required": true,
|
||||
"system": false,
|
||||
"type": "relation"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"hidden": false,
|
||||
"id": "text7739291048",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "name",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"autogeneratePattern": "",
|
||||
"hidden": false,
|
||||
"id": "text5528164482",
|
||||
"max": 0,
|
||||
"min": 0,
|
||||
"name": "health",
|
||||
"pattern": "",
|
||||
"presentable": false,
|
||||
"primaryKey": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "number8862034195",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "size",
|
||||
"onlyInt": true,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "number4418907321",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "alloc",
|
||||
"onlyInt": true,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "number2904183765",
|
||||
"max": null,
|
||||
"min": null,
|
||||
"name": "free",
|
||||
"onlyInt": true,
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "number"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "json4466109723",
|
||||
"maxSize": 0,
|
||||
"name": "scrub",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "json"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "json9012873456",
|
||||
"maxSize": 0,
|
||||
"name": "vdevs",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "json"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "json7182045639",
|
||||
"maxSize": 0,
|
||||
"name": "datasets",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "json"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "date9274163058",
|
||||
"max": "",
|
||||
"min": "",
|
||||
"name": "details_updated",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "date"
|
||||
},
|
||||
{
|
||||
"hidden": false,
|
||||
"id": "autodate3332085495",
|
||||
"name": "updated",
|
||||
"onCreate": true,
|
||||
"onUpdate": true,
|
||||
"presentable": false,
|
||||
"system": false,
|
||||
"type": "autodate"
|
||||
}
|
||||
],
|
||||
"id": "pbc_8441057391",
|
||||
"indexes": [
|
||||
"CREATE INDEX ` + "`" + `idx_zfsPoolsSystem` + "`" + ` ON ` + "`" + `zfs_pools` + "`" + ` (` + "`" + `system` + "`" + `)"
|
||||
],
|
||||
"listRule": null,
|
||||
"name": "zfs_pools",
|
||||
"system": false,
|
||||
"type": "base",
|
||||
"updateRule": null,
|
||||
"viewRule": null
|
||||
}
|
||||
]`
|
||||
|
||||
err := app.ImportCollectionsByMarshaledJSON([]byte(jsonData), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}, func(app core.App) error {
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -127,6 +127,7 @@ func (rm *RecordManager) CreateLongerRecords() {
|
||||
"created": shorterRecordPeriod,
|
||||
},
|
||||
)).
|
||||
OrderBy("created").
|
||||
All(&recordIds)
|
||||
|
||||
// continue if not enough shorter records
|
||||
@@ -196,6 +197,7 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
||||
tempCount := float64(0)
|
||||
var fanSums map[string]uint64
|
||||
fanCount := uint64(0)
|
||||
zfsPoolCounts := make(map[string]uint64)
|
||||
|
||||
// Accumulate totals
|
||||
for i := range records {
|
||||
@@ -336,6 +338,31 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
||||
}
|
||||
}
|
||||
|
||||
// Accumulate ZFS pool stats. Counts are tracked per entry so a pool
|
||||
// missing from some samples is not averaged as zero.
|
||||
if stats.ZfsPools != nil {
|
||||
if sum.ZfsPools == nil {
|
||||
sum.ZfsPools = make(map[string]*system.ZfsPool, len(stats.ZfsPools))
|
||||
}
|
||||
for name, value := range stats.ZfsPools {
|
||||
if value == nil {
|
||||
continue
|
||||
}
|
||||
pool := sum.ZfsPools[name]
|
||||
if pool == nil {
|
||||
pool = &system.ZfsPool{}
|
||||
sum.ZfsPools[name] = pool
|
||||
}
|
||||
pool.Total += value.Total
|
||||
pool.Used += value.Used
|
||||
pool.ReadBytes += value.ReadBytes
|
||||
pool.WriteBytes += value.WriteBytes
|
||||
if value.Health != "" {
|
||||
pool.Health = value.Health
|
||||
}
|
||||
zfsPoolCounts[name]++
|
||||
}
|
||||
}
|
||||
// Accumulate GPU data
|
||||
if stats.GPUData != nil {
|
||||
if sum.GPUData == nil {
|
||||
@@ -446,6 +473,14 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
|
||||
}
|
||||
}
|
||||
|
||||
// Average ZFS pool stats.
|
||||
for name, pool := range sum.ZfsPools {
|
||||
entryCount := zfsPoolCounts[name]
|
||||
pool.Total = twoDecimals(pool.Total / float64(entryCount))
|
||||
pool.Used = twoDecimals(pool.Used / float64(entryCount))
|
||||
pool.ReadBytes /= entryCount
|
||||
pool.WriteBytes /= entryCount
|
||||
}
|
||||
// Average GPU data
|
||||
if sum.GPUData != nil {
|
||||
for id := range sum.GPUData {
|
||||
|
||||
@@ -669,6 +669,33 @@ func TestAverageSystemStatsSlice_MixedOptionalFields(t *testing.T) {
|
||||
assert.Equal(t, 20.0, result.GPUData["gpu0"].Usage)
|
||||
}
|
||||
|
||||
func TestAverageSystemStatsSlice_Zfs(t *testing.T) {
|
||||
input := []system.Stats{
|
||||
{
|
||||
ZfsPools: map[string]*system.ZfsPool{
|
||||
"tank": {Total: 100, Used: 40, ReadBytes: 100, WriteBytes: 200, Health: "ONLINE"},
|
||||
},
|
||||
},
|
||||
{},
|
||||
{
|
||||
ZfsPools: map[string]*system.ZfsPool{
|
||||
"tank": {Total: 120, Used: 60, ReadBytes: 300, WriteBytes: 400, Health: "DEGRADED"},
|
||||
"backup": {Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result := records.AverageSystemStatsSlice(input)
|
||||
|
||||
require.Len(t, result.ZfsPools, 2)
|
||||
assert.Equal(t, &system.ZfsPool{
|
||||
Total: 110, Used: 50, ReadBytes: 200, WriteBytes: 300, Health: "DEGRADED",
|
||||
}, result.ZfsPools["tank"])
|
||||
assert.Equal(t, &system.ZfsPool{
|
||||
Total: 50, Used: 10, ReadBytes: 25, WriteBytes: 50, Health: "ONLINE",
|
||||
}, result.ZfsPools["backup"])
|
||||
}
|
||||
|
||||
// Tests with 10 records matching the common real-world case (10 x 1m -> 1 x 10m).
|
||||
func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
|
||||
input := make([]system.Stats, 10)
|
||||
|
||||
@@ -8,10 +8,11 @@ import { useSystemData } from "./system/use-system-data"
|
||||
import { CpuChart, ContainerCpuChart } from "./system/charts/cpu-charts"
|
||||
import { MemoryChart, ContainerMemoryChart, SwapChart } from "./system/charts/memory-charts"
|
||||
import { RootDiskCharts, ExtraFsCharts } from "./system/charts/disk-charts"
|
||||
import { ZfsCharts } from "./system/charts/zfs-charts"
|
||||
import { BandwidthChart, ContainerNetworkChart } from "./system/charts/network-charts"
|
||||
import { TemperatureChart, FanChart, BatteryChart } from "./system/charts/sensor-charts"
|
||||
import { GpuPowerChart, GpuDetailCharts } from "./system/charts/gpu-charts"
|
||||
import { LazyContainersTable, LazySmartTable, LazySystemdTable } from "./system/lazy-tables"
|
||||
import { LazyContainersTable, LazySmartTable, LazySystemdTable, LazyZfsTable } from "./system/lazy-tables"
|
||||
import { LoadAverageChart } from "./system/charts/load-average-chart"
|
||||
import { ContainerIcon, CpuIcon, HardDriveIcon, TerminalSquareIcon } from "lucide-react"
|
||||
import { GpuIcon } from "../ui/icons"
|
||||
@@ -63,6 +64,7 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
||||
const hasContainersTable = hasContainers && compareSemVer(chartData.agentVersion, SEMVER_0_14_0) >= 0
|
||||
const hasSystemd = system.info.sv
|
||||
const hasGpu = hasGpuData || hasGpuPowerData
|
||||
const hasZfs = Object.keys(systemStats.at(-1)?.stats?.z ?? {}).length > 0
|
||||
|
||||
// keep tabsRef in sync for keyboard navigation
|
||||
const tabs = ["core", "disk"]
|
||||
@@ -142,6 +144,10 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
||||
|
||||
<ExtraFsCharts systemData={systemData} />
|
||||
|
||||
{hasZfs && <ZfsCharts systemData={systemData} />}
|
||||
|
||||
{hasZfs && <LazyZfsTable systemId={system.id} />}
|
||||
|
||||
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
||||
|
||||
{hasContainersTable && <LazyContainersTable systemId={system.id} />}
|
||||
@@ -204,6 +210,8 @@ export default memo(function SystemDetail({ id }: { id: string }) {
|
||||
<RootDiskCharts systemData={systemData} />
|
||||
</div>
|
||||
<ExtraFsCharts systemData={systemData} />
|
||||
{hasZfs && <ZfsCharts systemData={systemData} />}
|
||||
{hasZfs && <LazyZfsTable systemId={system.id} />}
|
||||
{maybeHasSmartData && <LazySmartTable systemId={system.id} />}
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import { t } from "@lingui/core/macro"
|
||||
import AreaChartDefault from "@/components/charts/area-chart"
|
||||
import { decimalString, formatBytes, toFixedFloat } from "@/lib/utils"
|
||||
import type { SystemStatsRecord } from "@/types"
|
||||
import { ChartCard } from "../chart-card"
|
||||
import { Unit } from "@/lib/enums"
|
||||
import { useStore } from "@nanostores/react"
|
||||
import { $userSettings } from "@/lib/stores"
|
||||
import type { SystemData } from "../use-system-data"
|
||||
|
||||
// Accessors for ZFS metrics
|
||||
const poolUsage =
|
||||
(name: string) =>
|
||||
({ stats }: SystemStatsRecord) =>
|
||||
stats?.z?.[name]?.du ?? 0
|
||||
const poolRead =
|
||||
(name: string) =>
|
||||
({ stats }: SystemStatsRecord) =>
|
||||
stats?.z?.[name]?.rb ?? 0
|
||||
const poolWrite =
|
||||
(name: string) =>
|
||||
({ stats }: SystemStatsRecord) =>
|
||||
stats?.z?.[name]?.wb ?? 0
|
||||
|
||||
export function ZfsPoolUsageChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
|
||||
const { chartData, grid, dataEmpty } = systemData
|
||||
const latest = chartData.systemStats.at(-1)?.stats
|
||||
const pool = latest?.z?.[poolName]
|
||||
if (!pool) {
|
||||
return null
|
||||
}
|
||||
let poolTotal = pool.d
|
||||
// round to nearest GB
|
||||
if (poolTotal >= 100) {
|
||||
poolTotal = Math.round(poolTotal)
|
||||
}
|
||||
|
||||
return (
|
||||
<ChartCard
|
||||
empty={dataEmpty}
|
||||
grid={grid}
|
||||
title={`${poolName} ${t`Usage`}`}
|
||||
description={t`Usage of ZFS pool ${poolName}`}
|
||||
>
|
||||
<AreaChartDefault
|
||||
chartData={chartData}
|
||||
domain={[0, poolTotal]}
|
||||
showTotal={true}
|
||||
tickFormatter={(val) => {
|
||||
const { value, unit } = formatBytes(val * 1024, false, Unit.Bytes, true)
|
||||
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
|
||||
}}
|
||||
contentFormatter={({ value }) => {
|
||||
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
|
||||
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
|
||||
}}
|
||||
dataPoints={[
|
||||
{
|
||||
label: t`Pool Usage`,
|
||||
dataKey: poolUsage(poolName),
|
||||
color: 4,
|
||||
opacity: 0.4,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
</ChartCard>
|
||||
)
|
||||
}
|
||||
|
||||
export function ZfsPoolIOChart({ systemData, poolName }: { systemData: SystemData; poolName: string }) {
|
||||
const { chartData, grid, dataEmpty } = systemData
|
||||
const userSettings = useStore($userSettings)
|
||||
if (!chartData.systemStats?.length) {
|
||||
return null
|
||||
}
|
||||
return (
|
||||
<ChartCard
|
||||
empty={dataEmpty}
|
||||
grid={grid}
|
||||
title={`${poolName} I/O`}
|
||||
description={t`Throughput of ZFS pool ${poolName}`}
|
||||
>
|
||||
<AreaChartDefault
|
||||
chartData={chartData}
|
||||
showTotal={true}
|
||||
dataPoints={[
|
||||
{
|
||||
label: t({ message: "Write", comment: "Disk write" }),
|
||||
dataKey: poolWrite(poolName),
|
||||
color: 3,
|
||||
opacity: 0.3,
|
||||
},
|
||||
{
|
||||
label: t({ message: "Read", comment: "Disk read" }),
|
||||
dataKey: poolRead(poolName),
|
||||
color: 1,
|
||||
opacity: 0.3,
|
||||
},
|
||||
]}
|
||||
tickFormatter={(val) => {
|
||||
const { value, unit } = formatBytes(val, true, userSettings.unitDisk, false)
|
||||
return `${toFixedFloat(value, value >= 10 ? 0 : 1)} ${unit}`
|
||||
}}
|
||||
contentFormatter={({ value }) => {
|
||||
const { value: convertedValue, unit } = formatBytes(value, true, userSettings.unitDisk, false)
|
||||
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
|
||||
}}
|
||||
/>
|
||||
</ChartCard>
|
||||
)
|
||||
}
|
||||
|
||||
/** ZFS section: one stacked usage card per pool plus per-pool I/O cards. */
|
||||
export function ZfsCharts({ systemData }: { systemData: SystemData }) {
|
||||
const latest = systemData.chartData.systemStats?.at(-1)?.stats
|
||||
const pools = latest?.z ?? {}
|
||||
if (Object.keys(pools).length === 0) {
|
||||
return null
|
||||
}
|
||||
return (
|
||||
<div className="grid xl:grid-cols-2 gap-4">
|
||||
{Object.keys(pools).map((poolName) => (
|
||||
<div key={poolName} className="contents">
|
||||
<ZfsPoolUsageChart systemData={systemData} poolName={poolName} />
|
||||
<ZfsPoolIOChart systemData={systemData} poolName={poolName} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -24,6 +24,17 @@ export function LazySmartTable({ systemId }: { systemId: string }) {
|
||||
)
|
||||
}
|
||||
|
||||
const ZfsTable = lazy(() => import("./zfs-table"))
|
||||
|
||||
export function LazyZfsTable({ systemId }: { systemId: string }) {
|
||||
const { isIntersecting, ref } = useIntersectionObserver({ rootMargin: "90px" })
|
||||
return (
|
||||
<div ref={ref} className={cn(isIntersecting && "contents")}>
|
||||
{isIntersecting && <ZfsTable systemId={systemId} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const SystemdTable = lazy(() => import("../../systemd-table/systemd-table"))
|
||||
|
||||
export function LazySystemdTable({ systemId }: { systemId: string }) {
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Card, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"
|
||||
import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
|
||||
import { Input } from "@/components/ui/input"
|
||||
import { Separator } from "@/components/ui/separator"
|
||||
import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"
|
||||
import { isReadOnlyUser, pb } from "@/lib/api"
|
||||
import { cn, formatBytes, formatShortDate, hourWithSeconds, toFixedFloat } from "@/lib/utils"
|
||||
import type { ZfsDataset, ZfsPoolRecord, ZfsVdev } from "@/types"
|
||||
import { t } from "@lingui/core/macro"
|
||||
import { Trans } from "@lingui/react/macro"
|
||||
import type { Column, ColumnDef } from "@tanstack/react-table"
|
||||
import {
|
||||
flexRender,
|
||||
getCoreRowModel,
|
||||
getFilteredRowModel,
|
||||
getSortedRowModel,
|
||||
useReactTable,
|
||||
} from "@tanstack/react-table"
|
||||
import {
|
||||
ActivityIcon,
|
||||
BinaryIcon,
|
||||
CheckCircleIcon,
|
||||
CircleAlertIcon,
|
||||
ClockIcon,
|
||||
HardDriveDownloadIcon,
|
||||
HardDriveIcon,
|
||||
HardDriveUploadIcon,
|
||||
LoaderCircleIcon,
|
||||
MoreHorizontalIcon,
|
||||
RefreshCwIcon,
|
||||
RotateCwIcon,
|
||||
XCircleIcon,
|
||||
XIcon,
|
||||
} from "lucide-react"
|
||||
import { useCallback, useEffect, useMemo, useState } from "react"
|
||||
|
||||
const ZFS_POOL_FIELDS = "id,system,name,health,size,alloc,free,scrub,details_updated,updated"
|
||||
|
||||
/** Maps a zpool health string to a Badge variant. */
|
||||
function healthVariant(health: string): "success" | "warning" | "danger" | "outline" {
|
||||
switch (health) {
|
||||
case "ONLINE":
|
||||
return "success"
|
||||
case "DEGRADED":
|
||||
return "warning"
|
||||
case "FAULTED":
|
||||
case "OFFLINE":
|
||||
case "UNAVAIL":
|
||||
case "REMOVED":
|
||||
case "SUSPENDED":
|
||||
return "danger"
|
||||
default:
|
||||
return "outline"
|
||||
}
|
||||
}
|
||||
|
||||
function formatCapacity(bytes: number): string {
|
||||
if (!bytes) return "-"
|
||||
const { value, unit } = formatBytes(bytes)
|
||||
return `${toFixedFloat(value, value >= 10 ? 1 : 2)} ${unit}`
|
||||
}
|
||||
|
||||
function HeaderButton<T>({ column, name, Icon }: { column: Column<T>; name: string; Icon: React.ElementType }) {
|
||||
const isSorted = column.getIsSorted()
|
||||
return (
|
||||
<Button
|
||||
className={cn(
|
||||
"h-9 px-3 flex items-center gap-2 duration-50",
|
||||
isSorted && "bg-accent/70 light:bg-accent text-accent-foreground/90"
|
||||
)}
|
||||
variant="ghost"
|
||||
onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
|
||||
>
|
||||
<Icon className="size-4" />
|
||||
{name}
|
||||
</Button>
|
||||
)
|
||||
}
|
||||
|
||||
const columns: ColumnDef<ZfsPoolRecord>[] = [
|
||||
{
|
||||
accessorKey: "name",
|
||||
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Pool`} Icon={HardDriveIcon} />,
|
||||
cell: ({ getValue }) => <span className="font-medium ms-1.5">{getValue() as string}</span>,
|
||||
},
|
||||
{
|
||||
accessorKey: "health",
|
||||
sortingFn: (a, b) => a.original.health.localeCompare(b.original.health),
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Health`} Icon={ActivityIcon} />,
|
||||
cell: ({ getValue }) => {
|
||||
const health = (getValue() as string) || ""
|
||||
return <Badge variant={healthVariant(health)}>{health || t`Unknown`}</Badge>
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "size",
|
||||
accessorFn: (record) => record.size,
|
||||
invertSorting: true,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Size`} Icon={BinaryIcon} />,
|
||||
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||
},
|
||||
{
|
||||
id: "used",
|
||||
accessorFn: (record) => record.alloc,
|
||||
invertSorting: true,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
|
||||
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||
},
|
||||
{
|
||||
id: "free",
|
||||
accessorFn: (record) => record.free,
|
||||
invertSorting: true,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Free`} Icon={HardDriveUploadIcon} />,
|
||||
cell: ({ getValue }) => <span className="ms-1.5 tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||
},
|
||||
{
|
||||
id: "scrub",
|
||||
accessorFn: (record) => record.scrub?.state ?? "",
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Scrub`} Icon={RotateCwIcon} />,
|
||||
cell: ({ row }) => {
|
||||
const scrub = row.original.scrub
|
||||
if (!scrub?.state) return <span className="ms-1.5 text-muted-foreground">{t`None`}</span>
|
||||
return (
|
||||
<span className="ms-1.5 tabular-nums">
|
||||
{scrub.state}
|
||||
{scrub.progress ? ` (${scrub.progress})` : ""}
|
||||
</span>
|
||||
)
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "updated",
|
||||
invertSorting: true,
|
||||
accessorFn: (record) => record.details_updated || record.updated,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Updated`} Icon={ClockIcon} />,
|
||||
cell: ({ getValue }) => {
|
||||
const timestamp = getValue() as string
|
||||
if (!timestamp) return null
|
||||
const formatter =
|
||||
new Date(timestamp).toDateString() === new Date().toDateString() ? hourWithSeconds : formatShortDate
|
||||
return <span className="ms-1 tabular-nums">{formatter(timestamp)}</span>
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
function VdevTable({ vdevs }: { vdevs: ZfsVdev[] }) {
|
||||
if (!vdevs?.length) return null
|
||||
return (
|
||||
<div className="overflow-x-auto rounded-md border">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>{t`Vdev`}</TableHead>
|
||||
<TableHead>{t`State`}</TableHead>
|
||||
<TableHead className="text-right">{t`Read errors`}</TableHead>
|
||||
<TableHead className="text-right">{t`Write errors`}</TableHead>
|
||||
<TableHead className="text-right">{t`Checksum errors`}</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{vdevs.map((vdev) => (
|
||||
<TableRow key={vdev.name}>
|
||||
<TableCell className="font-mono text-xs">{vdev.name}</TableCell>
|
||||
<TableCell>
|
||||
<Badge variant={healthVariant(vdev.state ?? "")} className="font-normal">
|
||||
{vdev.state ?? "-"}
|
||||
</Badge>
|
||||
</TableCell>
|
||||
<TableCell
|
||||
className={cn("text-right tabular-nums", (vdev.readErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
|
||||
>
|
||||
{vdev.readErrs ?? 0}
|
||||
</TableCell>
|
||||
<TableCell
|
||||
className={cn("text-right tabular-nums", (vdev.writeErrs ?? 0) > 0 && "text-red-600 dark:text-red-400")}
|
||||
>
|
||||
{vdev.writeErrs ?? 0}
|
||||
</TableCell>
|
||||
<TableCell
|
||||
className={cn(
|
||||
"text-right tabular-nums",
|
||||
(vdev.checksumErrs ?? 0) > 0 && "text-red-600 dark:text-red-400"
|
||||
)}
|
||||
>
|
||||
{vdev.checksumErrs ?? 0}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const datasetColumns: ColumnDef<ZfsDataset>[] = [
|
||||
{
|
||||
accessorKey: "name",
|
||||
sortingFn: (a, b) => a.original.name.localeCompare(b.original.name),
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Dataset`} Icon={HardDriveIcon} />,
|
||||
cell: ({ getValue }) => <span className="font-mono text-xs">{getValue() as string}</span>,
|
||||
},
|
||||
{
|
||||
id: "used",
|
||||
accessorFn: (ds) => ds.used ?? 0,
|
||||
invertSorting: true,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Used`} Icon={HardDriveDownloadIcon} />,
|
||||
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||
},
|
||||
{
|
||||
id: "avail",
|
||||
accessorFn: (ds) => ds.avail ?? 0,
|
||||
invertSorting: true,
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Available`} Icon={HardDriveUploadIcon} />,
|
||||
cell: ({ getValue }) => <span className="text-right tabular-nums">{formatCapacity(getValue() as number)}</span>,
|
||||
},
|
||||
{
|
||||
accessorKey: "mount",
|
||||
sortingFn: (a, b) => (a.original.mount ?? "").localeCompare(b.original.mount ?? ""),
|
||||
header: ({ column }) => <HeaderButton column={column} name={t`Mountpoint`} Icon={HardDriveIcon} />,
|
||||
cell: ({ getValue }) => (
|
||||
<span className="font-mono text-xs text-muted-foreground">{(getValue() as string) || "-"}</span>
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
function DatasetTable({ datasets }: { datasets: ZfsDataset[] }) {
|
||||
const [filter, setFilter] = useState("")
|
||||
const filtered = useMemo(() => {
|
||||
if (!datasets) return []
|
||||
if (!filter) return datasets
|
||||
const needle = filter.toLowerCase()
|
||||
return datasets.filter((ds) => ds.name.toLowerCase().includes(needle))
|
||||
}, [datasets, filter])
|
||||
|
||||
const table = useReactTable({
|
||||
data: filtered,
|
||||
columns: datasetColumns,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
getSortedRowModel: getSortedRowModel(),
|
||||
})
|
||||
|
||||
if (!datasets?.length) return null
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-2 flex items-center justify-between gap-4">
|
||||
<h3 className="text-base font-semibold">
|
||||
<Trans>Datasets</Trans>
|
||||
</h3>
|
||||
<div className="relative w-64 max-w-full">
|
||||
<Input
|
||||
placeholder={t`Filter datasets...`}
|
||||
value={filter}
|
||||
onChange={(e) => setFilter(e.target.value)}
|
||||
className="px-4 w-full"
|
||||
/>
|
||||
{filter && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
aria-label={t`Clear`}
|
||||
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
|
||||
onClick={() => setFilter("")}
|
||||
>
|
||||
<XIcon className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="max-h-80 overflow-auto rounded-md border">
|
||||
<Table>
|
||||
<TableHeader className="sticky top-0 z-10">
|
||||
{table.getHeaderGroups().map((headerGroup) => (
|
||||
<TableRow key={headerGroup.id}>
|
||||
{headerGroup.headers.map((header) => (
|
||||
<TableHead key={header.id} className="px-2">
|
||||
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{table.getRowModel().rows.map((row) => (
|
||||
<TableRow key={row.id}>
|
||||
{row.getVisibleCells().map((cell) => (
|
||||
<TableCell key={cell.id} className="ps-5 whitespace-pre">
|
||||
{flexRender(cell.column.columnDef.cell, cell.getContext())}
|
||||
</TableCell>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function PoolSheet({
|
||||
poolId,
|
||||
open,
|
||||
onOpenChange,
|
||||
}: {
|
||||
poolId: string | null
|
||||
open: boolean
|
||||
onOpenChange: (open: boolean) => void
|
||||
}) {
|
||||
const [pool, setPool] = useState<ZfsPoolRecord | null>(null)
|
||||
const [isLoading, setIsLoading] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
let active = true
|
||||
if (!poolId) {
|
||||
setPool(null)
|
||||
return
|
||||
}
|
||||
// Only fetch when opening, not when closing (keeps data visible during close animation)
|
||||
if (!open) return
|
||||
setIsLoading(true)
|
||||
pb.collection("zfs_pools")
|
||||
.getOne(poolId)
|
||||
.then((record) => active && setPool(record as ZfsPoolRecord))
|
||||
.catch(() => active && setPool(null))
|
||||
.finally(() => active && setIsLoading(false))
|
||||
return () => {
|
||||
active = false
|
||||
}
|
||||
}, [open, poolId])
|
||||
|
||||
const health = pool?.health || ""
|
||||
const healthVariantValue = healthVariant(health)
|
||||
const HealthIcon =
|
||||
healthVariantValue === "success"
|
||||
? CheckCircleIcon
|
||||
: healthVariantValue === "warning"
|
||||
? CircleAlertIcon
|
||||
: XCircleIcon
|
||||
|
||||
return (
|
||||
<Sheet open={open} onOpenChange={onOpenChange}>
|
||||
<SheetContent className="w-full sm:max-w-220 gap-0 overflow-y-auto">
|
||||
<SheetHeader className="mb-0 border-b">
|
||||
<SheetTitle className="flex items-center gap-2">
|
||||
{pool ? pool.name : t`ZFS Pool`}
|
||||
{pool && <Badge variant={healthVariantValue}>{health}</Badge>}
|
||||
</SheetTitle>
|
||||
<SheetDescription className="flex flex-wrap items-center gap-x-2 gap-y-1">
|
||||
{pool?.size ? formatCapacity(pool.size) : null}
|
||||
{pool?.alloc ? (
|
||||
<>
|
||||
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||
<span>
|
||||
<Trans>Used</Trans>: {formatCapacity(pool.alloc)}
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
{pool?.free ? (
|
||||
<>
|
||||
<Separator orientation="vertical" className="h-2.5 bg-muted-foreground opacity-70" />
|
||||
<span>
|
||||
<Trans>Free</Trans>: {formatCapacity(pool.free)}
|
||||
</span>
|
||||
</>
|
||||
) : null}
|
||||
</SheetDescription>
|
||||
</SheetHeader>
|
||||
<div className="flex-1 p-4 flex flex-col gap-4">
|
||||
{isLoading ? (
|
||||
<div className="flex justify-center py-8">
|
||||
<LoaderCircleIcon className="animate-spin size-10 opacity-60" />
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
{pool && health && (
|
||||
<Alert className="pb-3 shrink-0">
|
||||
<HealthIcon className="size-4" />
|
||||
<AlertTitle>
|
||||
<Trans>Pool Health</Trans>: {health}
|
||||
</AlertTitle>
|
||||
{pool.scrub?.state && (
|
||||
<AlertDescription>
|
||||
<Trans>Scrub</Trans>: {pool.scrub.state}
|
||||
{pool.scrub.progress ? ` (${pool.scrub.progress})` : ""}
|
||||
{pool.scrub.errors ? `, ${pool.scrub.errors} errors` : ""}
|
||||
</AlertDescription>
|
||||
)}
|
||||
</Alert>
|
||||
)}
|
||||
{pool?.vdevs?.length || pool?.datasets?.length ? (
|
||||
<>
|
||||
{pool.vdevs?.length ? <VdevTable vdevs={pool.vdevs} /> : null}
|
||||
<DatasetTable datasets={pool.datasets ?? []} />
|
||||
</>
|
||||
) : (
|
||||
!isLoading && (
|
||||
<div className="py-8 text-center text-sm text-muted-foreground">
|
||||
<Trans>No detail data for this pool.</Trans>
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</SheetContent>
|
||||
</Sheet>
|
||||
)
|
||||
}
|
||||
|
||||
export default function ZfsTable({ systemId }: { systemId?: string }) {
|
||||
const [zfsPools, setZfsPools] = useState<ZfsPoolRecord[]>()
|
||||
const [globalFilter, setGlobalFilter] = useState("")
|
||||
const [activePoolId, setActivePoolId] = useState<string | null>(null)
|
||||
const [sheetOpen, setSheetOpen] = useState(false)
|
||||
const [refreshingId, setRefreshingId] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
let disposed = false
|
||||
let unsubscribe: () => void = () => {}
|
||||
// fetch initial records
|
||||
pb.collection<ZfsPoolRecord>("zfs_pools")
|
||||
.getFullList({
|
||||
filter: systemId ? pb.filter("system={:id}", { id: systemId }) : "",
|
||||
sort: "name",
|
||||
fields: ZFS_POOL_FIELDS,
|
||||
})
|
||||
.then((records) => !disposed && setZfsPools(records))
|
||||
.catch((error) => console.error("Failed to fetch ZFS pools:", error))
|
||||
|
||||
// subscribe to realtime updates
|
||||
const pbOptions = systemId ? { filter: `system="${systemId}"` } : undefined
|
||||
;(async () => {
|
||||
try {
|
||||
const unsubscribeNow = await pb.collection<ZfsPoolRecord>("zfs_pools").subscribe(
|
||||
"*",
|
||||
(event) => {
|
||||
const record = event.record as ZfsPoolRecord
|
||||
setZfsPools((current) => {
|
||||
const pools = current ?? []
|
||||
const matchesSystemScope = !systemId || record.system === systemId
|
||||
if (event.action === "delete") {
|
||||
return pools.filter((pool) => pool.id !== record.id)
|
||||
}
|
||||
if (!matchesSystemScope) {
|
||||
return pools.filter((pool) => pool.id !== record.id)
|
||||
}
|
||||
const existingIndex = pools.findIndex((pool) => pool.id === record.id)
|
||||
if (existingIndex === -1) {
|
||||
return [record, ...pools]
|
||||
}
|
||||
const next = [...pools]
|
||||
next[existingIndex] = record
|
||||
return next
|
||||
})
|
||||
},
|
||||
pbOptions
|
||||
)
|
||||
if (disposed) {
|
||||
unsubscribeNow()
|
||||
} else {
|
||||
unsubscribe = unsubscribeNow
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to subscribe to ZFS pool updates:", error)
|
||||
}
|
||||
})()
|
||||
|
||||
return () => {
|
||||
disposed = true
|
||||
unsubscribe?.()
|
||||
}
|
||||
}, [systemId])
|
||||
|
||||
const refreshSystem = useCallback(async (systemId: string) => {
|
||||
try {
|
||||
await pb.send("/api/beszel/zfs/refresh", {
|
||||
method: "POST",
|
||||
query: { system: systemId },
|
||||
})
|
||||
} catch (error) {
|
||||
console.error("Failed to refresh ZFS pools:", error)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const handleRowRefresh = useCallback(
|
||||
async (pool: ZfsPoolRecord) => {
|
||||
if (!pool.system) return
|
||||
setRefreshingId(pool.id)
|
||||
try {
|
||||
await refreshSystem(pool.system)
|
||||
} finally {
|
||||
setRefreshingId((id) => (id === pool.id ? null : id))
|
||||
}
|
||||
},
|
||||
[refreshSystem]
|
||||
)
|
||||
|
||||
const actionColumn = useMemo<ColumnDef<ZfsPoolRecord>>(
|
||||
() => ({
|
||||
id: "actions",
|
||||
enableSorting: false,
|
||||
header: () => (
|
||||
<span className="sr-only">
|
||||
<Trans>Actions</Trans>
|
||||
</span>
|
||||
),
|
||||
cell: ({ row }) => {
|
||||
const pool = row.original
|
||||
const isRowRefreshing = refreshingId === pool.id
|
||||
|
||||
return (
|
||||
<div className="flex justify-end">
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-10"
|
||||
onClick={(event) => event.stopPropagation()}
|
||||
onMouseDown={(event) => event.stopPropagation()}
|
||||
>
|
||||
<span className="sr-only">
|
||||
<Trans>Open menu</Trans>
|
||||
</span>
|
||||
<MoreHorizontalIcon className="w-5" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" onClick={(event) => event.stopPropagation()}>
|
||||
<DropdownMenuItem
|
||||
onClick={(event) => {
|
||||
event.stopPropagation()
|
||||
handleRowRefresh(pool)
|
||||
}}
|
||||
disabled={isRowRefreshing}
|
||||
>
|
||||
<RefreshCwIcon className={cn("me-2.5 size-4", isRowRefreshing && "animate-spin")} />
|
||||
<Trans>Refresh</Trans>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
)
|
||||
},
|
||||
}),
|
||||
[refreshingId, handleRowRefresh]
|
||||
)
|
||||
|
||||
const tableColumns = useMemo(() => {
|
||||
return isReadOnlyUser() ? columns : [...columns, actionColumn]
|
||||
}, [actionColumn])
|
||||
|
||||
const table = useReactTable({
|
||||
data: zfsPools || ([] as ZfsPoolRecord[]),
|
||||
columns: tableColumns,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
getSortedRowModel: getSortedRowModel(),
|
||||
getFilteredRowModel: getFilteredRowModel(),
|
||||
state: { globalFilter },
|
||||
onGlobalFilterChange: setGlobalFilter,
|
||||
globalFilterFn: (row, _columnId, filterValue) => {
|
||||
const pool = row.original
|
||||
const searchString = `${pool.name} ${pool.health ?? ""}`.toLowerCase()
|
||||
return (filterValue as string)
|
||||
.toLowerCase()
|
||||
.split(" ")
|
||||
.every((term) => searchString.includes(term))
|
||||
},
|
||||
})
|
||||
const rows = table.getRowModel().rows
|
||||
|
||||
// Hide the table on system pages if there's no data
|
||||
if (systemId && !zfsPools?.length && !globalFilter) {
|
||||
return null
|
||||
}
|
||||
|
||||
const openSheet = (pool: ZfsPoolRecord) => {
|
||||
setActivePoolId(pool.id)
|
||||
setSheetOpen(true)
|
||||
}
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Card className="@container w-full px-3 py-5 sm:py-6 sm:px-6">
|
||||
<CardHeader className="p-0 mb-3 sm:mb-4">
|
||||
<div className="grid md:flex gap-x-5 gap-y-3 w-full items-end">
|
||||
<div className="px-2 sm:px-1">
|
||||
<CardTitle className="mb-2">ZFS</CardTitle>
|
||||
<CardDescription className="flex">
|
||||
<Trans>Click on a pool to view vdev and dataset details.</Trans>
|
||||
</CardDescription>
|
||||
</div>
|
||||
<div className="relative ms-auto w-full max-w-full md:w-64">
|
||||
<Input
|
||||
placeholder={t`Filter...`}
|
||||
value={globalFilter}
|
||||
onChange={(event) => setGlobalFilter(event.target.value)}
|
||||
className="px-4 w-full max-w-full md:w-64"
|
||||
/>
|
||||
{globalFilter && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
aria-label={t`Clear`}
|
||||
className="absolute right-1 top-1/2 -translate-y-1/2 h-7 w-7 text-muted-foreground"
|
||||
onClick={() => setGlobalFilter("")}
|
||||
>
|
||||
<XIcon className="h-4 w-4" />
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<div className="h-min max-h-[calc(100dvh-17rem)] max-w-full relative overflow-auto rounded-md border">
|
||||
<Table>
|
||||
<TableHeader className="sticky top-0 z-50 w-full border-b-2">
|
||||
{table.getHeaderGroups().map((headerGroup) => (
|
||||
<TableRow key={headerGroup.id}>
|
||||
{headerGroup.headers.map((header) => (
|
||||
<TableHead key={header.id} className="px-2">
|
||||
{header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{rows.map((row) => (
|
||||
<TableRow
|
||||
key={row.id}
|
||||
data-state={row.getIsSelected() && "selected"}
|
||||
className="cursor-pointer"
|
||||
onClick={() => openSheet(row.original)}
|
||||
>
|
||||
{row.getVisibleCells().map((cell) => (
|
||||
<TableCell key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</TableCell>
|
||||
))}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
</Card>
|
||||
<PoolSheet poolId={activePoolId} open={sheetOpen} onOpenChange={setSheetOpen} />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Vendored
+52
@@ -151,6 +151,8 @@ export interface SystemStats {
|
||||
f?: Record<string, number>
|
||||
/** extra filesystems */
|
||||
efs?: Record<string, ExtraFsStats>
|
||||
/** ZFS pool metrics */
|
||||
z?: Record<string, ZfsPool>
|
||||
/** GPU data */
|
||||
g?: Record<string, GPUData>
|
||||
/** battery percent and state */
|
||||
@@ -178,6 +180,56 @@ export interface GPUData {
|
||||
e?: Record<string, number>
|
||||
}
|
||||
|
||||
export interface ZfsPool {
|
||||
/** total capacity (GiB) */
|
||||
d: number
|
||||
/** allocated (GiB) */
|
||||
du: number
|
||||
/** read throughput (bytes/s) */
|
||||
rb?: number
|
||||
/** write throughput (bytes/s) */
|
||||
wb?: number
|
||||
/** health: ONLINE, DEGRADED, FAULTED, ... */
|
||||
h?: string
|
||||
}
|
||||
|
||||
export interface ZfsScrub {
|
||||
/** NONE, SCANNING, FINISHED, CANCELED */
|
||||
state?: string
|
||||
/** progress while scanning, e.g. "10.00%" */
|
||||
progress?: string
|
||||
errors?: number
|
||||
}
|
||||
|
||||
export interface ZfsVdev {
|
||||
name: string
|
||||
state?: string
|
||||
readErrs?: number
|
||||
writeErrs?: number
|
||||
checksumErrs?: number
|
||||
}
|
||||
|
||||
export interface ZfsDataset {
|
||||
name: string
|
||||
used?: number
|
||||
avail?: number
|
||||
mount?: string
|
||||
}
|
||||
|
||||
export interface ZfsPoolRecord extends RecordModel {
|
||||
system: string
|
||||
name: string
|
||||
health: string
|
||||
size: number
|
||||
alloc: number
|
||||
free: number
|
||||
scrub: ZfsScrub | null
|
||||
vdevs: ZfsVdev[] | null
|
||||
datasets: ZfsDataset[] | null
|
||||
details_updated: string
|
||||
updated: string
|
||||
}
|
||||
|
||||
export interface ExtraFsStats {
|
||||
/** disk size (gb) */
|
||||
d: number
|
||||
|
||||
Reference in New Issue
Block a user