mirrors/beszel
1
0
Fork 0
mirror of https://github.com/henrygd/beszel.git synced 2026-10-22 06:12:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: collect memory and swap in bytes for low memory systems (#2359, #2360)

Browse Source 
Memory and swap were rounded to two decimal GB places before being sent
to the hub, which is inaccurate on low memory systems.

- agent: add MemBytes [total, used, buff/cache, zfs arc] and SwapBytes
[total, used]
- hub: migrate GB fields from older agents to the byte fields
- records: average byte fields, converting GB values from older records
- site: chart memory and swap from bytes, with fallback for older
records

Deprecated GB fields are still populated in agent payloads and stored
records for older hubs and third-party integrations.
This commit is contained in:
henrygd
2026-10-21 12:47:48 -04:00
parent aa431c5b21
commit 37c0547b02
10 changed files with 315 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/memory.go
+10 -4
View File
@@ -21,10 +21,13 @@ func (a *Agent) updateMemoryStats(stats *system.Stats) {
if err == nil {
hostTotal = v.Total
// Swap retains its existing /proc/meminfo source in both memory paths.
stats.Swap = utils.BytesToGigabytes(v.SwapTotal)
stats.SwapUsed = utils.BytesToGigabytes(saturatingSub(v.SwapTotal, v.SwapFree))
stats.SwapBytes = [2]uint64{v.SwapTotal, saturatingSub(v.SwapTotal, v.SwapFree)}
// TODO: stop populating deprecated GB fields once hubs < 0.21.1 are no longer supported
stats.Swap = utils.BytesToGigabytes(stats.SwapBytes[0])
stats.SwapUsed = utils.BytesToGigabytes(stats.SwapBytes[1])
}
var arcSize uint64
metrics, fromCgroup := containerMemoryMetrics(hostTotal, a.forceUseCgroup)
if !fromCgroup {
if err != nil {
@@ -34,16 +37,19 @@ func (a *Agent) updateMemoryStats(stats *system.Stats) {
metrics = memoryMetrics{Total: v.Total, Used: used, BuffCache: cacheBuff}
// Host ARC must not be subtracted from container-scoped accounting.
if a.zfs {
if arcSize, _ := zfs.ARCSize(); arcSize > 0 && arcSize < metrics.Used {
if size, _ := zfs.ARCSize(); size > 0 && size < metrics.Used {
arcSize = size
metrics.Used -= arcSize
stats.MemZfsArc = utils.BytesToGigabytes(arcSize)
}
}
}
stats.MemBytes = [4]uint64{metrics.Total, metrics.Used, metrics.BuffCache, arcSize}
// TODO: stop populating deprecated GB fields once hubs < 0.21.1 are no longer supported
stats.Mem = utils.BytesToGigabytes(metrics.Total)
stats.MemUsed = utils.BytesToGigabytes(metrics.Used)
stats.MemBuffCache = utils.BytesToGigabytes(metrics.BuffCache)
stats.MemZfsArc = utils.BytesToGigabytes(arcSize)
if metrics.Total > 0 {
stats.MemPct = utils.TwoDecimals(float64(metrics.Used) / float64(metrics.Total) * 100)
}
agent/memory_linux_test.go
+25
View File
@@ -146,6 +146,8 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Zero(t, stats.MemZfsArc)
assert.Equal(t, float64(2), stats.Swap)
assert.Equal(t, float64(1), stats.SwapUsed)
assert.Equal(t, [4]uint64{10 * gib, 4 * gib, 2 * gib, 0}, stats.MemBytes)
assert.Equal(t, [2]uint64{2 * gib, gib}, stats.SwapBytes)
assert.Equal(t, 10*gib, a.systemDetails.MemoryTotal)
assert.True(t, a.detailsDirty)
@@ -166,6 +168,7 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Equal(t, float64(12), stats.MemUsed)
assert.Equal(t, float64(4), stats.MemBuffCache)
assert.Equal(t, float64(60), stats.MemPct)
assert.Equal(t, [4]uint64{20 * gib, 12 * gib, 4 * gib, 0}, stats.MemBytes)
assert.Equal(t, 20*gib, a.systemDetails.MemoryTotal)
// Finite cgroup limits still work if /proc/meminfo cannot be read.
@@ -176,7 +179,9 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Equal(t, float64(8), stats.Mem)
assert.Equal(t, float64(6), stats.MemUsed)
assert.Equal(t, float64(75), stats.MemPct)
assert.Equal(t, [4]uint64{8 * gib, 6 * gib, 0, 0}, stats.MemBytes)
assert.Zero(t, stats.Swap)
assert.Zero(t, stats.SwapBytes)
}
func TestMemoryDetailsInitiallyUseCgroupLimit(t *testing.T) {
@@ -218,4 +223,24 @@ func TestMemoryStatsExcludeActiveFileCache(t *testing.T) {
assert.Equal(t, float64(17.82), stats.MemUsed)
assert.Equal(t, float64(108.73), stats.MemBuffCache)
assert.Equal(t, float64(13.92), stats.MemPct)
assert.Equal(t, [4]uint64{137438953472, usage - (cache - shmem), cache - shmem, 0}, stats.MemBytes)
}
// Low memory systems need byte precision because the GB fields round to 10 MiB.
func TestMemoryStatsLowMemoryPrecision(t *testing.T) {
dir := setupMemoryCgroup(t, true)
const mib = uint64(1 << 20)
writeMemoryFixture(t, dir, true, strconv.FormatUint(5*mib, 10), strconv.FormatUint(64*mib, 10),
"file "+strconv.FormatUint(2*mib, 10)+"\nshmem 0\n")
original := memoryVirtualMemory
t.Cleanup(func() { memoryVirtualMemory = original })
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
return &mem.VirtualMemoryStat{Total: 128 * mib, SwapTotal: 32 * mib, SwapFree: 31 * mib}, nil
}
var stats system.Stats
a := &Agent{forceUseCgroup: true}
a.updateMemoryStats(&stats)
assert.Equal(t, [4]uint64{64 * mib, 3 * mib, 2 * mib, 0}, stats.MemBytes)
assert.Equal(t, [2]uint64{32 * mib, mib}, stats.SwapBytes)
assert.Equal(t, float64(0), stats.MemUsed)
}
internal/entities/system/system.go
+10 -8
View File
@@ -22,14 +22,14 @@ type WiFi struct {
type Stats struct {
Cpu float64 `json:"cpu" cbor:"0,keyasint"`
MaxCpu float64 `json:"cpum,omitzero" cbor:"-"`
Mem float64 `json:"m" cbor:"2,keyasint"`
MaxMem float64 `json:"mm,omitzero" cbor:"-"`
MemUsed float64 `json:"mu" cbor:"3,keyasint"`
Mem float64 `json:"m" cbor:"2,keyasint"` // Deprecated: GB, use MemBytes. Memory GB fields are kept for older hubs and integrations.
MaxMem float64 `json:"mm,omitzero" cbor:"-"` // Deprecated: GB, use MaxMemBytes.
MemUsed float64 `json:"mu" cbor:"3,keyasint"` // Deprecated: GB, use MemBytes.
MemPct float64 `json:"mp" cbor:"4,keyasint"`
MemBuffCache float64 `json:"mb" cbor:"5,keyasint"`
MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // ZFS ARC memory
Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"`
SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"`
MemBuffCache float64 `json:"mb" cbor:"5,keyasint"` // Deprecated: GB, use MemBytes.
MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // Deprecated: GB, use MemBytes.
Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
DiskTotal float64 `json:"d" cbor:"9,keyasint"`
DiskUsed float64 `json:"du" cbor:"10,keyasint"`
DiskPct float64 `json:"dp" cbor:"11,keyasint"`
@@ -59,7 +59,9 @@ type Stats struct {
ZfsPools map[string]*ZfsPool `json:"z,omitempty" cbor:"39,keyasint,omitempty"` // ZFS pool metrics, keyed by pool name
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
WiFi map[string]int8 `json:"wf,omitempty" cbor:"40,keyasint,omitempty"` // RSSI dBm keyed by interface; unavailable readings omitted
MemBytes [4]uint64 `json:"mem,omitzero" cbor:"41,keyasint,omitzero"` // [total, used, buff/cache, zfs arc] bytes
MaxMemBytes uint64 `json:"memm,omitzero" cbor:"-"` // max used memory bytes
SwapBytes [2]uint64 `json:"swap,omitzero" cbor:"42,keyasint,omitzero"` // [total, used] bytes
}
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
internal/hub/systems/system.go
+15 -1
View File
@@ -1122,6 +1122,20 @@ func migrateDeprecatedFields(cd *system.CombinedData, createDetails bool) {
}
fs.DiskReadPs, fs.DiskWritePs = 0, 0
}
// migration added 0.21.1
const gb = 1024 * 1024 * 1024
if cd.Stats.MemBytes == [4]uint64{} {
cd.Stats.MemBytes = [4]uint64{
uint64(cd.Stats.Mem * gb),
uint64(cd.Stats.MemUsed * gb),
uint64(cd.Stats.MemBuffCache * gb),
uint64(cd.Stats.MemZfsArc * gb),
}
}
if cd.Stats.SwapBytes == [2]uint64{} {
cd.Stats.SwapBytes = [2]uint64{uint64(cd.Stats.Swap * gb), uint64(cd.Stats.SwapUsed * gb)}
}
// GB fields are kept in stored records for third-party integrations until they are removed
// migration added 0.19.0 - Move deprecated Info fields to Details struct
if cd.Details == nil && cd.Info.Hostname != "" {
if createDetails {
@@ -1133,7 +1147,7 @@ func migrateDeprecatedFields(cd *system.CombinedData, createDetails bool) {
CpuModel: cd.Info.CpuModel,
Podman: cd.Info.Podman,
Os: cd.Info.Os,
MemoryTotal: uint64(cd.Stats.Mem * 1024 * 1024 * 1024),
MemoryTotal: cd.Stats.MemBytes[0],
}
}
// zero the deprecated fields to prevent saving them in systems.info DB json payload
internal/hub/systems/system_test.go
+33
View File
@@ -102,6 +102,39 @@ func TestCombinedData_MigrateDeprecatedFields(t *testing.T) {
}
})
t.Run("Migrate memory GB fields to bytes", func(t *testing.T) {
const gb = 1024 * 1024 * 1024
cd := &system.CombinedData{
Stats: system.Stats{Mem: 16, MemUsed: 4.5, MemBuffCache: 2, MemZfsArc: 1, Swap: 4, SwapUsed: 0.5},
}
migrateDeprecatedFields(cd, true)
require.Equal(t, [4]uint64{16 * gb, 4.5 * gb, 2 * gb, 1 * gb}, cd.Stats.MemBytes)
require.Equal(t, [2]uint64{4 * gb, 0.5 * gb}, cd.Stats.SwapBytes)
// GB fields are kept for third-party integrations
require.Equal(t, 16.0, cd.Stats.Mem)
require.Equal(t, 4.5, cd.Stats.MemUsed)
require.Equal(t, 0.5, cd.Stats.SwapUsed)
})
t.Run("Keep memory bytes and GB fields from newer agents", func(t *testing.T) {
cd := &system.CombinedData{
Stats: system.Stats{
Mem: 0.06, MemUsed: 0.01, Swap: 0.03, SwapUsed: 0.01,
MemBytes: [4]uint64{64 << 20, 12 << 20, 3 << 20, 0},
SwapBytes: [2]uint64{32 << 20, 1 << 20},
},
}
migrateDeprecatedFields(cd, true)
require.Equal(t, [4]uint64{64 << 20, 12 << 20, 3 << 20, 0}, cd.Stats.MemBytes)
require.Equal(t, [2]uint64{32 << 20, 1 << 20}, cd.Stats.SwapBytes)
require.Equal(t, 0.06, cd.Stats.Mem)
require.Equal(t, 0.01, cd.Stats.MemUsed)
require.Equal(t, 0.03, cd.Stats.Swap)
require.Equal(t, 0.01, cd.Stats.SwapUsed)
})
t.Run("Migrate Info fields to Details struct", func(t *testing.T) {
cd := &system.CombinedData{
Stats: system.Stats{
internal/records/records.go
+50 -13
View File
@@ -302,13 +302,13 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
cpuBreakdownSums[j] += v
}
}
sum.Mem += stats.Mem
sum.MemUsed += stats.MemUsed
memBytes, swapBytes, maxMemBytes := memoryBytes(stats)
for j, v := range memBytes {
sum.MemBytes[j] += v
}
sum.SwapBytes[0] += swapBytes[0]
sum.SwapBytes[1] += swapBytes[1]
sum.MemPct += stats.MemPct
sum.MemBuffCache += stats.MemBuffCache
sum.MemZfsArc += stats.MemZfsArc
sum.Swap += stats.Swap
sum.SwapUsed += stats.SwapUsed
sum.DiskTotal += stats.DiskTotal
sum.DiskUsed += stats.DiskUsed
sum.DiskPct += stats.DiskPct
@@ -346,7 +346,7 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
}
// Set peak values
sum.MaxCpu = max(sum.MaxCpu, stats.MaxCpu, stats.Cpu)
sum.MaxMem = max(sum.MaxMem, stats.MaxMem, stats.MemUsed)
sum.MaxMemBytes = max(sum.MaxMemBytes, maxMemBytes, memBytes[1])
sum.MaxNetworkSent = max(sum.MaxNetworkSent, stats.MaxNetworkSent, stats.NetworkSent)
sum.MaxNetworkRecv = max(sum.MaxNetworkRecv, stats.MaxNetworkRecv, stats.NetworkRecv)
sum.MaxBandwidth[0] = max(sum.MaxBandwidth[0], stats.MaxBandwidth[0], stats.Bandwidth[0])
@@ -487,13 +487,20 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
// Compute averages
sum.Cpu = twoDecimals(sum.Cpu / count)
sum.Mem = twoDecimals(sum.Mem / count)
sum.MemUsed = twoDecimals(sum.MemUsed / count)
for i := range sum.MemBytes {
sum.MemBytes[i] = sum.MemBytes[i] / uint64(count)
}
sum.MemPct = twoDecimals(sum.MemPct / count)
sum.MemBuffCache = twoDecimals(sum.MemBuffCache / count)
sum.MemZfsArc = twoDecimals(sum.MemZfsArc / count)
sum.Swap = twoDecimals(sum.Swap / count)
sum.SwapUsed = twoDecimals(sum.SwapUsed / count)
sum.SwapBytes[0] = sum.SwapBytes[0] / uint64(count)
sum.SwapBytes[1] = sum.SwapBytes[1] / uint64(count)
// TODO: stop populating deprecated GB fields when they are removed from the agent
sum.Mem = bytesToGigabytes(sum.MemBytes[0])
sum.MemUsed = bytesToGigabytes(sum.MemBytes[1])
sum.MemBuffCache = bytesToGigabytes(sum.MemBytes[2])
sum.MemZfsArc = bytesToGigabytes(sum.MemBytes[3])
sum.MaxMem = bytesToGigabytes(sum.MaxMemBytes)
sum.Swap = bytesToGigabytes(sum.SwapBytes[0])
sum.SwapUsed = bytesToGigabytes(sum.SwapBytes[1])
sum.DiskTotal = twoDecimals(sum.DiskTotal / count)
sum.DiskUsed = twoDecimals(sum.DiskUsed / count)
sum.DiskPct = twoDecimals(sum.DiskPct / count)
@@ -711,6 +718,36 @@ func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordTy
}
/* Round float to two decimals */
const gb = 1024 * 1024 * 1024
// memoryBytes returns memory bytes, swap bytes, and max used memory bytes for a record,
// converting the deprecated GB fields of records created before 0.21.1.
//
// TODO: remove the GB conversion when the deprecated fields are removed. Hubs upgraded from
// versions before 0.21.1 average pre-upgrade records only in the first few hours after upgrade.
func memoryBytes(stats *system.Stats) (mem [4]uint64, swap [2]uint64, maxUsed uint64) {
mem, swap, maxUsed = stats.MemBytes, stats.SwapBytes, stats.MaxMemBytes
if mem == [4]uint64{} {
mem = [4]uint64{
uint64(stats.Mem * gb),
uint64(stats.MemUsed * gb),
uint64(stats.MemBuffCache * gb),
uint64(stats.MemZfsArc * gb),
}
}
if swap == [2]uint64{} {
swap = [2]uint64{uint64(stats.Swap * gb), uint64(stats.SwapUsed * gb)}
}
if maxUsed == 0 {
maxUsed = uint64(stats.MaxMem * gb)
}
return mem, swap, maxUsed
}
func bytesToGigabytes(b uint64) float64 {
return twoDecimals(float64(b) / gb)
}
func twoDecimals(value float64) float64 {
return math.Round(value*100) / 100
}
internal/records/records_averaging_test.go
+96 -80
View File
@@ -25,34 +25,28 @@ func TestAverageSystemStatsSlice_Empty(t *testing.T) {
func TestAverageSystemStatsSlice_SingleRecord(t *testing.T) {
input := []system.Stats{
{
Cpu: 45.67,
Mem: 16.0,
MemUsed: 8.5,
MemPct: 53.12,
MemBuffCache: 2.0,
Swap: 4.0,
SwapUsed: 1.0,
DiskTotal: 500.0,
DiskUsed: 250.0,
DiskPct: 50.0,
NetworkSent: 10.5,
NetworkRecv: 20.25,
LoadAvg: [3]float64{1.5, 2.0, 3.5},
Bandwidth: [2]uint64{1000, 2000},
DiskIO: [2]uint64{500, 600},
Battery: [2]uint8{80, 1},
Cpu: 45.67,
MemBytes: [4]uint64{16000, 8500, 2000, 0},
MemPct: 53.12,
SwapBytes: [2]uint64{4000, 1000},
DiskTotal: 500.0,
DiskUsed: 250.0,
DiskPct: 50.0,
NetworkSent: 10.5,
NetworkRecv: 20.25,
LoadAvg: [3]float64{1.5, 2.0, 3.5},
Bandwidth: [2]uint64{1000, 2000},
DiskIO: [2]uint64{500, 600},
Battery: [2]uint8{80, 1},
},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 45.67, result.Cpu)
assert.Equal(t, 16.0, result.Mem)
assert.Equal(t, 8.5, result.MemUsed)
assert.Equal(t, [4]uint64{16000, 8500, 2000, 0}, result.MemBytes)
assert.Equal(t, 53.12, result.MemPct)
assert.Equal(t, 2.0, result.MemBuffCache)
assert.Equal(t, 4.0, result.Swap)
assert.Equal(t, 1.0, result.SwapUsed)
assert.Equal(t, [2]uint64{4000, 1000}, result.SwapBytes)
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
assert.Equal(t, 50.0, result.DiskPct)
@@ -68,55 +62,43 @@ func TestAverageSystemStatsSlice_SingleRecord(t *testing.T) {
func TestAverageSystemStatsSlice_BasicAveraging(t *testing.T) {
input := []system.Stats{
{
Cpu: 20.0,
Mem: 16.0,
MemUsed: 6.0,
MemPct: 37.5,
MemBuffCache: 1.0,
MemZfsArc: 0.5,
Swap: 4.0,
SwapUsed: 1.0,
DiskTotal: 500.0,
DiskUsed: 200.0,
DiskPct: 40.0,
NetworkSent: 10.0,
NetworkRecv: 20.0,
LoadAvg: [3]float64{1.0, 2.0, 3.0},
Bandwidth: [2]uint64{1000, 2000},
DiskIO: [2]uint64{400, 600},
Battery: [2]uint8{80, 1},
Cpu: 20.0,
MemBytes: [4]uint64{16000, 6000, 1000, 500},
MemPct: 37.5,
SwapBytes: [2]uint64{4000, 1000},
DiskTotal: 500.0,
DiskUsed: 200.0,
DiskPct: 40.0,
NetworkSent: 10.0,
NetworkRecv: 20.0,
LoadAvg: [3]float64{1.0, 2.0, 3.0},
Bandwidth: [2]uint64{1000, 2000},
DiskIO: [2]uint64{400, 600},
Battery: [2]uint8{80, 1},
},
{
Cpu: 40.0,
Mem: 16.0,
MemUsed: 10.0,
MemPct: 62.5,
MemBuffCache: 3.0,
MemZfsArc: 1.5,
Swap: 4.0,
SwapUsed: 3.0,
DiskTotal: 500.0,
DiskUsed: 300.0,
DiskPct: 60.0,
NetworkSent: 30.0,
NetworkRecv: 40.0,
LoadAvg: [3]float64{3.0, 4.0, 5.0},
Bandwidth: [2]uint64{3000, 4000},
DiskIO: [2]uint64{600, 800},
Battery: [2]uint8{60, 1},
Cpu: 40.0,
MemBytes: [4]uint64{16000, 10000, 3000, 1500},
MemPct: 62.5,
SwapBytes: [2]uint64{4000, 3000},
DiskTotal: 500.0,
DiskUsed: 300.0,
DiskPct: 60.0,
NetworkSent: 30.0,
NetworkRecv: 40.0,
LoadAvg: [3]float64{3.0, 4.0, 5.0},
Bandwidth: [2]uint64{3000, 4000},
DiskIO: [2]uint64{600, 800},
Battery: [2]uint8{60, 1},
},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 30.0, result.Cpu)
assert.Equal(t, 16.0, result.Mem)
assert.Equal(t, 8.0, result.MemUsed)
assert.Equal(t, [4]uint64{16000, 8000, 2000, 1000}, result.MemBytes)
assert.Equal(t, 50.0, result.MemPct)
assert.Equal(t, 2.0, result.MemBuffCache)
assert.Equal(t, 1.0, result.MemZfsArc)
assert.Equal(t, 4.0, result.Swap)
assert.Equal(t, 2.0, result.SwapUsed)
assert.Equal(t, [2]uint64{4000, 2000}, result.SwapBytes)
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
assert.Equal(t, 50.0, result.DiskPct)
@@ -134,8 +116,8 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
{
Cpu: 20.0,
MaxCpu: 25.0,
MemUsed: 6.0,
MaxMem: 7.0,
MemBytes: [4]uint64{16000, 6000, 0, 0},
MaxMemBytes: 7000,
NetworkSent: 10.0,
MaxNetworkSent: 15.0,
NetworkRecv: 20.0,
@@ -150,8 +132,8 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
{
Cpu: 40.0,
MaxCpu: 50.0,
MemUsed: 10.0,
MaxMem: 12.0,
MemBytes: [4]uint64{16000, 10000, 0, 0},
MaxMemBytes: 12000,
NetworkSent: 30.0,
MaxNetworkSent: 35.0,
NetworkRecv: 40.0,
@@ -168,7 +150,7 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 50.0, result.MaxCpu)
assert.Equal(t, 12.0, result.MaxMem)
assert.Equal(t, uint64(12000), result.MaxMemBytes)
assert.Equal(t, 35.0, result.MaxNetworkSent)
assert.Equal(t, 45.0, result.MaxNetworkRecv)
assert.Equal(t, [2]uint64{3500, 4500}, result.MaxBandwidth)
@@ -236,14 +218,49 @@ func TestAverageSystemStatsSlice_DiskIoStatsPeakFromCurrentValues(t *testing.T)
// (i.e., current cpu > MaxCpu should still win).
func TestAverageSystemStatsSlice_PeakFromCurrentValues(t *testing.T) {
input := []system.Stats{
{Cpu: 95.0, MaxCpu: 80.0, MemUsed: 15.0, MaxMem: 10.0},
{Cpu: 10.0, MaxCpu: 20.0, MemUsed: 5.0, MaxMem: 8.0},
{Cpu: 95.0, MaxCpu: 80.0, MemBytes: [4]uint64{20000, 15000, 0, 0}, MaxMemBytes: 10000},
{Cpu: 10.0, MaxCpu: 20.0, MemBytes: [4]uint64{20000, 5000, 0, 0}, MaxMemBytes: 8000},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 95.0, result.MaxCpu)
assert.Equal(t, 15.0, result.MaxMem)
assert.Equal(t, uint64(15000), result.MaxMemBytes)
}
// Tests that GB memory fields from records created before 0.21.1 are converted to bytes,
// including when averaged together with newer records.
func TestAverageSystemStatsSlice_LegacyMemoryFields(t *testing.T) {
const gb = 1024 * 1024 * 1024
input := []system.Stats{
{Mem: 16, MemUsed: 6, MemBuffCache: 2, MemZfsArc: 1, MaxMem: 7, Swap: 4, SwapUsed: 1},
{MemBytes: [4]uint64{16 * gb, 10 * gb, 4 * gb, 3 * gb}, SwapBytes: [2]uint64{4 * gb, 3 * gb}},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, [4]uint64{16 * gb, 8 * gb, 3 * gb, 2 * gb}, result.MemBytes)
assert.Equal(t, [2]uint64{4 * gb, 2 * gb}, result.SwapBytes)
assert.Equal(t, uint64(10*gb), result.MaxMemBytes)
}
// Tests that deprecated GB fields are derived from averaged bytes for third-party integrations.
func TestAverageSystemStatsSlice_DeprecatedMemoryFields(t *testing.T) {
const gb = 1024 * 1024 * 1024
input := []system.Stats{
{MemBytes: [4]uint64{16 * gb, 6 * gb, 2 * gb, gb}, MaxMemBytes: 7 * gb, SwapBytes: [2]uint64{4 * gb, gb}},
{MemBytes: [4]uint64{16 * gb, 4 * gb, 3 * gb, 0}, SwapBytes: [2]uint64{4 * gb, 0}},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 16.0, result.Mem)
assert.Equal(t, 5.0, result.MemUsed)
assert.Equal(t, 2.5, result.MemBuffCache)
assert.Equal(t, 0.5, result.MemZfsArc)
assert.Equal(t, 7.0, result.MaxMem)
assert.Equal(t, 4.0, result.Swap)
assert.Equal(t, 0.5, result.SwapUsed)
}
// Tests that records without temperature data are excluded from the temperature average.
@@ -589,15 +606,15 @@ func TestAverageSystemStatsSlice_ZeroRepresentativeBattery(t *testing.T) {
func TestAverageSystemStatsSlice_ThreeRecordsRounding(t *testing.T) {
input := []system.Stats{
{Cpu: 10.0, Mem: 8.0},
{Cpu: 20.0, Mem: 8.0},
{Cpu: 30.0, Mem: 8.0},
{Cpu: 10.0, MemBytes: [4]uint64{8000, 1000, 0, 0}},
{Cpu: 20.0, MemBytes: [4]uint64{8000, 2000, 0, 0}},
{Cpu: 30.0, MemBytes: [4]uint64{8000, 2000, 0, 0}},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 20.0, result.Cpu)
assert.Equal(t, 8.0, result.Mem)
assert.Equal(t, [4]uint64{8000, 1666, 0, 0}, result.MemBytes)
}
// Tests records where some have optional fields and others don't.
@@ -664,10 +681,9 @@ func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
input := make([]system.Stats, 10)
for i := range input {
input[i] = system.Stats{
Cpu: float64(i * 10), // 0, 10, 20, ..., 90
Mem: 16.0,
MemUsed: float64(4 + i), // 4, 5, 6, ..., 13
MemPct: float64(25 + i), // 25, 26, ..., 34
Cpu: float64(i * 10), // 0, 10, 20, ..., 90
MemBytes: [4]uint64{16000, uint64(4000 + i*1000), 0, 0}, // used 4000, 5000, ..., 13000
MemPct: float64(25 + i), // 25, 26, ..., 34
DiskTotal: 500.0,
DiskUsed: 250.0,
DiskPct: 50.0,
@@ -680,9 +696,9 @@ func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 45.0, result.Cpu) // avg of 0..90
assert.Equal(t, 16.0, result.Mem) // constant
assert.Equal(t, 8.5, result.MemUsed) // avg of 4..13
assert.Equal(t, 45.0, result.Cpu) // avg of 0..90
assert.Equal(t, [4]uint64{16000, 8500, 0, 0}, result.MemBytes) // used is avg of 4000..13000
assert.Equal(t, uint64(13000), result.MaxMemBytes)
assert.Equal(t, 29.5, result.MemPct) // avg of 25..34
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
internal/site/src/components/routes/system/charts/memory-charts.tsx
+39 -24
View File
@@ -8,6 +8,23 @@ import { ChartCard, FilterBar, SelectAvgMax } from "../chart-card"
import { dockerOrPodman } from "../chart-data"
import { decimalString, formatBytes, toFixedFloat } from "@/lib/utils"
/**
* Converts deprecated GB memory values from records before 0.21.1 to bytes.
* TODO: remove when the deprecated fields are removed. Also needed for realtime data from
* agents < 0.21.1, which is not migrated by the hub, and for stored records (kept up to 30 days).
*/
const gbToBytes = (gb?: number) => (gb === undefined ? undefined : gb * 1024 ** 3)
const tickFormatter = (value: number) => {
const { value: convertedValue, unit } = formatBytes(value, false, Unit.Bytes)
return `${toFixedFloat(convertedValue, convertedValue >= 10 ? 0 : 1)} ${unit}`
}
const contentFormatter = ({ value }: { value: number }) => {
const { value: convertedValue, unit } = formatBytes(value, false, Unit.Bytes)
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
}
export function MemoryChart({
chartData,
grid,
@@ -24,7 +41,8 @@ export function MemoryChart({
maxValues: boolean
}) {
const maxValSelect = isLongerChart ? <SelectAvgMax max={maxValues} /> : null
const totalMem = toFixedFloat(chartData.systemStats.at(-1)?.stats.m ?? 0, 1)
const lastStats = chartData.systemStats.at(-1)?.stats
const totalMem = lastStats?.mem?.[0] ?? gbToBytes(lastStats?.m) ?? 0
return (
<ChartCard
@@ -40,18 +58,13 @@ export function MemoryChart({
itemSorter={(a, b) => a.order - b.order}
maxToggled={showMax}
showTotal={true}
tickFormatter={(value) => {
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
return `${toFixedFloat(convertedValue, value >= 10 ? 0 : 1)} ${unit}`
}}
contentFormatter={({ value }) => {
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
}}
tickFormatter={tickFormatter}
contentFormatter={contentFormatter}
dataPoints={[
{
label: t`Used`,
dataKey: ({ stats }) => (showMax ? stats?.mm : stats?.mu),
dataKey: ({ stats }) =>
showMax ? (stats?.memm ?? gbToBytes(stats?.mm)) : (stats?.mem?.[1] ?? gbToBytes(stats?.mu)),
color: 2,
opacity: 0.4,
stackId: "1",
@@ -59,14 +72,16 @@ export function MemoryChart({
},
{
label: "ZFS ARC",
dataKey: ({ stats }) => (showMax ? null : stats?.mz),
// undefined when zero to keep ZFS ARC out of the tooltip on systems without ZFS
dataKey: ({ stats }) => (showMax ? null : (stats?.mem?.[3] ?? gbToBytes(stats?.mz)) || undefined),
color: "hsla(175 60% 45% / 0.8)",
opacity: 0.5,
stackId: "1",
order: 2,
},
{
label: t`Cache / Buffers`,
dataKey: ({ stats }) => (showMax ? null : stats?.mb),
dataKey: ({ stats }) => (showMax ? null : (stats?.mem?.[2] ?? gbToBytes(stats?.mb))),
color: "hsla(160 60% 45% / 0.5)",
opacity: 0.4,
stackId: "1",
@@ -136,7 +151,8 @@ export function SwapChart({
}) {
// const userSettings = useStore($userSettings)
const hasSwapData = (systemStats.at(-1)?.stats.su ?? 0) > 0
const lastStats = systemStats.at(-1)?.stats
const hasSwapData = (lastStats?.swap?.[1] ?? gbToBytes(lastStats?.su) ?? 0) > 0
if (!hasSwapData) {
return null
}
@@ -144,20 +160,19 @@ export function SwapChart({
<ChartCard empty={dataEmpty} grid={grid} title={t`Swap Usage`} description={t`Swap space used by the system`}>
<AreaChartDefault
chartData={chartData}
domain={[0, () => toFixedFloat(chartData.systemStats.at(-1)?.stats.s ?? 0.04, 2)]}
contentFormatter={({ value }) => {
// mem values are supplied as GB
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
}}
tickFormatter={(value) => {
const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
return `${toFixedFloat(convertedValue, value >= 10 ? 0 : 1)} ${unit}`
}}
domain={[
0,
() => {
const stats = chartData.systemStats.at(-1)?.stats
return stats?.swap?.[0] ?? gbToBytes(stats?.s) ?? 0
},
]}
contentFormatter={contentFormatter}
tickFormatter={tickFormatter}
dataPoints={[
{
label: t`Used`,
dataKey: ({ stats }) => stats?.su,
dataKey: ({ stats }) => stats?.swap?.[1] ?? gbToBytes(stats?.su),
color: 2,
opacity: 0.4,
},
internal/site/src/types.d.ts
Vendored
+18 -12
View File
@@ -104,22 +104,28 @@ export interface SystemStats {
cpus?: number[]
/** load average */
la?: [number, number, number]
/** total memory (gb) */
m: number
/** memory used (gb) */
mu: number
/** memory bytes [total, used, buffer + cache, zfs arc] */
mem?: [number, number, number, number]
/** max used memory (bytes) */
memm?: number
/** swap bytes [total, used] */
swap?: [number, number]
/** @deprecated total memory (gb) - records before 0.21.1 */
m?: number
/** @deprecated memory used (gb) - records before 0.21.1 */
mu?: number
/** memory percent */
mp: number
/** memory buffer + cache (gb) */
mb: number
/** max used memory (gb) */
/** @deprecated memory buffer + cache (gb) - records before 0.21.1 */
mb?: number
/** @deprecated max used memory (gb) - records before 0.21.1 */
mm?: number
/** zfs arc memory (gb) */
/** @deprecated zfs arc memory (gb) - records before 0.21.1 */
mz?: number
/** swap space (gb) */
s: number
/** swap used (gb) */
su: number
/** @deprecated swap space (gb) - records before 0.21.1 */
s?: number
/** @deprecated swap used (gb) - records before 0.21.1 */
su?: number
/** disk size (gb) */
d: number
/** disk used (gb) */
supplemental/CHANGELOG.md
+19
View File
@@ -2,6 +2,25 @@
For developers using the agent as a Go package, `Agent.Start` now requires a `context.Context`.
For authors of integrations, please note that existing memory fields other than memory percentage are deprecated in favor of new fields which provide raw byte values. The deprecated fields are still populated, but will be removed in a future version.
```go
type Stats struct {
// Deprecated fields (populated on hubs <= 0.21.1; will be removed in a future version)
Mem float64 `json:"m" cbor:"2,keyasint"` // Deprecated: GB, use MemBytes. Memory GB fields are kept for older hubs and integrations.
MaxMem float64 `json:"mm,omitzero" cbor:"-"` // Deprecated: GB, use MaxMemBytes.
MemUsed float64 `json:"mu" cbor:"3,keyasint"` // Deprecated: GB, use MemBytes.
MemBuffCache float64 `json:"mb" cbor:"5,keyasint"` // Deprecated: GB, use MemBytes.
MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // Deprecated: GB, use MemBytes.
Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
// New fields (populated on hub version 0.21.1+ for all agents)
MemBytes [4]uint64 `json:"mem,omitzero" cbor:"41,keyasint,omitzero"` // [total, used, buff/cache, zfs arc] bytes
MaxMemBytes uint64 `json:"memm,omitzero" cbor:"-"` // max used memory bytes
SwapBytes [2]uint64 `json:"swap,omitzero" cbor:"42,keyasint,omitzero"` // [total, used] bytes
}
```
- Add opt-in cgroup CPU and memory accounting (#2505)
- Add `HUB_HEADERS` environment variable to set custom WebSocket headers on the agent (#2585)