diff --git a/agent/memory.go b/agent/memory.go
index 4d21b7041..ebe2ad9d6 100644
--- a/agent/memory.go
+++ b/agent/memory.go
@@ -21,10 +21,13 @@ func (a *Agent) updateMemoryStats(stats *system.Stats) {
if err == nil {
hostTotal = v.Total
// Swap retains its existing /proc/meminfo source in both memory paths.
- stats.Swap = utils.BytesToGigabytes(v.SwapTotal)
- stats.SwapUsed = utils.BytesToGigabytes(saturatingSub(v.SwapTotal, v.SwapFree))
+ stats.SwapBytes = [2]uint64{v.SwapTotal, saturatingSub(v.SwapTotal, v.SwapFree)}
+ // TODO: stop populating deprecated GB fields once hubs < 0.21.1 are no longer supported
+ stats.Swap = utils.BytesToGigabytes(stats.SwapBytes[0])
+ stats.SwapUsed = utils.BytesToGigabytes(stats.SwapBytes[1])
}
+ var arcSize uint64
metrics, fromCgroup := containerMemoryMetrics(hostTotal, a.forceUseCgroup)
if !fromCgroup {
if err != nil {
@@ -34,16 +37,19 @@ func (a *Agent) updateMemoryStats(stats *system.Stats) {
metrics = memoryMetrics{Total: v.Total, Used: used, BuffCache: cacheBuff}
// Host ARC must not be subtracted from container-scoped accounting.
if a.zfs {
- if arcSize, _ := zfs.ARCSize(); arcSize > 0 && arcSize < metrics.Used {
+ if size, _ := zfs.ARCSize(); size > 0 && size < metrics.Used {
+ arcSize = size
metrics.Used -= arcSize
- stats.MemZfsArc = utils.BytesToGigabytes(arcSize)
}
}
}
+ stats.MemBytes = [4]uint64{metrics.Total, metrics.Used, metrics.BuffCache, arcSize}
+ // TODO: stop populating deprecated GB fields once hubs < 0.21.1 are no longer supported
stats.Mem = utils.BytesToGigabytes(metrics.Total)
stats.MemUsed = utils.BytesToGigabytes(metrics.Used)
stats.MemBuffCache = utils.BytesToGigabytes(metrics.BuffCache)
+ stats.MemZfsArc = utils.BytesToGigabytes(arcSize)
if metrics.Total > 0 {
stats.MemPct = utils.TwoDecimals(float64(metrics.Used) / float64(metrics.Total) * 100)
}
diff --git a/agent/memory_linux_test.go b/agent/memory_linux_test.go
index 40ec3e4d9..93615af1c 100644
--- a/agent/memory_linux_test.go
+++ b/agent/memory_linux_test.go
@@ -146,6 +146,8 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Zero(t, stats.MemZfsArc)
assert.Equal(t, float64(2), stats.Swap)
assert.Equal(t, float64(1), stats.SwapUsed)
+ assert.Equal(t, [4]uint64{10 * gib, 4 * gib, 2 * gib, 0}, stats.MemBytes)
+ assert.Equal(t, [2]uint64{2 * gib, gib}, stats.SwapBytes)
assert.Equal(t, 10*gib, a.systemDetails.MemoryTotal)
assert.True(t, a.detailsDirty)
@@ -166,6 +168,7 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Equal(t, float64(12), stats.MemUsed)
assert.Equal(t, float64(4), stats.MemBuffCache)
assert.Equal(t, float64(60), stats.MemPct)
+ assert.Equal(t, [4]uint64{20 * gib, 12 * gib, 4 * gib, 0}, stats.MemBytes)
assert.Equal(t, 20*gib, a.systemDetails.MemoryTotal)
// Finite cgroup limits still work if /proc/meminfo cannot be read.
@@ -176,7 +179,9 @@ func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
assert.Equal(t, float64(8), stats.Mem)
assert.Equal(t, float64(6), stats.MemUsed)
assert.Equal(t, float64(75), stats.MemPct)
+ assert.Equal(t, [4]uint64{8 * gib, 6 * gib, 0, 0}, stats.MemBytes)
assert.Zero(t, stats.Swap)
+ assert.Zero(t, stats.SwapBytes)
}
func TestMemoryDetailsInitiallyUseCgroupLimit(t *testing.T) {
@@ -218,4 +223,24 @@ func TestMemoryStatsExcludeActiveFileCache(t *testing.T) {
assert.Equal(t, float64(17.82), stats.MemUsed)
assert.Equal(t, float64(108.73), stats.MemBuffCache)
assert.Equal(t, float64(13.92), stats.MemPct)
+ assert.Equal(t, [4]uint64{137438953472, usage - (cache - shmem), cache - shmem, 0}, stats.MemBytes)
+}
+
+// Low memory systems need byte precision because the GB fields round to 10 MiB.
+func TestMemoryStatsLowMemoryPrecision(t *testing.T) {
+ dir := setupMemoryCgroup(t, true)
+ const mib = uint64(1 << 20)
+ writeMemoryFixture(t, dir, true, strconv.FormatUint(5*mib, 10), strconv.FormatUint(64*mib, 10),
+ "file "+strconv.FormatUint(2*mib, 10)+"\nshmem 0\n")
+ original := memoryVirtualMemory
+ t.Cleanup(func() { memoryVirtualMemory = original })
+ memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
+ return &mem.VirtualMemoryStat{Total: 128 * mib, SwapTotal: 32 * mib, SwapFree: 31 * mib}, nil
+ }
+ var stats system.Stats
+ a := &Agent{forceUseCgroup: true}
+ a.updateMemoryStats(&stats)
+ assert.Equal(t, [4]uint64{64 * mib, 3 * mib, 2 * mib, 0}, stats.MemBytes)
+ assert.Equal(t, [2]uint64{32 * mib, mib}, stats.SwapBytes)
+ assert.Equal(t, float64(0), stats.MemUsed)
}
diff --git a/internal/entities/system/system.go b/internal/entities/system/system.go
index b6b6168f0..002bdc8fc 100644
--- a/internal/entities/system/system.go
+++ b/internal/entities/system/system.go
@@ -22,14 +22,14 @@ type WiFi struct {
type Stats struct {
Cpu float64 `json:"cpu" cbor:"0,keyasint"`
MaxCpu float64 `json:"cpum,omitzero" cbor:"-"`
- Mem float64 `json:"m" cbor:"2,keyasint"`
- MaxMem float64 `json:"mm,omitzero" cbor:"-"`
- MemUsed float64 `json:"mu" cbor:"3,keyasint"`
+ Mem float64 `json:"m" cbor:"2,keyasint"` // Deprecated: GB, use MemBytes. Memory GB fields are kept for older hubs and integrations.
+ MaxMem float64 `json:"mm,omitzero" cbor:"-"` // Deprecated: GB, use MaxMemBytes.
+ MemUsed float64 `json:"mu" cbor:"3,keyasint"` // Deprecated: GB, use MemBytes.
MemPct float64 `json:"mp" cbor:"4,keyasint"`
- MemBuffCache float64 `json:"mb" cbor:"5,keyasint"`
- MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // ZFS ARC memory
- Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"`
- SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"`
+ MemBuffCache float64 `json:"mb" cbor:"5,keyasint"` // Deprecated: GB, use MemBytes.
+ MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // Deprecated: GB, use MemBytes.
+ Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
+ SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
DiskTotal float64 `json:"d" cbor:"9,keyasint"`
DiskUsed float64 `json:"du" cbor:"10,keyasint"`
DiskPct float64 `json:"dp" cbor:"11,keyasint"`
@@ -59,7 +59,9 @@ type Stats struct {
ZfsPools map[string]*ZfsPool `json:"z,omitempty" cbor:"39,keyasint,omitempty"` // ZFS pool metrics, keyed by pool name
DiskIOTotal [2]uint64 `json:"diot,omitzero" cbor:"38,keyasint,omitzero"` // [total read bytes, total write bytes] cumulative device counters
WiFi map[string]int8 `json:"wf,omitempty" cbor:"40,keyasint,omitempty"` // RSSI dBm keyed by interface; unavailable readings omitted
-
+ MemBytes [4]uint64 `json:"mem,omitzero" cbor:"41,keyasint,omitzero"` // [total, used, buff/cache, zfs arc] bytes
+ MaxMemBytes uint64 `json:"memm,omitzero" cbor:"-"` // max used memory bytes
+ SwapBytes [2]uint64 `json:"swap,omitzero" cbor:"42,keyasint,omitzero"` // [total, used] bytes
}
// ZfsPool holds per-pool ZFS metrics for a single collection interval.
diff --git a/internal/hub/systems/system.go b/internal/hub/systems/system.go
index 75917e096..86b17cda2 100644
--- a/internal/hub/systems/system.go
+++ b/internal/hub/systems/system.go
@@ -1122,6 +1122,20 @@ func migrateDeprecatedFields(cd *system.CombinedData, createDetails bool) {
}
fs.DiskReadPs, fs.DiskWritePs = 0, 0
}
+ // migration added 0.21.1
+ const gb = 1024 * 1024 * 1024
+ if cd.Stats.MemBytes == [4]uint64{} {
+ cd.Stats.MemBytes = [4]uint64{
+ uint64(cd.Stats.Mem * gb),
+ uint64(cd.Stats.MemUsed * gb),
+ uint64(cd.Stats.MemBuffCache * gb),
+ uint64(cd.Stats.MemZfsArc * gb),
+ }
+ }
+ if cd.Stats.SwapBytes == [2]uint64{} {
+ cd.Stats.SwapBytes = [2]uint64{uint64(cd.Stats.Swap * gb), uint64(cd.Stats.SwapUsed * gb)}
+ }
+ // GB fields are kept in stored records for third-party integrations until they are removed
// migration added 0.19.0 - Move deprecated Info fields to Details struct
if cd.Details == nil && cd.Info.Hostname != "" {
if createDetails {
@@ -1133,7 +1147,7 @@ func migrateDeprecatedFields(cd *system.CombinedData, createDetails bool) {
CpuModel: cd.Info.CpuModel,
Podman: cd.Info.Podman,
Os: cd.Info.Os,
- MemoryTotal: uint64(cd.Stats.Mem * 1024 * 1024 * 1024),
+ MemoryTotal: cd.Stats.MemBytes[0],
}
}
// zero the deprecated fields to prevent saving them in systems.info DB json payload
diff --git a/internal/hub/systems/system_test.go b/internal/hub/systems/system_test.go
index c8c207b0f..f01ee7449 100644
--- a/internal/hub/systems/system_test.go
+++ b/internal/hub/systems/system_test.go
@@ -102,6 +102,39 @@ func TestCombinedData_MigrateDeprecatedFields(t *testing.T) {
}
})
+ t.Run("Migrate memory GB fields to bytes", func(t *testing.T) {
+ const gb = 1024 * 1024 * 1024
+ cd := &system.CombinedData{
+ Stats: system.Stats{Mem: 16, MemUsed: 4.5, MemBuffCache: 2, MemZfsArc: 1, Swap: 4, SwapUsed: 0.5},
+ }
+ migrateDeprecatedFields(cd, true)
+
+ require.Equal(t, [4]uint64{16 * gb, 4.5 * gb, 2 * gb, 1 * gb}, cd.Stats.MemBytes)
+ require.Equal(t, [2]uint64{4 * gb, 0.5 * gb}, cd.Stats.SwapBytes)
+ // GB fields are kept for third-party integrations
+ require.Equal(t, 16.0, cd.Stats.Mem)
+ require.Equal(t, 4.5, cd.Stats.MemUsed)
+ require.Equal(t, 0.5, cd.Stats.SwapUsed)
+ })
+
+ t.Run("Keep memory bytes and GB fields from newer agents", func(t *testing.T) {
+ cd := &system.CombinedData{
+ Stats: system.Stats{
+ Mem: 0.06, MemUsed: 0.01, Swap: 0.03, SwapUsed: 0.01,
+ MemBytes: [4]uint64{64 << 20, 12 << 20, 3 << 20, 0},
+ SwapBytes: [2]uint64{32 << 20, 1 << 20},
+ },
+ }
+ migrateDeprecatedFields(cd, true)
+
+ require.Equal(t, [4]uint64{64 << 20, 12 << 20, 3 << 20, 0}, cd.Stats.MemBytes)
+ require.Equal(t, [2]uint64{32 << 20, 1 << 20}, cd.Stats.SwapBytes)
+ require.Equal(t, 0.06, cd.Stats.Mem)
+ require.Equal(t, 0.01, cd.Stats.MemUsed)
+ require.Equal(t, 0.03, cd.Stats.Swap)
+ require.Equal(t, 0.01, cd.Stats.SwapUsed)
+ })
+
t.Run("Migrate Info fields to Details struct", func(t *testing.T) {
cd := &system.CombinedData{
Stats: system.Stats{
diff --git a/internal/records/records.go b/internal/records/records.go
index 991e16b79..94d58a2cb 100644
--- a/internal/records/records.go
+++ b/internal/records/records.go
@@ -302,13 +302,13 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
cpuBreakdownSums[j] += v
}
}
- sum.Mem += stats.Mem
- sum.MemUsed += stats.MemUsed
+ memBytes, swapBytes, maxMemBytes := memoryBytes(stats)
+ for j, v := range memBytes {
+ sum.MemBytes[j] += v
+ }
+ sum.SwapBytes[0] += swapBytes[0]
+ sum.SwapBytes[1] += swapBytes[1]
sum.MemPct += stats.MemPct
- sum.MemBuffCache += stats.MemBuffCache
- sum.MemZfsArc += stats.MemZfsArc
- sum.Swap += stats.Swap
- sum.SwapUsed += stats.SwapUsed
sum.DiskTotal += stats.DiskTotal
sum.DiskUsed += stats.DiskUsed
sum.DiskPct += stats.DiskPct
@@ -346,7 +346,7 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
}
// Set peak values
sum.MaxCpu = max(sum.MaxCpu, stats.MaxCpu, stats.Cpu)
- sum.MaxMem = max(sum.MaxMem, stats.MaxMem, stats.MemUsed)
+ sum.MaxMemBytes = max(sum.MaxMemBytes, maxMemBytes, memBytes[1])
sum.MaxNetworkSent = max(sum.MaxNetworkSent, stats.MaxNetworkSent, stats.NetworkSent)
sum.MaxNetworkRecv = max(sum.MaxNetworkRecv, stats.MaxNetworkRecv, stats.NetworkRecv)
sum.MaxBandwidth[0] = max(sum.MaxBandwidth[0], stats.MaxBandwidth[0], stats.Bandwidth[0])
@@ -487,13 +487,20 @@ func AverageSystemStatsSlice(records []system.Stats) system.Stats {
// Compute averages
sum.Cpu = twoDecimals(sum.Cpu / count)
- sum.Mem = twoDecimals(sum.Mem / count)
- sum.MemUsed = twoDecimals(sum.MemUsed / count)
+ for i := range sum.MemBytes {
+ sum.MemBytes[i] = sum.MemBytes[i] / uint64(count)
+ }
sum.MemPct = twoDecimals(sum.MemPct / count)
- sum.MemBuffCache = twoDecimals(sum.MemBuffCache / count)
- sum.MemZfsArc = twoDecimals(sum.MemZfsArc / count)
- sum.Swap = twoDecimals(sum.Swap / count)
- sum.SwapUsed = twoDecimals(sum.SwapUsed / count)
+ sum.SwapBytes[0] = sum.SwapBytes[0] / uint64(count)
+ sum.SwapBytes[1] = sum.SwapBytes[1] / uint64(count)
+ // TODO: stop populating deprecated GB fields when they are removed from the agent
+ sum.Mem = bytesToGigabytes(sum.MemBytes[0])
+ sum.MemUsed = bytesToGigabytes(sum.MemBytes[1])
+ sum.MemBuffCache = bytesToGigabytes(sum.MemBytes[2])
+ sum.MemZfsArc = bytesToGigabytes(sum.MemBytes[3])
+ sum.MaxMem = bytesToGigabytes(sum.MaxMemBytes)
+ sum.Swap = bytesToGigabytes(sum.SwapBytes[0])
+ sum.SwapUsed = bytesToGigabytes(sum.SwapBytes[1])
sum.DiskTotal = twoDecimals(sum.DiskTotal / count)
sum.DiskUsed = twoDecimals(sum.DiskUsed / count)
sum.DiskPct = twoDecimals(sum.DiskPct / count)
@@ -711,6 +718,36 @@ func (rm *RecordManager) AverageMonitorStats(db dbx.Builder, monitorID, recordTy
}
/* Round float to two decimals */
+const gb = 1024 * 1024 * 1024
+
+// memoryBytes returns memory bytes, swap bytes, and max used memory bytes for a record,
+// converting the deprecated GB fields of records created before 0.21.1.
+//
+// TODO: remove the GB conversion when the deprecated fields are removed. Hubs upgraded from
+// versions before 0.21.1 average pre-upgrade records only in the first few hours after upgrade.
+func memoryBytes(stats *system.Stats) (mem [4]uint64, swap [2]uint64, maxUsed uint64) {
+ mem, swap, maxUsed = stats.MemBytes, stats.SwapBytes, stats.MaxMemBytes
+ if mem == [4]uint64{} {
+ mem = [4]uint64{
+ uint64(stats.Mem * gb),
+ uint64(stats.MemUsed * gb),
+ uint64(stats.MemBuffCache * gb),
+ uint64(stats.MemZfsArc * gb),
+ }
+ }
+ if swap == [2]uint64{} {
+ swap = [2]uint64{uint64(stats.Swap * gb), uint64(stats.SwapUsed * gb)}
+ }
+ if maxUsed == 0 {
+ maxUsed = uint64(stats.MaxMem * gb)
+ }
+ return mem, swap, maxUsed
+}
+
+func bytesToGigabytes(b uint64) float64 {
+ return twoDecimals(float64(b) / gb)
+}
+
func twoDecimals(value float64) float64 {
return math.Round(value*100) / 100
}
diff --git a/internal/records/records_averaging_test.go b/internal/records/records_averaging_test.go
index 1ab1b5a42..b48b38ced 100644
--- a/internal/records/records_averaging_test.go
+++ b/internal/records/records_averaging_test.go
@@ -25,34 +25,28 @@ func TestAverageSystemStatsSlice_Empty(t *testing.T) {
func TestAverageSystemStatsSlice_SingleRecord(t *testing.T) {
input := []system.Stats{
{
- Cpu: 45.67,
- Mem: 16.0,
- MemUsed: 8.5,
- MemPct: 53.12,
- MemBuffCache: 2.0,
- Swap: 4.0,
- SwapUsed: 1.0,
- DiskTotal: 500.0,
- DiskUsed: 250.0,
- DiskPct: 50.0,
- NetworkSent: 10.5,
- NetworkRecv: 20.25,
- LoadAvg: [3]float64{1.5, 2.0, 3.5},
- Bandwidth: [2]uint64{1000, 2000},
- DiskIO: [2]uint64{500, 600},
- Battery: [2]uint8{80, 1},
+ Cpu: 45.67,
+ MemBytes: [4]uint64{16000, 8500, 2000, 0},
+ MemPct: 53.12,
+ SwapBytes: [2]uint64{4000, 1000},
+ DiskTotal: 500.0,
+ DiskUsed: 250.0,
+ DiskPct: 50.0,
+ NetworkSent: 10.5,
+ NetworkRecv: 20.25,
+ LoadAvg: [3]float64{1.5, 2.0, 3.5},
+ Bandwidth: [2]uint64{1000, 2000},
+ DiskIO: [2]uint64{500, 600},
+ Battery: [2]uint8{80, 1},
},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 45.67, result.Cpu)
- assert.Equal(t, 16.0, result.Mem)
- assert.Equal(t, 8.5, result.MemUsed)
+ assert.Equal(t, [4]uint64{16000, 8500, 2000, 0}, result.MemBytes)
assert.Equal(t, 53.12, result.MemPct)
- assert.Equal(t, 2.0, result.MemBuffCache)
- assert.Equal(t, 4.0, result.Swap)
- assert.Equal(t, 1.0, result.SwapUsed)
+ assert.Equal(t, [2]uint64{4000, 1000}, result.SwapBytes)
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
assert.Equal(t, 50.0, result.DiskPct)
@@ -68,55 +62,43 @@ func TestAverageSystemStatsSlice_SingleRecord(t *testing.T) {
func TestAverageSystemStatsSlice_BasicAveraging(t *testing.T) {
input := []system.Stats{
{
- Cpu: 20.0,
- Mem: 16.0,
- MemUsed: 6.0,
- MemPct: 37.5,
- MemBuffCache: 1.0,
- MemZfsArc: 0.5,
- Swap: 4.0,
- SwapUsed: 1.0,
- DiskTotal: 500.0,
- DiskUsed: 200.0,
- DiskPct: 40.0,
- NetworkSent: 10.0,
- NetworkRecv: 20.0,
- LoadAvg: [3]float64{1.0, 2.0, 3.0},
- Bandwidth: [2]uint64{1000, 2000},
- DiskIO: [2]uint64{400, 600},
- Battery: [2]uint8{80, 1},
+ Cpu: 20.0,
+ MemBytes: [4]uint64{16000, 6000, 1000, 500},
+ MemPct: 37.5,
+ SwapBytes: [2]uint64{4000, 1000},
+ DiskTotal: 500.0,
+ DiskUsed: 200.0,
+ DiskPct: 40.0,
+ NetworkSent: 10.0,
+ NetworkRecv: 20.0,
+ LoadAvg: [3]float64{1.0, 2.0, 3.0},
+ Bandwidth: [2]uint64{1000, 2000},
+ DiskIO: [2]uint64{400, 600},
+ Battery: [2]uint8{80, 1},
},
{
- Cpu: 40.0,
- Mem: 16.0,
- MemUsed: 10.0,
- MemPct: 62.5,
- MemBuffCache: 3.0,
- MemZfsArc: 1.5,
- Swap: 4.0,
- SwapUsed: 3.0,
- DiskTotal: 500.0,
- DiskUsed: 300.0,
- DiskPct: 60.0,
- NetworkSent: 30.0,
- NetworkRecv: 40.0,
- LoadAvg: [3]float64{3.0, 4.0, 5.0},
- Bandwidth: [2]uint64{3000, 4000},
- DiskIO: [2]uint64{600, 800},
- Battery: [2]uint8{60, 1},
+ Cpu: 40.0,
+ MemBytes: [4]uint64{16000, 10000, 3000, 1500},
+ MemPct: 62.5,
+ SwapBytes: [2]uint64{4000, 3000},
+ DiskTotal: 500.0,
+ DiskUsed: 300.0,
+ DiskPct: 60.0,
+ NetworkSent: 30.0,
+ NetworkRecv: 40.0,
+ LoadAvg: [3]float64{3.0, 4.0, 5.0},
+ Bandwidth: [2]uint64{3000, 4000},
+ DiskIO: [2]uint64{600, 800},
+ Battery: [2]uint8{60, 1},
},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 30.0, result.Cpu)
- assert.Equal(t, 16.0, result.Mem)
- assert.Equal(t, 8.0, result.MemUsed)
+ assert.Equal(t, [4]uint64{16000, 8000, 2000, 1000}, result.MemBytes)
assert.Equal(t, 50.0, result.MemPct)
- assert.Equal(t, 2.0, result.MemBuffCache)
- assert.Equal(t, 1.0, result.MemZfsArc)
- assert.Equal(t, 4.0, result.Swap)
- assert.Equal(t, 2.0, result.SwapUsed)
+ assert.Equal(t, [2]uint64{4000, 2000}, result.SwapBytes)
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
assert.Equal(t, 50.0, result.DiskPct)
@@ -134,8 +116,8 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
{
Cpu: 20.0,
MaxCpu: 25.0,
- MemUsed: 6.0,
- MaxMem: 7.0,
+ MemBytes: [4]uint64{16000, 6000, 0, 0},
+ MaxMemBytes: 7000,
NetworkSent: 10.0,
MaxNetworkSent: 15.0,
NetworkRecv: 20.0,
@@ -150,8 +132,8 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
{
Cpu: 40.0,
MaxCpu: 50.0,
- MemUsed: 10.0,
- MaxMem: 12.0,
+ MemBytes: [4]uint64{16000, 10000, 0, 0},
+ MaxMemBytes: 12000,
NetworkSent: 30.0,
MaxNetworkSent: 35.0,
NetworkRecv: 40.0,
@@ -168,7 +150,7 @@ func TestAverageSystemStatsSlice_PeakValues(t *testing.T) {
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 50.0, result.MaxCpu)
- assert.Equal(t, 12.0, result.MaxMem)
+ assert.Equal(t, uint64(12000), result.MaxMemBytes)
assert.Equal(t, 35.0, result.MaxNetworkSent)
assert.Equal(t, 45.0, result.MaxNetworkRecv)
assert.Equal(t, [2]uint64{3500, 4500}, result.MaxBandwidth)
@@ -236,14 +218,49 @@ func TestAverageSystemStatsSlice_DiskIoStatsPeakFromCurrentValues(t *testing.T)
// (i.e., current cpu > MaxCpu should still win).
func TestAverageSystemStatsSlice_PeakFromCurrentValues(t *testing.T) {
input := []system.Stats{
- {Cpu: 95.0, MaxCpu: 80.0, MemUsed: 15.0, MaxMem: 10.0},
- {Cpu: 10.0, MaxCpu: 20.0, MemUsed: 5.0, MaxMem: 8.0},
+ {Cpu: 95.0, MaxCpu: 80.0, MemBytes: [4]uint64{20000, 15000, 0, 0}, MaxMemBytes: 10000},
+ {Cpu: 10.0, MaxCpu: 20.0, MemBytes: [4]uint64{20000, 5000, 0, 0}, MaxMemBytes: 8000},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 95.0, result.MaxCpu)
- assert.Equal(t, 15.0, result.MaxMem)
+ assert.Equal(t, uint64(15000), result.MaxMemBytes)
+}
+
+// Tests that GB memory fields from records created before 0.21.1 are converted to bytes,
+// including when averaged together with newer records.
+func TestAverageSystemStatsSlice_LegacyMemoryFields(t *testing.T) {
+ const gb = 1024 * 1024 * 1024
+ input := []system.Stats{
+ {Mem: 16, MemUsed: 6, MemBuffCache: 2, MemZfsArc: 1, MaxMem: 7, Swap: 4, SwapUsed: 1},
+ {MemBytes: [4]uint64{16 * gb, 10 * gb, 4 * gb, 3 * gb}, SwapBytes: [2]uint64{4 * gb, 3 * gb}},
+ }
+
+ result := records.AverageSystemStatsSlice(input)
+
+ assert.Equal(t, [4]uint64{16 * gb, 8 * gb, 3 * gb, 2 * gb}, result.MemBytes)
+ assert.Equal(t, [2]uint64{4 * gb, 2 * gb}, result.SwapBytes)
+ assert.Equal(t, uint64(10*gb), result.MaxMemBytes)
+}
+
+// Tests that deprecated GB fields are derived from averaged bytes for third-party integrations.
+func TestAverageSystemStatsSlice_DeprecatedMemoryFields(t *testing.T) {
+ const gb = 1024 * 1024 * 1024
+ input := []system.Stats{
+ {MemBytes: [4]uint64{16 * gb, 6 * gb, 2 * gb, gb}, MaxMemBytes: 7 * gb, SwapBytes: [2]uint64{4 * gb, gb}},
+ {MemBytes: [4]uint64{16 * gb, 4 * gb, 3 * gb, 0}, SwapBytes: [2]uint64{4 * gb, 0}},
+ }
+
+ result := records.AverageSystemStatsSlice(input)
+
+ assert.Equal(t, 16.0, result.Mem)
+ assert.Equal(t, 5.0, result.MemUsed)
+ assert.Equal(t, 2.5, result.MemBuffCache)
+ assert.Equal(t, 0.5, result.MemZfsArc)
+ assert.Equal(t, 7.0, result.MaxMem)
+ assert.Equal(t, 4.0, result.Swap)
+ assert.Equal(t, 0.5, result.SwapUsed)
}
// Tests that records without temperature data are excluded from the temperature average.
@@ -589,15 +606,15 @@ func TestAverageSystemStatsSlice_ZeroRepresentativeBattery(t *testing.T) {
func TestAverageSystemStatsSlice_ThreeRecordsRounding(t *testing.T) {
input := []system.Stats{
- {Cpu: 10.0, Mem: 8.0},
- {Cpu: 20.0, Mem: 8.0},
- {Cpu: 30.0, Mem: 8.0},
+ {Cpu: 10.0, MemBytes: [4]uint64{8000, 1000, 0, 0}},
+ {Cpu: 20.0, MemBytes: [4]uint64{8000, 2000, 0, 0}},
+ {Cpu: 30.0, MemBytes: [4]uint64{8000, 2000, 0, 0}},
}
result := records.AverageSystemStatsSlice(input)
assert.Equal(t, 20.0, result.Cpu)
- assert.Equal(t, 8.0, result.Mem)
+ assert.Equal(t, [4]uint64{8000, 1666, 0, 0}, result.MemBytes)
}
// Tests records where some have optional fields and others don't.
@@ -664,10 +681,9 @@ func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
input := make([]system.Stats, 10)
for i := range input {
input[i] = system.Stats{
- Cpu: float64(i * 10), // 0, 10, 20, ..., 90
- Mem: 16.0,
- MemUsed: float64(4 + i), // 4, 5, 6, ..., 13
- MemPct: float64(25 + i), // 25, 26, ..., 34
+ Cpu: float64(i * 10), // 0, 10, 20, ..., 90
+ MemBytes: [4]uint64{16000, uint64(4000 + i*1000), 0, 0}, // used 4000, 5000, ..., 13000
+ MemPct: float64(25 + i), // 25, 26, ..., 34
DiskTotal: 500.0,
DiskUsed: 250.0,
DiskPct: 50.0,
@@ -680,9 +696,9 @@ func TestAverageSystemStatsSlice_TenRecords(t *testing.T) {
result := records.AverageSystemStatsSlice(input)
- assert.Equal(t, 45.0, result.Cpu) // avg of 0..90
- assert.Equal(t, 16.0, result.Mem) // constant
- assert.Equal(t, 8.5, result.MemUsed) // avg of 4..13
+ assert.Equal(t, 45.0, result.Cpu) // avg of 0..90
+ assert.Equal(t, [4]uint64{16000, 8500, 0, 0}, result.MemBytes) // used is avg of 4000..13000
+ assert.Equal(t, uint64(13000), result.MaxMemBytes)
assert.Equal(t, 29.5, result.MemPct) // avg of 25..34
assert.Equal(t, 500.0, result.DiskTotal)
assert.Equal(t, 250.0, result.DiskUsed)
diff --git a/internal/site/src/components/routes/system/charts/memory-charts.tsx b/internal/site/src/components/routes/system/charts/memory-charts.tsx
index efdd35754..671a4cf48 100644
--- a/internal/site/src/components/routes/system/charts/memory-charts.tsx
+++ b/internal/site/src/components/routes/system/charts/memory-charts.tsx
@@ -8,6 +8,23 @@ import { ChartCard, FilterBar, SelectAvgMax } from "../chart-card"
import { dockerOrPodman } from "../chart-data"
import { decimalString, formatBytes, toFixedFloat } from "@/lib/utils"
+/**
+ * Converts deprecated GB memory values from records before 0.21.1 to bytes.
+ * TODO: remove when the deprecated fields are removed. Also needed for realtime data from
+ * agents < 0.21.1, which is not migrated by the hub, and for stored records (kept up to 30 days).
+ */
+const gbToBytes = (gb?: number) => (gb === undefined ? undefined : gb * 1024 ** 3)
+
+const tickFormatter = (value: number) => {
+ const { value: convertedValue, unit } = formatBytes(value, false, Unit.Bytes)
+ return `${toFixedFloat(convertedValue, convertedValue >= 10 ? 0 : 1)} ${unit}`
+}
+
+const contentFormatter = ({ value }: { value: number }) => {
+ const { value: convertedValue, unit } = formatBytes(value, false, Unit.Bytes)
+ return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
+}
+
export function MemoryChart({
chartData,
grid,
@@ -24,7 +41,8 @@ export function MemoryChart({
maxValues: boolean
}) {
const maxValSelect = isLongerChart ? : null
- const totalMem = toFixedFloat(chartData.systemStats.at(-1)?.stats.m ?? 0, 1)
+ const lastStats = chartData.systemStats.at(-1)?.stats
+ const totalMem = lastStats?.mem?.[0] ?? gbToBytes(lastStats?.m) ?? 0
return (
a.order - b.order}
maxToggled={showMax}
showTotal={true}
- tickFormatter={(value) => {
- const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
- return `${toFixedFloat(convertedValue, value >= 10 ? 0 : 1)} ${unit}`
- }}
- contentFormatter={({ value }) => {
- const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
- return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
- }}
+ tickFormatter={tickFormatter}
+ contentFormatter={contentFormatter}
dataPoints={[
{
label: t`Used`,
- dataKey: ({ stats }) => (showMax ? stats?.mm : stats?.mu),
+ dataKey: ({ stats }) =>
+ showMax ? (stats?.memm ?? gbToBytes(stats?.mm)) : (stats?.mem?.[1] ?? gbToBytes(stats?.mu)),
color: 2,
opacity: 0.4,
stackId: "1",
@@ -59,14 +72,16 @@ export function MemoryChart({
},
{
label: "ZFS ARC",
- dataKey: ({ stats }) => (showMax ? null : stats?.mz),
+ // undefined when zero to keep ZFS ARC out of the tooltip on systems without ZFS
+ dataKey: ({ stats }) => (showMax ? null : (stats?.mem?.[3] ?? gbToBytes(stats?.mz)) || undefined),
color: "hsla(175 60% 45% / 0.8)",
opacity: 0.5,
+ stackId: "1",
order: 2,
},
{
label: t`Cache / Buffers`,
- dataKey: ({ stats }) => (showMax ? null : stats?.mb),
+ dataKey: ({ stats }) => (showMax ? null : (stats?.mem?.[2] ?? gbToBytes(stats?.mb))),
color: "hsla(160 60% 45% / 0.5)",
opacity: 0.4,
stackId: "1",
@@ -136,7 +151,8 @@ export function SwapChart({
}) {
// const userSettings = useStore($userSettings)
- const hasSwapData = (systemStats.at(-1)?.stats.su ?? 0) > 0
+ const lastStats = systemStats.at(-1)?.stats
+ const hasSwapData = (lastStats?.swap?.[1] ?? gbToBytes(lastStats?.su) ?? 0) > 0
if (!hasSwapData) {
return null
}
@@ -144,20 +160,19 @@ export function SwapChart({
toFixedFloat(chartData.systemStats.at(-1)?.stats.s ?? 0.04, 2)]}
- contentFormatter={({ value }) => {
- // mem values are supplied as GB
- const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
- return `${decimalString(convertedValue, convertedValue >= 100 ? 1 : 2)} ${unit}`
- }}
- tickFormatter={(value) => {
- const { value: convertedValue, unit } = formatBytes(value * 1024, false, Unit.Bytes, true)
- return `${toFixedFloat(convertedValue, value >= 10 ? 0 : 1)} ${unit}`
- }}
+ domain={[
+ 0,
+ () => {
+ const stats = chartData.systemStats.at(-1)?.stats
+ return stats?.swap?.[0] ?? gbToBytes(stats?.s) ?? 0
+ },
+ ]}
+ contentFormatter={contentFormatter}
+ tickFormatter={tickFormatter}
dataPoints={[
{
label: t`Used`,
- dataKey: ({ stats }) => stats?.su,
+ dataKey: ({ stats }) => stats?.swap?.[1] ?? gbToBytes(stats?.su),
color: 2,
opacity: 0.4,
},
diff --git a/internal/site/src/types.d.ts b/internal/site/src/types.d.ts
index 92b920c9e..cb44d0816 100644
--- a/internal/site/src/types.d.ts
+++ b/internal/site/src/types.d.ts
@@ -104,22 +104,28 @@ export interface SystemStats {
cpus?: number[]
/** load average */
la?: [number, number, number]
- /** total memory (gb) */
- m: number
- /** memory used (gb) */
- mu: number
+ /** memory bytes [total, used, buffer + cache, zfs arc] */
+ mem?: [number, number, number, number]
+ /** max used memory (bytes) */
+ memm?: number
+ /** swap bytes [total, used] */
+ swap?: [number, number]
+ /** @deprecated total memory (gb) - records before 0.21.1 */
+ m?: number
+ /** @deprecated memory used (gb) - records before 0.21.1 */
+ mu?: number
/** memory percent */
mp: number
- /** memory buffer + cache (gb) */
- mb: number
- /** max used memory (gb) */
+ /** @deprecated memory buffer + cache (gb) - records before 0.21.1 */
+ mb?: number
+ /** @deprecated max used memory (gb) - records before 0.21.1 */
mm?: number
- /** zfs arc memory (gb) */
+ /** @deprecated zfs arc memory (gb) - records before 0.21.1 */
mz?: number
- /** swap space (gb) */
- s: number
- /** swap used (gb) */
- su: number
+ /** @deprecated swap space (gb) - records before 0.21.1 */
+ s?: number
+ /** @deprecated swap used (gb) - records before 0.21.1 */
+ su?: number
/** disk size (gb) */
d: number
/** disk used (gb) */
diff --git a/supplemental/CHANGELOG.md b/supplemental/CHANGELOG.md
index 4ff6e19dc..a899e2a34 100644
--- a/supplemental/CHANGELOG.md
+++ b/supplemental/CHANGELOG.md
@@ -2,6 +2,25 @@
For developers using the agent as a Go package, `Agent.Start` now requires a `context.Context`.
+For authors of integrations, please note that existing memory fields other than memory percentage are deprecated in favor of new fields which provide raw byte values. The deprecated fields are still populated, but will be removed in a future version.
+
+```go
+type Stats struct {
+ // Deprecated fields (populated on hubs <= 0.21.1; will be removed in a future version)
+ Mem float64 `json:"m" cbor:"2,keyasint"` // Deprecated: GB, use MemBytes. Memory GB fields are kept for older hubs and integrations.
+ MaxMem float64 `json:"mm,omitzero" cbor:"-"` // Deprecated: GB, use MaxMemBytes.
+ MemUsed float64 `json:"mu" cbor:"3,keyasint"` // Deprecated: GB, use MemBytes.
+ MemBuffCache float64 `json:"mb" cbor:"5,keyasint"` // Deprecated: GB, use MemBytes.
+ MemZfsArc float64 `json:"mz,omitzero" cbor:"6,keyasint,omitempty"` // Deprecated: GB, use MemBytes.
+ Swap float64 `json:"s,omitzero" cbor:"7,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
+ SwapUsed float64 `json:"su,omitzero" cbor:"8,keyasint,omitempty"` // Deprecated: GB, use SwapBytes.
+ // New fields (populated on hub version 0.21.1+ for all agents)
+ MemBytes [4]uint64 `json:"mem,omitzero" cbor:"41,keyasint,omitzero"` // [total, used, buff/cache, zfs arc] bytes
+ MaxMemBytes uint64 `json:"memm,omitzero" cbor:"-"` // max used memory bytes
+ SwapBytes [2]uint64 `json:"swap,omitzero" cbor:"42,keyasint,omitzero"` // [total, used] bytes
+}
+```
+
- Add opt-in cgroup CPU and memory accounting (#2505)
- Add `HUB_HEADERS` environment variable to set custom WebSocket headers on the agent (#2585)