mirror of
https://github.com/henrygd/beszel.git
synced 2026-10-22 06:12:33 +00:00
henrygd
37c0547b02
Memory and swap were rounded to two decimal GB places before being sent to the hub, which is inaccurate on low memory systems. - agent: add MemBytes [total, used, buff/cache, zfs arc] and SwapBytes [total, used] - hub: migrate GB fields from older agents to the byte fields - records: average byte fields, converting GB values from older records - site: chart memory and swap from bytes, with fallback for older records Deprecated GB fields are still populated in agent payloads and stored records for older hubs and third-party integrations.
247 lines
10 KiB
Go
247 lines
10 KiB
Go
//go:build testing && linux
|
|
|
|
package agent
|
|
|
|
import (
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/henrygd/beszel/internal/entities/system"
|
|
"github.com/shirou/gopsutil/v4/mem"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func setupMemoryCgroup(t *testing.T, v2 bool) string {
|
|
t.Helper()
|
|
swapCpuContainerSeams(t)
|
|
if v2 {
|
|
writeCpuFixture(t, procSelfCgroup, "0::/system.slice/agent.service\n")
|
|
writeCpuFixture(t, cgroupMountinfo, "")
|
|
return cgroupRoot
|
|
}
|
|
dir := filepath.Join(filepath.Dir(cgroupRoot), "memory")
|
|
writeCpuFixture(t, procSelfCgroup, "2:memory:/system.slice/agent.service\n")
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"30 25 0:26 / "+dir+" rw - cgroup cgroup rw,memory\n")
|
|
return dir
|
|
}
|
|
|
|
func writeMemoryFixture(t *testing.T, dir string, v2 bool, usage, limit, stat string) {
|
|
t.Helper()
|
|
usageName, limitName := "memory.usage_in_bytes", "memory.limit_in_bytes"
|
|
if v2 {
|
|
usageName, limitName = "memory.current", "memory.max"
|
|
}
|
|
writeCpuFixture(t, filepath.Join(dir, usageName), usage)
|
|
writeCpuFixture(t, filepath.Join(dir, limitName), limit)
|
|
writeCpuFixture(t, filepath.Join(dir, "memory.stat"), stat)
|
|
}
|
|
|
|
func TestContainerMemoryMetrics(t *testing.T) {
|
|
for _, v2 := range []bool{true, false} {
|
|
t.Run(strconv.FormatBool(v2), func(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, v2)
|
|
key, shmemKey := "total_cache", "total_shmem"
|
|
if v2 {
|
|
key, shmemKey = "file", "shmem"
|
|
}
|
|
stat := func(cache, shmem string) string {
|
|
return key + " " + cache + "\n" + shmemKey + " " + shmem + "\n"
|
|
}
|
|
for _, tt := range []struct {
|
|
name, usage, limit, stat string
|
|
host uint64
|
|
want memoryMetrics
|
|
ok bool
|
|
}{
|
|
{"file cache excluded", "600", "1000", stat("200", "0"), 2000, memoryMetrics{1000, 400, 200}, true},
|
|
{"shared memory retained", "600", "1000", stat("200", "50"), 2000, memoryMetrics{1000, 450, 150}, true},
|
|
{"shared memory larger than cache", "600", "1000", stat("200", "300"), 2000, memoryMetrics{1000, 600, 0}, true},
|
|
{"zero usage", "0", "1000", stat("0", "0"), 2000, memoryMetrics{1000, 0, 0}, true},
|
|
{"cache larger than charge", "100", "1000", stat("200", "0"), 2000, memoryMetrics{1000, 0, 100}, true},
|
|
{"host caps limit", "600", "3000", stat("200", "0"), 2000, memoryMetrics{2000, 400, 200}, true},
|
|
{"finite without host", "600", "1000", stat("200", "0"), 0, memoryMetrics{1000, 400, 200}, true},
|
|
{"missing cache key", "600", "1000", shmemKey + " 0\n", 2000, memoryMetrics{}, false},
|
|
{"missing shared memory key", "600", "1000", key + " 200\n", 2000, memoryMetrics{}, false},
|
|
{"invalid cache", "600", "1000", stat("-1", "0"), 2000, memoryMetrics{}, false},
|
|
{"invalid shared memory", "600", "1000", stat("200", "-1"), 2000, memoryMetrics{}, false},
|
|
{"invalid usage", "bad", "1000", stat("200", "0"), 2000, memoryMetrics{}, false},
|
|
{"invalid limit", "600", "bad", stat("200", "0"), 2000, memoryMetrics{}, false},
|
|
{"zero limit", "600", "0", stat("200", "0"), 2000, memoryMetrics{}, false},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
writeMemoryFixture(t, dir, v2, tt.usage, tt.limit, tt.stat)
|
|
metrics, ok := containerMemoryMetrics(tt.host, true)
|
|
assert.Equal(t, tt.ok, ok)
|
|
assert.Equal(t, tt.want, metrics)
|
|
})
|
|
}
|
|
limit := "9223372036854771712"
|
|
if v2 {
|
|
limit = "max"
|
|
}
|
|
writeMemoryFixture(t, dir, v2, "600", limit, stat("200", "0"))
|
|
writeMemoryFixture(t, filepath.Join(dir, "system.slice/agent.service"), v2, "1", "100", stat("0", "0"))
|
|
metrics, ok := containerMemoryMetrics(2000, true)
|
|
require.True(t, ok)
|
|
assert.Equal(t, memoryMetrics{2000, 400, 200}, metrics)
|
|
_, ok = containerMemoryMetrics(0, true)
|
|
assert.False(t, ok)
|
|
if !v2 {
|
|
writeMemoryFixture(t, dir, false, "600", "2147479552", stat("200", "0"))
|
|
metrics, ok = containerMemoryMetrics(2000, true)
|
|
require.True(t, ok)
|
|
assert.EqualValues(t, 2000, metrics.Total)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestContainerMemoryScopeAndEnablement(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, true)
|
|
// A discovered non-default mount must win over the default root.
|
|
mount := filepath.Join(t.TempDir(), "unified")
|
|
writeCpuFixture(t, cgroupMountinfo,
|
|
"30 25 0:26 /guest "+mount+" rw - cgroup2 cgroup2 rw\n")
|
|
writeMemoryFixture(t, dir, true, "1", "1000", "file 0\nshmem 0\n")
|
|
writeMemoryFixture(t, mount, true, "600", "1000", "file 200\nshmem 0\n")
|
|
writeMemoryFixture(t, filepath.Join(mount, "system.slice/agent.service"), true, "10", "100", "file 0\nshmem 0\n")
|
|
_, ok := containerMemoryMetrics(2000, false)
|
|
assert.False(t, ok) // Docker/host agents keep the existing path by default.
|
|
markLxc(t)
|
|
inLxc = sync.OnceValue(detectLxc) // Simulate a fresh process in LXC.
|
|
_, ok = containerMemoryMetrics(2000, false)
|
|
assert.False(t, ok) // lxcfs already serves container-scoped /proc/meminfo.
|
|
metrics, ok := containerMemoryMetrics(2000, true)
|
|
require.True(t, ok)
|
|
assert.Equal(t, memoryMetrics{1000, 400, 200}, metrics)
|
|
require.NoError(t, os.Remove(filepath.Join(mount, "memory.stat")))
|
|
_, ok = containerMemoryMetrics(2000, true)
|
|
assert.False(t, ok)
|
|
}
|
|
|
|
func TestUpdateMemoryStatsCgroupAndFallback(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, true)
|
|
const gib = uint64(1 << 30)
|
|
writeMemoryFixture(t, dir, true, strconv.FormatUint(6*gib, 10), strconv.FormatUint(10*gib, 10),
|
|
"file "+strconv.FormatUint(2*gib, 10)+"\nshmem 0\n")
|
|
original := memoryVirtualMemory
|
|
t.Cleanup(func() { memoryVirtualMemory = original })
|
|
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
|
|
return &mem.VirtualMemoryStat{Total: 20 * gib, Used: 12 * gib, Free: 4 * gib,
|
|
Cached: 4 * gib, SwapTotal: 2 * gib, SwapFree: gib}, nil
|
|
}
|
|
a := &Agent{forceUseCgroup: true, memCalc: "htop", zfs: true}
|
|
var stats system.Stats
|
|
a.updateMemoryStats(&stats)
|
|
assert.Equal(t, float64(10), stats.Mem)
|
|
assert.Equal(t, float64(4), stats.MemUsed)
|
|
assert.Equal(t, float64(2), stats.MemBuffCache)
|
|
assert.Equal(t, float64(40), stats.MemPct)
|
|
assert.Zero(t, stats.MemZfsArc)
|
|
assert.Equal(t, float64(2), stats.Swap)
|
|
assert.Equal(t, float64(1), stats.SwapUsed)
|
|
assert.Equal(t, [4]uint64{10 * gib, 4 * gib, 2 * gib, 0}, stats.MemBytes)
|
|
assert.Equal(t, [2]uint64{2 * gib, gib}, stats.SwapBytes)
|
|
assert.Equal(t, 10*gib, a.systemDetails.MemoryTotal)
|
|
assert.True(t, a.detailsDirty)
|
|
|
|
a.detailsDirty = false
|
|
a.updateMemoryStats(&system.Stats{})
|
|
assert.False(t, a.detailsDirty)
|
|
writeCpuFixture(t, filepath.Join(dir, "memory.max"), strconv.FormatUint(8*gib, 10))
|
|
a.updateMemoryStats(&system.Stats{})
|
|
assert.Equal(t, 8*gib, a.systemDetails.MemoryTotal)
|
|
assert.True(t, a.detailsDirty)
|
|
|
|
// A missing required field falls back to the original htop calculation.
|
|
writeCpuFixture(t, filepath.Join(dir, "memory.stat"), "file 123\n")
|
|
a.zfs = false
|
|
stats = system.Stats{}
|
|
a.updateMemoryStats(&stats)
|
|
assert.Equal(t, float64(20), stats.Mem)
|
|
assert.Equal(t, float64(12), stats.MemUsed)
|
|
assert.Equal(t, float64(4), stats.MemBuffCache)
|
|
assert.Equal(t, float64(60), stats.MemPct)
|
|
assert.Equal(t, [4]uint64{20 * gib, 12 * gib, 4 * gib, 0}, stats.MemBytes)
|
|
assert.Equal(t, 20*gib, a.systemDetails.MemoryTotal)
|
|
|
|
// Finite cgroup limits still work if /proc/meminfo cannot be read.
|
|
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) { return nil, errors.New("unreadable") }
|
|
writeCpuFixture(t, filepath.Join(dir, "memory.stat"), "file 0\nshmem 0\n")
|
|
stats = system.Stats{}
|
|
a.updateMemoryStats(&stats)
|
|
assert.Equal(t, float64(8), stats.Mem)
|
|
assert.Equal(t, float64(6), stats.MemUsed)
|
|
assert.Equal(t, float64(75), stats.MemPct)
|
|
assert.Equal(t, [4]uint64{8 * gib, 6 * gib, 0, 0}, stats.MemBytes)
|
|
assert.Zero(t, stats.Swap)
|
|
assert.Zero(t, stats.SwapBytes)
|
|
}
|
|
|
|
func TestMemoryDetailsInitiallyUseCgroupLimit(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, true)
|
|
writeMemoryFixture(t, dir, true, "600", "1000", "file 200\nshmem 0\n")
|
|
original := memoryVirtualMemory
|
|
t.Cleanup(func() { memoryVirtualMemory = original })
|
|
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
|
|
return &mem.VirtualMemoryStat{Total: 2000}, nil
|
|
}
|
|
a := &Agent{forceUseCgroup: true}
|
|
a.refreshSystemDetails()
|
|
assert.EqualValues(t, 1000, a.systemDetails.MemoryTotal)
|
|
}
|
|
|
|
// Captured from this container: active file cache made the working-set
|
|
// calculation report ~80 GiB despite only ~17 GiB of anonymous RSS.
|
|
func TestMemoryStatsExcludeActiveFileCache(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, false)
|
|
const usage = uint64(135881601024)
|
|
const cache = uint64(116823166976)
|
|
const shmem = uint64(77180928)
|
|
writeMemoryFixture(t, dir, false, strconv.FormatUint(usage, 10), "137438953472",
|
|
"total_cache 116823166976\ntotal_shmem 77180928\n"+
|
|
"total_inactive_file 49902608384\ntotal_active_file 67684671488\n")
|
|
original := memoryVirtualMemory
|
|
t.Cleanup(func() { memoryVirtualMemory = original })
|
|
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
|
|
return &mem.VirtualMemoryStat{Total: 1580621720 * 1024}, nil
|
|
}
|
|
metrics, ok := containerMemoryMetrics(1580621720*1024, true)
|
|
require.True(t, ok)
|
|
assert.Equal(t, usage-(cache-shmem), metrics.Used)
|
|
assert.Equal(t, cache-shmem, metrics.BuffCache)
|
|
assert.Equal(t, usage, metrics.Used+metrics.BuffCache)
|
|
var stats system.Stats
|
|
a := &Agent{forceUseCgroup: true}
|
|
a.updateMemoryStats(&stats)
|
|
assert.Equal(t, float64(17.82), stats.MemUsed)
|
|
assert.Equal(t, float64(108.73), stats.MemBuffCache)
|
|
assert.Equal(t, float64(13.92), stats.MemPct)
|
|
assert.Equal(t, [4]uint64{137438953472, usage - (cache - shmem), cache - shmem, 0}, stats.MemBytes)
|
|
}
|
|
|
|
// Low memory systems need byte precision because the GB fields round to 10 MiB.
|
|
func TestMemoryStatsLowMemoryPrecision(t *testing.T) {
|
|
dir := setupMemoryCgroup(t, true)
|
|
const mib = uint64(1 << 20)
|
|
writeMemoryFixture(t, dir, true, strconv.FormatUint(5*mib, 10), strconv.FormatUint(64*mib, 10),
|
|
"file "+strconv.FormatUint(2*mib, 10)+"\nshmem 0\n")
|
|
original := memoryVirtualMemory
|
|
t.Cleanup(func() { memoryVirtualMemory = original })
|
|
memoryVirtualMemory = func() (*mem.VirtualMemoryStat, error) {
|
|
return &mem.VirtualMemoryStat{Total: 128 * mib, SwapTotal: 32 * mib, SwapFree: 31 * mib}, nil
|
|
}
|
|
var stats system.Stats
|
|
a := &Agent{forceUseCgroup: true}
|
|
a.updateMemoryStats(&stats)
|
|
assert.Equal(t, [4]uint64{64 * mib, 3 * mib, 2 * mib, 0}, stats.MemBytes)
|
|
assert.Equal(t, [2]uint64{32 * mib, mib}, stats.SwapBytes)
|
|
assert.Equal(t, float64(0), stats.MemUsed)
|
|
}
|