diff --git a/agent/docker.go b/agent/docker.go index 0bac2439..f10fffa5 100644 --- a/agent/docker.go +++ b/agent/docker.go @@ -544,11 +544,18 @@ func (dm *dockerManager) updateContainerStats(ctr *container.ApiInfo, cacheTimeM // Get previous CPU values prevCpuContainer, prevCpuSystem := dm.getCpuPreviousValues(cacheTimeMs, ctr.IdShort) - // Calculate CPU percentage based on platform + // Calculate CPU percentage based on platform. + // Podman reports system_cpu_usage from cgroup cpu.stat (not /proc/stat), so it reflects + // only cgroup-tracked activity rather than total host capacity. Use a time-based method + // instead so the result is comparable to host CPU utilization. See: + // https://github.com/henrygd/beszel/issues/2049 var cpuPct float64 if dm.isWindows { prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort] cpuPct = res.CalculateCpuPercentWindows(prevCpuContainer, prevRead) + } else if dm.usingPodman && res.CPUStats.OnlineCPUs > 0 { + prevRead := dm.lastCpuReadTime[cacheTimeMs][ctr.IdShort] + cpuPct = res.CalculateCpuPercentPodman(prevCpuContainer, prevRead) } else { cpuPct = res.CalculateCpuPercentLinux(prevCpuContainer, prevCpuSystem) } diff --git a/agent/docker_test.go b/agent/docker_test.go index 77a965f3..0ba67aff 100644 --- a/agent/docker_test.go +++ b/agent/docker_test.go @@ -1059,6 +1059,162 @@ func TestCpuPercentageWindowsHandlesCounterRollback(t *testing.T) { assert.Greater(t, stats.CalculateCpuPercentWindows(500_000, prevRead), 0.0) } +func TestCalculateCpuPercentPodman(t *testing.T) { + baseTime := time.Date(2026, 3, 15, 12, 0, 0, 0, time.UTC) + + tests := []struct { + name string + prevCpuContainer uint64 + prevRead time.Time + currentUsage uint64 + currentRead time.Time + onlineCPUs uint32 + expectedPct float64 + }{ + { + name: "normal calculation", + // container used 2ms of CPU over 1s with 2 CPUs → 0.1% + prevCpuContainer: 1_000_000_000, + prevRead: baseTime, + currentUsage: 1_002_000_000, // +2ms CPU time + currentRead: baseTime.Add(time.Second), + onlineCPUs: 2, + expectedPct: 0.1, // 2e6 / (1e9 * 2) * 100 + }, + { + name: "first run returns zero", + prevCpuContainer: 0, + prevRead: baseTime, + currentUsage: 5_000_000, + currentRead: baseTime.Add(time.Second), + onlineCPUs: 4, + expectedPct: 0.0, + }, + { + name: "zero online cpus returns zero", + prevCpuContainer: 1_000_000_000, + prevRead: baseTime, + currentUsage: 1_010_000_000, + currentRead: baseTime.Add(time.Second), + onlineCPUs: 0, + expectedPct: 0.0, + }, + { + name: "same read time returns zero", + prevCpuContainer: 1_000_000_000, + prevRead: baseTime, + currentUsage: 1_010_000_000, + currentRead: baseTime, // no elapsed time + onlineCPUs: 2, + expectedPct: 0.0, + }, + { + name: "counter rollback returns zero", + prevCpuContainer: 2_000_000_000, + prevRead: baseTime, + currentUsage: 1_000_000_000, + currentRead: baseTime.Add(time.Second), + onlineCPUs: 2, + expectedPct: 0.0, + }, + { + name: "100% single cpu", + // container consumed a full CPU-second over 1s on a 1-CPU host → 100% + prevCpuContainer: 1_000_000_000, + prevRead: baseTime, + currentUsage: 2_000_000_000, // +1s CPU time + currentRead: baseTime.Add(time.Second), + onlineCPUs: 1, + expectedPct: 100.0, // 1e9 / (1e9 * 1) * 100 + }, + { + name: "high utilization on multi-cpu host", + // container used 800ms on a 4-CPU host over 1s → 20% + prevCpuContainer: 10_000_000_000, + prevRead: baseTime, + currentUsage: 10_800_000_000, + currentRead: baseTime.Add(time.Second), + onlineCPUs: 4, + expectedPct: 20.0, // 800e6 / (1e9 * 4) * 100 + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &container.ApiStats{ + Read: tt.currentRead, + CPUStats: container.CPUStats{ + CPUUsage: container.CPUUsage{TotalUsage: tt.currentUsage}, + OnlineCPUs: tt.onlineCPUs, + }, + } + got := s.CalculateCpuPercentPodman(tt.prevCpuContainer, tt.prevRead) + assert.InDelta(t, tt.expectedPct, got, 0.001, "test %q", tt.name) + }) + } +} + +func TestUpdateContainerStatsPodmanCpuCalculation(t *testing.T) { + // Verify that Podman containers use the time-based CPU calculation + // when online_cpus is provided in the stats response. + // container used 20ms CPU over 1s with 2 CPUs → 1% + prevReadTime := time.Date(2026, 3, 15, 21, 26, 58, 0, time.UTC) // 1 second before stats read + const prevCpuUsage = uint64(5_000_000_000) + + dm := &dockerManager{ + client: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + switch req.URL.EscapedPath() { + case "/containers/0123456789ab/stats": + return &http.Response{ + StatusCode: http.StatusOK, + Status: "200 OK", + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{ + "read":"2026-03-15T21:26:59Z", + "cpu_stats":{"cpu_usage":{"total_usage":5020000000},"system_cpu_usage":9999999,"online_cpus":2}, + "memory_stats":{"usage":1048576,"stats":{"inactive_file":262144}}, + "networks":{"eth0":{"rx_bytes":0,"tx_bytes":0}} + }`)), + Request: req, + }, nil + default: + return nil, fmt.Errorf("unexpected path: %s", req.URL.EscapedPath()) + } + })}, + containerStatsMap: make(map[string]*container.Stats), + apiStats: &container.ApiStats{}, + usingPodman: true, + lastCpuContainer: map[uint16]map[string]uint64{ + defaultCacheTimeMs: {"0123456789ab": prevCpuUsage}, + }, + lastCpuSystem: map[uint16]map[string]uint64{ + defaultCacheTimeMs: {"0123456789ab": 1}, // intentionally tiny — should NOT be used + }, + lastCpuReadTime: map[uint16]map[string]time.Time{ + defaultCacheTimeMs: {"0123456789ab": prevReadTime}, + }, + networkSentTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), + networkRecvTrackers: make(map[uint16]*deltatracker.DeltaTracker[string, uint64]), + lastNetworkReadTime: make(map[uint16]map[string]time.Time), + } + + ctr := &container.ApiInfo{ + IdShort: "0123456789ab", + Names: []string{"/myapp"}, + Status: "Up 5 minutes", + Image: "myapp:latest", + } + + err := dm.updateContainerStats(ctr, defaultCacheTimeMs) + require.NoError(t, err) + + // cpu delta = 5020000000 - 5000000000 = 20000000 ns (20ms) + // elapsed = 1s = 1000000000 ns, online_cpus = 2 + // expected = 20000000 / (1000000000 * 2) * 100 = 1.0% + expectedCpu := 1.0 + assert.InDelta(t, expectedCpu, dm.containerStatsMap[ctr.IdShort].Cpu, 0.01) +} + func TestNetworkStatsCalculationWithRealData(t *testing.T) { // Create synthetic test data to avoid timing issues apiStats1 := &container.ApiStats{ diff --git a/internal/entities/container/container.go b/internal/entities/container/container.go index 989d4b2b..ef309017 100644 --- a/internal/entities/container/container.go +++ b/internal/entities/container/container.go @@ -74,6 +74,30 @@ func (s *ApiStats) CalculateCpuPercentLinux(prevCpuContainer uint64, prevCpuSyst return float64(cpuDelta) / float64(systemDelta) * 100.0 } +// CalculateCpuPercentPodman calculates CPU percentage for Podman containers. +// Podman populates system_cpu_usage from cgroup cpu.stat rather than /proc/stat, so it +// represents only cgroup-accounted activity, not total host CPU capacity. Using it as +// a denominator inflates the result. Instead we use elapsed wall-clock time × online_cpus, +// matching the approach used for Windows and recommended in: +// https://github.com/henrygd/beszel/issues/2049 +func (s *ApiStats) CalculateCpuPercentPodman(prevCpuContainer uint64, prevRead time.Time) float64 { + if prevCpuContainer == 0 || s.CPUStats.OnlineCPUs == 0 { + return 0.0 + } + // Treat a reset or out-of-order counter as a new baseline instead of + // allowing unsigned subtraction to wrap to an enormous percentage. + if s.CPUStats.CPUUsage.TotalUsage < prevCpuContainer { + return 0.0 + } + cpuDelta := s.CPUStats.CPUUsage.TotalUsage - prevCpuContainer + elapsedNs := uint64(s.Read.Sub(prevRead).Nanoseconds()) + systemCapacity := elapsedNs * uint64(s.CPUStats.OnlineCPUs) + if systemCapacity == 0 { + return 0.0 + } + return float64(cpuDelta) / float64(systemCapacity) * 100.0 +} + // from: https://github.com/docker/cli/blob/master/cli/command/container/stats_helpers.go#L185 func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time.Time) float64 { // Max number of 100ns intervals between the previous time read and now @@ -98,8 +122,10 @@ func (s *ApiStats) CalculateCpuPercentWindows(prevCpuUsage uint64, prevRead time type CPUStats struct { // CPU Usage. Linux and Windows. CPUUsage CPUUsage `json:"cpu_usage"` - // System Usage. Linux only. + // System Usage. Linux only. Populated from /proc/stat on Docker; from cgroup cpu.stat on Podman. SystemUsage uint64 `json:"system_cpu_usage,omitempty"` + // Number of online CPUs. Linux only. Used by Podman for time-based CPU calculation. + OnlineCPUs uint32 `json:"online_cpus,omitempty"` } type CPUUsage struct {