diff --git a/agent/disk.go b/agent/disk.go index 9e89f9cd..13bb1b1c 100644 --- a/agent/disk.go +++ b/agent/disk.go @@ -3,6 +3,7 @@ package agent import ( "context" "log/slog" + "math" "os" "path/filepath" "runtime" @@ -686,25 +687,27 @@ func (a *Agent) updateDiskIo(cacheTimeMs uint16, systemStats *system.Stats) { // This is the total number of milliseconds spent by all reads (as // measured from __make_request() to end_that_request_last()). // https://www.kernel.org/doc/Documentation/iostats.txt (fields 4, 8) - diskReadTime := utils.TwoDecimals(float64(d.ReadTime-prev.readTime) / float64(msElapsed) * 100) - diskWriteTime := utils.TwoDecimals(float64(d.WriteTime-prev.writeTime) / float64(msElapsed) * 100) + deltaReadTime := ioTimeDelta(d.ReadTime, prev.readTime) + deltaWriteTime := ioTimeDelta(d.WriteTime, prev.writeTime) + diskReadTime := utils.TwoDecimals(float64(deltaReadTime) / float64(msElapsed) * 100) + diskWriteTime := utils.TwoDecimals(float64(deltaWriteTime) / float64(msElapsed) * 100) // I/O utilization %: fraction of wall time the device had any I/O in progress (0-100). - diskIoUtilPct := utils.TwoDecimals(float64(d.IoTime-prev.ioTime) / float64(msElapsed) * 100) + diskIoUtilPct := utils.TwoDecimals(float64(ioTimeDelta(d.IoTime, prev.ioTime)) / float64(msElapsed) * 100) // Weighted I/O: queue-depth weighted I/O time, normalized to interval (can exceed 100%). // Linux kernel field 11: incremented by iops_in_progress × ms_since_last_update. // Used to display queue depth. Multipled by 100 to increase accuracy of digit truncation (divided by 100 in UI). - diskWeightedIO := utils.TwoDecimals(float64(d.WeightedIO-prev.weightedIO) / float64(msElapsed) * 100) + diskWeightedIO := utils.TwoDecimals(float64(ioTimeDelta(d.WeightedIO, prev.weightedIO)) / float64(msElapsed) * 100) // r_await / w_await: average time per read/write operation in milliseconds. // Equivalent to r_await and w_await in iostat. var rAwait, wAwait float64 if deltaReadCount := d.ReadCount - prev.readCount; deltaReadCount > 0 { - rAwait = utils.TwoDecimals(float64(d.ReadTime-prev.readTime) / float64(deltaReadCount)) + rAwait = utils.TwoDecimals(float64(deltaReadTime) / float64(deltaReadCount)) } if deltaWriteCount := d.WriteCount - prev.writeCount; deltaWriteCount > 0 { - wAwait = utils.TwoDecimals(float64(d.WriteTime-prev.writeTime) / float64(deltaWriteCount)) + wAwait = utils.TwoDecimals(float64(deltaWriteTime) / float64(deltaWriteCount)) } // Update global fsStats baseline for cross-interval correctness @@ -740,6 +743,21 @@ func (a *Agent) updateDiskIo(cacheTimeMs uint16, systemStats *system.Stats) { } } +// ioTimeDelta returns the increase of a cumulative millisecond counter from +// the disk I/O stats. Linux prints these fields of /proc/diskstats as 32-bit +// unsigned ints, so they wrap to zero at 2^32. A busy disk reaches that in +// days for the weighted I/O time. Other platforms report 64-bit counters, +// so a lower value there is a reset. +func ioTimeDelta(current, previous uint64) uint64 { + if current >= previous { + return current - previous + } + if runtime.GOOS == "linux" && previous <= math.MaxUint32 { + return current + (math.MaxUint32 + 1 - previous) + } + return 0 +} + // getRootMountPoint returns the appropriate root mount point for the system. // On Windows it returns the system drive (e.g. "C:"). // For immutable systems like Fedora Silverblue, it returns /sysroot instead of / diff --git a/agent/disk_io_linux_test.go b/agent/disk_io_linux_test.go new file mode 100644 index 00000000..90e462a0 --- /dev/null +++ b/agent/disk_io_linux_test.go @@ -0,0 +1,75 @@ +//go:build linux + +package agent + +import ( + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/henrygd/beszel/internal/entities/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Linux prints four millisecond fields of /proc/diskstats as 32-bit unsigned ints: +// read time, write time, io time and weighted io time. They wrap to zero at 2^32. +func TestUpdateDiskIoTimeCounterWrap(t *testing.T) { + const wrap = uint64(1) << 32 + + tests := []struct { + name string + base uint64 // added to every previous time counter + }{ + {"no wrap", 0}, + {"32-bit wrap", wrap - 1000}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Deltas over 60s: read 300ms / 10 ops, write 400ms / 20 ops, + // io time 1200ms, weighted io 3000ms. + prev := prevDisk{ + readBytes: 20000 * 512, + writeBytes: 10000 * 512, + readTime: tt.base + 900, + writeTime: tt.base + 700, + ioTime: tt.base + 400, + weightedIO: tt.base, + readCount: 1000, + writeCount: 500, + at: time.Now().Add(-60 * time.Second), + } + cur := func(v uint64) uint64 { return v % wrap } + line := fmt.Sprintf(" 8 0 sda %d 0 %d %d %d 0 %d %d 0 %d %d\n", + 1010, 21200, cur(prev.readTime+300), + 520, 10400, cur(prev.writeTime+400), + cur(prev.ioTime+1200), cur(prev.weightedIO+3000)) + + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "diskstats"), []byte(line), 0o644)) + t.Setenv("HOST_PROC", dir) + t.Setenv("HOST_SYS", dir) + t.Setenv("HOST_DEV", dir) + t.Setenv("HOST_RUN", dir) + + fs := &system.FsStats{Root: true} + a := &Agent{ + fsNames: []string{"sda"}, + fsStats: map[string]*system.FsStats{"sda": fs}, + diskPrev: map[uint16]map[string]prevDisk{60000: {"sda": prev}}, + } + var stats system.Stats + a.updateDiskIo(60000, &stats) + + // Same order as DiskIoStats in system.FsStats. + want := [6]float64{0.5, 0.67, 2, 30, 20, 5} + for i := range want { + assert.InDelta(t, want[i], fs.DiskIoStats[i], 0.01, "DiskIoStats[%d]", i) + assert.InDelta(t, want[i], stats.DiskIoStats[i], 0.01, "system DiskIoStats[%d]", i) + } + }) + } +} diff --git a/agent/disk_test.go b/agent/disk_test.go index 9e8b3d2b..70fa2083 100644 --- a/agent/disk_test.go +++ b/agent/disk_test.go @@ -3,7 +3,9 @@ package agent import ( + "math" "os" + "runtime" "strings" "testing" "time" @@ -1041,3 +1043,17 @@ func TestInitializeDiskIoStatsResetsTrackedDevices(t *testing.T) { assert.Equal(t, uint64(50), agent.fsStats["sdb"].TotalRead) assert.Equal(t, uint64(60), agent.fsStats["sdb"].TotalWrite) } + +func TestIoTimeDelta(t *testing.T) { + assert.Equal(t, uint64(300), ioTimeDelta(1200, 900)) + + // A lower value is a 32-bit wrap only on Linux. Other platforms + // report 64-bit counters, so there it is a reset. + var want uint64 + if runtime.GOOS == "linux" { + want = 1200 + } + assert.Equal(t, want, ioTimeDelta(200, math.MaxUint32+1-1000)) + + assert.Equal(t, uint64(0), ioTimeDelta(200, math.MaxUint32+1000)) +}