-
diff --git a/telemetry/server/go.mod b/telemetry/server/go.mod
index 8c5f7ecbf..4f0778b3d 100644
--- a/telemetry/server/go.mod
+++ b/telemetry/server/go.mod
@@ -1,6 +1,6 @@
module github.com/seaweedfs/seaweedfs/telemetry/server
-go 1.25.8
+go 1.26
require (
github.com/prometheus/client_golang v1.24.1
diff --git a/telemetry/server/storage/confirmed_test.go b/telemetry/server/storage/confirmed_test.go
index 4670d1433..51e5d2872 100644
--- a/telemetry/server/storage/confirmed_test.go
+++ b/telemetry/server/storage/confirmed_test.go
@@ -44,13 +44,18 @@ func TestConfirmedClusters(t *testing.T) {
t.Fatalf("fallback distribution missing active cluster: %v", v)
}
- // Give cluster A a sample from yesterday: now seen on 2 distinct days.
+ // Give cluster A samples from the six previous days: now seen on 7
+ // distinct days.
s.mu.Lock()
id := "aaaaaaaa-0000-0000-0000-000000000001"
- s.histories[id] = append([]HistorySample{{
- Ts: time.Now().AddDate(0, 0, -1).Unix(),
- TotalDiskBytes: 50,
- }}, s.histories[id]...)
+ var older []HistorySample
+ for offset := -6; offset < 0; offset++ {
+ older = append(older, HistorySample{
+ Ts: time.Now().AddDate(0, 0, offset).Unix(),
+ TotalDiskBytes: 50,
+ })
+ }
+ s.histories[id] = append(older, s.histories[id]...)
s.mu.Unlock()
// A one-shot cluster B arrives (like an injected report): it counts as
@@ -60,7 +65,7 @@ func TestConfirmedClusters(t *testing.T) {
}
stats = statsOf(t, s)
if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 {
- t.Fatalf("day two: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
+ t.Fatalf("day seven: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
}
v := stats["versions"].(map[string]int)
if v["4.40"] != 1 {
diff --git a/telemetry/server/storage/history.go b/telemetry/server/storage/history.go
index e51d9a786..9a25ba19f 100644
--- a/telemetry/server/storage/history.go
+++ b/telemetry/server/storage/history.go
@@ -8,7 +8,7 @@ import (
// confirmDays is how many distinct UTC days a cluster must have reported
// on before it counts as confirmed in the aggregated stats.
-const confirmDays = 2
+const confirmDays = 7
// activeDays is how recently a cluster must have reported to count as active.
const activeDays = 7
@@ -43,9 +43,9 @@ func (s *PrometheusStorage) appendHistory(data *proto.TelemetryData, receivedAt
}
// seriesHistories picks the clusters the fleet-wide series are built from: the
-// confirmed ones. A cluster that only ever reported on one day is usually a CI
-// or test cluster that lived for a minute, and those arrive faster than they
-// age out, so counting them makes every fleet total climb forever. Falls back to
+// confirmed ones. A cluster that reported for less than a week is usually a CI
+// or test cluster, and those arrive faster than they age out, so counting them
+// makes every fleet total climb forever. Falls back to
// all clusters while none is confirmed yet, so a fresh server still draws its
// charts. Callers must hold s.mu.
func (s *PrometheusStorage) seriesHistories() map[string][]HistorySample {
diff --git a/telemetry/server/storage/metrics_test.go b/telemetry/server/storage/metrics_test.go
index 185b631af..566c66924 100644
--- a/telemetry/server/storage/metrics_test.go
+++ b/telemetry/server/storage/metrics_test.go
@@ -15,7 +15,8 @@ func TestGetMetricsSumsEachDay(t *testing.T) {
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
// Stopped reporting past the active window: counts on its own days only.
- seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
+ seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+ -14, -13, -12, -11, -10, -9, -8)
metrics, err := s.GetMetrics(10)
if err != nil {
@@ -40,7 +41,8 @@ func TestGetMetricsSumsEachDay(t *testing.T) {
func TestGetMetricsCarriesSkippedDaysForward(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
- seedSamples(s, "gappy", HistorySample{TotalDiskBytes: 500, VolumeServerCount: 5}, -3, -1)
+ seedSamples(s, "gappy", HistorySample{TotalDiskBytes: 500, VolumeServerCount: 5},
+ -8, -7, -6, -5, -4, -3, -1)
metrics, err := s.GetMetrics(4)
if err != nil {
@@ -56,21 +58,23 @@ func TestGetMetricsCarriesSkippedDaysForward(t *testing.T) {
// out of nothing on its first day of data.
func TestGetMetricsWindowStartsAtOldestSample(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
- seedSamples(s, "recent", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 1}, -2, -1, 0)
+ seedSamples(s, "recent", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 1},
+ -6, -5, -4, -3, -2, -1, 0)
metrics, err := s.GetMetrics(30)
if err != nil {
t.Fatal(err)
}
- if got := metrics["dates"].([]string); len(got) != 3 {
- t.Errorf("dates = %v, want the 3 days with history, not 30", got)
+ if got := metrics["dates"].([]string); len(got) != 7 {
+ t.Errorf("dates = %v, want the 7 days with history, not 30", got)
}
- if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{100, 100, 100}) {
+ if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{100, 100, 100, 100, 100, 100, 100}) {
t.Errorf("disk_usage = %v, want no leading zero days", got)
}
// History reaching past the requested window still clips to the window.
- seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1}, -40, -39)
+ seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1},
+ -45, -44, -43, -42, -41, -40, -39)
metrics, err = s.GetMetrics(10)
if err != nil {
t.Fatal(err)
@@ -86,8 +90,10 @@ func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
- seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
- seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
+ seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
+ -8, -7, -6, -5, -4, -3, -2)
+ seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+ -14, -13, -12, -11, -10, -9, -8)
metrics, err := s.GetMetrics(10)
if err != nil {
@@ -111,7 +117,8 @@ func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
- seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3}, -3, -2, -1, 0)
+ seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
+ -6, -5, -4, -3, -2, -1, 0)
for _, id := range []string{"ci-1", "ci-2", "ci-3"} {
seedSamples(s, id, HistorySample{TotalDiskBytes: 5, VolumeServerCount: 14}, -1)
}
@@ -128,7 +135,7 @@ func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
}
}
-// Until any cluster has two days of history the charts fall back to every
+// Until any cluster has a week of history the charts fall back to every
// cluster, so a fresh server doesn't serve empty series.
func TestGetMetricsFallsBackWhenNoneConfirmed(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.go
index 8c664e090..4cc90fab1 100644
--- a/telemetry/server/storage/prometheus.go
+++ b/telemetry/server/storage/prometheus.go
@@ -53,7 +53,7 @@ func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
}),
confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_confirmed_clusters",
- Help: "Active clusters seen on at least 2 distinct days (last 7 days)",
+ Help: "Active clusters seen on at least 7 distinct days (last 7 days)",
}),
volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_volume_servers",
@@ -220,9 +220,9 @@ func (s *PrometheusStorage) updateStats() {
activeInstances++
versionsAll[instance.TelemetryData.Version]++
osAll[instance.TelemetryData.Os]++
- // A cluster is confirmed once seen on >=2 distinct UTC days
- // (histories hold one sample per day), so one-shot reports
- // can't skew the distributions below.
+ // A cluster is confirmed once seen on confirmDays distinct UTC
+ // days (histories hold one sample per day), so short-lived
+ // clusters can't skew the distributions below.
if len(s.histories[instance.TelemetryData.TopologyId]) >= confirmDays {
confirmedInstances++
versionsConfirmed[instance.TelemetryData.Version]++
@@ -231,7 +231,7 @@ func (s *PrometheusStorage) updateStats() {
}
}
- // Before any cluster has two days of history (fresh server with no
+ // Before any cluster has a week of history (fresh server with no
// prior state), fall back to all active clusters so the dashboard
// distributions aren't empty.
versions, osDistribution := versionsConfirmed, osConfirmed
diff --git a/telemetry/server/storage/sizes_test.go b/telemetry/server/storage/sizes_test.go
index b2ea82c06..c4bec1663 100644
--- a/telemetry/server/storage/sizes_test.go
+++ b/telemetry/server/storage/sizes_test.go
@@ -25,11 +25,13 @@ func TestClusterSizeSeries(t *testing.T) {
// Reported every day of the window.
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
- // Reported two days ago and not since: still active, so its size is held
- // to the right edge instead of dropping out of the stack.
- seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
+ // Stopped reporting two days ago: still active, so its size is held to
+ // the right edge instead of dropping out of the stack.
+ seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
+ -8, -7, -6, -5, -4, -3, -2)
// Stopped reporting past the active window: its own days only.
- seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
+ seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
+ -14, -13, -12, -11, -10, -9, -8)
// One day of history only: unconfirmed, so it stays out of the stack.
seedSamples(s, "oneshot", HistorySample{TotalDiskBytes: 400, VolumeServerCount: 4}, -1)
@@ -48,7 +50,7 @@ func TestClusterSizeSeries(t *testing.T) {
if got := byId["daily"].Disk; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
t.Errorf("daily = %v, want 300 every day", got)
}
- if got := byId["lagging"].Disk; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 200, 200, 200, 200}) {
+ if got := byId["lagging"].Disk; !equal(got, []uint64{0, 200, 200, 200, 200, 200, 200, 200, 200, 200}) {
t.Errorf("lagging = %v, want its size carried to the right edge", got)
}
if got := byId["gone"].Disk; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
@@ -62,7 +64,7 @@ func TestClusterSizeSeries(t *testing.T) {
if got := byId["daily"].Servers; !equal(got, []uint64{3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) {
t.Errorf("daily servers = %v, want 3 every day", got)
}
- if got := byId["lagging"].Servers; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 2, 2, 2, 2}) {
+ if got := byId["lagging"].Servers; !equal(got, []uint64{0, 2, 2, 2, 2, 2, 2, 2, 2, 2}) {
t.Errorf("lagging servers = %v, want carried to the right edge", got)
}
if got := byId["gone"].Servers; !equal(got, []uint64{9, 9, 0, 0, 0, 0, 0, 0, 0, 0}) {
@@ -89,10 +91,10 @@ func TestClusterSizeSeries(t *testing.T) {
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 2 clusters", series.Other)
}
- if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 200, 200, 200, 200}) {
+ if !equal(series.Other.Disk, []uint64{900, 1100, 200, 200, 200, 200, 200, 200, 200, 200}) {
t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
}
- if !equal(series.Other.Servers, []uint64{9, 9, 0, 0, 0, 0, 2, 2, 2, 2}) {
+ if !equal(series.Other.Servers, []uint64{9, 11, 2, 2, 2, 2, 2, 2, 2, 2}) {
t.Errorf("other servers = %v, want lagging+gone summed per day", series.Other.Servers)
}
if series.ClusterCount != 3 || series.TotalDisk != 500 || series.TotalServers != 5 {
diff --git a/telemetry/server/storage/versions_test.go b/telemetry/server/storage/versions_test.go
index 828770019..9fc012cc5 100644
--- a/telemetry/server/storage/versions_test.go
+++ b/telemetry/server/storage/versions_test.go
@@ -10,31 +10,31 @@ func TestVersionSeries(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
// Upgraded mid-window: its band leaves the old version for the new one.
- seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -4, -3)
+ seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -6, -5, -4, -3)
seedSamples(s, "upgraded", HistorySample{Version: "4.40"}, -2, -1, 0)
// Reported every day on the same version.
- seedSamples(s, "steady", HistorySample{Version: "4.40"}, -4, -3, -2, -1, 0)
+ seedSamples(s, "steady", HistorySample{Version: "4.40"}, -6, -5, -4, -3, -2, -1, 0)
// One day of history only: unconfirmed, so it stays out of the stack.
seedSamples(s, "oneshot", HistorySample{Version: "4.40"}, -1)
// Confirmed, but its samples predate versions being recorded.
- seedSamples(s, "versionless", HistorySample{}, -9, -8)
+ seedSamples(s, "versionless", HistorySample{}, -13, -12, -11, -10, -9, -8, -7)
series := s.GetVersionSeries(10, 0)
// The versionless days are dropped before the axis is built, so the chart
// spans the days a version is known for instead of climbing out of blanks.
- if len(series.Dates) != 5 {
- t.Fatalf("dates = %v, want the 5 days with versions", series.Dates)
+ if len(series.Dates) != 7 {
+ t.Fatalf("dates = %v, want the 7 days with versions", series.Dates)
}
if len(series.Versions) != 2 ||
series.Versions[0].Version != "4.39" || series.Versions[1].Version != "4.40" {
t.Fatalf("versions = %+v, want 4.39 then 4.40", series.Versions)
}
- if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 0, 0, 0}) {
- t.Errorf("4.39 = %v, want the upgraded cluster's first two days", got)
+ if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 1, 1, 0, 0, 0}) {
+ t.Errorf("4.39 = %v, want the upgraded cluster's first four days", got)
}
- if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 2, 2, 2}) {
- t.Errorf("4.40 = %v, want steady plus upgraded from day 3", got)
+ if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 1, 1, 2, 2, 2}) {
+ t.Errorf("4.40 = %v, want steady plus upgraded from day 5", got)
}
if series.TotalClusters != 2 {
t.Errorf("total_clusters = %d, want 2", series.TotalClusters)
@@ -48,12 +48,12 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
// Stopped reporting past the active window: its own days only.
- seedSamples(s, "gone", HistorySample{Version: "3.97"}, -9, -8)
+ seedSamples(s, "gone", HistorySample{Version: "3.97"}, -14, -13, -12, -11, -10, -9, -8)
// Reported every day of the window.
seedSamples(s, "daily", HistorySample{Version: "4.40"}, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
- // Reported three days ago and not since: still active, so it holds its
- // version to the right edge instead of dropping out of the stack.
- seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -3, -2)
+ // Stopped reporting two days ago: still active, so it holds its version
+ // to the right edge instead of dropping out of the stack.
+ seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -8, -7, -6, -5, -4, -3, -2)
series := s.GetVersionSeries(10, 0)
if len(series.Dates) != 10 {
@@ -66,7 +66,7 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
if got := byVersion["3.97"]; !equal(got, []uint64{1, 1, 0, 0, 0, 0, 0, 0, 0, 0}) {
t.Errorf("3.97 = %v, want nothing after its last report", got)
}
- if got := byVersion["4.30"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 1, 1, 1, 1}) {
+ if got := byVersion["4.30"]; !equal(got, []uint64{0, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
t.Errorf("4.30 = %v, want carried to the right edge", got)
}
if got := byVersion["4.40"]; !equal(got, []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
@@ -85,7 +85,7 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 2 versions", series.Other)
}
- if !equal(series.Other.Clusters, []uint64{1, 1, 0, 0, 0, 0, 1, 1, 1, 1}) {
+ if !equal(series.Other.Clusters, []uint64{1, 2, 1, 1, 1, 1, 1, 1, 1, 1}) {
t.Errorf("other = %v, want 3.97+4.30 summed per day", series.Other.Clusters)
}
if series.TotalClusters != 2 {