mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-08-28 11:56:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: confirm a cluster after a week of reports, not two days (#10899)

Browse Source 
* telemetry: sync the server module to go 1.26

The root module moved to go 1.26 but the telemetry server module, which
replaces seaweedfs with the repo root, stayed on 1.25.8, so go refuses
to build or test it until the directive catches up.

* telemetry: confirm a cluster after a week of reports, not two days

Two days of history still lets recurring CI and demo clusters into the
confirmed fleet: anything torn down and rebuilt across a UTC midnight
counts. Requiring seven distinct UTC days keeps the fleet charts and the
version/OS distributions to clusters that actually stay up; real
clusters qualify after their first week, and the fallback to all active
clusters while none is confirmed is unchanged.
This commit is contained in:
Chris Lu
2026-08-23 11:14:17 -07:00
committed by GitHub
parent 0f85d005ad
commit c167af541e
9 changed files with 66 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
telemetry/README.md
+1 -1
View File
@@ -139,7 +139,7 @@ The telemetry server exposes these Prometheus metrics:
### Cluster Metrics
- `seaweedfs_telemetry_total_clusters`: Total unique clusters (30 days)
- `seaweedfs_telemetry_active_clusters`: Active clusters (7 days)
- `seaweedfs_telemetry_confirmed_clusters`: Active clusters seen on 2+ distinct days — one-shot reports don't count, and the version/OS distributions in `/api/stats` are computed over these
- `seaweedfs_telemetry_confirmed_clusters`: Active clusters seen on 7+ distinct days — one-shot reports don't count, and the version/OS distributions in `/api/stats` are computed over these
### Per-Cluster Metrics
- `seaweedfs_telemetry_volume_servers{cluster_id}`: Volume servers per cluster
telemetry/server/dashboard/dashboard.go
+1 -1
View File
@@ -129,7 +129,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
</div>
<div class="stat-card">
<div class="stat-value" id="confirmedInstances">-</div>
<div class="stat-label">Confirmed Clusters (2+ days)</div>
<div class="stat-label">Confirmed Clusters (7+ days)</div>
</div>
<div class="stat-card">
<div class="stat-value" id="totalVersions">-</div>
telemetry/server/go.mod
+1 -1
View File
@@ -1,6 +1,6 @@
module github.com/seaweedfs/seaweedfs/telemetry/server
go 1.25.8
go 1.26
require (
github.com/prometheus/client_golang v1.24.1
telemetry/server/storage/confirmed_test.go
+11 -6
View File
@@ -44,13 +44,18 @@ func TestConfirmedClusters(t *testing.T) {
t.Fatalf("fallback distribution missing active cluster: %v", v)
}
// Give cluster A a sample from yesterday: now seen on 2 distinct days.
// Give cluster A samples from the six previous days: now seen on 7
// distinct days.
s.mu.Lock()
id := "aaaaaaaa-0000-0000-0000-000000000001"
s.histories[id] = append([]HistorySample{{
Ts: time.Now().AddDate(0, 0, -1).Unix(),
TotalDiskBytes: 50,
}}, s.histories[id]...)
var older []HistorySample
for offset := -6; offset < 0; offset++ {
older = append(older, HistorySample{
Ts: time.Now().AddDate(0, 0, offset).Unix(),
TotalDiskBytes: 50,
})
}
s.histories[id] = append(older, s.histories[id]...)
s.mu.Unlock()
// A one-shot cluster B arrives (like an injected report): it counts as
@@ -60,7 +65,7 @@ func TestConfirmedClusters(t *testing.T) {
}
stats = statsOf(t, s)
if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 {
t.Fatalf("day two: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
t.Fatalf("day seven: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
}
v := stats["versions"].(map[string]int)
if v["4.40"] != 1 {
telemetry/server/storage/history.go
+4 -4
View File
@@ -8,7 +8,7 @@ import (
// confirmDays is how many distinct UTC days a cluster must have reported
// on before it counts as confirmed in the aggregated stats.
const confirmDays = 2
const confirmDays = 7
// activeDays is how recently a cluster must have reported to count as active.
const activeDays = 7
@@ -43,9 +43,9 @@ func (s *PrometheusStorage) appendHistory(data *proto.TelemetryData, receivedAt
}
// seriesHistories picks the clusters the fleet-wide series are built from: the
// confirmed ones. A cluster that only ever reported on one day is usually a CI
// or test cluster that lived for a minute, and those arrive faster than they
// age out, so counting them makes every fleet total climb forever. Falls back to
// confirmed ones. A cluster that reported for less than a week is usually a CI
// or test cluster, and those arrive faster than they age out, so counting them
// makes every fleet total climb forever. Falls back to
// all clusters while none is confirmed yet, so a fresh server still draws its
// charts. Callers must hold s.mu.
func (s *PrometheusStorage) seriesHistories() map[string][]HistorySample {
telemetry/server/storage/metrics_test.go
+18 -11
View File
@@ -15,7 +15,8 @@ func TestGetMetricsSumsEachDay(t *testing.T) {
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
// Stopped reporting past the active window: counts on its own days only.
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
-14, -13, -12, -11, -10, -9, -8)
metrics, err := s.GetMetrics(10)
if err != nil {
@@ -40,7 +41,8 @@ func TestGetMetricsSumsEachDay(t *testing.T) {
func TestGetMetricsCarriesSkippedDaysForward(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "gappy", HistorySample{TotalDiskBytes: 500, VolumeServerCount: 5}, -3, -1)
seedSamples(s, "gappy", HistorySample{TotalDiskBytes: 500, VolumeServerCount: 5},
-8, -7, -6, -5, -4, -3, -1)
metrics, err := s.GetMetrics(4)
if err != nil {
@@ -56,21 +58,23 @@ func TestGetMetricsCarriesSkippedDaysForward(t *testing.T) {
// out of nothing on its first day of data.
func TestGetMetricsWindowStartsAtOldestSample(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "recent", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 1}, -2, -1, 0)
seedSamples(s, "recent", HistorySample{TotalDiskBytes: 100, VolumeServerCount: 1},
-6, -5, -4, -3, -2, -1, 0)
metrics, err := s.GetMetrics(30)
if err != nil {
t.Fatal(err)
}
if got := metrics["dates"].([]string); len(got) != 3 {
t.Errorf("dates = %v, want the 3 days with history, not 30", got)
if got := metrics["dates"].([]string); len(got) != 7 {
t.Errorf("dates = %v, want the 7 days with history, not 30", got)
}
if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{100, 100, 100}) {
if got := metrics["disk_usage"].([]uint64); !equal(got, []uint64{100, 100, 100, 100, 100, 100, 100}) {
t.Errorf("disk_usage = %v, want no leading zero days", got)
}
// History reaching past the requested window still clips to the window.
seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1}, -40, -39)
seedSamples(s, "old", HistorySample{TotalDiskBytes: 50, VolumeServerCount: 1},
-45, -44, -43, -42, -41, -40, -39)
metrics, err = s.GetMetrics(10)
if err != nil {
t.Fatal(err)
@@ -86,8 +90,10 @@ func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
-8, -7, -6, -5, -4, -3, -2)
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
-14, -13, -12, -11, -10, -9, -8)
metrics, err := s.GetMetrics(10)
if err != nil {
@@ -111,7 +117,8 @@ func TestGetMetricsAgreesWithClusterSizes(t *testing.T) {
func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3}, -3, -2, -1, 0)
seedSamples(s, "real", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-6, -5, -4, -3, -2, -1, 0)
for _, id := range []string{"ci-1", "ci-2", "ci-3"} {
seedSamples(s, id, HistorySample{TotalDiskBytes: 5, VolumeServerCount: 14}, -1)
}
@@ -128,7 +135,7 @@ func TestGetMetricsExcludesUnconfirmedClusters(t *testing.T) {
}
}
// Until any cluster has two days of history the charts fall back to every
// Until any cluster has a week of history the charts fall back to every
// cluster, so a fresh server doesn't serve empty series.
func TestGetMetricsFallsBackWhenNoneConfirmed(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
telemetry/server/storage/prometheus.go
+5 -5
View File
@@ -53,7 +53,7 @@ func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
}),
confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_confirmed_clusters",
Help: "Active clusters seen on at least 2 distinct days (last 7 days)",
Help: "Active clusters seen on at least 7 distinct days (last 7 days)",
}),
volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_volume_servers",
@@ -220,9 +220,9 @@ func (s *PrometheusStorage) updateStats() {
activeInstances++
versionsAll[instance.TelemetryData.Version]++
osAll[instance.TelemetryData.Os]++
// A cluster is confirmed once seen on >=2 distinct UTC days
// (histories hold one sample per day), so one-shot reports
// can't skew the distributions below.
// A cluster is confirmed once seen on confirmDays distinct UTC
// days (histories hold one sample per day), so short-lived
// clusters can't skew the distributions below.
if len(s.histories[instance.TelemetryData.TopologyId]) >= confirmDays {
confirmedInstances++
versionsConfirmed[instance.TelemetryData.Version]++
@@ -231,7 +231,7 @@ func (s *PrometheusStorage) updateStats() {
}
}
// Before any cluster has two days of history (fresh server with no
// Before any cluster has a week of history (fresh server with no
// prior state), fall back to all active clusters so the dashboard
// distributions aren't empty.
versions, osDistribution := versionsConfirmed, osConfirmed
telemetry/server/storage/sizes_test.go
+10 -8
View File
@@ -25,11 +25,13 @@ func TestClusterSizeSeries(t *testing.T) {
// Reported every day of the window.
seedSamples(s, "daily", HistorySample{TotalDiskBytes: 300, VolumeServerCount: 3},
-9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
// Reported two days ago and not since: still active, so its size is held
// to the right edge instead of dropping out of the stack.
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2}, -3, -2)
// Stopped reporting two days ago: still active, so its size is held to
// the right edge instead of dropping out of the stack.
seedSamples(s, "lagging", HistorySample{TotalDiskBytes: 200, VolumeServerCount: 2},
-8, -7, -6, -5, -4, -3, -2)
// Stopped reporting past the active window: its own days only.
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9}, -9, -8)
seedSamples(s, "gone", HistorySample{TotalDiskBytes: 900, VolumeServerCount: 9},
-14, -13, -12, -11, -10, -9, -8)
// One day of history only: unconfirmed, so it stays out of the stack.
seedSamples(s, "oneshot", HistorySample{TotalDiskBytes: 400, VolumeServerCount: 4}, -1)
@@ -48,7 +50,7 @@ func TestClusterSizeSeries(t *testing.T) {
if got := byId["daily"].Disk; !equal(got, []uint64{300, 300, 300, 300, 300, 300, 300, 300, 300, 300}) {
t.Errorf("daily = %v, want 300 every day", got)
}
if got := byId["lagging"].Disk; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 200, 200, 200, 200}) {
if got := byId["lagging"].Disk; !equal(got, []uint64{0, 200, 200, 200, 200, 200, 200, 200, 200, 200}) {
t.Errorf("lagging = %v, want its size carried to the right edge", got)
}
if got := byId["gone"].Disk; !equal(got, []uint64{900, 900, 0, 0, 0, 0, 0, 0, 0, 0}) {
@@ -62,7 +64,7 @@ func TestClusterSizeSeries(t *testing.T) {
if got := byId["daily"].Servers; !equal(got, []uint64{3, 3, 3, 3, 3, 3, 3, 3, 3, 3}) {
t.Errorf("daily servers = %v, want 3 every day", got)
}
if got := byId["lagging"].Servers; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 2, 2, 2, 2}) {
if got := byId["lagging"].Servers; !equal(got, []uint64{0, 2, 2, 2, 2, 2, 2, 2, 2, 2}) {
t.Errorf("lagging servers = %v, want carried to the right edge", got)
}
if got := byId["gone"].Servers; !equal(got, []uint64{9, 9, 0, 0, 0, 0, 0, 0, 0, 0}) {
@@ -89,10 +91,10 @@ func TestClusterSizeSeries(t *testing.T) {
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 2 clusters", series.Other)
}
if !equal(series.Other.Disk, []uint64{900, 900, 0, 0, 0, 0, 200, 200, 200, 200}) {
if !equal(series.Other.Disk, []uint64{900, 1100, 200, 200, 200, 200, 200, 200, 200, 200}) {
t.Errorf("other = %v, want lagging+gone summed per day", series.Other.Disk)
}
if !equal(series.Other.Servers, []uint64{9, 9, 0, 0, 0, 0, 2, 2, 2, 2}) {
if !equal(series.Other.Servers, []uint64{9, 11, 2, 2, 2, 2, 2, 2, 2, 2}) {
t.Errorf("other servers = %v, want lagging+gone summed per day", series.Other.Servers)
}
if series.ClusterCount != 3 || series.TotalDisk != 500 || series.TotalServers != 5 {
telemetry/server/storage/versions_test.go
+15 -15
View File
@@ -10,31 +10,31 @@ func TestVersionSeries(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
// Upgraded mid-window: its band leaves the old version for the new one.
seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -4, -3)
seedSamples(s, "upgraded", HistorySample{Version: "4.39"}, -6, -5, -4, -3)
seedSamples(s, "upgraded", HistorySample{Version: "4.40"}, -2, -1, 0)
// Reported every day on the same version.
seedSamples(s, "steady", HistorySample{Version: "4.40"}, -4, -3, -2, -1, 0)
seedSamples(s, "steady", HistorySample{Version: "4.40"}, -6, -5, -4, -3, -2, -1, 0)
// One day of history only: unconfirmed, so it stays out of the stack.
seedSamples(s, "oneshot", HistorySample{Version: "4.40"}, -1)
// Confirmed, but its samples predate versions being recorded.
seedSamples(s, "versionless", HistorySample{}, -9, -8)
seedSamples(s, "versionless", HistorySample{}, -13, -12, -11, -10, -9, -8, -7)
series := s.GetVersionSeries(10, 0)
// The versionless days are dropped before the axis is built, so the chart
// spans the days a version is known for instead of climbing out of blanks.
if len(series.Dates) != 5 {
t.Fatalf("dates = %v, want the 5 days with versions", series.Dates)
if len(series.Dates) != 7 {
t.Fatalf("dates = %v, want the 7 days with versions", series.Dates)
}
if len(series.Versions) != 2 ||
series.Versions[0].Version != "4.39" || series.Versions[1].Version != "4.40" {
t.Fatalf("versions = %+v, want 4.39 then 4.40", series.Versions)
}
if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 0, 0, 0}) {
t.Errorf("4.39 = %v, want the upgraded cluster's first two days", got)
if got := series.Versions[0].Clusters; !equal(got, []uint64{1, 1, 1, 1, 0, 0, 0}) {
t.Errorf("4.39 = %v, want the upgraded cluster's first four days", got)
}
if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 2, 2, 2}) {
t.Errorf("4.40 = %v, want steady plus upgraded from day 3", got)
if got := series.Versions[1].Clusters; !equal(got, []uint64{1, 1, 1, 1, 2, 2, 2}) {
t.Errorf("4.40 = %v, want steady plus upgraded from day 5", got)
}
if series.TotalClusters != 2 {
t.Errorf("total_clusters = %d, want 2", series.TotalClusters)
@@ -48,12 +48,12 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
s := newPrometheusStorage(prometheus.NewRegistry())
// Stopped reporting past the active window: its own days only.
seedSamples(s, "gone", HistorySample{Version: "3.97"}, -9, -8)
seedSamples(s, "gone", HistorySample{Version: "3.97"}, -14, -13, -12, -11, -10, -9, -8)
// Reported every day of the window.
seedSamples(s, "daily", HistorySample{Version: "4.40"}, -9, -8, -7, -6, -5, -4, -3, -2, -1, 0)
// Reported three days ago and not since: still active, so it holds its
// version to the right edge instead of dropping out of the stack.
seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -3, -2)
// Stopped reporting two days ago: still active, so it holds its version
// to the right edge instead of dropping out of the stack.
seedSamples(s, "lagging", HistorySample{Version: "4.30"}, -8, -7, -6, -5, -4, -3, -2)
series := s.GetVersionSeries(10, 0)
if len(series.Dates) != 10 {
@@ -66,7 +66,7 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
if got := byVersion["3.97"]; !equal(got, []uint64{1, 1, 0, 0, 0, 0, 0, 0, 0, 0}) {
t.Errorf("3.97 = %v, want nothing after its last report", got)
}
if got := byVersion["4.30"]; !equal(got, []uint64{0, 0, 0, 0, 0, 0, 1, 1, 1, 1}) {
if got := byVersion["4.30"]; !equal(got, []uint64{0, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
t.Errorf("4.30 = %v, want carried to the right edge", got)
}
if got := byVersion["4.40"]; !equal(got, []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1}) {
@@ -85,7 +85,7 @@ func TestVersionSeriesHoldsForwardAndLimits(t *testing.T) {
if series.Other == nil || series.Other.Count != 2 {
t.Fatalf("other = %+v, want 2 versions", series.Other)
}
if !equal(series.Other.Clusters, []uint64{1, 1, 0, 0, 0, 0, 1, 1, 1, 1}) {
if !equal(series.Other.Clusters, []uint64{1, 2, 1, 1, 1, 1, 1, 1, 1, 1}) {
t.Errorf("other = %v, want 3.97+4.30 summed per day", series.Other.Clusters)
}
if series.TotalClusters != 2 {