From 8e8b4c4f34e7f9e40d75217ef62435557dbe5a4a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 22 Jul 2026 20:51:59 -0700 Subject: [PATCH] telemetry: confirmed-cluster stats (2+ distinct days) (#10402) telemetry: confirmed-cluster stats Count a cluster as confirmed once it has reported on >=2 distinct UTC days (per-cluster history makes this a length check). Version/OS distributions in /api/stats are computed over confirmed clusters, so a one-shot injected report can't appear in them; falls back to all active clusters while no confirmed ones exist (fresh server). Adds the seaweedfs_telemetry_confirmed_clusters gauge and a dashboard card. --- telemetry/README.md | 1 + telemetry/server/dashboard/dashboard.go | 5 ++ telemetry/server/storage/confirmed_test.go | 72 ++++++++++++++++++++++ telemetry/server/storage/history.go | 4 ++ telemetry/server/storage/prometheus.go | 42 ++++++++++--- 5 files changed, 116 insertions(+), 8 deletions(-) create mode 100644 telemetry/server/storage/confirmed_test.go diff --git a/telemetry/README.md b/telemetry/README.md index b9ea6ac74..6e1db0aeb 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -136,6 +136,7 @@ The telemetry server exposes these Prometheus metrics: ### Cluster Metrics - `seaweedfs_telemetry_total_clusters`: Total unique clusters (30 days) - `seaweedfs_telemetry_active_clusters`: Active clusters (7 days) +- `seaweedfs_telemetry_confirmed_clusters`: Active clusters seen on 2+ distinct days — one-shot reports don't count, and the version/OS distributions in `/api/stats` are computed over these ### Per-Cluster Metrics - `seaweedfs_telemetry_volume_servers{cluster_id}`: Volume servers per cluster diff --git a/telemetry/server/dashboard/dashboard.go b/telemetry/server/dashboard/dashboard.go index 9e8af595f..78301acc5 100644 --- a/telemetry/server/dashboard/dashboard.go +++ b/telemetry/server/dashboard/dashboard.go @@ -123,6 +123,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
-
Active Instances (7 days)
+
+
-
+
Confirmed Clusters (2+ days)
+
-
Different Versions
@@ -198,6 +202,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) { function updateStats(stats) { document.getElementById('totalInstances').textContent = stats.total_instances || 0; document.getElementById('activeInstances').textContent = stats.active_instances || 0; + document.getElementById('confirmedInstances').textContent = stats.confirmed_instances || 0; document.getElementById('totalVersions').textContent = Object.keys(stats.versions || {}).length; document.getElementById('totalOS').textContent = Object.keys(stats.os_distribution || {}).length; } diff --git a/telemetry/server/storage/confirmed_test.go b/telemetry/server/storage/confirmed_test.go new file mode 100644 index 000000000..4670d1433 --- /dev/null +++ b/telemetry/server/storage/confirmed_test.go @@ -0,0 +1,72 @@ +package storage + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/seaweedfs/seaweedfs/telemetry/proto" +) + +func report(id, version string) *proto.TelemetryData { + return &proto.TelemetryData{ + TopologyId: id, + Version: version, + Os: "linux/amd64", + VolumeServerCount: 1, + TotalDiskBytes: 100, + TotalVolumeCount: 1, + } +} + +func statsOf(t *testing.T, s *PrometheusStorage) map[string]interface{} { + t.Helper() + stats, err := s.GetStats() + if err != nil { + t.Fatalf("stats: %v", err) + } + return stats +} + +func TestConfirmedClusters(t *testing.T) { + s := newPrometheusStorage(prometheus.NewRegistry()) + + // A single-day cluster is active but not confirmed; with no confirmed + // clusters yet, distributions fall back to all active clusters. + if err := s.StoreTelemetry(report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")); err != nil { + t.Fatal(err) + } + stats := statsOf(t, s) + if stats["active_instances"] != 1 || stats["confirmed_instances"] != 0 { + t.Fatalf("day one: active=%v confirmed=%v, want 1/0", stats["active_instances"], stats["confirmed_instances"]) + } + if v := stats["versions"].(map[string]int); v["4.40"] != 1 { + t.Fatalf("fallback distribution missing active cluster: %v", v) + } + + // Give cluster A a sample from yesterday: now seen on 2 distinct days. + s.mu.Lock() + id := "aaaaaaaa-0000-0000-0000-000000000001" + s.histories[id] = append([]HistorySample{{ + Ts: time.Now().AddDate(0, 0, -1).Unix(), + TotalDiskBytes: 50, + }}, s.histories[id]...) + s.mu.Unlock() + + // A one-shot cluster B arrives (like an injected report): it counts as + // active, but the distributions now only reflect confirmed clusters. + if err := s.StoreTelemetry(report("bbbbbbbb-0000-0000-0000-000000000002", "9.99")); err != nil { + t.Fatal(err) + } + stats = statsOf(t, s) + if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 { + t.Fatalf("day two: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"]) + } + v := stats["versions"].(map[string]int) + if v["4.40"] != 1 { + t.Errorf("confirmed cluster missing from distribution: %v", v) + } + if _, ok := v["9.99"]; ok { + t.Errorf("one-shot cluster polluted the distribution: %v", v) + } +} diff --git a/telemetry/server/storage/history.go b/telemetry/server/storage/history.go index 8f5e26f45..eff68bc03 100644 --- a/telemetry/server/storage/history.go +++ b/telemetry/server/storage/history.go @@ -6,6 +6,10 @@ import ( "github.com/seaweedfs/seaweedfs/telemetry/proto" ) +// confirmDays is how many distinct UTC days a cluster must have reported +// on before it counts as confirmed in the aggregated stats. +const confirmDays = 2 + // HistorySample is one retained data point of a cluster's daily reports. // Tags are kept short because thousands of samples end up in the state file. type HistorySample struct { diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.go index 3c7869c82..15e040f9d 100644 --- a/telemetry/server/storage/prometheus.go +++ b/telemetry/server/storage/prometheus.go @@ -14,6 +14,7 @@ type PrometheusStorage struct { // Prometheus metrics totalClusters prometheus.Gauge activeClusters prometheus.Gauge + confirmedClusters prometheus.Gauge volumeServerCount *prometheus.GaugeVec totalDiskBytes *prometheus.GaugeVec totalVolumeCount *prometheus.GaugeVec @@ -51,6 +52,10 @@ func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage { Name: "seaweedfs_telemetry_active_clusters", Help: "Number of active SeaweedFS clusters (last 7 days)", }), + confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "seaweedfs_telemetry_confirmed_clusters", + Help: "Active clusters seen on at least 2 distinct days (last 7 days)", + }), volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_volume_servers", Help: "Number of volume servers per cluster", @@ -207,8 +212,11 @@ func (s *PrometheusStorage) updateStats() { totalInstances := 0 activeInstances := 0 - versions := make(map[string]int) - osDistribution := make(map[string]int) + confirmedInstances := 0 + versionsAll := make(map[string]int) + osAll := make(map[string]int) + versionsConfirmed := make(map[string]int) + osConfirmed := make(map[string]int) for _, instance := range s.instances { if instance.ReceivedAt.After(last30Days) { @@ -216,21 +224,39 @@ func (s *PrometheusStorage) updateStats() { } if instance.ReceivedAt.After(last7Days) { activeInstances++ - versions[instance.TelemetryData.Version]++ - osDistribution[instance.TelemetryData.Os]++ + versionsAll[instance.TelemetryData.Version]++ + osAll[instance.TelemetryData.Os]++ + // A cluster is confirmed once seen on >=2 distinct UTC days + // (histories hold one sample per day), so one-shot reports + // can't skew the distributions below. + if len(s.histories[instance.TelemetryData.TopologyId]) >= confirmDays { + confirmedInstances++ + versionsConfirmed[instance.TelemetryData.Version]++ + osConfirmed[instance.TelemetryData.Os]++ + } } } + // Before any cluster has two days of history (fresh server with no + // prior state), fall back to all active clusters so the dashboard + // distributions aren't empty. + versions, osDistribution := versionsConfirmed, osConfirmed + if confirmedInstances == 0 { + versions, osDistribution = versionsAll, osAll + } + // Update Prometheus gauges s.totalClusters.Set(float64(totalInstances)) s.activeClusters.Set(float64(activeInstances)) + s.confirmedClusters.Set(float64(confirmedInstances)) // Update cached stats for API s.stats = map[string]interface{}{ - "total_instances": totalInstances, - "active_instances": activeInstances, - "versions": versions, - "os_distribution": osDistribution, + "total_instances": totalInstances, + "active_instances": activeInstances, + "confirmed_instances": confirmedInstances, + "versions": versions, + "os_distribution": osDistribution, } }