telemetry: confirmed-cluster stats (2+ distinct days) (#10402)

telemetry: confirmed-cluster stats

Count a cluster as confirmed once it has reported on >=2 distinct UTC
days (per-cluster history makes this a length check). Version/OS
distributions in /api/stats are computed over confirmed clusters, so a
one-shot injected report can't appear in them; falls back to all active
clusters while no confirmed ones exist (fresh server). Adds the
seaweedfs_telemetry_confirmed_clusters gauge and a dashboard card.

telemetry/README.md

@@ -136,6 +136,7 @@ The telemetry server exposes these Prometheus metrics:
 ### Cluster Metrics
 - `seaweedfs_telemetry_total_clusters`: Total unique clusters (30 days)
 - `seaweedfs_telemetry_active_clusters`: Active clusters (7 days)
+- `seaweedfs_telemetry_confirmed_clusters`: Active clusters seen on 2+ distinct days — one-shot reports don't count, and the version/OS distributions in `/api/stats` are computed over these
 
 ### Per-Cluster Metrics
 - `seaweedfs_telemetry_volume_servers{cluster_id}`: Volume servers per cluster

telemetry/server/dashboard/dashboard.go

@@ -123,6 +123,10 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
                     <div class="stat-value" id="activeInstances">-</div>
                     <div class="stat-label">Active Instances (7 days)</div>
                 </div>
+                <div class="stat-card">
+                    <div class="stat-value" id="confirmedInstances">-</div>
+                    <div class="stat-label">Confirmed Clusters (2+ days)</div>
+                </div>
                 <div class="stat-card">
                     <div class="stat-value" id="totalVersions">-</div>
                     <div class="stat-label">Different Versions</div>
@@ -198,6 +202,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
         function updateStats(stats) {
             document.getElementById('totalInstances').textContent = stats.total_instances || 0;
             document.getElementById('activeInstances').textContent = stats.active_instances || 0;
+            document.getElementById('confirmedInstances').textContent = stats.confirmed_instances || 0;
             document.getElementById('totalVersions').textContent = Object.keys(stats.versions || {}).length;
             document.getElementById('totalOS').textContent = Object.keys(stats.os_distribution || {}).length;
         }

telemetry/server/storage/confirmed_test.go

@@ -0,0 +1,72 @@
+package storage
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func report(id, version string) *proto.TelemetryData {
+	return &proto.TelemetryData{
+		TopologyId:        id,
+		Version:           version,
+		Os:                "linux/amd64",
+		VolumeServerCount: 1,
+		TotalDiskBytes:    100,
+		TotalVolumeCount:  1,
+	}
+}
+
+func statsOf(t *testing.T, s *PrometheusStorage) map[string]interface{} {
+	t.Helper()
+	stats, err := s.GetStats()
+	if err != nil {
+		t.Fatalf("stats: %v", err)
+	}
+	return stats
+}
+
+func TestConfirmedClusters(t *testing.T) {
+	s := newPrometheusStorage(prometheus.NewRegistry())
+
+	// A single-day cluster is active but not confirmed; with no confirmed
+	// clusters yet, distributions fall back to all active clusters.
+	if err := s.StoreTelemetry(report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")); err != nil {
+		t.Fatal(err)
+	}
+	stats := statsOf(t, s)
+	if stats["active_instances"] != 1 || stats["confirmed_instances"] != 0 {
+		t.Fatalf("day one: active=%v confirmed=%v, want 1/0", stats["active_instances"], stats["confirmed_instances"])
+	}
+	if v := stats["versions"].(map[string]int); v["4.40"] != 1 {
+		t.Fatalf("fallback distribution missing active cluster: %v", v)
+	}
+
+	// Give cluster A a sample from yesterday: now seen on 2 distinct days.
+	s.mu.Lock()
+	id := "aaaaaaaa-0000-0000-0000-000000000001"
+	s.histories[id] = append([]HistorySample{{
+		Ts:             time.Now().AddDate(0, 0, -1).Unix(),
+		TotalDiskBytes: 50,
+	}}, s.histories[id]...)
+	s.mu.Unlock()
+
+	// A one-shot cluster B arrives (like an injected report): it counts as
+	// active, but the distributions now only reflect confirmed clusters.
+	if err := s.StoreTelemetry(report("bbbbbbbb-0000-0000-0000-000000000002", "9.99")); err != nil {
+		t.Fatal(err)
+	}
+	stats = statsOf(t, s)
+	if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 {
+		t.Fatalf("day two: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
+	}
+	v := stats["versions"].(map[string]int)
+	if v["4.40"] != 1 {
+		t.Errorf("confirmed cluster missing from distribution: %v", v)
+	}
+	if _, ok := v["9.99"]; ok {
+		t.Errorf("one-shot cluster polluted the distribution: %v", v)
+	}
+}

telemetry/server/storage/history.go

@@ -6,6 +6,10 @@ import (
 	"github.com/seaweedfs/seaweedfs/telemetry/proto"
 )
 
+// confirmDays is how many distinct UTC days a cluster must have reported
+// on before it counts as confirmed in the aggregated stats.
+const confirmDays = 2
+
 // HistorySample is one retained data point of a cluster's daily reports.
 // Tags are kept short because thousands of samples end up in the state file.
 type HistorySample struct {

telemetry/server/storage/prometheus.go

@@ -14,6 +14,7 @@ type PrometheusStorage struct {
 	// Prometheus metrics
 	totalClusters     prometheus.Gauge
 	activeClusters    prometheus.Gauge
+	confirmedClusters prometheus.Gauge
 	volumeServerCount *prometheus.GaugeVec
 	totalDiskBytes    *prometheus.GaugeVec
 	totalVolumeCount  *prometheus.GaugeVec
@@ -51,6 +52,10 @@ func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
 			Name: "seaweedfs_telemetry_active_clusters",
 			Help: "Number of active SeaweedFS clusters (last 7 days)",
 		}),
+		confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "seaweedfs_telemetry_confirmed_clusters",
+			Help: "Active clusters seen on at least 2 distinct days (last 7 days)",
+		}),
 		volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "seaweedfs_telemetry_volume_servers",
 			Help: "Number of volume servers per cluster",
@@ -207,8 +212,11 @@ func (s *PrometheusStorage) updateStats() {
 
 	totalInstances := 0
 	activeInstances := 0
-	versions := make(map[string]int)
-	osDistribution := make(map[string]int)
+	confirmedInstances := 0
+	versionsAll := make(map[string]int)
+	osAll := make(map[string]int)
+	versionsConfirmed := make(map[string]int)
+	osConfirmed := make(map[string]int)
 
 	for _, instance := range s.instances {
 		if instance.ReceivedAt.After(last30Days) {
@@ -216,21 +224,39 @@ func (s *PrometheusStorage) updateStats() {
 		}
 		if instance.ReceivedAt.After(last7Days) {
 			activeInstances++
-			versions[instance.TelemetryData.Version]++
-			osDistribution[instance.TelemetryData.Os]++
+			versionsAll[instance.TelemetryData.Version]++
+			osAll[instance.TelemetryData.Os]++
+			// A cluster is confirmed once seen on >=2 distinct UTC days
+			// (histories hold one sample per day), so one-shot reports
+			// can't skew the distributions below.
+			if len(s.histories[instance.TelemetryData.TopologyId]) >= confirmDays {
+				confirmedInstances++
+				versionsConfirmed[instance.TelemetryData.Version]++
+				osConfirmed[instance.TelemetryData.Os]++
+			}
 		}
 	}
 
+	// Before any cluster has two days of history (fresh server with no
+	// prior state), fall back to all active clusters so the dashboard
+	// distributions aren't empty.
+	versions, osDistribution := versionsConfirmed, osConfirmed
+	if confirmedInstances == 0 {
+		versions, osDistribution = versionsAll, osAll
+	}
+
 	// Update Prometheus gauges
 	s.totalClusters.Set(float64(totalInstances))
 	s.activeClusters.Set(float64(activeInstances))
+	s.confirmedClusters.Set(float64(confirmedInstances))
 
 	// Update cached stats for API
 	s.stats = map[string]interface{}{
-		"total_instances":  totalInstances,
-		"active_instances": activeInstances,
-		"versions":         versions,
-		"os_distribution":  osDistribution,
+		"total_instances":     totalInstances,
+		"active_instances":    activeInstances,
+		"confirmed_instances": confirmedInstances,
+		"versions":            versions,
+		"os_distribution":     osDistribution,
 	}
 }