-
Different Versions
@@ -198,6 +202,7 @@ func (h *Handler) ServeIndex(w http.ResponseWriter, r *http.Request) {
function updateStats(stats) {
document.getElementById('totalInstances').textContent = stats.total_instances || 0;
document.getElementById('activeInstances').textContent = stats.active_instances || 0;
+ document.getElementById('confirmedInstances').textContent = stats.confirmed_instances || 0;
document.getElementById('totalVersions').textContent = Object.keys(stats.versions || {}).length;
document.getElementById('totalOS').textContent = Object.keys(stats.os_distribution || {}).length;
}
diff --git a/telemetry/server/storage/confirmed_test.go b/telemetry/server/storage/confirmed_test.go
new file mode 100644
index 000000000..4670d1433
--- /dev/null
+++ b/telemetry/server/storage/confirmed_test.go
@@ -0,0 +1,72 @@
+package storage
+
+import (
+ "testing"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/seaweedfs/seaweedfs/telemetry/proto"
+)
+
+func report(id, version string) *proto.TelemetryData {
+ return &proto.TelemetryData{
+ TopologyId: id,
+ Version: version,
+ Os: "linux/amd64",
+ VolumeServerCount: 1,
+ TotalDiskBytes: 100,
+ TotalVolumeCount: 1,
+ }
+}
+
+func statsOf(t *testing.T, s *PrometheusStorage) map[string]interface{} {
+ t.Helper()
+ stats, err := s.GetStats()
+ if err != nil {
+ t.Fatalf("stats: %v", err)
+ }
+ return stats
+}
+
+func TestConfirmedClusters(t *testing.T) {
+ s := newPrometheusStorage(prometheus.NewRegistry())
+
+ // A single-day cluster is active but not confirmed; with no confirmed
+ // clusters yet, distributions fall back to all active clusters.
+ if err := s.StoreTelemetry(report("aaaaaaaa-0000-0000-0000-000000000001", "4.40")); err != nil {
+ t.Fatal(err)
+ }
+ stats := statsOf(t, s)
+ if stats["active_instances"] != 1 || stats["confirmed_instances"] != 0 {
+ t.Fatalf("day one: active=%v confirmed=%v, want 1/0", stats["active_instances"], stats["confirmed_instances"])
+ }
+ if v := stats["versions"].(map[string]int); v["4.40"] != 1 {
+ t.Fatalf("fallback distribution missing active cluster: %v", v)
+ }
+
+ // Give cluster A a sample from yesterday: now seen on 2 distinct days.
+ s.mu.Lock()
+ id := "aaaaaaaa-0000-0000-0000-000000000001"
+ s.histories[id] = append([]HistorySample{{
+ Ts: time.Now().AddDate(0, 0, -1).Unix(),
+ TotalDiskBytes: 50,
+ }}, s.histories[id]...)
+ s.mu.Unlock()
+
+ // A one-shot cluster B arrives (like an injected report): it counts as
+ // active, but the distributions now only reflect confirmed clusters.
+ if err := s.StoreTelemetry(report("bbbbbbbb-0000-0000-0000-000000000002", "9.99")); err != nil {
+ t.Fatal(err)
+ }
+ stats = statsOf(t, s)
+ if stats["active_instances"] != 2 || stats["confirmed_instances"] != 1 {
+ t.Fatalf("day two: active=%v confirmed=%v, want 2/1", stats["active_instances"], stats["confirmed_instances"])
+ }
+ v := stats["versions"].(map[string]int)
+ if v["4.40"] != 1 {
+ t.Errorf("confirmed cluster missing from distribution: %v", v)
+ }
+ if _, ok := v["9.99"]; ok {
+ t.Errorf("one-shot cluster polluted the distribution: %v", v)
+ }
+}
diff --git a/telemetry/server/storage/history.go b/telemetry/server/storage/history.go
index 8f5e26f45..eff68bc03 100644
--- a/telemetry/server/storage/history.go
+++ b/telemetry/server/storage/history.go
@@ -6,6 +6,10 @@ import (
"github.com/seaweedfs/seaweedfs/telemetry/proto"
)
+// confirmDays is how many distinct UTC days a cluster must have reported
+// on before it counts as confirmed in the aggregated stats.
+const confirmDays = 2
+
// HistorySample is one retained data point of a cluster's daily reports.
// Tags are kept short because thousands of samples end up in the state file.
type HistorySample struct {
diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.go
index 3c7869c82..15e040f9d 100644
--- a/telemetry/server/storage/prometheus.go
+++ b/telemetry/server/storage/prometheus.go
@@ -14,6 +14,7 @@ type PrometheusStorage struct {
// Prometheus metrics
totalClusters prometheus.Gauge
activeClusters prometheus.Gauge
+ confirmedClusters prometheus.Gauge
volumeServerCount *prometheus.GaugeVec
totalDiskBytes *prometheus.GaugeVec
totalVolumeCount *prometheus.GaugeVec
@@ -51,6 +52,10 @@ func newPrometheusStorage(reg prometheus.Registerer) *PrometheusStorage {
Name: "seaweedfs_telemetry_active_clusters",
Help: "Number of active SeaweedFS clusters (last 7 days)",
}),
+ confirmedClusters: promauto.NewGauge(prometheus.GaugeOpts{
+ Name: "seaweedfs_telemetry_confirmed_clusters",
+ Help: "Active clusters seen on at least 2 distinct days (last 7 days)",
+ }),
volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_volume_servers",
Help: "Number of volume servers per cluster",
@@ -207,8 +212,11 @@ func (s *PrometheusStorage) updateStats() {
totalInstances := 0
activeInstances := 0
- versions := make(map[string]int)
- osDistribution := make(map[string]int)
+ confirmedInstances := 0
+ versionsAll := make(map[string]int)
+ osAll := make(map[string]int)
+ versionsConfirmed := make(map[string]int)
+ osConfirmed := make(map[string]int)
for _, instance := range s.instances {
if instance.ReceivedAt.After(last30Days) {
@@ -216,21 +224,39 @@ func (s *PrometheusStorage) updateStats() {
}
if instance.ReceivedAt.After(last7Days) {
activeInstances++
- versions[instance.TelemetryData.Version]++
- osDistribution[instance.TelemetryData.Os]++
+ versionsAll[instance.TelemetryData.Version]++
+ osAll[instance.TelemetryData.Os]++
+ // A cluster is confirmed once seen on >=2 distinct UTC days
+ // (histories hold one sample per day), so one-shot reports
+ // can't skew the distributions below.
+ if len(s.histories[instance.TelemetryData.TopologyId]) >= confirmDays {
+ confirmedInstances++
+ versionsConfirmed[instance.TelemetryData.Version]++
+ osConfirmed[instance.TelemetryData.Os]++
+ }
}
}
+ // Before any cluster has two days of history (fresh server with no
+ // prior state), fall back to all active clusters so the dashboard
+ // distributions aren't empty.
+ versions, osDistribution := versionsConfirmed, osConfirmed
+ if confirmedInstances == 0 {
+ versions, osDistribution = versionsAll, osAll
+ }
+
// Update Prometheus gauges
s.totalClusters.Set(float64(totalInstances))
s.activeClusters.Set(float64(activeInstances))
+ s.confirmedClusters.Set(float64(confirmedInstances))
// Update cached stats for API
s.stats = map[string]interface{}{
- "total_instances": totalInstances,
- "active_instances": activeInstances,
- "versions": versions,
- "os_distribution": osDistribution,
+ "total_instances": totalInstances,
+ "active_instances": activeInstances,
+ "confirmed_instances": confirmedInstances,
+ "versions": versions,
+ "os_distribution": osDistribution,
}
}