From 6832b76529731b4f238309f37fab333fb7f0cb86 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 22 Jul 2026 17:41:27 -0700 Subject: [PATCH] telemetry: key per-cluster value gauges by cluster_id only (#10397) telemetry: key value gauges by cluster_id only The value gauges were labeled {cluster_id, version, os}, so a cluster reporting back after an upgrade started a new series while the old one kept its last value forever: sum() double-counted every upgraded cluster, and per-cluster history broke at every version change. Key the five value gauges by cluster_id alone so each cluster keeps one continuous series across upgrades; version/os metadata stays on cluster_info (deleted and re-set on change), available to value queries via 'on(cluster_id) group_left' joins. Update README accordingly. --- telemetry/README.md | 27 ++++++----- telemetry/server/storage/prometheus.go | 65 +++++++++++++++----------- 2 files changed, 55 insertions(+), 37 deletions(-) diff --git a/telemetry/README.md b/telemetry/README.md index f2d1f1ccf..6f1e4797b 100644 --- a/telemetry/README.md +++ b/telemetry/README.md @@ -136,13 +136,18 @@ The telemetry server exposes these Prometheus metrics: - `seaweedfs_telemetry_active_clusters`: Active clusters (7 days) ### Per-Cluster Metrics -- `seaweedfs_telemetry_volume_servers{cluster_id, version, os}`: Volume servers per cluster -- `seaweedfs_telemetry_disk_bytes{cluster_id, version, os}`: Disk usage per cluster -- `seaweedfs_telemetry_volume_count{cluster_id, version, os}`: Volume count per cluster -- `seaweedfs_telemetry_filer_count{cluster_id, version, os}`: Filer servers per cluster -- `seaweedfs_telemetry_broker_count{cluster_id, version, os}`: Broker servers per cluster +- `seaweedfs_telemetry_volume_servers{cluster_id}`: Volume servers per cluster +- `seaweedfs_telemetry_disk_bytes{cluster_id}`: Disk usage per cluster +- `seaweedfs_telemetry_volume_count{cluster_id}`: Volume count per cluster +- `seaweedfs_telemetry_filer_count{cluster_id}`: Filer servers per cluster +- `seaweedfs_telemetry_broker_count{cluster_id}`: Broker servers per cluster - `seaweedfs_telemetry_cluster_info{cluster_id, version, os}`: Cluster metadata +Value gauges are keyed by `cluster_id` only, so a cluster keeps one continuous +series across upgrades. To slice values by version or OS, join with +`cluster_info`, e.g. +`seaweedfs_telemetry_disk_bytes * on(cluster_id) group_left(version, os) seaweedfs_telemetry_cluster_info`. + ### Server Metrics - `seaweedfs_telemetry_reports_received_total`: Total telemetry reports received @@ -274,14 +279,14 @@ The included Grafana dashboard provides: # Total active clusters seaweedfs_telemetry_active_clusters -# Disk usage by version -sum by (version) (seaweedfs_telemetry_disk_bytes) +# Disk usage per cluster +seaweedfs_telemetry_disk_bytes{cluster_id=""} + +# Disk usage by version (join with cluster_info for version/os) +sum by (version) (seaweedfs_telemetry_disk_bytes * on(cluster_id) group_left(version) seaweedfs_telemetry_cluster_info) # Volume servers by operating system -sum by (os) (seaweedfs_telemetry_volume_servers) - -# Filer servers by version -sum by (version) (seaweedfs_telemetry_filer_count) +sum by (os) (seaweedfs_telemetry_volume_servers * on(cluster_id) group_left(os) seaweedfs_telemetry_cluster_info) # Broker servers across all clusters sum(seaweedfs_telemetry_broker_count) diff --git a/telemetry/server/storage/prometheus.go b/telemetry/server/storage/prometheus.go index 6bef9bd98..a4103ff9e 100644 --- a/telemetry/server/storage/prometheus.go +++ b/telemetry/server/storage/prometheus.go @@ -47,23 +47,23 @@ func NewPrometheusStorage() *PrometheusStorage { volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_volume_servers", Help: "Number of volume servers per cluster", - }, []string{"cluster_id", "version", "os"}), + }, []string{"cluster_id"}), totalDiskBytes: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_disk_bytes", Help: "Total disk usage in bytes per cluster", - }, []string{"cluster_id", "version", "os"}), + }, []string{"cluster_id"}), totalVolumeCount: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_volume_count", Help: "Total number of volumes per cluster", - }, []string{"cluster_id", "version", "os"}), + }, []string{"cluster_id"}), filerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_filer_count", Help: "Number of filer servers per cluster", - }, []string{"cluster_id", "version", "os"}), + }, []string{"cluster_id"}), brokerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_broker_count", Help: "Number of broker servers per cluster", - }, []string{"cluster_id", "version", "os"}), + }, []string{"cluster_id"}), clusterInfo: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "seaweedfs_telemetry_cluster_info", Help: "Cluster information (always 1, labels contain metadata)", @@ -81,11 +81,11 @@ func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error { s.mu.Lock() defer s.mu.Unlock() - // Update Prometheus metrics + // Update Prometheus metrics. Value gauges are keyed by cluster_id only so + // a cluster's series continues across upgrades; version/os metadata lives + // on cluster_info (join with `* on(cluster_id) group_left(version, os)`). labels := prometheus.Labels{ "cluster_id": data.TopologyId, - "version": data.Version, - "os": data.Os, } s.volumeServerCount.With(labels).Set(float64(data.VolumeServerCount)) @@ -94,12 +94,14 @@ func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error { s.filerCount.With(labels).Set(float64(data.FilerCount)) s.brokerCount.With(labels).Set(float64(data.BrokerCount)) - infoLabels := prometheus.Labels{ - "cluster_id": data.TopologyId, - "version": data.Version, - "os": data.Os, + // Drop the cluster_info series recorded under the previous label set when + // a cluster reports back with a different version or OS, so it is not + // counted under two versions at once. + if prev, ok := s.instances[data.TopologyId]; ok && + (prev.TelemetryData.Version != data.Version || prev.TelemetryData.Os != data.Os) { + s.clusterInfo.Delete(infoLabels(prev.TelemetryData)) } - s.clusterInfo.With(infoLabels).Set(1) + s.clusterInfo.With(infoLabels(data)).Set(1) s.telemetryReceived.Inc() @@ -227,21 +229,32 @@ func (s *PrometheusStorage) CleanupOldInstances(maxAge time.Duration) { for instanceID, instance := range s.instances { if instance.ReceivedAt.Before(cutoff) { delete(s.instances, instanceID) - - // Remove from Prometheus metrics - labels := prometheus.Labels{ - "cluster_id": instance.TelemetryData.TopologyId, - "version": instance.TelemetryData.Version, - "os": instance.TelemetryData.Os, - } - s.volumeServerCount.Delete(labels) - s.totalDiskBytes.Delete(labels) - s.totalVolumeCount.Delete(labels) - s.filerCount.Delete(labels) - s.brokerCount.Delete(labels) - s.clusterInfo.Delete(labels) + s.deleteClusterMetrics(instance.TelemetryData) } } s.updateStats() } + +// deleteClusterMetrics removes all gauges stored for the given report's +// cluster. Callers must hold s.mu. +func (s *PrometheusStorage) deleteClusterMetrics(data *proto.TelemetryData) { + labels := prometheus.Labels{ + "cluster_id": data.TopologyId, + } + s.volumeServerCount.Delete(labels) + s.totalDiskBytes.Delete(labels) + s.totalVolumeCount.Delete(labels) + s.filerCount.Delete(labels) + s.brokerCount.Delete(labels) + s.clusterInfo.Delete(infoLabels(data)) +} + +// infoLabels is the full label set used by the cluster_info metric. +func infoLabels(data *proto.TelemetryData) prometheus.Labels { + return prometheus.Labels{ + "cluster_id": data.TopologyId, + "version": data.Version, + "os": data.Os, + } +}