mirrors/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-12 19:16:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

telemetry: key per-cluster value gauges by cluster_id only (#10397)

Browse Source 
telemetry: key value gauges by cluster_id only

The value gauges were labeled {cluster_id, version, os}, so a cluster
reporting back after an upgrade started a new series while the old one
kept its last value forever: sum() double-counted every upgraded
cluster, and per-cluster history broke at every version change.

Key the five value gauges by cluster_id alone so each cluster keeps one
continuous series across upgrades; version/os metadata stays on
cluster_info (deleted and re-set on change), available to value queries
via 'on(cluster_id) group_left' joins. Update README accordingly.
This commit is contained in:
Chris Lu
2026-07-22 17:41:27 -07:00
committed by GitHub
parent 87ecaf1afa
commit 6832b76529
2 changed files with 55 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
telemetry/README.md
+16 -11
View File
@@ -136,13 +136,18 @@ The telemetry server exposes these Prometheus metrics:
- `seaweedfs_telemetry_active_clusters`: Active clusters (7 days)
### Per-Cluster Metrics
- `seaweedfs_telemetry_volume_servers{cluster_id, version, os}`: Volume servers per cluster
- `seaweedfs_telemetry_disk_bytes{cluster_id, version, os}`: Disk usage per cluster
- `seaweedfs_telemetry_volume_count{cluster_id, version, os}`: Volume count per cluster
- `seaweedfs_telemetry_filer_count{cluster_id, version, os}`: Filer servers per cluster
- `seaweedfs_telemetry_broker_count{cluster_id, version, os}`: Broker servers per cluster
- `seaweedfs_telemetry_volume_servers{cluster_id}`: Volume servers per cluster
- `seaweedfs_telemetry_disk_bytes{cluster_id}`: Disk usage per cluster
- `seaweedfs_telemetry_volume_count{cluster_id}`: Volume count per cluster
- `seaweedfs_telemetry_filer_count{cluster_id}`: Filer servers per cluster
- `seaweedfs_telemetry_broker_count{cluster_id}`: Broker servers per cluster
- `seaweedfs_telemetry_cluster_info{cluster_id, version, os}`: Cluster metadata
Value gauges are keyed by `cluster_id` only, so a cluster keeps one continuous
series across upgrades. To slice values by version or OS, join with
`cluster_info`, e.g.
`seaweedfs_telemetry_disk_bytes * on(cluster_id) group_left(version, os) seaweedfs_telemetry_cluster_info`.
### Server Metrics
- `seaweedfs_telemetry_reports_received_total`: Total telemetry reports received
@@ -274,14 +279,14 @@ The included Grafana dashboard provides:
# Total active clusters
seaweedfs_telemetry_active_clusters
# Disk usage by version
sum by (version) (seaweedfs_telemetry_disk_bytes)
# Disk usage per cluster
seaweedfs_telemetry_disk_bytes{cluster_id="<uuid>"}
# Disk usage by version (join with cluster_info for version/os)
sum by (version) (seaweedfs_telemetry_disk_bytes * on(cluster_id) group_left(version) seaweedfs_telemetry_cluster_info)
# Volume servers by operating system
sum by (os) (seaweedfs_telemetry_volume_servers)
# Filer servers by version
sum by (version) (seaweedfs_telemetry_filer_count)
sum by (os) (seaweedfs_telemetry_volume_servers * on(cluster_id) group_left(os) seaweedfs_telemetry_cluster_info)
# Broker servers across all clusters
sum(seaweedfs_telemetry_broker_count)
telemetry/server/storage/prometheus.go
+39 -26
View File
@@ -47,23 +47,23 @@ func NewPrometheusStorage() *PrometheusStorage {
volumeServerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_volume_servers",
Help: "Number of volume servers per cluster",
}, []string{"cluster_id", "version", "os"}),
}, []string{"cluster_id"}),
totalDiskBytes: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_disk_bytes",
Help: "Total disk usage in bytes per cluster",
}, []string{"cluster_id", "version", "os"}),
}, []string{"cluster_id"}),
totalVolumeCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_volume_count",
Help: "Total number of volumes per cluster",
}, []string{"cluster_id", "version", "os"}),
}, []string{"cluster_id"}),
filerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_filer_count",
Help: "Number of filer servers per cluster",
}, []string{"cluster_id", "version", "os"}),
}, []string{"cluster_id"}),
brokerCount: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_broker_count",
Help: "Number of broker servers per cluster",
}, []string{"cluster_id", "version", "os"}),
}, []string{"cluster_id"}),
clusterInfo: promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "seaweedfs_telemetry_cluster_info",
Help: "Cluster information (always 1, labels contain metadata)",
@@ -81,11 +81,11 @@ func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error {
s.mu.Lock()
defer s.mu.Unlock()
// Update Prometheus metrics
// Update Prometheus metrics. Value gauges are keyed by cluster_id only so
// a cluster's series continues across upgrades; version/os metadata lives
// on cluster_info (join with `* on(cluster_id) group_left(version, os)`).
labels := prometheus.Labels{
"cluster_id": data.TopologyId,
"version": data.Version,
"os": data.Os,
}
s.volumeServerCount.With(labels).Set(float64(data.VolumeServerCount))
@@ -94,12 +94,14 @@ func (s *PrometheusStorage) StoreTelemetry(data *proto.TelemetryData) error {
s.filerCount.With(labels).Set(float64(data.FilerCount))
s.brokerCount.With(labels).Set(float64(data.BrokerCount))
infoLabels := prometheus.Labels{
"cluster_id": data.TopologyId,
"version": data.Version,
"os": data.Os,
// Drop the cluster_info series recorded under the previous label set when
// a cluster reports back with a different version or OS, so it is not
// counted under two versions at once.
if prev, ok := s.instances[data.TopologyId]; ok &&
(prev.TelemetryData.Version != data.Version || prev.TelemetryData.Os != data.Os) {
s.clusterInfo.Delete(infoLabels(prev.TelemetryData))
}
s.clusterInfo.With(infoLabels).Set(1)
s.clusterInfo.With(infoLabels(data)).Set(1)
s.telemetryReceived.Inc()
@@ -227,21 +229,32 @@ func (s *PrometheusStorage) CleanupOldInstances(maxAge time.Duration) {
for instanceID, instance := range s.instances {
if instance.ReceivedAt.Before(cutoff) {
delete(s.instances, instanceID)
// Remove from Prometheus metrics
labels := prometheus.Labels{
"cluster_id": instance.TelemetryData.TopologyId,
"version": instance.TelemetryData.Version,
"os": instance.TelemetryData.Os,
}
s.volumeServerCount.Delete(labels)
s.totalDiskBytes.Delete(labels)
s.totalVolumeCount.Delete(labels)
s.filerCount.Delete(labels)
s.brokerCount.Delete(labels)
s.clusterInfo.Delete(labels)
s.deleteClusterMetrics(instance.TelemetryData)
}
}
s.updateStats()
}
// deleteClusterMetrics removes all gauges stored for the given report's
// cluster. Callers must hold s.mu.
func (s *PrometheusStorage) deleteClusterMetrics(data *proto.TelemetryData) {
labels := prometheus.Labels{
"cluster_id": data.TopologyId,
}
s.volumeServerCount.Delete(labels)
s.totalDiskBytes.Delete(labels)
s.totalVolumeCount.Delete(labels)
s.filerCount.Delete(labels)
s.brokerCount.Delete(labels)
s.clusterInfo.Delete(infoLabels(data))
}
// infoLabels is the full label set used by the cluster_info metric.
func infoLabels(data *proto.TelemetryData) prometheus.Labels {
return prometheus.Labels{
"cluster_id": data.TopologyId,
"version": data.Version,
"os": data.Os,
}
}