Files
seaweedfs/weed/stats/metrics_replication_test.go
df1a25fd3e feat: add Prometheus metrics for replication operations (#10006)
* feat: add Prometheus metrics for replication operations

Adds 5 metrics to instrument volume server replication (write/delete):
- Operations counter with success/failure labels
- Duration histogram for latency tracking
- Targets gauge for replica fanout
- Failures counter with error reason labels
- Under-replicated volumes gauge on master

* fix: record replication duration histogram only when replicaCount > 0

* fix: update replication targets gauge for all operations including zero

* fix: ensure symmetric replication success/failure counting and proper metrics updates

* fix: change VolumeServerReplicationTargets from Gauge to Histogram

- Replace .Set() with .Observe() in store_replicate.go (2 occurrences)
- Update test to use CollectAndCount for histogram assertion
- Rename TestReplicationTargetsGauge -> TestReplicationTargetsHistogram
- Update documentation to reflect Histogram type and PromQL examples

* Add comments to replication metrics and improve test coverage

* metrics: add replication panels to grafana dashboard

Master row gets an under-replicated volumes timeseries; Volume Servers
row gets replication operations, failures-by-reason, p99 duration, and
average fan-out panels for the new replication metrics.

* metrics: name the replication duration histogram replication_seconds

Match the volumeServer convention (request_seconds, vacuuming_seconds)
rather than the admin/lifecycle _duration_seconds spelling.

* metrics: guard replication fan-out panel against divide-by-zero

clamp_min the _count rate so the avg-targets ratio reads 0 instead of
NaN when there are no replication events in the window.

---------

Co-authored-by: Ubuntu User <ubuntu@example.com>
Co-authored-by: Chris Lu <chris.lu@gmail.com>
2026-06-19 11:05:43 -07:00

124 lines
4.3 KiB
Go

package stats
import (
"testing"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/testutil"
)
func TestReplicationMetricsRegistered(t *testing.T) {
VolumeServerReplicationCounter.Reset()
VolumeServerReplicationHistogram.Reset()
VolumeServerReplicationFailures.Reset()
MasterUnderReplicatedVolumes.Reset()
t.Cleanup(func() {
VolumeServerReplicationCounter.Reset()
VolumeServerReplicationHistogram.Reset()
VolumeServerReplicationFailures.Reset()
MasterUnderReplicatedVolumes.Reset()
})
// Seed CounterVec and HistogramVec with a value so collection returns them
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
VolumeServerReplicationHistogram.WithLabelValues(ReplicationOpWrite).Observe(0.1)
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
VolumeServerReplicationTargets.Observe(1)
MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(1)
metrics := []struct {
name string
c prometheus.Collector
}{
{"VolumeServerReplicationCounter", VolumeServerReplicationCounter},
{"VolumeServerReplicationHistogram", VolumeServerReplicationHistogram},
{"VolumeServerReplicationTargets", VolumeServerReplicationTargets},
{"VolumeServerReplicationFailures", VolumeServerReplicationFailures},
{"MasterUnderReplicatedVolumes", MasterUnderReplicatedVolumes},
}
for _, m := range metrics {
count := testutil.CollectAndCount(m.c)
if count < 1 {
t.Errorf("%s: expected at least 1 collection, got %d", m.name, count)
}
}
}
func TestReplicationCounterIncrement(t *testing.T) {
VolumeServerReplicationCounter.Reset()
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure).Inc()
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess).Inc()
got := testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess))
if got != 1 {
t.Errorf("expected 1.0, got %f", got)
}
got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess))
if got != 1 {
t.Errorf("expected 1.0, got %f", got)
}
got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure))
if got != 1 {
t.Errorf("expected 1.0, got %f", got)
}
}
func TestReplicationTargetsHistogram(t *testing.T) {
// VolumeServerReplicationTargets is a plain prometheus.Histogram
// (not a HistogramVec) and does not expose a Reset() method, so the
// histogram state is cumulative across tests that share the Gather
// registry. We check >= to tolerate prior observations.
VolumeServerReplicationTargets.Observe(3)
metrics, err := Gather.Gather()
if err != nil {
t.Fatalf("Failed to gather metrics: %v", err)
}
found := false
for _, mf := range metrics {
if mf.GetName() == "SeaweedFS_volumeServer_replication_targets" {
found = true
h := mf.GetMetric()[0].GetHistogram()
if h.GetSampleCount() < 1 {
t.Errorf("expected histogram_count>=1, got %d", h.GetSampleCount())
}
if h.GetSampleSum() < 3 {
t.Errorf("expected histogram_sum>=3, got %f", h.GetSampleSum())
}
}
}
if !found {
t.Error("SeaweedFS_volumeServer_replication_targets not found in gathered metrics")
}
}
func TestUnderReplicatedVolumesGauge(t *testing.T) {
MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(5)
got := testutil.ToFloat64(MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", ""))
if got != 5 {
t.Errorf("expected 5.0, got %f", got)
}
}
func TestReplicationFailuresCounter(t *testing.T) {
VolumeServerReplicationFailures.Reset()
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused).Inc()
got := testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout))
if got != 1 {
t.Errorf("expected 1.0, got %f", got)
}
got = testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused))
if got != 1 {
t.Errorf("expected 1.0, got %f", got)
}
}