mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-08-16 20:26:45 +00:00
* feat: add Prometheus metrics for replication operations Adds 5 metrics to instrument volume server replication (write/delete): - Operations counter with success/failure labels - Duration histogram for latency tracking - Targets gauge for replica fanout - Failures counter with error reason labels - Under-replicated volumes gauge on master * fix: record replication duration histogram only when replicaCount > 0 * fix: update replication targets gauge for all operations including zero * fix: ensure symmetric replication success/failure counting and proper metrics updates * fix: change VolumeServerReplicationTargets from Gauge to Histogram - Replace .Set() with .Observe() in store_replicate.go (2 occurrences) - Update test to use CollectAndCount for histogram assertion - Rename TestReplicationTargetsGauge -> TestReplicationTargetsHistogram - Update documentation to reflect Histogram type and PromQL examples * Add comments to replication metrics and improve test coverage * metrics: add replication panels to grafana dashboard Master row gets an under-replicated volumes timeseries; Volume Servers row gets replication operations, failures-by-reason, p99 duration, and average fan-out panels for the new replication metrics. * metrics: name the replication duration histogram replication_seconds Match the volumeServer convention (request_seconds, vacuuming_seconds) rather than the admin/lifecycle _duration_seconds spelling. * metrics: guard replication fan-out panel against divide-by-zero clamp_min the _count rate so the avg-targets ratio reads 0 instead of NaN when there are no replication events in the window. --------- Co-authored-by: Ubuntu User <ubuntu@example.com> Co-authored-by: Chris Lu <chris.lu@gmail.com>
124 lines
4.3 KiB
Go
124 lines
4.3 KiB
Go
package stats
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/testutil"
|
|
)
|
|
|
|
func TestReplicationMetricsRegistered(t *testing.T) {
|
|
|
|
VolumeServerReplicationCounter.Reset()
|
|
VolumeServerReplicationHistogram.Reset()
|
|
VolumeServerReplicationFailures.Reset()
|
|
MasterUnderReplicatedVolumes.Reset()
|
|
|
|
t.Cleanup(func() {
|
|
VolumeServerReplicationCounter.Reset()
|
|
VolumeServerReplicationHistogram.Reset()
|
|
VolumeServerReplicationFailures.Reset()
|
|
MasterUnderReplicatedVolumes.Reset()
|
|
})
|
|
|
|
// Seed CounterVec and HistogramVec with a value so collection returns them
|
|
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
|
|
VolumeServerReplicationHistogram.WithLabelValues(ReplicationOpWrite).Observe(0.1)
|
|
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
|
|
VolumeServerReplicationTargets.Observe(1)
|
|
MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(1)
|
|
|
|
metrics := []struct {
|
|
name string
|
|
c prometheus.Collector
|
|
}{
|
|
{"VolumeServerReplicationCounter", VolumeServerReplicationCounter},
|
|
{"VolumeServerReplicationHistogram", VolumeServerReplicationHistogram},
|
|
{"VolumeServerReplicationTargets", VolumeServerReplicationTargets},
|
|
{"VolumeServerReplicationFailures", VolumeServerReplicationFailures},
|
|
{"MasterUnderReplicatedVolumes", MasterUnderReplicatedVolumes},
|
|
}
|
|
for _, m := range metrics {
|
|
count := testutil.CollectAndCount(m.c)
|
|
if count < 1 {
|
|
t.Errorf("%s: expected at least 1 collection, got %d", m.name, count)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReplicationCounterIncrement(t *testing.T) {
|
|
VolumeServerReplicationCounter.Reset()
|
|
|
|
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess).Inc()
|
|
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure).Inc()
|
|
VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess).Inc()
|
|
|
|
got := testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationSuccess))
|
|
if got != 1 {
|
|
t.Errorf("expected 1.0, got %f", got)
|
|
}
|
|
got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpDelete, ReplicationSuccess))
|
|
if got != 1 {
|
|
t.Errorf("expected 1.0, got %f", got)
|
|
}
|
|
got = testutil.ToFloat64(VolumeServerReplicationCounter.WithLabelValues(ReplicationOpWrite, ReplicationFailure))
|
|
if got != 1 {
|
|
t.Errorf("expected 1.0, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestReplicationTargetsHistogram(t *testing.T) {
|
|
// VolumeServerReplicationTargets is a plain prometheus.Histogram
|
|
// (not a HistogramVec) and does not expose a Reset() method, so the
|
|
// histogram state is cumulative across tests that share the Gather
|
|
// registry. We check >= to tolerate prior observations.
|
|
VolumeServerReplicationTargets.Observe(3)
|
|
|
|
metrics, err := Gather.Gather()
|
|
if err != nil {
|
|
t.Fatalf("Failed to gather metrics: %v", err)
|
|
}
|
|
|
|
found := false
|
|
for _, mf := range metrics {
|
|
if mf.GetName() == "SeaweedFS_volumeServer_replication_targets" {
|
|
found = true
|
|
h := mf.GetMetric()[0].GetHistogram()
|
|
if h.GetSampleCount() < 1 {
|
|
t.Errorf("expected histogram_count>=1, got %d", h.GetSampleCount())
|
|
}
|
|
if h.GetSampleSum() < 3 {
|
|
t.Errorf("expected histogram_sum>=3, got %f", h.GetSampleSum())
|
|
}
|
|
}
|
|
}
|
|
if !found {
|
|
t.Error("SeaweedFS_volumeServer_replication_targets not found in gathered metrics")
|
|
}
|
|
}
|
|
|
|
func TestUnderReplicatedVolumesGauge(t *testing.T) {
|
|
MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", "").Set(5)
|
|
got := testutil.ToFloat64(MasterUnderReplicatedVolumes.WithLabelValues("default", "ssd", "1", ""))
|
|
if got != 5 {
|
|
t.Errorf("expected 5.0, got %f", got)
|
|
}
|
|
}
|
|
|
|
func TestReplicationFailuresCounter(t *testing.T) {
|
|
VolumeServerReplicationFailures.Reset()
|
|
|
|
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout).Inc()
|
|
VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused).Inc()
|
|
|
|
got := testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpWrite, FailureTimeout))
|
|
if got != 1 {
|
|
t.Errorf("expected 1.0, got %f", got)
|
|
}
|
|
|
|
got = testutil.ToFloat64(VolumeServerReplicationFailures.WithLabelValues(ReplicationOpDelete, FailureConnectionRefused))
|
|
if got != 1 {
|
|
t.Errorf("expected 1.0, got %f", got)
|
|
}
|
|
}
|