From 6c3d81a146ceda77d73150bc58e63a5c8329b0cf Mon Sep 17 00:00:00 2001 From: Quang Ngo Date: Mon, 2 Mar 2026 10:19:16 +1100 Subject: [PATCH 1/2] Add schedule_expected_interval_seconds metric Add a new Prometheus gauge metric that exposes the expected interval between consecutive scheduled backups. This enables dynamic alerting thresholds per schedule backups. Signed-off-by: Quang Ngo --- pkg/controller/schedule_controller.go | 7 +++ pkg/metrics/metrics.go | 22 +++++++ pkg/metrics/metrics_test.go | 84 +++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index ec8894571..443b3c08b 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -129,6 +129,13 @@ func (c *scheduleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } else { schedule.Status.Phase = velerov1.SchedulePhaseEnabled schedule.Status.ValidationErrors = nil + + // Compute expected interval between consecutive scheduled backup runs. + // Only meaningful when the cron expression is valid. + now := c.clock.Now() + nextRun := cronSchedule.Next(now) + nextNextRun := cronSchedule.Next(nextRun) + c.metrics.SetScheduleExpectedIntervalSeconds(schedule.Name, nextNextRun.Sub(nextRun).Seconds()) } scheduleNeedsPatch := false diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 30e67a7b6..86d78028c 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -80,6 +80,9 @@ const ( DataDownloadFailureTotal = "data_download_failure_total" DataDownloadCancelTotal = "data_download_cancel_total" + // schedule metrics + scheduleExpectedIntervalSeconds = "schedule_expected_interval_seconds" + // repo maintenance metrics repoMaintenanceSuccessTotal = "repo_maintenance_success_total" repoMaintenanceFailureTotal = "repo_maintenance_failure_total" @@ -347,6 +350,14 @@ func NewServerMetrics() *ServerMetrics { }, []string{scheduleLabel, backupNameLabel}, ), + scheduleExpectedIntervalSeconds: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricNamespace, + Name: scheduleExpectedIntervalSeconds, + Help: "Expected interval between consecutive scheduled backups, in seconds", + }, + []string{scheduleLabel}, + ), repoMaintenanceSuccessTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: metricNamespace, @@ -644,6 +655,9 @@ func (m *ServerMetrics) RemoveSchedule(scheduleName string) { if c, ok := m.metrics[csiSnapshotFailureTotal].(*prometheus.CounterVec); ok { c.DeleteLabelValues(scheduleName, "") } + if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { + g.DeleteLabelValues(scheduleName) + } } // InitMetricsForNode initializes counter metrics for a node. @@ -758,6 +772,14 @@ func (m *ServerMetrics) SetBackupLastSuccessfulTimestamp(backupSchedule string, } } +// SetScheduleExpectedIntervalSeconds records the expected interval in seconds, +// between consecutive backups for a schedule. +func (m *ServerMetrics) SetScheduleExpectedIntervalSeconds(scheduleName string, seconds float64) { + if g, ok := m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec); ok { + g.WithLabelValues(scheduleName).Set(seconds) + } +} + // SetBackupTotal records the current number of existent backups. func (m *ServerMetrics) SetBackupTotal(numberOfBackups int64) { if g, ok := m.metrics[backupTotal].(prometheus.Gauge); ok { diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 184e496ab..a24f2bf33 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -259,6 +259,90 @@ func TestMultipleAdhocBackupsShareMetrics(t *testing.T) { assert.Equal(t, float64(1), validationFailureMetric, "All adhoc validation failures should be counted together") } +// TestSetScheduleExpectedIntervalSeconds verifies that the expected interval metric +// is properly recorded for schedules. +func TestSetScheduleExpectedIntervalSeconds(t *testing.T) { + tests := []struct { + name string + scheduleName string + intervalSeconds float64 + description string + }{ + { + name: "every 5 minutes schedule", + scheduleName: "frequent-backup", + intervalSeconds: 300, + description: "Expected interval should be 5m in seconds", + }, + { + name: "daily schedule", + scheduleName: "daily-backup", + intervalSeconds: 86400, + description: "Expected interval should be 24h in seconds", + }, + { + name: "monthly schedule", + scheduleName: "monthly-backup", + intervalSeconds: 2678400, // 31 days in seconds + description: "Expected interval should be 31 days in seconds", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewServerMetrics() + m.SetScheduleExpectedIntervalSeconds(tc.scheduleName, tc.intervalSeconds) + + metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), tc.scheduleName) + assert.Equal(t, tc.intervalSeconds, metric, tc.description) + }) + } +} + +// TestScheduleExpectedIntervalNotInitializedByDefault verifies that the expected +// interval metric is not initialized by InitSchedule, so it only appears for +// schedules with a valid cron expression. +func TestScheduleExpectedIntervalNotInitializedByDefault(t *testing.T) { + m := NewServerMetrics() + m.InitSchedule("test-schedule") + + // The metric should not have any values after InitSchedule + ch := make(chan prometheus.Metric, 1) + m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) + close(ch) + + count := 0 + for range ch { + count++ + } + assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should not be initialized by InitSchedule") +} + +// TestRemoveScheduleCleansUpExpectedInterval verifies that RemoveSchedule +// cleans up the expected interval metric. +func TestRemoveScheduleCleansUpExpectedInterval(t *testing.T) { + m := NewServerMetrics() + m.InitSchedule("test-schedule") + m.SetScheduleExpectedIntervalSeconds("test-schedule", 3600) + + // Verify metric exists + metric := getMetricValue(t, m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec), "test-schedule") + assert.Equal(t, float64(3600), metric) + + // Remove schedule and verify metric is cleaned up + m.RemoveSchedule("test-schedule") + + ch := make(chan prometheus.Metric, 1) + m.metrics[scheduleExpectedIntervalSeconds].(*prometheus.GaugeVec).Collect(ch) + close(ch) + + count := 0 + for range ch { + count++ + } + assert.Equal(t, 0, count, "scheduleExpectedIntervalSeconds should be removed after RemoveSchedule") +} + // TestInitScheduleWithEmptyName verifies that InitSchedule works correctly // with an empty schedule name (for adhoc backups). func TestInitScheduleWithEmptyName(t *testing.T) { From 1c08af84614e7f1197d9fe5b9a117668851c1afd Mon Sep 17 00:00:00 2001 From: Quang Ngo Date: Mon, 2 Mar 2026 10:49:14 +1100 Subject: [PATCH 2/2] Add changelog for #9570 Signed-off-by: Quang Ngo --- changelogs/unreleased/9570-H-M-Quang-Ngo | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/9570-H-M-Quang-Ngo diff --git a/changelogs/unreleased/9570-H-M-Quang-Ngo b/changelogs/unreleased/9570-H-M-Quang-Ngo new file mode 100644 index 000000000..603cd75e5 --- /dev/null +++ b/changelogs/unreleased/9570-H-M-Quang-Ngo @@ -0,0 +1 @@ +Add schedule_expected_interval_seconds metric for dynamic backup alerting thresholds (#9559)