From 59289fba76ad39096543c79bf493ed551f795a10 Mon Sep 17 00:00:00 2001 From: Shubham Pampattiwar Date: Mon, 15 Sep 2025 16:01:33 -0700 Subject: [PATCH 1/2] Fix Schedule Backup Queue Accumulation During Extended Blocking Scenarios Signed-off-by: Shubham Pampattiwar --- pkg/controller/schedule_controller.go | 2 +- pkg/controller/schedule_controller_test.go | 26 ++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go index 799a8c77a..ec8894571 100644 --- a/pkg/controller/schedule_controller.go +++ b/pkg/controller/schedule_controller.go @@ -229,7 +229,7 @@ func (c *scheduleReconciler) checkIfBackupInNewOrProgress(schedule *velerov1.Sch } for _, backup := range backupList.Items { - if backup.Status.Phase == velerov1.BackupPhaseNew || backup.Status.Phase == velerov1.BackupPhaseInProgress { + if backup.Status.Phase == "" || backup.Status.Phase == velerov1.BackupPhaseNew || backup.Status.Phase == velerov1.BackupPhaseInProgress { log.Debugf("%s/%s still has backups that are in InProgress or New...", schedule.Namespace, schedule.Name) return true } diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go index ab0a3f66d..f4585763c 100644 --- a/pkg/controller/schedule_controller_test.go +++ b/pkg/controller/schedule_controller_test.go @@ -149,6 +149,13 @@ func TestReconcileOfSchedule(t *testing.T) { expectedPhase: string(velerov1.SchedulePhaseEnabled), backup: builder.ForBackup("ns", "name-20220905120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Phase(velerov1.BackupPhaseNew).Result(), }, + { + name: "schedule already has backup with empty phase (not yet reconciled).", + schedule: newScheduleBuilder(velerov1.SchedulePhaseEnabled).CronSchedule("@every 5m").LastBackupTime("2000-01-01 00:00:00").Result(), + fakeClockTime: "2017-01-01 12:00:00", + expectedPhase: string(velerov1.SchedulePhaseEnabled), + backup: builder.ForBackup("ns", "name-20220905120000").ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")).Phase("").Result(), + }, } for _, test := range tests { @@ -215,10 +222,10 @@ func TestReconcileOfSchedule(t *testing.T) { backups := &velerov1.BackupList{} require.NoError(t, client.List(ctx, backups)) - // If backup associated with schedule's status is in New or InProgress, + // If backup associated with schedule's status is in New or InProgress or empty phase, // new backup shouldn't be submitted. if test.backup != nil && - (test.backup.Status.Phase == velerov1.BackupPhaseNew || test.backup.Status.Phase == velerov1.BackupPhaseInProgress) { + (test.backup.Status.Phase == "" || test.backup.Status.Phase == velerov1.BackupPhaseNew || test.backup.Status.Phase == velerov1.BackupPhaseInProgress) { assert.Len(t, backups.Items, 1) require.NoError(t, client.Delete(ctx, test.backup)) } @@ -479,4 +486,19 @@ func TestCheckIfBackupInNewOrProgress(t *testing.T) { reconciler = NewScheduleReconciler("namespace", logger, client, metrics.NewServerMetrics(), false) result = reconciler.checkIfBackupInNewOrProgress(testSchedule) assert.True(t, result) + + // Clean backup in InProgress phase. + err = client.Delete(ctx, inProgressBackup) + require.NoError(t, err, "fail to delete backup in InProgress phase in TestCheckIfBackupInNewOrProgress: %v", err) + + // Create backup with empty phase (not yet reconciled). + emptyPhaseBackup := builder.ForBackup("ns", "backup-3"). + ObjectMeta(builder.WithLabels(velerov1.ScheduleNameLabel, "name")). + Phase("").Result() + err = client.Create(ctx, emptyPhaseBackup) + require.NoError(t, err, "fail to create backup with empty phase in TestCheckIfBackupInNewOrProgress: %v", err) + + reconciler = NewScheduleReconciler("namespace", logger, client, metrics.NewServerMetrics(), false) + result = reconciler.checkIfBackupInNewOrProgress(testSchedule) + assert.True(t, result) } From a7b2985c8396c5db9a259e62abcd47acc4da7343 Mon Sep 17 00:00:00 2001 From: Shubham Pampattiwar Date: Mon, 15 Sep 2025 16:07:40 -0700 Subject: [PATCH 2/2] add changelog file Signed-off-by: Shubham Pampattiwar --- changelogs/unreleased/9264-shubham-pampattiwar | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelogs/unreleased/9264-shubham-pampattiwar diff --git a/changelogs/unreleased/9264-shubham-pampattiwar b/changelogs/unreleased/9264-shubham-pampattiwar new file mode 100644 index 000000000..711ea4b57 --- /dev/null +++ b/changelogs/unreleased/9264-shubham-pampattiwar @@ -0,0 +1 @@ +Fix schedule controller to prevent backup queue accumulation during extended blocking scenarios by properly handling empty backup phases \ No newline at end of file