cancel pvb/pvr on velero server restarts

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
2026-09-07 16:47:11 +00:00 · 2025-06-23 15:29:09 +08:00
parent 9500f0b82f
commit cded6bd207
7 changed files with 101 additions and 10 deletions
@@ -961,6 +961,7 @@ func markInProgressBackupsFailed(ctx context.Context, client ctrlclient.Client,
 		}
 		log.WithField("backup", backup.GetName()).Warn(updated.Status.FailureReason)
 		markDataUploadsCancel(ctx, client, backup, log)
+		markPodVolumeBackupsCancel(ctx, client, backup, log)
 	}
 }

@@ -983,8 +984,10 @@ func markInProgressRestoresFailed(ctx context.Context, client ctrlclient.Client,
 			log.WithError(errors.WithStack(err)).Errorf("failed to patch restore %q", restore.GetName())
 			continue
 		}
+
 		log.WithField("restore", restore.GetName()).Warn(updated.Status.FailureReason)
 		markDataDownloadsCancel(ctx, client, restore, log)
+		markPodVolumeRestoresCancel(ctx, client, restore, log)
 	}
 }

@@ -1069,3 +1072,90 @@ func markDataDownloadsCancel(ctx context.Context, client ctrlclient.Client, rest
 		}
 	}
 }
+
+func markPodVolumeBackupsCancel(ctx context.Context, client ctrlclient.Client, backup velerov1api.Backup, log logrus.FieldLogger) {
+	pvbs := &velerov1api.PodVolumeBackupList{}
+
+	if err := client.List(ctx, pvbs, &ctrlclient.ListOptions{
+		Namespace: backup.GetNamespace(),
+		LabelSelector: labels.Set(map[string]string{
+			velerov1api.BackupUIDLabel: string(backup.GetUID()),
+		}).AsSelector(),
+	}); err != nil {
+		log.WithError(errors.WithStack(err)).Error("failed to list PVBs")
+		return
+	}
+
+	for i := range pvbs.Items {
+		pvb := pvbs.Items[i]
+		if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseAccepted ||
+			pvb.Status.Phase == velerov1api.PodVolumeBackupPhasePrepared ||
+			pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseInProgress ||
+			pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseNew ||
+			pvb.Status.Phase == "" {
+			err := controller.UpdatePVBWithRetry(ctx, client, types.NamespacedName{Namespace: pvb.Namespace, Name: pvb.Name}, log.WithField("PVB", pvb.Name),
+				func(pvb *velerov1api.PodVolumeBackup) bool {
+					if pvb.Spec.Cancel {
+						return false
+					}
+
+					pvb.Spec.Cancel = true
+					pvb.Status.Message = fmt.Sprintf("PVB is in status %q during the velero server starting, mark it as cancel", pvb.Status.Phase)
+
+					return true
+				})
+
+			if err != nil {
+				log.WithError(errors.WithStack(err)).Errorf("failed to mark PVB %q cancel", pvb.GetName())
+				continue
+			}
+			log.WithField("PVB is mark for cancel due to server restart", pvb.GetName()).Warn(pvb.Status.Message)
+		}
+	}
+}
+
+func markPodVolumeRestoresCancel(ctx context.Context, client ctrlclient.Client, restore velerov1api.Restore, log logrus.FieldLogger) {
+	pvrs := &velerov1api.PodVolumeRestoreList{}
+
+	if err := client.List(ctx, pvrs, &ctrlclient.ListOptions{
+		Namespace: restore.GetNamespace(),
+		LabelSelector: labels.Set(map[string]string{
+			velerov1api.RestoreUIDLabel: string(restore.GetUID()),
+		}).AsSelector(),
+	}); err != nil {
+		log.WithError(errors.WithStack(err)).Error("failed to list PVRs")
+		return
+	}
+
+	for i := range pvrs.Items {
+		pvr := pvrs.Items[i]
+		if controller.IsLegacyPVR(&pvr) {
+			log.WithField("PVR", pvr.GetName()).Warn("Found a legacy PVR during velero server restart, cannot stop it")
+			continue
+		}
+
+		if pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseAccepted ||
+			pvr.Status.Phase == velerov1api.PodVolumeRestorePhasePrepared ||
+			pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseInProgress ||
+			pvr.Status.Phase == velerov1api.PodVolumeRestorePhaseNew ||
+			pvr.Status.Phase == "" {
+			err := controller.UpdatePVRWithRetry(ctx, client, types.NamespacedName{Namespace: pvr.Namespace, Name: pvr.Name}, log.WithField("PVR", pvr.Name),
+				func(pvr *velerov1api.PodVolumeRestore) bool {
+					if pvr.Spec.Cancel {
+						return false
+					}
+
+					pvr.Spec.Cancel = true
+					pvr.Status.Message = fmt.Sprintf("PVR is in status %q during the velero server starting, mark it as cancel", pvr.Status.Phase)
+
+					return true
+				})
+
+			if err != nil {
+				log.WithError(errors.WithStack(err)).Errorf("failed to mark PVR %q cancel", pvr.GetName())
+				continue
+			}
+			log.WithField("PVR is mark for cancel due to server restart", pvr.GetName()).Warn(pvr.Status.Message)
+		}
+	}
+}