diff --git a/changelogs/unreleased/5829-kaovilai b/changelogs/unreleased/5829-kaovilai new file mode 100644 index 000000000..5342bb565 --- /dev/null +++ b/changelogs/unreleased/5829-kaovilai @@ -0,0 +1 @@ +Correct PVB/PVR Failed Phase patching during startup \ No newline at end of file diff --git a/pkg/cmd/cli/restic/server.go b/pkg/cmd/cli/restic/server.go index 77c1b4188..c870fe9da 100644 --- a/pkg/cmd/cli/restic/server.go +++ b/pkg/cmd/cli/restic/server.go @@ -300,12 +300,11 @@ func (s *resticServer) markInProgressPVBsFailed(client ctrlclient.Client) { log.Debugf("the node of podvolumebackup %q is %q, not %q, skip", pvb.GetName(), pvb.Spec.Node, s.nodeName) continue } - original := pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed - pvb.Status.Message = fmt.Sprintf("get a podvolumebackup with status %q during the server starting, mark it as %q", velerov1api.PodVolumeBackupPhaseInProgress, pvb.Status.Phase) - pvb.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()} - if err := client.Patch(s.ctx, &pvbs.Items[i], ctrlclient.MergeFrom(original)); err != nil { - log.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumebackup %q", pvb.GetName()) + + if err := controller.UpdatePVBStatusToFailed(client, s.ctx, &pvbs.Items[i], + fmt.Sprintf("get a podvolumebackup with status %q during the server starting, mark it as %q", velerov1api.PodVolumeBackupPhaseInProgress, velerov1api.PodVolumeBackupPhaseFailed), + time.Now()); err != nil { + s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumebackup %q", pvb.GetName()) continue } log.WithField("podvolumebackup", pvb.GetName()).Warn(pvb.Status.Message) @@ -338,12 +337,10 @@ func (s *resticServer) markInProgressPVRsFailed(client ctrlclient.Client) { continue } - original := pvr.DeepCopy() - pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed - pvr.Status.Message = fmt.Sprintf("get a podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, pvr.Status.Phase) - pvr.Status.CompletionTimestamp = &metav1.Time{Time: time.Now()} - if err := client.Patch(s.ctx, &pvrs.Items[i], ctrlclient.MergeFrom(original)); err != nil { - log.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName()) + if err := controller.UpdatePVRStatusToFailed(client, s.ctx, &pvrs.Items[i], + fmt.Sprintf("get a podvolumerestore with status %q during the server starting, mark it as %q", velerov1api.PodVolumeRestorePhaseInProgress, velerov1api.PodVolumeRestorePhaseFailed), + time.Now()); err != nil { + s.logger.WithError(errors.WithStack(err)).Errorf("failed to patch podvolumerestore %q", pvr.GetName()) continue } log.WithField("podvolumerestore", pvr.GetName()).Warn(pvr.Status.Message) diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 420e3ca9c..8f7985245 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -276,19 +276,22 @@ func (r *PodVolumeBackupReconciler) updateBackupProgressFunc(pvb *velerov1api.Po } func (r *PodVolumeBackupReconciler) updateStatusToFailed(ctx context.Context, pvb *velerov1api.PodVolumeBackup, err error, msg string, log logrus.FieldLogger) (ctrl.Result, error) { - original := pvb.DeepCopy() - pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed - pvb.Status.Message = errors.WithMessage(err, msg).Error() - pvb.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()} - - if err = r.Client.Patch(ctx, pvb, client.MergeFrom(original)); err != nil { + if err = UpdatePVBStatusToFailed(r.Client, ctx, pvb, errors.WithMessage(err, msg).Error(), r.Clock.Now()); err != nil { log.WithError(err).Error("error updating PodVolumeBackup status") return ctrl.Result{}, err } - return ctrl.Result{}, nil } +func UpdatePVBStatusToFailed(c client.Client, ctx context.Context, pvb *velerov1api.PodVolumeBackup, errString string, time time.Time) error { + original := pvb.DeepCopy() + pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseFailed + pvb.Status.Message = errString + pvb.Status.CompletionTimestamp = &metav1.Time{Time: time} + + return c.Patch(ctx, pvb, client.MergeFrom(original)) +} + type resticDetails struct { credsFile, caCertFile string envs []string diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index b911fffbd..2b46d0750 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "path/filepath" + "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -110,11 +111,7 @@ func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req } if err = c.processRestore(ctx, pvr, pod, log); err != nil { - original = pvr.DeepCopy() - pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed - pvr.Status.Message = err.Error() - pvr.Status.CompletionTimestamp = &metav1.Time{Time: c.clock.Now()} - if e := c.Patch(ctx, pvr, client.MergeFrom(original)); e != nil { + if e := UpdatePVRStatusToFailed(c, ctx, pvr, err.Error(), c.clock.Now()); e != nil { log.WithError(err).Error("Unable to update status to failed") } @@ -133,6 +130,15 @@ func (c *PodVolumeRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{}, nil } +func UpdatePVRStatusToFailed(c client.Client, ctx context.Context, pvr *velerov1api.PodVolumeRestore, errString string, time time.Time) error { + original := pvr.DeepCopy() + pvr.Status.Phase = velerov1api.PodVolumeRestorePhaseFailed + pvr.Status.Message = errString + pvr.Status.CompletionTimestamp = &metav1.Time{Time: time} + + return c.Patch(ctx, pvr, client.MergeFrom(original)) +} + func (c *PodVolumeRestoreReconciler) shouldProcess(ctx context.Context, log logrus.FieldLogger, pvr *velerov1api.PodVolumeRestore) (bool, *corev1api.Pod, error) { if !isPVRNew(pvr) { log.Debug("PodVolumeRestore is not new, skip")