Mark in-progress CRs as failed when starting the server

Mark in-progress CRs as failed when starting the server

Fixes #4953

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
This commit is contained in:
Wenkai Yin(尹文开)
2022-06-07 17:02:17 +08:00
parent 9af031b84a
commit 6fa4d7d606
10 changed files with 204 additions and 186 deletions
+4 -17
View File
@@ -144,12 +144,13 @@ func NewRestoreController(
restore := obj.(*api.Restore)
switch restore.Status.Phase {
case "", api.RestorePhaseNew, api.RestorePhaseInProgress:
case "", api.RestorePhaseNew:
// only process new restores
default:
c.logger.WithFields(logrus.Fields{
"restore": kubeutil.NamespaceAndName(restore),
"phase": restore.Status.Phase,
}).Debug("Restore is not new or in-progress, skipping")
}).Debug("Restore is not new, skipping")
return
}
@@ -201,21 +202,7 @@ func (c *restoreController) processQueueItem(key string) error {
// is ("" | New)
switch restore.Status.Phase {
case "", api.RestorePhaseNew:
case api.RestorePhaseInProgress:
// A restore may stay in-progress forever because of
// 1) the controller restarts during the processing of a restore
// 2) the restore with in-progress status isn't updated to completed or failed status successfully
// So we try to mark such restores as failed to avoid it
updated := restore.DeepCopy()
updated.Status.Phase = api.RestorePhaseFailed
updated.Status.FailureReason = fmt.Sprintf("got a Restore with unexpected status %q, this may be due to a restart of the controller during the restore, mark it as %q",
api.RestorePhaseInProgress, updated.Status.Phase)
_, err = patchRestore(restore, updated, c.restoreClient)
if err != nil {
return errors.Wrapf(err, "error updating Restore status to %s", updated.Status.Phase)
}
log.Warn(updated.Status.FailureReason)
return nil
// only process new restores
default:
return nil
}