Make in-progress backup/restore as failed when doing the reconcile

Make in-progress backup/restore as failed when doing the reconcile to avoid hanging in in-progress status

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>

pkg/controller/restore_controller.go

@@ -144,13 +144,12 @@ func NewRestoreController(
 				restore := obj.(*api.Restore)
 
 				switch restore.Status.Phase {
-				case "", api.RestorePhaseNew:
-					// only process new restores
+				case "", api.RestorePhaseNew, api.RestorePhaseInProgress:
 				default:
 					c.logger.WithFields(logrus.Fields{
 						"restore": kubeutil.NamespaceAndName(restore),
 						"phase":   restore.Status.Phase,
-					}).Debug("Restore is not new, skipping")
+					}).Debug("Restore is not new or in-progress, skipping")
 					return
 				}
 
@@ -202,7 +201,21 @@ func (c *restoreController) processQueueItem(key string) error {
 	// is ("" | New)
 	switch restore.Status.Phase {
 	case "", api.RestorePhaseNew:
-		// only process new restores
+	case api.RestorePhaseInProgress:
+		// A restore may stay in-progress forever because of
+		// 1) the controller restarts during the processing of a restore
+		// 2) the restore with in-progress status isn't updated to completed or failed status successfully
+		// So we try to mark such restores as failed to avoid it
+		updated := restore.DeepCopy()
+		updated.Status.Phase = api.RestorePhaseFailed
+		updated.Status.FailureReason = fmt.Sprintf("got a Restore with unexpected status %q, this may be due to a restart of the controller during the restore, mark it as %q",
+			api.RestorePhaseInProgress, updated.Status.Phase)
+		_, err = patchRestore(restore, updated, c.restoreClient)
+		if err != nil {
+			return errors.Wrapf(err, "error updating Restore status to %s", updated.Status.Phase)
+		}
+		log.Warn(updated.Status.FailureReason)
+		return nil
 	default:
 		return nil
 	}