RIAv2 async operations controller work

Signed-off-by: Scott Seago <sseago@redhat.com>
2026-09-20 06:54:32 +00:00 · 2023-03-17 14:30:39 -04:00
parent 117d5e846f
commit 2155b2b215
33 changed files with 1478 additions and 79 deletions
@@ -40,6 +40,7 @@ import (

 	"github.com/vmware-tanzu/velero/internal/hook"
 	api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	"github.com/vmware-tanzu/velero/pkg/itemoperation"
 	"github.com/vmware-tanzu/velero/pkg/label"
 	"github.com/vmware-tanzu/velero/pkg/metrics"
 	"github.com/vmware-tanzu/velero/pkg/persistence"
@@ -84,15 +85,16 @@ var nonRestorableResources = []string{
 }

 type restoreReconciler struct {
-	ctx             context.Context
-	namespace       string
-	restorer        pkgrestore.Restorer
-	kbClient        client.Client
-	restoreLogLevel logrus.Level
-	logger          logrus.FieldLogger
-	metrics         *metrics.ServerMetrics
-	logFormat       logging.Format
-	clock           clock.WithTickerAndDelayedExecution
+	ctx                         context.Context
+	namespace                   string
+	restorer                    pkgrestore.Restorer
+	kbClient                    client.Client
+	restoreLogLevel             logrus.Level
+	logger                      logrus.FieldLogger
+	metrics                     *metrics.ServerMetrics
+	logFormat                   logging.Format
+	clock                       clock.WithTickerAndDelayedExecution
+	defaultItemOperationTimeout time.Duration

 	newPluginManager  func(logger logrus.FieldLogger) clientmgmt.Manager
 	backupStoreGetter persistence.ObjectBackupStoreGetter
@@ -114,17 +116,19 @@ func NewRestoreReconciler(
 	backupStoreGetter persistence.ObjectBackupStoreGetter,
 	metrics *metrics.ServerMetrics,
 	logFormat logging.Format,
+	defaultItemOperationTimeout time.Duration,
 ) *restoreReconciler {
 	r := &restoreReconciler{
-		ctx:             ctx,
-		namespace:       namespace,
-		restorer:        restorer,
-		kbClient:        kbClient,
-		logger:          logger,
-		restoreLogLevel: restoreLogLevel,
-		metrics:         metrics,
-		logFormat:       logFormat,
-		clock:           &clock.RealClock{},
+		ctx:                         ctx,
+		namespace:                   namespace,
+		restorer:                    restorer,
+		kbClient:                    kbClient,
+		logger:                      logger,
+		restoreLogLevel:             restoreLogLevel,
+		metrics:                     metrics,
+		logFormat:                   logFormat,
+		clock:                       &clock.RealClock{},
+		defaultItemOperationTimeout: defaultItemOperationTimeout,

 		// use variables to refer to these functions so they can be
 		// replaced with fakes for testing.
@@ -172,6 +176,10 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		restore.Status.StartTimestamp = &metav1.Time{Time: r.clock.Now()}
 		restore.Status.Phase = api.RestorePhaseInProgress
 	}
+	if restore.Spec.ItemOperationTimeout.Duration == 0 {
+		// set default item operation timeout
+		restore.Spec.ItemOperationTimeout.Duration = r.defaultItemOperationTimeout
+	}

 	// patch to update status and persist to API
 	err = kubeutil.PatchResource(original, restore, r.kbClient)
@@ -194,17 +202,14 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
 		restore.Status.Phase = api.RestorePhaseFailed
 		restore.Status.FailureReason = err.Error()
 		r.metrics.RegisterRestoreFailed(backupScheduleName)
-	} else if restore.Status.Errors > 0 {
-		log.Debug("Restore partially failed")
-		restore.Status.Phase = api.RestorePhasePartiallyFailed
-		r.metrics.RegisterRestorePartialFailure(backupScheduleName)
-	} else {
-		log.Debug("Restore completed")
-		restore.Status.Phase = api.RestorePhaseCompleted
-		r.metrics.RegisterRestoreSuccess(backupScheduleName)
 	}

-	restore.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()}
+	// mark completion if in terminal phase
+	if restore.Status.Phase == api.RestorePhaseFailed ||
+		restore.Status.Phase == api.RestorePhasePartiallyFailed ||
+		restore.Status.Phase == api.RestorePhaseCompleted {
+		restore.Status.CompletionTimestamp = &metav1.Time{Time: r.clock.Now()}
+	}
 	log.Debug("Updating restore's final status")
 	if err = kubeutil.PatchResource(original, restore, r.kbClient); err != nil {
 		log.WithError(errors.WithStack(err)).Info("Error updating restore's final status")
@@ -375,15 +380,19 @@ func mostRecentCompletedBackup(backups []api.Backup) api.Backup {
 // fetchBackupInfo checks the backup lister for a backup that matches the given name. If it doesn't
 // find it, it returns an error.
 func (r *restoreReconciler) fetchBackupInfo(backupName string) (backupInfo, error) {
+	return fetchBackupInfoInternal(r.kbClient, r.namespace, backupName)
+}
+
+func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName string) (backupInfo, error) {
 	backup := &api.Backup{}
-	err := r.kbClient.Get(context.Background(), types.NamespacedName{Namespace: r.namespace, Name: backupName}, backup)
+	err := kbClient.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: backupName}, backup)
 	if err != nil {
-		return backupInfo{}, err
+		return backupInfo{}, errors.Wrap(err, fmt.Sprintf("can't find backup %s/%s", namespace, backupName))
 	}

 	location := &api.BackupStorageLocation{}
-	if err := r.kbClient.Get(context.Background(), client.ObjectKey{
-		Namespace: r.namespace,
+	if err := kbClient.Get(context.Background(), client.ObjectKey{
+		Namespace: namespace,
 		Name:      backup.Spec.StorageLocation,
 	}, location); err != nil {
 		return backupInfo{}, errors.WithStack(err)
@@ -469,6 +478,21 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
 	restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, snapshotItemResolver,
 		pluginManager)

+	// Iterate over restore item operations and update progress.
+	// Any errors on operations at this point should be added to restore errors.
+	// If any operations are still not complete, then restore will not be set to
+	// Completed yet.
+	inProgressOperations, _, opsCompleted, opsFailed, errs := getRestoreItemOperationProgress(restoreReq.Restore, pluginManager, *restoreReq.GetItemOperationsList())
+	if len(errs) > 0 {
+		for err := range errs {
+			restoreLog.Error(err)
+		}
+	}
+
+	restore.Status.RestoreItemOperationsAttempted = len(*restoreReq.GetItemOperationsList())
+	restore.Status.RestoreItemOperationsCompleted = opsCompleted
+	restore.Status.RestoreItemOperationsFailed = opsFailed
+
 	// log errors and warnings to the restore log
 	for _, msg := range restoreErrors.Velero {
 		restoreLog.Errorf("Velero restore error: %v", msg)
@@ -537,6 +561,29 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
 		r.logger.WithError(err).Error("Error uploading restored resource list to backup storage")
 	}

+	if err := putOperationsForRestore(restore, *restoreReq.GetItemOperationsList(), backupStore); err != nil {
+		r.logger.WithError(err).Error("Error uploading restore item action operation resource list to backup storage")
+	}
+
+	if restore.Status.Errors > 0 {
+		if inProgressOperations {
+			r.logger.Debug("Restore WaitingForPluginOperationsPartiallyFailed")
+			restore.Status.Phase = api.RestorePhaseWaitingForPluginOperationsPartiallyFailed
+		} else {
+			r.logger.Debug("Restore partially failed")
+			restore.Status.Phase = api.RestorePhasePartiallyFailed
+			r.metrics.RegisterRestorePartialFailure(restore.Spec.ScheduleName)
+		}
+	} else {
+		if inProgressOperations {
+			r.logger.Debug("Restore WaitingForPluginOperations")
+			restore.Status.Phase = api.RestorePhaseWaitingForPluginOperations
+		} else {
+			r.logger.Debug("Restore completed")
+			restore.Status.Phase = api.RestorePhaseCompleted
+			r.metrics.RegisterRestoreSuccess(restore.Spec.ScheduleName)
+		}
+	}
 	return nil
 }

@@ -603,6 +650,26 @@ func putRestoredResourceList(restore *api.Restore, list map[string][]string, bac
 	return nil
 }

+func putOperationsForRestore(restore *api.Restore, operations []*itemoperation.RestoreOperation, backupStore persistence.BackupStore) error {
+	buf := new(bytes.Buffer)
+	gzw := gzip.NewWriter(buf)
+	defer gzw.Close()
+
+	if err := json.NewEncoder(gzw).Encode(operations); err != nil {
+		return errors.Wrap(err, "error encoding restore item operations list to JSON")
+	}
+
+	if err := gzw.Close(); err != nil {
+		return errors.Wrap(err, "error closing gzip writer")
+	}
+
+	if err := backupStore.PutRestoreItemOperations(restore.Name, buf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func downloadToTempFile(backupName string, backupStore persistence.BackupStore, logger logrus.FieldLogger) (*os.File, error) {
 	readCloser, err := backupStore.GetBackupContents(backupName)
 	if err != nil {