Support update the backup VolumeInfos by the Async ops result.

1. Add PutBackupVolumeInfos method.
2. Add CompletionTimestamp in VolumeInfo.
3. Add Size in SnapshotDataMovementInfo.
4. Update CompletionTimpstmap, SnapshotHandle, RetainedSnapshot
   and Size in VolumeInfo on DataUpload Operation completes.

Signed-off-by: Xun Jiang <blackpigletbruce@gmail.com>

pkg/controller/backup_controller_test.go

@@ -44,6 +44,7 @@ import (
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 	fakeClient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 
+	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/builder"
@@ -78,9 +79,15 @@ func (b *fakeBackupper) BackupWithResolvers(logger logrus.FieldLogger, backup *p
 	return args.Error(0)
 }
 
-func (b *fakeBackupper) FinalizeBackup(logger logrus.FieldLogger, backup *pkgbackup.Request, inBackupFile io.Reader, outBackupFile io.Writer,
+func (b *fakeBackupper) FinalizeBackup(
+	logger logrus.FieldLogger,
+	backup *pkgbackup.Request,
+	inBackupFile io.Reader,
+	outBackupFile io.Writer,
 	backupItemActionResolver framework.BackupItemActionResolverV2,
-	asyncBIAOperations []*itemoperation.BackupOperation) error {
+	asyncBIAOperations []*itemoperation.BackupOperation,
+	volumeInfos []*volume.VolumeInfo,
+) error {
 	args := b.Called(logger, backup, inBackupFile, outBackupFile, backupItemActionResolver, asyncBIAOperations)
 	return args.Error(0)
 }

pkg/controller/backup_finalizer_controller.go

@@ -18,7 +18,9 @@ package controller
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
+	"encoding/json"
 	"os"
 
 	"github.com/pkg/errors"
@@ -29,6 +31,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
 	"github.com/vmware-tanzu/velero/pkg/itemoperation"
@@ -111,7 +114,11 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	original := backup.DeepCopy()
 	defer func() {
 		switch backup.Status.Phase {
-		case velerov1api.BackupPhaseCompleted, velerov1api.BackupPhasePartiallyFailed, velerov1api.BackupPhaseFailed, velerov1api.BackupPhaseFailedValidation:
+		case
+			velerov1api.BackupPhaseCompleted,
+			velerov1api.BackupPhasePartiallyFailed,
+			velerov1api.BackupPhaseFailed,
+			velerov1api.BackupPhaseFailedValidation:
 			r.backupTracker.Delete(backup.Namespace, backup.Name)
 		}
 		// Always attempt to Patch the backup object and status after each reconciliation.
@@ -150,8 +157,14 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		SkippedPVTracker: pkgbackup.NewSkipPVTracker(),
 	}
 	var outBackupFile *os.File
+	var volumeInfos []*volume.VolumeInfo
 	if len(operations) > 0 {
-		// Call itemBackupper.BackupItem for the list of items updated by async operations
+		volumeInfos, err = backupStore.GetBackupVolumeInfos(backup.Name)
+		if err != nil {
+			log.WithError(err).Error("error getting backup VolumeInfos")
+			return ctrl.Result{}, errors.WithStack(err)
+		}
+
 		log.Info("Setting up finalized backup temp file")
 		inBackupFile, err := downloadToTempFile(backup.Name, backupStore, log)
 		if err != nil {
@@ -172,7 +185,17 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 			return ctrl.Result{}, errors.WithStack(err)
 		}
 		backupItemActionsResolver := framework.NewBackupItemActionResolverV2(actions)
-		err = r.backupper.FinalizeBackup(log, backupRequest, inBackupFile, outBackupFile, backupItemActionsResolver, operations)
+
+		// Call itemBackupper.BackupItem for the list of items updated by async operations
+		err = r.backupper.FinalizeBackup(
+			log,
+			backupRequest,
+			inBackupFile,
+			outBackupFile,
+			backupItemActionsResolver,
+			operations,
+			volumeInfos,
+		)
 		if err != nil {
 			log.WithError(err).Error("error finalizing Backup")
 			return ctrl.Result{}, errors.WithStack(err)
@@ -209,6 +232,24 @@ func (r *backupFinalizerReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 		if err != nil {
 			return ctrl.Result{}, errors.Wrap(err, "error uploading backup final contents")
 		}
+
+		// Update the backup's VolumeInfos
+		backupVolumeInfoBuf := new(bytes.Buffer)
+		gzw := gzip.NewWriter(backupVolumeInfoBuf)
+		defer gzw.Close()
+
+		if err := json.NewEncoder(gzw).Encode(volumeInfos); err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "error encoding restore results to JSON")
+		}
+
+		if err := gzw.Close(); err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "error closing gzip writer")
+		}
+
+		err = backupStore.PutBackupVolumeInfos(backup.Name, backupVolumeInfoBuf)
+		if err != nil {
+			return ctrl.Result{}, errors.Wrap(err, "fail to upload backup VolumeInfos")
+		}
 	}
 	return ctrl.Result{}, nil
 }

pkg/controller/backup_finalizer_controller_test.go

@@ -222,6 +222,8 @@ func TestBackupFinalizerReconcile(t *testing.T) {
 			backupStore.On("GetBackupContents", mock.Anything).Return(io.NopCloser(bytes.NewReader([]byte("hello world"))), nil)
 			backupStore.On("PutBackupContents", mock.Anything, mock.Anything).Return(nil)
 			backupStore.On("PutBackupMetadata", mock.Anything, mock.Anything).Return(nil)
+			backupStore.On("GetBackupVolumeInfos", mock.Anything).Return(nil, nil)
+			backupStore.On("PutBackupVolumeInfos", mock.Anything, mock.Anything).Return(nil)
 			pluginManager.On("GetBackupItemActionsV2").Return(nil, nil)
 			backupper.On("FinalizeBackup", mock.Anything, mock.Anything, mock.Anything, mock.Anything, framework.BackupItemActionResolverV2{}, mock.Anything).Return(nil)
 			_, err := reconciler.Reconcile(context.TODO(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: test.backup.Namespace, Name: test.backup.Name}})