Merge pull request #9354 from Lyndon-Li/snapshot-size-support

Snapshot size support

changelogs/unreleased/9354-Lyndon-Li

@@ -0,0 +1 @@
+Add snapshotSize for DataDownload, PodVolumeRestore

config/crd/v1/bases/velero.io_podvolumerestores.yaml

@@ -133,6 +133,10 @@ spec:
               snapshotID:
                 description: SnapshotID is the ID of the volume snapshot to be restored.
                 type: string
+              snapshotSize:
+                description: SnapshotSize is the logical size in Bytes of the snapshot.
+                format: int64
+                type: integer
               sourceNamespace:
                 description: SourceNamespace is the original namespace for namaspace
                   mapping.

config/crd/v2alpha1/bases/velero.io_datadownloads.yaml

@@ -108,6 +108,10 @@ spec:
                 description: SnapshotID is the ID of the Velero backup snapshot to
                   be restored from.
                 type: string
+              snapshotSize:
+                description: SnapshotSize is the logical size in Bytes of the snapshot.
+                format: int64
+                type: integer
               sourceNamespace:
                 description: |-
                   SourceNamespace is the original namespace where the volume is backed up from.

pkg/apis/velero/v1/pod_volume_restore_type.go

@@ -58,6 +58,10 @@ type PodVolumeRestoreSpec struct {
 	// Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
 	// when the PodVolumeRestore is in InProgress phase
 	Cancel bool `json:"cancel,omitempty"`
+
+	// SnapshotSize is the logical size in Bytes of the snapshot.
+	// +optional
+	SnapshotSize int64 `json:"snapshotSize,omitempty"`
 }
 
 // PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore.

pkg/apis/velero/v2alpha1/data_download_types.go

@@ -58,6 +58,10 @@ type DataDownloadSpec struct {
 	// NodeOS is OS of the node where the DataDownload is processed.
 	// +optional
 	NodeOS NodeOS `json:"nodeOS,omitempty"`
+
+	// SnapshotSize is the logical size in Bytes of the snapshot.
+	// +optional
+	SnapshotSize int64 `json:"snapshotSize,omitempty"`
 }
 
 // TargetVolumeSpec is the specification for a target PVC.

pkg/apis/velero/v2alpha1/data_upload_types.go

@@ -244,4 +244,8 @@ type DataUploadResult struct {
 	// NodeOS is OS of the node where the DataUpload is processed.
 	// +optional
 	NodeOS NodeOS `json:"nodeOS,omitempty"`
+
+	// SnapshotSize is the logical size in Bytes of the snapshot.
+	// +optional
+	SnapshotSize int64 `json:"snapshotSize,omitempty"`
 }

pkg/builder/data_upload_builder.go

@@ -180,3 +180,9 @@ func (d *DataUploadBuilder) Message(msg string) *DataUploadBuilder {
 	d.object.Status.Message = msg
 	return d
 }
+
+// TotalBytes sets the DataUpload's TotalBytes.
+func (d *DataUploadBuilder) TotalBytes(size int64) *DataUploadBuilder {
+	d.object.Status.Progress.TotalBytes = size
+	return d
+}

pkg/controller/backup_repository_controller.go

@@ -608,7 +608,7 @@ func getBackupRepositoryConfig(ctx context.Context, ctrlClient client.Client, co
 
 	jsonData, found := loc.Data[repoType]
 	if !found {
-		log.Info("No data for repo type %s in config map %s", repoType, configName)
+		log.Infof("No data for repo type %s in config map %s", repoType, configName)
 		return nil, nil
 	}
 

pkg/controller/data_upload_controller_test.go

@@ -850,11 +850,20 @@ func TestOnDataUploadCompleted(t *testing.T) {
 	// Add the DataUpload object to the fake client
 	require.NoError(t, r.client.Create(ctx, du))
 	r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())}
-	r.OnDataUploadCompleted(ctx, namespace, duName, datapath.Result{})
+	r.OnDataUploadCompleted(ctx, namespace, duName, datapath.Result{
+		Backup: datapath.BackupResult{
+			SnapshotID: "fake-id",
+			Source: datapath.AccessPoint{
+				ByPath: "fake-path",
+			},
+		},
+	})
 	updatedDu := &velerov2alpha1api.DataUpload{}
 	require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu))
 	assert.Equal(t, velerov2alpha1api.DataUploadPhaseCompleted, updatedDu.Status.Phase)
 	assert.False(t, updatedDu.Status.CompletionTimestamp.IsZero())
+	assert.Equal(t, "fake-id", updatedDu.Status.SnapshotID)
+	assert.Equal(t, "fake-path", updatedDu.Status.Path)
 }
 
 func TestFindDataUploadForPod(t *testing.T) {

pkg/podvolume/restorer.go

@@ -179,7 +179,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo
 			}
 		}
 
-		volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
+		volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
 		if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil {
 			errs = append(errs, errors.WithStack(err))
 			continue
@@ -252,7 +252,7 @@ ForEachVolume:
 	return errs
 }
 
-func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore {
+func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot string, size int64, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore {
 	pvr := &velerov1api.PodVolumeRestore{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace:    restore.Namespace,
@@ -281,6 +281,7 @@ func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backu
 			},
 			Volume:                volume,
 			SnapshotID:            snapshot,
+			SnapshotSize:          size,
 			BackupStorageLocation: backupLocation,
 			RepoIdentifier:        repoIdentifier,
 			UploaderType:          uploaderType,

pkg/podvolume/restorer_test.go

@@ -57,34 +57,34 @@ func TestGetVolumesRepositoryType(t *testing.T) {
 		{
 			name: "empty repository type, first one",
 			volumes: map[string]volumeBackupInfo{
-				"volume1": {"fake-snapshot-id-1", "fake-uploader-1", ""},
-				"volume2": {"", "", "fake-type"},
+				"volume1": {"fake-snapshot-id-1", 0, "fake-uploader-1", ""},
+				"volume2": {"", 0, "", "fake-type"},
 			},
 			expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-1, uploader fake-uploader-1",
 		},
 		{
 			name: "empty repository type, last one",
 			volumes: map[string]volumeBackupInfo{
-				"volume1": {"", "", "fake-type"},
-				"volume2": {"", "", "fake-type"},
-				"volume3": {"fake-snapshot-id-3", "fake-uploader-3", ""},
+				"volume1": {"", 0, "", "fake-type"},
+				"volume2": {"", 0, "", "fake-type"},
+				"volume3": {"fake-snapshot-id-3", 0, "fake-uploader-3", ""},
 			},
 			expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-3, uploader fake-uploader-3",
 		},
 		{
 			name: "empty repository type, middle one",
 			volumes: map[string]volumeBackupInfo{
-				"volume1": {"", "", "fake-type"},
-				"volume2": {"fake-snapshot-id-2", "fake-uploader-2", ""},
-				"volume3": {"", "", "fake-type"},
+				"volume1": {"", 0, "", "fake-type"},
+				"volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", ""},
+				"volume3": {"", 0, "", "fake-type"},
 			},
 			expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-2, uploader fake-uploader-2",
 		},
 		{
 			name: "mismatch repository type",
 			volumes: map[string]volumeBackupInfo{
-				"volume1": {"", "", "fake-type1"},
-				"volume2": {"fake-snapshot-id-2", "fake-uploader-2", "fake-type2"},
+				"volume1": {"", 0, "", "fake-type1"},
+				"volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", "fake-type2"},
 			},
 			prefixOnly:  true,
 			expectedErr: "multiple repository type in one backup",
@@ -92,9 +92,9 @@ func TestGetVolumesRepositoryType(t *testing.T) {
 		{
 			name: "success",
 			volumes: map[string]volumeBackupInfo{
-				"volume1": {"", "", "fake-type"},
-				"volume2": {"", "", "fake-type"},
-				"volume3": {"", "", "fake-type"},
+				"volume1": {"", 0, "", "fake-type"},
+				"volume2": {"", 0, "", "fake-type"},
+				"volume3": {"", 0, "", "fake-type"},
 			},
 			expected: "fake-type",
 		},

pkg/podvolume/util.go

@@ -39,6 +39,7 @@ const (
 // volumeBackupInfo describes the backup info of a volume backed up by PodVolumeBackups
 type volumeBackupInfo struct {
 	snapshotID     string
+	snapshotSize   int64
 	uploaderType   string
 	repositoryType string
 }
@@ -94,6 +95,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup,
 
 		volumes[pvb.Spec.Volume] = volumeBackupInfo{
 			snapshotID:     pvb.Status.SnapshotID,
+			snapshotSize:   pvb.Status.Progress.TotalBytes,
 			uploaderType:   getUploaderTypeOrDefault(pvb.Spec.UploaderType),
 			repositoryType: getRepositoryType(pvb.Spec.UploaderType),
 		}
@@ -109,7 +111,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup,
 	}
 
 	for k, v := range fromAnnntation {
-		volumes[k] = volumeBackupInfo{v, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic}
+		volumes[k] = volumeBackupInfo{v, 0, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic}
 	}
 
 	return volumes

pkg/restore/actions/csi/pvc_action.go

@@ -431,6 +431,7 @@ func newDataDownload(
 			BackupStorageLocation: dataUploadResult.BackupStorageLocation,
 			DataMover:             dataUploadResult.DataMover,
 			SnapshotID:            dataUploadResult.SnapshotID,
+			SnapshotSize:          dataUploadResult.SnapshotSize,
 			SourceNamespace:       dataUploadResult.SourceNamespace,
 			OperationTimeout:      backup.Spec.CSISnapshotTimeout,
 			NodeOS:                dataUploadResult.NodeOS,

pkg/restore/actions/dataupload_retrieve_action.go

@@ -76,6 +76,7 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut
 		BackupStorageLocation: backup.Spec.StorageLocation,
 		DataMover:             dataUpload.Spec.DataMover,
 		SnapshotID:            dataUpload.Status.SnapshotID,
+		SnapshotSize:          dataUpload.Status.Progress.TotalBytes,
 		SourceNamespace:       dataUpload.Spec.SourceNamespace,
 		DataMoverResult:       dataUpload.Status.DataMoverResult,
 		NodeOS:                dataUpload.Status.NodeOS,

pkg/restore/actions/dataupload_retrieve_action_test.go

@@ -58,13 +58,13 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) {
 		},
 		{
 			name:          "DataUploadRetrieve Action test",
-			dataUpload:    builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").Result(),
+			dataUpload:    builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").SnapshotID("fake-id").TotalBytes(1000).Result(),
 			restore:       builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(),
 			runtimeScheme: scheme,
 			veleroObjs: []runtime.Object{
 				builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(),
 			},
-			expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","sourceNamespace":"testNamespace"}`).Result(),
+			expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","snapshotID":"fake-id","sourceNamespace":"testNamespace","snapshotSize":1000}`).Result(),
 		},
 		{
 			name:          "Long source namespace and PVC name should also work",