mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-08 22:23:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

add snapshot size to data mover CRs

Browse Source 
Signed-off-by: Lyndon-Li <lyonghui@vmware.com>
This commit is contained in:
Lyndon-Li
2025-10-21 14:13:59 +08:00
parent 9b02402631
commit 166f50d776
21 changed files with 110 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
config/crd/v1/bases/velero.io_podvolumebackups.yaml
View File

@@ -225,6 +225,10 @@ spec:
description: SnapshotID is the identifier for the snapshot of the
pod volume.
type: string
snapshotSize:
description: SnapshotSize is the logical size of the snapshot.
format: int64
type: integer
startTimestamp:
description: |-
StartTimestamp records the time a backup was started.

4
config/crd/v1/bases/velero.io_podvolumerestores.yaml
View File

@@ -133,6 +133,10 @@ spec:
snapshotID:
description: SnapshotID is the ID of the volume snapshot to be restored.
type: string
snapshotSize:
description: SnapshotSize is the logical size of the snapshot.
format: int64
type: integer
sourceNamespace:
description: SourceNamespace is the original namespace for namaspace
mapping.

4
config/crd/v1/crds/crds.go
View File

File diff suppressed because one or more lines are too long

4
config/crd/v2alpha1/bases/velero.io_datadownloads.yaml
View File

@@ -108,6 +108,10 @@ spec:
description: SnapshotID is the ID of the Velero backup snapshot to
be restored from.
type: string
snapshotSize:
description: SnapshotSize is the logical size of the snapshot.
format: int64
type: integer
sourceNamespace:
description: |-
SourceNamespace is the original namespace where the volume is backed up from.

4
config/crd/v2alpha1/bases/velero.io_datauploads.yaml
View File

@@ -219,6 +219,10 @@ spec:
description: SnapshotID is the identifier for the snapshot in the
backup repository.
type: string
snapshotSize:
description: SnapshotSize is the logical size of the snapshot.
format: int64
type: integer
startTimestamp:
description: |-
StartTimestamp records the time a backup was started.

4
config/crd/v2alpha1/crds/crds.go
View File

File diff suppressed because one or more lines are too long

4
pkg/apis/velero/v1/pod_volume_backup_types.go
View File

@@ -123,6 +123,10 @@ type PodVolumeBackupStatus struct {
// +optional
// +nullable
AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"`
// SnapshotSize is the logical size of the snapshot.
// +optional
SnapshotSize int64 `json:"snapshotSize,omitempty"`
}
// TODO(2.0) After converting all resources to use the runttime-controller client,

4
pkg/apis/velero/v1/pod_volume_restore_type.go
View File

@@ -58,6 +58,10 @@ type PodVolumeRestoreSpec struct {
// Cancel indicates request to cancel the ongoing PodVolumeRestore. It can be set
// when the PodVolumeRestore is in InProgress phase
Cancel bool `json:"cancel,omitempty"`
// SnapshotSize is the logical size of the snapshot.
// +optional
SnapshotSize int64 `json:"snapshotSize,omitempty"`
}
// PodVolumeRestorePhase represents the lifecycle phase of a PodVolumeRestore.

4
pkg/apis/velero/v2alpha1/data_download_types.go
View File

@@ -58,6 +58,10 @@ type DataDownloadSpec struct {
// NodeOS is OS of the node where the DataDownload is processed.
// +optional
NodeOS NodeOS `json:"nodeOS,omitempty"`
// SnapshotSize is the logical size of the snapshot.
// +optional
SnapshotSize int64 `json:"snapshotSize,omitempty"`
}
// TargetVolumeSpec is the specification for a target PVC.

8
pkg/apis/velero/v2alpha1/data_upload_types.go
View File

@@ -172,6 +172,10 @@ type DataUploadStatus struct {
// +optional
// +nullable
AcceptedTimestamp *metav1.Time `json:"acceptedTimestamp,omitempty"`
// SnapshotSize is the logical size of the snapshot.
// +optional
SnapshotSize int64 `json:"snapshotSize,omitempty"`
}
// TODO(2.0) After converting all resources to use the runttime-controller client,
@@ -244,4 +248,8 @@ type DataUploadResult struct {
// NodeOS is OS of the node where the DataUpload is processed.
// +optional
NodeOS NodeOS `json:"nodeOS,omitempty"`
// SnapshotSize is the logical size of the snapshot.
// +optional
SnapshotSize int64 `json:"snapshotSize,omitempty"`
}

12
pkg/builder/data_upload_builder.go
View File

@@ -180,3 +180,15 @@ func (d *DataUploadBuilder) Message(msg string) *DataUploadBuilder {
d.object.Status.Message = msg
return d
}
// SnapshotSize sets the DataUpload's SnapshotSize.
func (d *DataUploadBuilder) SnapshotSize(size int64) *DataUploadBuilder {
d.object.Status.SnapshotSize = size
return d
}
// TotalBytes sets the DataUpload's TotalBytes.
func (d *DataUploadBuilder) TotalBytes(size int64) *DataUploadBuilder {
d.object.Status.SnapshotSize = size
return d
}

2
pkg/controller/backup_repository_controller.go
View File

@@ -608,7 +608,7 @@ func getBackupRepositoryConfig(ctx context.Context, ctrlClient client.Client, co
jsonData, found := loc.Data[repoType]
if !found {
log.Info("No data for repo type %s in config map %s", repoType, configName)
log.Infof("No data for repo type %s in config map %s", repoType, configName)
return nil, nil
}

1
pkg/controller/data_upload_controller.go
View File

@@ -493,6 +493,7 @@ func (r *DataUploadReconciler) OnDataUploadCompleted(ctx context.Context, namesp
du.Status.Path = result.Backup.Source.ByPath
du.Status.Phase = velerov2alpha1api.DataUploadPhaseCompleted
du.Status.SnapshotID = result.Backup.SnapshotID
du.Status.SnapshotSize = result.Backup.TotalBytes
du.Status.CompletionTimestamp = &metav1.Time{Time: r.Clock.Now()}
if result.Backup.EmptySnapshot {
du.Status.Message = "volume was empty so no data was upload"

13
pkg/controller/data_upload_controller_test.go
View File

@@ -850,11 +850,22 @@ func TestOnDataUploadCompleted(t *testing.T) {
// Add the DataUpload object to the fake client
require.NoError(t, r.client.Create(ctx, du))
r.snapshotExposerList = map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer{velerov2alpha1api.SnapshotTypeCSI: exposer.NewCSISnapshotExposer(r.kubeClient, r.csiSnapshotClient, velerotest.NewLogger())}
r.OnDataUploadCompleted(ctx, namespace, duName, datapath.Result{})
r.OnDataUploadCompleted(ctx, namespace, duName, datapath.Result{
Backup: datapath.BackupResult{
SnapshotID: "fake-id",
Source: datapath.AccessPoint{
ByPath: "fake-path",
},
TotalBytes: int64(1000),
},
})
updatedDu := &velerov2alpha1api.DataUpload{}
require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu))
assert.Equal(t, velerov2alpha1api.DataUploadPhaseCompleted, updatedDu.Status.Phase)
assert.False(t, updatedDu.Status.CompletionTimestamp.IsZero())
assert.Equal(t, "fake-id", updatedDu.Status.SnapshotID)
assert.Equal(t, "fake-path", updatedDu.Status.Path)
assert.Equal(t, int64(1000), updatedDu.Status.SnapshotSize)
}
func TestFindDataUploadForPod(t *testing.T) {

1
pkg/controller/pod_volume_backup_controller.go
View File

@@ -525,6 +525,7 @@ func (r *PodVolumeBackupReconciler) OnDataPathCompleted(ctx context.Context, nam
pvb.Status.Path = result.Backup.Source.ByPath
pvb.Status.Phase = velerov1api.PodVolumeBackupPhaseCompleted
pvb.Status.SnapshotID = result.Backup.SnapshotID
pvb.Status.SnapshotSize = result.Backup.TotalBytes
pvb.Status.CompletionTimestamp = &completionTime
if result.Backup.EmptySnapshot {
pvb.Status.Message = "volume was empty so no snapshot was taken"

5
pkg/podvolume/restorer.go
View File

@@ -179,7 +179,7 @@ func (r *restorer) RestorePodVolumes(data RestoreData, tracker *volume.RestoreVo
}
}
volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
volumeRestore := newPodVolumeRestore(data.Restore, data.Pod, data.BackupLocation, volume, backupInfo.snapshotID, backupInfo.snapshotSize, repoIdentifier, backupInfo.uploaderType, data.SourceNamespace, pvc)
if err := veleroclient.CreateRetryGenerateName(r.crClient, r.ctx, volumeRestore); err != nil {
errs = append(errs, errors.WithStack(err))
continue
@@ -252,7 +252,7 @@ ForEachVolume:
return errs
}
func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore {
func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backupLocation, volume, snapshot string, size int64, repoIdentifier, uploaderType, sourceNamespace string, pvc *corev1api.PersistentVolumeClaim) *velerov1api.PodVolumeRestore {
pvr := &velerov1api.PodVolumeRestore{
ObjectMeta: metav1.ObjectMeta{
Namespace: restore.Namespace,
@@ -281,6 +281,7 @@ func newPodVolumeRestore(restore *velerov1api.Restore, pod *corev1api.Pod, backu
},
Volume: volume,
SnapshotID: snapshot,
SnapshotSize: size,
BackupStorageLocation: backupLocation,
RepoIdentifier: repoIdentifier,
UploaderType: uploaderType,

26
pkg/podvolume/restorer_test.go
View File

@@ -57,34 +57,34 @@ func TestGetVolumesRepositoryType(t *testing.T) {
{
name: "empty repository type, first one",
volumes: map[string]volumeBackupInfo{
"volume1": {"fake-snapshot-id-1", "fake-uploader-1", ""},
"volume2": {"", "", "fake-type"},
"volume1": {"fake-snapshot-id-1", 0, "fake-uploader-1", ""},
"volume2": {"", 0, "", "fake-type"},
},
expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-1, uploader fake-uploader-1",
},
{
name: "empty repository type, last one",
volumes: map[string]volumeBackupInfo{
"volume1": {"", "", "fake-type"},
"volume2": {"", "", "fake-type"},
"volume3": {"fake-snapshot-id-3", "fake-uploader-3", ""},
"volume1": {"", 0, "", "fake-type"},
"volume2": {"", 0, "", "fake-type"},
"volume3": {"fake-snapshot-id-3", 0, "fake-uploader-3", ""},
},
expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-3, uploader fake-uploader-3",
},
{
name: "empty repository type, middle one",
volumes: map[string]volumeBackupInfo{
"volume1": {"", "", "fake-type"},
"volume2": {"fake-snapshot-id-2", "fake-uploader-2", ""},
"volume3": {"", "", "fake-type"},
"volume1": {"", 0, "", "fake-type"},
"volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", ""},
"volume3": {"", 0, "", "fake-type"},
},
expectedErr: "empty repository type found among volume snapshots, snapshot ID fake-snapshot-id-2, uploader fake-uploader-2",
},
{
name: "mismatch repository type",
volumes: map[string]volumeBackupInfo{
"volume1": {"", "", "fake-type1"},
"volume2": {"fake-snapshot-id-2", "fake-uploader-2", "fake-type2"},
"volume1": {"", 0, "", "fake-type1"},
"volume2": {"fake-snapshot-id-2", 0, "fake-uploader-2", "fake-type2"},
},
prefixOnly: true,
expectedErr: "multiple repository type in one backup",
@@ -92,9 +92,9 @@ func TestGetVolumesRepositoryType(t *testing.T) {
{
name: "success",
volumes: map[string]volumeBackupInfo{
"volume1": {"", "", "fake-type"},
"volume2": {"", "", "fake-type"},
"volume3": {"", "", "fake-type"},
"volume1": {"", 0, "", "fake-type"},
"volume2": {"", 0, "", "fake-type"},
"volume3": {"", 0, "", "fake-type"},
},
expected: "fake-type",
},

9
pkg/podvolume/util.go
View File

@@ -39,6 +39,7 @@ const (
// volumeBackupInfo describes the backup info of a volume backed up by PodVolumeBackups
type volumeBackupInfo struct {
snapshotID string
snapshotSize int64
uploaderType string
repositoryType string
}
@@ -92,8 +93,14 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup,
continue
}
snapshotSize := pvb.Status.SnapshotSize
if snapshotSize == 0 {
snapshotSize = pvb.Status.Progress.TotalBytes
}
volumes[pvb.Spec.Volume] = volumeBackupInfo{
snapshotID: pvb.Status.SnapshotID,
snapshotSize: snapshotSize,
uploaderType: getUploaderTypeOrDefault(pvb.Spec.UploaderType),
repositoryType: getRepositoryType(pvb.Spec.UploaderType),
}
@@ -109,7 +116,7 @@ func getVolumeBackupInfoForPod(podVolumeBackups []*velerov1api.PodVolumeBackup,
}
for k, v := range fromAnnntation {
volumes[k] = volumeBackupInfo{v, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic}
volumes[k] = volumeBackupInfo{v, 0, uploader.ResticType, velerov1api.BackupRepositoryTypeRestic}
}
return volumes

1
pkg/restore/actions/csi/pvc_action.go
View File

@@ -431,6 +431,7 @@ func newDataDownload(
BackupStorageLocation: dataUploadResult.BackupStorageLocation,
DataMover: dataUploadResult.DataMover,
SnapshotID: dataUploadResult.SnapshotID,
SnapshotSize: dataUploadResult.SnapshotSize,
SourceNamespace: dataUploadResult.SourceNamespace,
OperationTimeout: backup.Spec.CSISnapshotTimeout,
NodeOS: dataUploadResult.NodeOS,

6
pkg/restore/actions/dataupload_retrieve_action.go
View File

@@ -72,10 +72,16 @@ func (d *DataUploadRetrieveAction) Execute(input *velero.RestoreItemActionExecut
return nil, errors.Wrapf(err, "error to get backup for restore %s", input.Restore.Name)
}
snapshotSize := dataUpload.Status.SnapshotSize
if snapshotSize == 0 {
snapshotSize = dataUpload.Status.Progress.TotalBytes
}
dataUploadResult := velerov2alpha1.DataUploadResult{
BackupStorageLocation: backup.Spec.StorageLocation,
DataMover: dataUpload.Spec.DataMover,
SnapshotID: dataUpload.Status.SnapshotID,
SnapshotSize: snapshotSize,
SourceNamespace: dataUpload.Spec.SourceNamespace,
DataMoverResult: dataUpload.Status.DataMoverResult,
NodeOS: dataUpload.Status.NodeOS,

14
pkg/restore/actions/dataupload_retrieve_action_test.go
View File

@@ -58,13 +58,13 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) {
},
{
name: "DataUploadRetrieve Action test",
dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").Result(),
dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").SnapshotID("fake-id").SnapshotSize(1000).Result(),
restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(),
runtimeScheme: scheme,
veleroObjs: []runtime.Object{
builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(),
},
expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","sourceNamespace":"testNamespace"}`).Result(),
expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","snapshotID":"fake-id","sourceNamespace":"testNamespace","snapshotSize":1000}`).Result(),
},
{
name: "Long source namespace and PVC name should also work",
@@ -76,6 +76,16 @@ func TestDataUploadRetrieveActionExectue(t *testing.T) {
},
expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "migre209d0da-49c7-45ba-8d5a-3e59fd591ec1.kibishii-data-ki152333", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","sourceNamespace":"migre209d0da-49c7-45ba-8d5a-3e59fd591ec1"}`).Result(),
},
{
name: "snapshotSize is zero",
dataUpload: builder.ForDataUpload("velero", "testDU").SourceNamespace("testNamespace").SourcePVC("testPVC").TotalBytes(2000).Result(),
restore: builder.ForRestore("velero", "testRestore").ObjectMeta(builder.WithUID("testingUID")).Backup("testBackup").Result(),
runtimeScheme: scheme,
veleroObjs: []runtime.Object{
builder.ForBackup("velero", "testBackup").StorageLocation("testLocation").Result(),
},
expectedDataUploadResult: builder.ForConfigMap("velero", "").ObjectMeta(builder.WithGenerateName("testDU-"), builder.WithLabels(velerov1.PVCNamespaceNameLabel, "testNamespace.testPVC", velerov1.RestoreUIDLabel, "testingUID", velerov1.ResourceUsageLabel, string(velerov1.VeleroResourceUsageDataUploadResult))).Data("testingUID", `{"backupStorageLocation":"testLocation","sourceNamespace":"testNamespace","snapshotSize":2000}`).Result(),
},
}
for _, tc := range tests {