From 0a2f5278b2cd339205fee8b37712217499a9907e Mon Sep 17 00:00:00 2001 From: lyndon-li <98304688+Lyndon-Li@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:24:05 +0800 Subject: [PATCH] Move the progress messages to activities (#10552) * move the progress messages to activities Signed-off-by: Lyndon-Li * update doc for activities in data mover CR Signed-off-by: Lyndon-Li --------- Signed-off-by: Lyndon-Li --- pkg/apis/velero/v1/pod_volume_backup_types.go | 7 ++++++- pkg/apis/velero/v1/pod_volume_restore_type.go | 7 ++++++- pkg/apis/velero/v2alpha1/data_download_types.go | 7 ++++++- pkg/apis/velero/v2alpha1/data_upload_types.go | 7 ++++++- pkg/controller/data_download_controller.go | 5 ++--- pkg/controller/data_download_controller_test.go | 9 +++++++-- pkg/controller/data_upload_controller.go | 5 ++--- pkg/controller/data_upload_controller_test.go | 9 +++++++-- pkg/controller/pod_volume_backup_controller.go | 5 ++--- .../pod_volume_backup_controller_test.go | 9 +++++++-- pkg/controller/pod_volume_restore_controller.go | 5 ++--- .../pod_volume_restore_controller_test.go | 9 +++++++-- .../docs/main/csi-snapshot-data-movement.md | 17 ++++++++++++++++- 13 files changed, 76 insertions(+), 25 deletions(-) diff --git a/pkg/apis/velero/v1/pod_volume_backup_types.go b/pkg/apis/velero/v1/pod_volume_backup_types.go index b8daead83..826b1a5cb 100644 --- a/pkg/apis/velero/v1/pod_volume_backup_types.go +++ b/pkg/apis/velero/v1/pod_volume_backup_types.go @@ -98,7 +98,7 @@ type PodVolumeBackupStatus struct { // +optional SnapshotID string `json:"snapshotID,omitempty"` - // Message is a message about the pod volume backup's status. + // Message is a message describing the pod volume backup when it reaches to a terminal status. // +optional Message string `json:"message,omitempty"` @@ -144,6 +144,11 @@ type PodVolumeBackupStatus struct { // FallbackFull indicates whether the incremental backup has fallen back to full backup FallbackFull bool `json:"fallbackFull,omitempty"` + + // Activities contains one or more messages about what have been done for this pod volume backup. + // +optional + // +nullable + Activities []string `json:"activities,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, diff --git a/pkg/apis/velero/v1/pod_volume_restore_type.go b/pkg/apis/velero/v1/pod_volume_restore_type.go index d0b537a87..a56dc09cf 100644 --- a/pkg/apis/velero/v1/pod_volume_restore_type.go +++ b/pkg/apis/velero/v1/pod_volume_restore_type.go @@ -88,7 +88,7 @@ type PodVolumeRestoreStatus struct { // +optional Phase PodVolumeRestorePhase `json:"phase,omitempty"` - // Message is a message about the pod volume restore's status. + // Message is a message describing the pod volume restore when it reaches to a terminal status. // +optional Message string `json:"message,omitempty"` @@ -127,6 +127,11 @@ type PodVolumeRestoreStatus struct { // FallbackFull indicates whether the incremental restore has fallen back to full restore FallbackFull bool `json:"fallbackFull,omitempty"` + + // Activities contains one or more messages about what have been done for this pod volume restore. + // +optional + // +nullable + Activities []string `json:"activities,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. diff --git a/pkg/apis/velero/v2alpha1/data_download_types.go b/pkg/apis/velero/v2alpha1/data_download_types.go index 3b7bcbe38..02a3c8060 100644 --- a/pkg/apis/velero/v2alpha1/data_download_types.go +++ b/pkg/apis/velero/v2alpha1/data_download_types.go @@ -109,7 +109,7 @@ type DataDownloadStatus struct { // +optional Phase DataDownloadPhase `json:"phase,omitempty"` - // Message is a message about the DataDownload's status. + // Message is a message describing the DataDownload when it reaches to a terminal status. // +optional Message string `json:"message,omitempty"` @@ -152,6 +152,11 @@ type DataDownloadStatus struct { // FallbackFull indicates whether the incremental restore has fallen back to full restore FallbackFull bool `json:"fallbackFull,omitempty"` + + // Activities contains one or more messages about what have been done for this DataDownload. + // +optional + // +nullable + Activities []string `json:"activities,omitempty"` } // TODO(2.0) After converting all resources to use the runtime-controller client, the genclient and k8s:deepcopy markers will no longer be needed and should be removed. diff --git a/pkg/apis/velero/v2alpha1/data_upload_types.go b/pkg/apis/velero/v2alpha1/data_upload_types.go index f6911de6e..ea5f51da1 100644 --- a/pkg/apis/velero/v2alpha1/data_upload_types.go +++ b/pkg/apis/velero/v2alpha1/data_upload_types.go @@ -147,7 +147,7 @@ type DataUploadStatus struct { // +nullable DataMoverResult *map[string]string `json:"dataMoverResult,omitempty"` - // Message is a message about the DataUpload's status. + // Message is a message describing the DataUpload when it reaches to a terminal status. // +optional Message string `json:"message,omitempty"` @@ -205,6 +205,11 @@ type DataUploadStatus struct { // FallbackFull indicates whether the incremental backup has fallen back to full backup FallbackFull bool `json:"fallbackFull,omitempty"` + + // Activities contains one or more messages about what have been done for this DataUpload. + // +optional + // +nullable + Activities []string `json:"activities,omitempty"` } // TODO(2.0) After converting all resources to use the runttime-controller client, diff --git a/pkg/controller/data_download_controller.go b/pkg/controller/data_download_controller.go index 513a3eefc..7943b8569 100644 --- a/pkg/controller/data_download_controller.go +++ b/pkg/controller/data_download_controller.go @@ -621,9 +621,8 @@ func (r *DataDownloadReconciler) OnDataDownloadProgress(ctx context.Context, nam } if progress.Message != "" { - message := progress.Message + ";" - if !strings.HasSuffix(dd.Status.Message, message) { - dd.Status.Message += message + if len(dd.Status.Activities) == 0 || dd.Status.Activities[len(dd.Status.Activities)-1] != progress.Message { + dd.Status.Activities = append(dd.Status.Activities, progress.Message) } } diff --git a/pkg/controller/data_download_controller_test.go b/pkg/controller/data_download_controller_test.go index 7fcbb8978..53961606b 100644 --- a/pkg/controller/data_download_controller_test.go +++ b/pkg/controller/data_download_controller_test.go @@ -829,7 +829,7 @@ func TestOnDataDownloadProgress(t *testing.T) { // Call the OnDataDownloadProgress function r.OnDataDownloadProgress(ctx, namespace, duName, progress) - if len(test.needErrs) != 0 && !test.needErrs[0] { + if len(test.needErrs) == 0 { // Get the updated DataDownload object from the fake client updatedDd := &velerov2alpha1api.DataDownload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDd)) @@ -845,7 +845,12 @@ func TestOnDataDownloadProgress(t *testing.T) { assert.Equal(t, int64(0), updatedDd.Status.Progress.BytesDone) // assuming default or original value } if progress.Message != "" { - assert.Contains(t, updatedDd.Status.Message, progress.Message) + assert.Contains(t, updatedDd.Status.Activities, progress.Message) + + // Call with the same message again to verify deduplication + r.OnDataDownloadProgress(ctx, namespace, duName, progress) + require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDd)) + assert.Equal(t, []string{progress.Message}, updatedDd.Status.Activities) } } }) diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index 5752eb0a4..c7c22320b 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -648,9 +648,8 @@ func (r *DataUploadReconciler) OnDataUploadProgress(ctx context.Context, namespa } if progress.Message != "" { - message := progress.Message + ";" - if !strings.HasSuffix(du.Status.Message, message) { - du.Status.Message += message + if len(du.Status.Activities) == 0 || du.Status.Activities[len(du.Status.Activities)-1] != progress.Message { + du.Status.Activities = append(du.Status.Activities, progress.Message) } } diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 5b4fc7780..e6e66d107 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -851,7 +851,7 @@ func TestOnDataUploadProgress(t *testing.T) { // Call the OnDataUploadProgress function r.OnDataUploadProgress(ctx, namespace, duName, progress) - if len(test.needErrs) != 0 && !test.needErrs[0] { + if len(test.needErrs) == 0 { // Get the updated DataUpload object from the fake client updatedDu := &velerov2alpha1api.DataUpload{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) @@ -867,7 +867,12 @@ func TestOnDataUploadProgress(t *testing.T) { assert.Equal(t, int64(0), updatedDu.Status.Progress.BytesDone) // assuming default or original value } if progress.Message != "" { - assert.Contains(t, updatedDu.Status.Message, progress.Message) + assert.Contains(t, updatedDu.Status.Activities, progress.Message) + + // Call with the same message again to verify deduplication + r.OnDataUploadProgress(ctx, namespace, duName, progress) + require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: duName, Namespace: namespace}, updatedDu)) + assert.Equal(t, []string{progress.Message}, updatedDu.Status.Activities) } } }) diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index 2bf0b1fd2..c952227f5 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -641,9 +641,8 @@ func (r *PodVolumeBackupReconciler) OnDataPathProgress(ctx context.Context, name } if progress.Message != "" { - message := progress.Message + ";" - if !strings.HasSuffix(pvb.Status.Message, message) { - pvb.Status.Message += message + if len(pvb.Status.Activities) == 0 || pvb.Status.Activities[len(pvb.Status.Activities)-1] != progress.Message { + pvb.Status.Activities = append(pvb.Status.Activities, progress.Message) } } diff --git a/pkg/controller/pod_volume_backup_controller_test.go b/pkg/controller/pod_volume_backup_controller_test.go index 9c8240951..354102055 100644 --- a/pkg/controller/pod_volume_backup_controller_test.go +++ b/pkg/controller/pod_volume_backup_controller_test.go @@ -666,7 +666,7 @@ func TestOnPVBProgress(t *testing.T) { progress := &test.progress r.OnDataPathProgress(ctx, namespace, pvbName, progress) - if len(test.needErrs) != 0 && !test.needErrs[0] { + if len(test.needErrs) == 0 { updatedPvb := &velerov1api.PodVolumeBackup{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) if progress.TotalBytes != -1 { @@ -680,7 +680,12 @@ func TestOnPVBProgress(t *testing.T) { assert.Equal(t, int64(0), updatedPvb.Status.Progress.BytesDone) // assuming default or original value } if progress.Message != "" { - assert.Contains(t, updatedPvb.Status.Message, progress.Message) + assert.Contains(t, updatedPvb.Status.Activities, progress.Message) + + // Call with the same message again to verify deduplication + r.OnDataPathProgress(ctx, namespace, pvbName, progress) + require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvbName, Namespace: namespace}, updatedPvb)) + assert.Equal(t, []string{progress.Message}, updatedPvb.Status.Activities) } } }) diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 7da739d2f..a3fa9b209 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -917,9 +917,8 @@ func (r *PodVolumeRestoreReconciler) OnDataPathProgress(ctx context.Context, nam } if progress.Message != "" { - message := progress.Message + ";" - if !strings.HasSuffix(pvr.Status.Message, message) { - pvr.Status.Message += message + if len(pvr.Status.Activities) == 0 || pvr.Status.Activities[len(pvr.Status.Activities)-1] != progress.Message { + pvr.Status.Activities = append(pvr.Status.Activities, progress.Message) } } diff --git a/pkg/controller/pod_volume_restore_controller_test.go b/pkg/controller/pod_volume_restore_controller_test.go index 03b2841d1..7996077d0 100644 --- a/pkg/controller/pod_volume_restore_controller_test.go +++ b/pkg/controller/pod_volume_restore_controller_test.go @@ -1512,7 +1512,7 @@ func TestOnPodVolumeRestoreProgress(t *testing.T) { progress := &test.progress r.OnDataPathProgress(ctx, namespace, pvrName, progress) - if len(test.needErrs) != 0 && !test.needErrs[0] { + if len(test.needErrs) == 0 { updatedPVR := &velerov1api.PodVolumeRestore{} require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) if progress.TotalBytes != -1 { @@ -1526,7 +1526,12 @@ func TestOnPodVolumeRestoreProgress(t *testing.T) { assert.Equal(t, int64(0), updatedPVR.Status.Progress.BytesDone) // assuming default or original value } if progress.Message != "" { - assert.Contains(t, updatedPVR.Status.Message, progress.Message) + assert.Contains(t, updatedPVR.Status.Activities, progress.Message) + + // Call with the same message again to verify deduplication + r.OnDataPathProgress(ctx, namespace, pvrName, progress) + require.NoError(t, r.client.Get(ctx, types.NamespacedName{Name: pvrName, Namespace: namespace}, updatedPVR)) + assert.Equal(t, []string{progress.Message}, updatedPVR.Status.Activities) } } }) diff --git a/site/content/docs/main/csi-snapshot-data-movement.md b/site/content/docs/main/csi-snapshot-data-movement.md index c05666f45..02eaf8605 100644 --- a/site/content/docs/main/csi-snapshot-data-movement.md +++ b/site/content/docs/main/csi-snapshot-data-movement.md @@ -281,7 +281,7 @@ For incremental backups using `velero-block`, Velero will automatically fall bac - The critical information is missing or cannot be retrieved from the parent snapshot. - The parent snapshot is missing in the backup repository. -When fallback occurs, unallocated regions are still skipped and identical data blocks remain deduplicated by the backup repository. The backup description will clearly indicate that a fallback took place: `Backup Type: Incremental (fallen back to Full)`. +When fallback occurs, unallocated regions are still skipped and identical data blocks remain deduplicated by the backup repository. The backup description will clearly indicate that a fallback took place: `Backup Type: Incremental (fallen back to Full)`. In addition, the reason for the fallback is captured in the `status.activities` field of the corresponding `DataUpload` CR. ### Monitoring Backup Progress @@ -331,6 +331,10 @@ You can also view the full `DataUpload` custom resource: kubectl -n velero get datauploads -l velero.io/backup-name=YOUR_BACKUP_NAME -o yaml ``` +In the `DataUpload` status: +- `status.activities`: Lists operational activity and progress messages encountered during data movement, such as incremental fallback reasons. +- `status.message`: Describes the terminal status if the operation failed or was cancelled. + ## To restore You do not need to specify data mover information when creating a restore. Velero automatically retrieves the configurations (data mover type, backup mode, uploader) from the backup metadata. @@ -371,6 +375,8 @@ You can also view the `DataDownload` custom resources directly: kubectl -n velero get datadownloads -l velero.io/restore-name=YOUR_RESTORE_NAME -o yaml ``` +Similar to `DataUpload`, `status.activities` captures operational activities during data download, while `status.message` describes terminal failure or cancellation details. + ## Limitations - **[Velero Block Data Mover] Linux Node Execution**: Block data mover pods mount raw block devices. Because Windows containers do not support raw block mode volumes, block data mover pods can only run on Linux nodes. However, Windows workloads backed by block storage can still be backed up and restored using the block data mover as long as the data mover pods run on Linux nodes. @@ -407,6 +413,10 @@ velero restore logs RESTORE_NAME When reviewing backup details, check whether an incremental backup fell back to full: - Look for `Backup Type: Incremental (fallen back to Full)` under `Data Movement`. This indicates that CBT metadata retrieval was unsuccessful or the parent snapshot was missing. +- Inspect the specific reason for fallback from `status.activities` on the `DataUpload` CR: + ```bash + kubectl -n velero get datauploads -l velero.io/backup-name=BACKUP_NAME -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.activities}{" - "}{.}{"\n"}{end}{end}' + ``` What is the status of your `DataUpload` and `DataDownload`? @@ -416,6 +426,11 @@ kubectl -n velero get datauploads -l velero.io/backup-name=BACKUP_NAME -o yaml kubectl -n velero get datadownloads -l velero.io/restore-name=RESTORE_NAME -o yaml ``` +Key fields to check in the CR status: +- `status.phase`: Displays the current lifecycle phase (e.g., `Accepted`, `Prepared`, `InProgress`, `Completed`, `Failed`, `Cancelled`). +- `status.activities`: Lists events and progress messages that occurred during data movement (such as fallback details or warnings). +- `status.message`: Details the reason when the CR reaches a terminal failure or cancelled status. + Is there any useful information in the Velero server or data mover pod logs? ```bash