diff --git a/changelogs/unreleased/9026-Lyndon-Li b/changelogs/unreleased/9026-Lyndon-Li new file mode 100644 index 000000000..96300961b --- /dev/null +++ b/changelogs/unreleased/9026-Lyndon-Li @@ -0,0 +1 @@ +Fix issue #8965, support PVB/PVR's cancel state in the backup/restore \ No newline at end of file diff --git a/pkg/backup/backup.go b/pkg/backup/backup.go index 3375b7b55..755eaea56 100644 --- a/pkg/backup/backup.go +++ b/pkg/backup/backup.go @@ -823,7 +823,8 @@ func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log l } for _, pvb := range pvbs { pvbMap[pvb] = pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || - pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || + pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled } } @@ -840,7 +841,8 @@ func (kb *kubernetesBackupper) waitUntilPVBsProcessed(ctx context.Context, log l continue } if updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCompleted || - updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { + updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || + updatedPVB.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { pvbMap[pvb] = true continue } diff --git a/pkg/cmd/util/output/backup_describer.go b/pkg/cmd/util/output/backup_describer.go index c2afca726..528117985 100644 --- a/pkg/cmd/util/output/backup_describer.go +++ b/pkg/cmd/util/output/backup_describer.go @@ -786,7 +786,11 @@ func describePodVolumeBackups(d *Describer, details bool, podVolumeBackups []vel for _, phase := range []string{ string(velerov1api.PodVolumeBackupPhaseCompleted), string(velerov1api.PodVolumeBackupPhaseFailed), + string(velerov1api.PodVolumeBackupPhaseCanceled), "In Progress", + string(velerov1api.PodVolumeBackupPhaseCanceling), + string(velerov1api.PodVolumeBackupPhasePrepared), + string(velerov1api.PodVolumeBackupPhaseAccepted), string(velerov1api.PodVolumeBackupPhaseNew), } { if len(backupsByPhase[phase]) == 0 { @@ -822,7 +826,11 @@ func groupByPhase(backups []velerov1api.PodVolumeBackup) map[string][]velerov1ap phaseToGroup := map[velerov1api.PodVolumeBackupPhase]string{ velerov1api.PodVolumeBackupPhaseCompleted: string(velerov1api.PodVolumeBackupPhaseCompleted), velerov1api.PodVolumeBackupPhaseFailed: string(velerov1api.PodVolumeBackupPhaseFailed), + velerov1api.PodVolumeBackupPhaseCanceled: string(velerov1api.PodVolumeBackupPhaseCanceled), velerov1api.PodVolumeBackupPhaseInProgress: "In Progress", + velerov1api.PodVolumeBackupPhaseCanceling: string(velerov1api.PodVolumeBackupPhaseCanceling), + velerov1api.PodVolumeBackupPhasePrepared: string(velerov1api.PodVolumeBackupPhasePrepared), + velerov1api.PodVolumeBackupPhaseAccepted: string(velerov1api.PodVolumeBackupPhaseAccepted), velerov1api.PodVolumeBackupPhaseNew: string(velerov1api.PodVolumeBackupPhaseNew), "": string(velerov1api.PodVolumeBackupPhaseNew), } diff --git a/pkg/cmd/util/output/backup_describer_test.go b/pkg/cmd/util/output/backup_describer_test.go index c4b4b9b8f..bc5c17796 100644 --- a/pkg/cmd/util/output/backup_describer_test.go +++ b/pkg/cmd/util/output/backup_describer_test.go @@ -572,6 +572,54 @@ func TestDescribePodVolumeBackups(t *testing.T) { PodName("pod-2"). PodNamespace("pod-ns-1"). SnapshotID("snap-2").Result() + pvb3 := builder.ForPodVolumeBackup("test-ns1", "test-pvb3"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseFailed). + BackupStorageLocation("bsl-1"). + Volume("vol-3"). + PodName("pod-3"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-3").Result() + pvb4 := builder.ForPodVolumeBackup("test-ns1", "test-pvb4"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseCanceled). + BackupStorageLocation("bsl-1"). + Volume("vol-4"). + PodName("pod-4"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-4").Result() + pvb5 := builder.ForPodVolumeBackup("test-ns1", "test-pvb5"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseInProgress). + BackupStorageLocation("bsl-1"). + Volume("vol-5"). + PodName("pod-5"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-5").Result() + pvb6 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseCanceling). + BackupStorageLocation("bsl-1"). + Volume("vol-6"). + PodName("pod-6"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-6").Result() + pvb7 := builder.ForPodVolumeBackup("test-ns1", "test-pvb7"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhasePrepared). + BackupStorageLocation("bsl-1"). + Volume("vol-7"). + PodName("pod-7"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-7").Result() + pvb8 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseAccepted). + BackupStorageLocation("bsl-1"). + Volume("vol-8"). + PodName("pod-8"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-8").Result() testcases := []struct { name string @@ -602,6 +650,28 @@ func TestDescribePodVolumeBackups(t *testing.T) { Completed: pod-ns-1/pod-1: vol-1 pod-ns-1/pod-2: vol-2 +`, + }, + { + name: "all phases with details", + inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2, *pvb3, *pvb4, *pvb5, *pvb6, *pvb7, *pvb8}, + inputDetails: true, + expect: ` Pod Volume Backups - kopia: + Completed: + pod-ns-1/pod-1: vol-1 + pod-ns-1/pod-2: vol-2 + Failed: + pod-ns-1/pod-3: vol-3 + Canceled: + pod-ns-1/pod-4: vol-4 + In Progress: + pod-ns-1/pod-5: vol-5 + Canceling: + pod-ns-1/pod-6: vol-6 + Prepared: + pod-ns-1/pod-7: vol-7 + Accepted: + pod-ns-1/pod-8: vol-8 `, }, } diff --git a/pkg/cmd/util/output/backup_structured_describer.go b/pkg/cmd/util/output/backup_structured_describer.go index fbb9e354b..f27526718 100644 --- a/pkg/cmd/util/output/backup_structured_describer.go +++ b/pkg/cmd/util/output/backup_structured_describer.go @@ -499,7 +499,11 @@ func describePodVolumeBackupsInSF(backups []velerov1api.PodVolumeBackup, details for _, phase := range []string{ string(velerov1api.PodVolumeBackupPhaseCompleted), string(velerov1api.PodVolumeBackupPhaseFailed), + string(velerov1api.PodVolumeBackupPhaseCanceled), "In Progress", + string(velerov1api.PodVolumeBackupPhaseCanceling), + string(velerov1api.PodVolumeBackupPhasePrepared), + string(velerov1api.PodVolumeBackupPhaseAccepted), string(velerov1api.PodVolumeBackupPhaseNew), } { if len(backupsByPhase[phase]) == 0 { diff --git a/pkg/cmd/util/output/backup_structured_describer_test.go b/pkg/cmd/util/output/backup_structured_describer_test.go index 0e12b82e7..c5ede1b36 100644 --- a/pkg/cmd/util/output/backup_structured_describer_test.go +++ b/pkg/cmd/util/output/backup_structured_describer_test.go @@ -240,6 +240,55 @@ func TestDescribePodVolumeBackupsInSF(t *testing.T) { PodNamespace("pod-ns-1"). SnapshotID("snap-2").Result() + pvb3 := builder.ForPodVolumeBackup("test-ns1", "test-pvb3"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseFailed). + BackupStorageLocation("bsl-1"). + Volume("vol-3"). + PodName("pod-3"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-3").Result() + pvb4 := builder.ForPodVolumeBackup("test-ns1", "test-pvb4"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseCanceled). + BackupStorageLocation("bsl-1"). + Volume("vol-4"). + PodName("pod-4"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-4").Result() + pvb5 := builder.ForPodVolumeBackup("test-ns1", "test-pvb5"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseInProgress). + BackupStorageLocation("bsl-1"). + Volume("vol-5"). + PodName("pod-5"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-5").Result() + pvb6 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseCanceling). + BackupStorageLocation("bsl-1"). + Volume("vol-6"). + PodName("pod-6"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-6").Result() + pvb7 := builder.ForPodVolumeBackup("test-ns1", "test-pvb7"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhasePrepared). + BackupStorageLocation("bsl-1"). + Volume("vol-7"). + PodName("pod-7"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-7").Result() + pvb8 := builder.ForPodVolumeBackup("test-ns1", "test-pvb6"). + UploaderType("kopia"). + Phase(velerov1api.PodVolumeBackupPhaseAccepted). + BackupStorageLocation("bsl-1"). + Volume("vol-8"). + PodName("pod-8"). + PodNamespace("pod-ns-1"). + SnapshotID("snap-8").Result() + testcases := []struct { name string inputPVBList []velerov1api.PodVolumeBackup @@ -268,6 +317,40 @@ func TestDescribePodVolumeBackupsInSF(t *testing.T) { }, }, }, + { + name: "all phases", + inputPVBList: []velerov1api.PodVolumeBackup{*pvb1, *pvb2, *pvb3, *pvb4, *pvb5, *pvb6, *pvb7, *pvb8}, + inputDetails: true, + expect: map[string]any{ + "podVolumeBackups": map[string]any{ + "podVolumeBackupsDetails": map[string]any{ + "Completed": []map[string]string{ + {"pod-ns-1/pod-1": "vol-1"}, + {"pod-ns-1/pod-2": "vol-2"}, + }, + "Failed": []map[string]string{ + {"pod-ns-1/pod-3": "vol-3"}, + }, + "Canceled": []map[string]string{ + {"pod-ns-1/pod-4": "vol-4"}, + }, + "In Progress": []map[string]string{ + {"pod-ns-1/pod-5": "vol-5"}, + }, + "Canceling": []map[string]string{ + {"pod-ns-1/pod-6": "vol-6"}, + }, + "Prepared": []map[string]string{ + {"pod-ns-1/pod-7": "vol-7"}, + }, + "Accepted": []map[string]string{ + {"pod-ns-1/pod-8": "vol-8"}, + }, + }, + "uploderType": "kopia", + }, + }, + }, } for _, tc := range testcases { t.Run(tc.name, func(tt *testing.T) { diff --git a/pkg/cmd/util/output/restore_describer.go b/pkg/cmd/util/output/restore_describer.go index f6ead5c61..76522d49f 100644 --- a/pkg/cmd/util/output/restore_describer.go +++ b/pkg/cmd/util/output/restore_describer.go @@ -362,8 +362,11 @@ func describePodVolumeRestores(d *Describer, restores []velerov1api.PodVolumeRes // go through phases in a specific order for _, phase := range []string{ string(velerov1api.PodVolumeRestorePhaseCompleted), + string(velerov1api.PodVolumeRestorePhaseCanceled), string(velerov1api.PodVolumeRestorePhaseFailed), "In Progress", + string(velerov1api.PodVolumeRestorePhasePrepared), + string(velerov1api.PodVolumeRestorePhaseAccepted), string(velerov1api.PodVolumeRestorePhaseNew), } { if len(restoresByPhase[phase]) == 0 { @@ -442,8 +445,11 @@ func groupRestoresByPhase(restores []velerov1api.PodVolumeRestore) map[string][] phaseToGroup := map[velerov1api.PodVolumeRestorePhase]string{ velerov1api.PodVolumeRestorePhaseCompleted: string(velerov1api.PodVolumeRestorePhaseCompleted), + velerov1api.PodVolumeRestorePhaseCanceled: string(velerov1api.PodVolumeRestorePhaseCanceled), velerov1api.PodVolumeRestorePhaseFailed: string(velerov1api.PodVolumeRestorePhaseFailed), velerov1api.PodVolumeRestorePhaseInProgress: "In Progress", + velerov1api.PodVolumeRestorePhasePrepared: string(velerov1api.PodVolumeRestorePhasePrepared), + velerov1api.PodVolumeRestorePhaseAccepted: string(velerov1api.PodVolumeRestorePhaseAccepted), velerov1api.PodVolumeRestorePhaseNew: string(velerov1api.PodVolumeRestorePhaseNew), "": string(velerov1api.PodVolumeRestorePhaseNew), } diff --git a/pkg/podvolume/backupper.go b/pkg/podvolume/backupper.go index 1c71c668b..fba3cb19a 100644 --- a/pkg/podvolume/backupper.go +++ b/pkg/podvolume/backupper.go @@ -426,8 +426,10 @@ func (b *backupper) WaitAllPodVolumesProcessed(log logrus.FieldLogger) []*velero continue } podVolumeBackups = append(podVolumeBackups, pvb) - if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed || pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { + if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseFailed { log.Errorf("pod volume backup failed: %s", pvb.Status.Message) + } else if pvb.Status.Phase == velerov1api.PodVolumeBackupPhaseCanceled { + log.Errorf("pod volume backup canceled: %s", pvb.Status.Message) } } } diff --git a/pkg/podvolume/restorer.go b/pkg/podvolume/restorer.go index c23ca417c..e2ca579de 100644 --- a/pkg/podvolume/restorer.go +++ b/pkg/podvolume/restorer.go @@ -228,8 +228,10 @@ ForEachVolume: errs = append(errs, errors.New("timed out waiting for all PodVolumeRestores to complete")) break ForEachVolume case res := <-resultsChan: - if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed || res.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { + if res.Status.Phase == velerov1api.PodVolumeRestorePhaseFailed { errs = append(errs, errors.Errorf("pod volume restore failed: %s", res.Status.Message)) + } else if res.Status.Phase == velerov1api.PodVolumeRestorePhaseCanceled { + errs = append(errs, errors.Errorf("pod volume restore canceled: %s", res.Status.Message)) } tracker.TrackPodVolume(res) case err := <-r.nodeAgentCheck: