From 59aff1275732bb51799248bcbd8f06b87ca1c885 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Wed, 6 Aug 2025 14:08:18 +0800 Subject: [PATCH] Modify [Basic][Restic] case check PVB and PVR count logic. PVB and PVR used to print related pod namespace in output. In v1.17, the behavior changed. Use backup or restore name to filter them. Shorten the timeout context from 1h to 5m, because AWS was not covered anymore. Remove an used if branch for vSphere. Signed-off-by: Xun Jiang --- test/util/common/common.go | 28 ++++++++++- test/util/kibishii/kibishii_utils.go | 69 +++++++++++----------------- test/util/velero/velero_utils.go | 45 ++++++++---------- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/test/util/common/common.go b/test/util/common/common.go index 4b0041e44..e22a25662 100644 --- a/test/util/common/common.go +++ b/test/util/common/common.go @@ -9,6 +9,7 @@ import ( "io" "os" "os/exec" + "strings" ) const ( @@ -21,12 +22,12 @@ type OsCommandLine struct { Args []string } -func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string, error) { +func GetListByCmdPipes(ctx context.Context, cmdLines []*OsCommandLine) ([]string, error) { var buf bytes.Buffer var err error var cmds []*exec.Cmd - for _, cmdline := range cmdlines { + for _, cmdline := range cmdLines { cmd := exec.Command(cmdline.Cmd, cmdline.Args...) cmds = append(cmds, cmd) } @@ -64,6 +65,29 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string return ret, nil } +func GetResourceWithLabel(ctx context.Context, namespace, resourceName string, labels map[string]string) ([]string, error) { + labelStr := "" + + for key, value := range labels { + strings.Join([]string{labelStr, key + "=" + value}, ",") + } + + cmds := []*OsCommandLine{} + cmd := &OsCommandLine{ + Cmd: "kubectl", + Args: []string{"get", resourceName, "--no-headers", "-n", namespace, "-l", labelStr}, + } + cmds = append(cmds, cmd) + + cmd = &OsCommandLine{ + Cmd: "awk", + Args: []string{"{print $1}"}, + } + cmds = append(cmds, cmd) + + return GetListByCmdPipes(ctx, cmds) +} + func CMDExecWithOutput(checkCMD *exec.Cmd) (*[]byte, error) { stdoutPipe, err := checkCMD.StdoutPipe() if err != nil { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 9ed6080b5..f75f2c363 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -83,7 +83,7 @@ func RunKibishiiTests( ) error { pvCount := len(KibishiiPVCNameList) client := *veleroCfg.ClientToInstallVelero - oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) + timeOutContext, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() veleroCLI := veleroCfg.VeleroCLI providerName := veleroCfg.CloudProvider @@ -97,7 +97,7 @@ func RunKibishiiTests( fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } } - if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { + if err := CreateNamespace(timeOutContext, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { @@ -109,7 +109,7 @@ func RunKibishiiTests( }() fmt.Printf("KibishiiPrepareBeforeBackup %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := KibishiiPrepareBeforeBackup( - oneHourTimeout, + timeOutContext, client, providerName, kibishiiNamespace, @@ -134,7 +134,7 @@ func RunKibishiiTests( BackupCfg.ProvideSnapshotsVolumeParam = veleroCfg.ProvideSnapshotsVolumeParam fmt.Printf("VeleroBackupNamespace %s\n", time.Now().Format("2006-01-02 15:04:05")) - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { + if err := VeleroBackupNamespace(timeOutContext, veleroCLI, veleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -161,7 +161,7 @@ func RunKibishiiTests( if veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete fmt.Println("Waiting for vSphere uploads to complete") - if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil { + if err := WaitForVSphereUploadCompletion(timeOutContext, time.Hour, kibishiiNamespace, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } @@ -180,25 +180,14 @@ func RunKibishiiTests( return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } else { - pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) + pvbNum, err := BackupPVBNum(timeOutContext, veleroCfg.VeleroNamespace, backupName) if err != nil { return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) } - if len(pvbs) != pvCount { - return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", len(pvbs), pvCount, kibishiiNamespace)) + if pvbNum != pvCount { + return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", pvbNum, pvCount, kibishiiNamespace)) } - if providerName == Vsphere { - // Wait for uploads started by the Velero Plugin for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - - // TODO[High] - uncomment code block below when vSphere plugin PR #500 is included in release version. - // https://github.com/vmware-tanzu/velero-plugin-for-vsphere/pull/500/ - - // fmt.Println("Make sure no vSphere snapshot uploads created") - // if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 0); err != nil { - // return errors.Wrapf(err, "Error get vSphere snapshot uploads") - // } - } else { + if providerName != Vsphere { // wait for a period to confirm no snapshots content exist for the backup time.Sleep(1 * time.Minute) if strings.EqualFold(veleroFeatures, FeatureCSI) { @@ -227,11 +216,11 @@ func RunKibishiiTests( fmt.Printf("Re-populate volume %s\n", time.Now().Format("2006-01-02 15:04:05")) for _, pod := range KibishiiPodNameList { // To ensure Kibishii verification result is accurate - ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data") + ClearKibishiiData(timeOutContext, kibishiiNamespace, pod, "kibishii", "data") CreateFileContent := fileBaseContent + pod err := CreateFileToPod( - oneHourTimeout, + timeOutContext, kibishiiNamespace, pod, "kibishii", @@ -244,13 +233,13 @@ func RunKibishiiTests( return errors.Wrapf(err, "failed to create file %s", fileName) } } - fmt.Printf("Re-poulate volume done %s\n", time.Now().Format("2006-01-02 15:04:05")) + fmt.Printf("Re-populate volume done %s\n", time.Now().Format("2006-01-02 15:04:05")) pvList := []string{} if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") { // Get leftover PV list for PV cleanup for _, pvc := range KibishiiPVCNameList { - pv, err := GetPvName(oneHourTimeout, client, pvc, kibishiiNamespace) + pv, err := GetPvName(timeOutContext, client, pvc, kibishiiNamespace) if err != nil { errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } @@ -259,7 +248,7 @@ func RunKibishiiTests( } fmt.Printf("Simulating a disaster by removing namespace %s %s\n", kibishiiNamespace, time.Now().Format("2006-01-02 15:04:05")) - if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + if err := DeleteNamespace(timeOutContext, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } @@ -267,35 +256,35 @@ func RunKibishiiTests( // In scenario of CSI PV-retain-policy test, to restore PV of the backed up resource, we should make sure // there are no PVs of the same name left, because in previous test step, PV's reclaim policy is retain, // so PVs are not deleted although workload namespace is destroyed. - if err := DeletePVs(oneHourTimeout, *veleroCfg.ClientToInstallVelero, pvList); err != nil { + if err := DeletePVs(timeOutContext, *veleroCfg.ClientToInstallVelero, pvList); err != nil { return errors.Wrapf(err, "failed to delete PVs %v", pvList) } } if useVolumeSnapshots { - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) + // NativeSnapshot is not the focus of Velero roadmap now, + // and AWS EBS is not covered by E2E test. + // Don't need to wait up to 5 minutes. + fmt.Println("Waiting 30 seconds to make sure the snapshots are ready...") + time.Sleep(30 * time.Second) } fmt.Printf("VeleroRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) - if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil { + if err := VeleroRestore(timeOutContext, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } if !useVolumeSnapshots && providerName != Vsphere { - pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) + pvrNum, err := RestorePVRNum(timeOutContext, veleroCfg.VeleroNamespace, restoreName) if err != nil { return errors.Wrapf(err, "failed to get PVR for namespace %s", kibishiiNamespace) - } else if len(pvrs) != pvCount { - return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", len(pvrs), pvCount)) + } else if pvrNum != pvCount { + return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", pvrNum, pvCount)) } } fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) - if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName, veleroCfg.WorkerOS); err != nil { + if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, timeOutContext, DefaultKibishiiData, fileName, veleroCfg.WorkerOS); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } @@ -329,19 +318,13 @@ func installKibishii( return errors.New("vSphere needs to download the Kibishii repository first because it needs to inject some image patch file to work.") } - // TODO: blackpiglet debug - fmt.Printf("targetKustomizeDir %s, workerOS: %s, WorkerOSWindows: %s.\n", targetKustomizeDir, workerOS, common.WorkerOSWindows) - if workerOS == common.WorkerOSWindows { targetKustomizeDir += "-windows" - - // TODO: blackpiglet debug - fmt.Printf("targetKustomizeDir for windows %s\n", targetKustomizeDir) } fmt.Printf("The installed Kibishii Kustomize package directory is %s.\n", targetKustomizeDir) } - // update kibishi images with image registry proxy if it is set + // update kibishii images with image registry proxy if it is set baseDir := resolveBasePath(kibishiiDirectory) fmt.Printf("Using image registry proxy %s to patch Kibishii images. Base Dir: %s\n", imageRegistryProxy, baseDir) diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 59212861e..ab4ce5724 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -1528,35 +1528,30 @@ func ListVeleroPods(ctx context.Context, veleroNamespace string) ([]string, erro return common.GetListByCmdPipes(ctx, cmds) } -func GetVeleroResource(ctx context.Context, veleroNamespace, namespace, resourceName string) ([]string, error) { - cmds := []*common.OsCommandLine{} - cmd := &common.OsCommandLine{ - Cmd: "kubectl", - Args: []string{"get", resourceName, "-n", veleroNamespace}, - } - cmds = append(cmds, cmd) +func BackupPVBNum(ctx context.Context, veleroNamespace, backupName string) (int, error) { + outputList, err := common.GetResourceWithLabel( + ctx, + veleroNamespace, + "PodVolumeBackup", + map[string]string{ + velerov1api.BackupNameLabel: backupName, + }, + ) - cmd = &common.OsCommandLine{ - Cmd: "grep", - Args: []string{namespace}, - } - cmds = append(cmds, cmd) - - cmd = &common.OsCommandLine{ - Cmd: "awk", - Args: []string{"{print $1}"}, - } - cmds = append(cmds, cmd) - - return common.GetListByCmdPipes(ctx, cmds) + return len(outputList), err } -func GetPVB(ctx context.Context, veleroNamespace, namespace string) ([]string, error) { - return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumebackup") -} +func RestorePVRNum(ctx context.Context, veleroNamespace, restoreName string) (int, error) { + outputList, err := common.GetResourceWithLabel( + ctx, + veleroNamespace, + "PodVolumeRestore", + map[string]string{ + velerov1api.RestoreNameLabel: restoreName, + }, + ) -func GetPVR(ctx context.Context, veleroNamespace, namespace string) ([]string, error) { - return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumerestore") + return len(outputList), err } func IsSupportUploaderType(version string) (bool, error) {