Merge pull request #9160 from blackpiglet/fix_main_e2e_issues_basic_restic

Modify [Basic][Restic] case check PVB and PVR count logic.

test/util/common/common.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 )
 
 const (
@@ -21,12 +22,12 @@ type OsCommandLine struct {
 	Args []string
 }
 
-func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string, error) {
+func GetListByCmdPipes(ctx context.Context, cmdLines []*OsCommandLine) ([]string, error) {
 	var buf bytes.Buffer
 	var err error
 	var cmds []*exec.Cmd
 
-	for _, cmdline := range cmdlines {
+	for _, cmdline := range cmdLines {
 		cmd := exec.Command(cmdline.Cmd, cmdline.Args...)
 		cmds = append(cmds, cmd)
 	}
@@ -64,6 +65,29 @@ func GetListByCmdPipes(ctx context.Context, cmdlines []*OsCommandLine) ([]string
 	return ret, nil
 }
 
+func GetResourceWithLabel(ctx context.Context, namespace, resourceName string, labels map[string]string) ([]string, error) {
+	labelStr := ""
+
+	for key, value := range labels {
+		strings.Join([]string{labelStr, key + "=" + value}, ",")
+	}
+
+	cmds := []*OsCommandLine{}
+	cmd := &OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", resourceName, "--no-headers", "-n", namespace, "-l", labelStr},
+	}
+	cmds = append(cmds, cmd)
+
+	cmd = &OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+	cmds = append(cmds, cmd)
+
+	return GetListByCmdPipes(ctx, cmds)
+}
+
 func CMDExecWithOutput(checkCMD *exec.Cmd) (*[]byte, error) {
 	stdoutPipe, err := checkCMD.StdoutPipe()
 	if err != nil {

test/util/kibishii/kibishii_utils.go

@@ -83,7 +83,7 @@ func RunKibishiiTests(
 ) error {
 	pvCount := len(KibishiiPVCNameList)
 	client := *veleroCfg.ClientToInstallVelero
-	oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
+	timeOutContext, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 	defer ctxCancel()
 	veleroCLI := veleroCfg.VeleroCLI
 	providerName := veleroCfg.CloudProvider
@@ -97,7 +97,7 @@ func RunKibishiiTests(
 			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
 		}
 	}
-	if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil {
+	if err := CreateNamespace(timeOutContext, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
 	}
 	defer func() {
@@ -109,7 +109,7 @@ func RunKibishiiTests(
 	}()
 	fmt.Printf("KibishiiPrepareBeforeBackup %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	if err := KibishiiPrepareBeforeBackup(
-		oneHourTimeout,
+		timeOutContext,
 		client,
 		providerName,
 		kibishiiNamespace,
@@ -134,7 +134,7 @@ func RunKibishiiTests(
 	BackupCfg.ProvideSnapshotsVolumeParam = veleroCfg.ProvideSnapshotsVolumeParam
 
 	fmt.Printf("VeleroBackupNamespace %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
+	if err := VeleroBackupNamespace(timeOutContext, veleroCLI, veleroNamespace, BackupCfg); err != nil {
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}
@@ -161,7 +161,7 @@ func RunKibishiiTests(
 		if veleroCfg.HasVspherePlugin {
 			// Wait for uploads started by the Velero Plugin for vSphere to complete
 			fmt.Println("Waiting for vSphere uploads to complete")
-			if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil {
+			if err := WaitForVSphereUploadCompletion(timeOutContext, time.Hour, kibishiiNamespace, 2); err != nil {
 				return errors.Wrapf(err, "Error waiting for uploads to complete")
 			}
 		}
@@ -180,25 +180,14 @@ func RunKibishiiTests(
 			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	} else {
-		pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace)
+		pvbNum, err := BackupPVBNum(timeOutContext, veleroCfg.VeleroNamespace, backupName)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace)
 		}
-		if len(pvbs) != pvCount {
-			return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", len(pvbs), pvCount, kibishiiNamespace))
+		if pvbNum != pvCount {
+			return errors.New(fmt.Sprintf("PVB count %d should be %d in namespace %s", pvbNum, pvCount, kibishiiNamespace))
 		}
-		if providerName == Vsphere {
-			// Wait for uploads started by the Velero Plugin for vSphere to complete
-			// TODO - remove after upload progress monitoring is implemented
-
-			// TODO[High] - uncomment code block below when vSphere plugin PR #500 is included in release version.
-			//   https://github.com/vmware-tanzu/velero-plugin-for-vsphere/pull/500/
-
-			// fmt.Println("Make sure no vSphere snapshot uploads created")
-			// if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 0); err != nil {
-			// 	return errors.Wrapf(err, "Error get vSphere snapshot uploads")
-			// }
-		} else {
+		if providerName != Vsphere {
 			// wait for a period to confirm no snapshots content exist for the backup
 			time.Sleep(1 * time.Minute)
 			if strings.EqualFold(veleroFeatures, FeatureCSI) {
@@ -227,11 +216,11 @@ func RunKibishiiTests(
 	fmt.Printf("Re-populate volume  %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	for _, pod := range KibishiiPodNameList {
 		// To ensure Kibishii verification result is accurate
-		ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data")
+		ClearKibishiiData(timeOutContext, kibishiiNamespace, pod, "kibishii", "data")
 
 		CreateFileContent := fileBaseContent + pod
 		err := CreateFileToPod(
-			oneHourTimeout,
+			timeOutContext,
 			kibishiiNamespace,
 			pod,
 			"kibishii",
@@ -244,13 +233,13 @@ func RunKibishiiTests(
 			return errors.Wrapf(err, "failed to create file %s", fileName)
 		}
 	}
-	fmt.Printf("Re-poulate volume done %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	fmt.Printf("Re-populate volume done %s\n", time.Now().Format("2006-01-02 15:04:05"))
 
 	pvList := []string{}
 	if strings.Contains(veleroCfg.KibishiiDirectory, "sc-reclaim-policy") {
 		// Get leftover PV list for PV cleanup
 		for _, pvc := range KibishiiPVCNameList {
-			pv, err := GetPvName(oneHourTimeout, client, pvc, kibishiiNamespace)
+			pv, err := GetPvName(timeOutContext, client, pvc, kibishiiNamespace)
 			if err != nil {
 				errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 			}
@@ -259,7 +248,7 @@ func RunKibishiiTests(
 	}
 
 	fmt.Printf("Simulating a disaster by removing namespace %s %s\n", kibishiiNamespace, time.Now().Format("2006-01-02 15:04:05"))
-	if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
+	if err := DeleteNamespace(timeOutContext, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
 
@@ -267,35 +256,35 @@ func RunKibishiiTests(
 		// In scenario of CSI PV-retain-policy test, to restore PV of the backed up resource, we should make sure
 		// there are no PVs of the same name left, because in previous test step, PV's reclaim policy is retain,
 		// so PVs are not deleted although workload namespace is destroyed.
-		if err := DeletePVs(oneHourTimeout, *veleroCfg.ClientToInstallVelero, pvList); err != nil {
+		if err := DeletePVs(timeOutContext, *veleroCfg.ClientToInstallVelero, pvList); err != nil {
 			return errors.Wrapf(err, "failed to delete PVs %v", pvList)
 		}
 	}
 
 	if useVolumeSnapshots {
-		// the snapshots of AWS may be still in pending status when do the restore, wait for a while
-		// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
-		// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
-		fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
-		time.Sleep(5 * time.Minute)
+		// NativeSnapshot is not the focus of Velero roadmap now,
+		// and AWS EBS is not covered by E2E test.
+		// Don't need to wait up to 5 minutes.
+		fmt.Println("Waiting 30 seconds to make sure the snapshots are ready...")
+		time.Sleep(30 * time.Second)
 	}
 
 	fmt.Printf("VeleroRestore %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil {
+	if err := VeleroRestore(timeOutContext, veleroCLI, veleroNamespace, restoreName, backupName, ""); err != nil {
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
 		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
 	}
 	if !useVolumeSnapshots && providerName != Vsphere {
-		pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace)
+		pvrNum, err := RestorePVRNum(timeOutContext, veleroCfg.VeleroNamespace, restoreName)
 		if err != nil {
 			return errors.Wrapf(err, "failed to get PVR for namespace %s", kibishiiNamespace)
-		} else if len(pvrs) != pvCount {
-			return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", len(pvrs), pvCount))
+		} else if pvrNum != pvCount {
+			return errors.New(fmt.Sprintf("PVR count %d is not as expected %d", pvrNum, pvCount))
 		}
 	}
 
 	fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05"))
-	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName, veleroCfg.WorkerOS); err != nil {
+	if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, timeOutContext, DefaultKibishiiData, fileName, veleroCfg.WorkerOS); err != nil {
 		return errors.Wrapf(err, "Error verifying kibishii after restore")
 	}
 
@@ -329,19 +318,13 @@ func installKibishii(
 			return errors.New("vSphere needs to download the Kibishii repository first because it needs to inject some image patch file to work.")
 		}
 
-		// TODO: blackpiglet debug
-		fmt.Printf("targetKustomizeDir %s, workerOS: %s, WorkerOSWindows: %s.\n", targetKustomizeDir, workerOS, common.WorkerOSWindows)
-
 		if workerOS == common.WorkerOSWindows {
 			targetKustomizeDir += "-windows"
-
-			// TODO: blackpiglet debug
-			fmt.Printf("targetKustomizeDir for windows %s\n", targetKustomizeDir)
 		}
 		fmt.Printf("The installed Kibishii Kustomize package directory is %s.\n", targetKustomizeDir)
 	}
 
-	// update kibishi images with image registry proxy if it is set
+	// update kibishii images with image registry proxy if it is set
 	baseDir := resolveBasePath(kibishiiDirectory)
 	fmt.Printf("Using image registry proxy %s to patch Kibishii images. Base Dir: %s\n", imageRegistryProxy, baseDir)
 

test/util/velero/velero_utils.go

@@ -1528,35 +1528,30 @@ func ListVeleroPods(ctx context.Context, veleroNamespace string) ([]string, erro
 	return common.GetListByCmdPipes(ctx, cmds)
 }
 
-func GetVeleroResource(ctx context.Context, veleroNamespace, namespace, resourceName string) ([]string, error) {
-	cmds := []*common.OsCommandLine{}
-	cmd := &common.OsCommandLine{
-		Cmd:  "kubectl",
-		Args: []string{"get", resourceName, "-n", veleroNamespace},
-	}
-	cmds = append(cmds, cmd)
+func BackupPVBNum(ctx context.Context, veleroNamespace, backupName string) (int, error) {
+	outputList, err := common.GetResourceWithLabel(
+		ctx,
+		veleroNamespace,
+		"PodVolumeBackup",
+		map[string]string{
+			velerov1api.BackupNameLabel: backupName,
+		},
+	)
 
-	cmd = &common.OsCommandLine{
-		Cmd:  "grep",
-		Args: []string{namespace},
-	}
-	cmds = append(cmds, cmd)
-
-	cmd = &common.OsCommandLine{
-		Cmd:  "awk",
-		Args: []string{"{print $1}"},
-	}
-	cmds = append(cmds, cmd)
-
-	return common.GetListByCmdPipes(ctx, cmds)
+	return len(outputList), err
 }
 
-func GetPVB(ctx context.Context, veleroNamespace, namespace string) ([]string, error) {
-	return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumebackup")
-}
+func RestorePVRNum(ctx context.Context, veleroNamespace, restoreName string) (int, error) {
+	outputList, err := common.GetResourceWithLabel(
+		ctx,
+		veleroNamespace,
+		"PodVolumeRestore",
+		map[string]string{
+			velerov1api.RestoreNameLabel: restoreName,
+		},
+	)
 
-func GetPVR(ctx context.Context, veleroNamespace, namespace string) ([]string, error) {
-	return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumerestore")
+	return len(outputList), err
 }
 
 func IsSupportUploaderType(version string) (bool, error) {