Add E2E test of bsl deletion

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/util/common/common.go

@@ -0,0 +1,54 @@
+package common
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"os/exec"
+)
+
+type OsCommandLine struct {
+	Cmd  string
+	Args []string
+}
+
+func GetListBy2Pipes(ctx context.Context, cmdline1, cmdline2, cmdline3 OsCommandLine) ([]string, error) {
+	var b2 bytes.Buffer
+	var errVelero, errAwk error
+
+	c1 := exec.CommandContext(ctx, cmdline1.Cmd, cmdline1.Args...)
+	c2 := exec.Command(cmdline2.Cmd, cmdline2.Args...)
+	c3 := exec.Command(cmdline3.Cmd, cmdline3.Args...)
+	fmt.Println(c1)
+	fmt.Println(c2)
+	fmt.Println(c3)
+	c2.Stdin, errVelero = c1.StdoutPipe()
+	if errVelero != nil {
+		return nil, errVelero
+	}
+	c3.Stdin, errAwk = c2.StdoutPipe()
+	if errAwk != nil {
+		return nil, errAwk
+	}
+	c3.Stdout = &b2
+	_ = c3.Start()
+	_ = c2.Start()
+	_ = c1.Run()
+	_ = c2.Wait()
+	_ = c3.Wait()
+
+	fmt.Println(&b2)
+	scanner := bufio.NewScanner(&b2)
+	var ret []string
+	for scanner.Scan() {
+		fmt.Printf("line: %s\n", scanner.Text())
+		ret = append(ret, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}

test/e2e/util/k8s/common.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
+	common "github.com/vmware-tanzu/velero/test/e2e/util/common"
 )
 
 // ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on.
@@ -77,3 +78,61 @@ func WaitForPods(ctx context.Context, client TestClient, namespace string, pods
 	}
 	return nil
 }
+
+func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, error) {
+	// Example:
+	//    NAME                                  STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS             AGE
+	//    kibishii-data-kibishii-deployment-0   Bound    pvc-94b9fdf2-c30f-4a7b-87bf-06eadca0d5b6   1Gi        RWO            kibishii-storage-class   115s
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", "pvc", "-n", namespace},
+	}
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{podName},
+	}
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}
+
+func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) {
+	// Example:
+	// 	  NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                              STORAGECLASS             REASON   AGE
+	//    pvc-3f784366-58db-40b2-8fec-77307807e74b   1Gi        RWO            Delete           Bound    bsl-deletion/kibishii-data-kibishii-deployment-0   kibishii-storage-class            6h41m
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", "pv"},
+	}
+
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{pvc},
+	}
+
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}
+
+func AddLabelToPv(ctx context.Context, pv, label string) error {
+	return exec.CommandContext(ctx, "kubectl", "label", "pv", pv, label).Run()
+}
+
+func AddLabelToPvc(ctx context.Context, pvc, namespace, label string) error {
+	args := []string{"label", "pvc", pvc, "-n", namespace, label}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}
+
+func AddLabelToPod(ctx context.Context, podName, namespace, label string) error {
+	args := []string{"label", "pod", podName, "-n", namespace, label}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -51,7 +51,7 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace)
 	}
 
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil {
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil {
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}

test/e2e/util/providers/aws_utils.go

@@ -29,7 +29,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
-	. "github.com/vmware-tanzu/velero/test/e2e"
+	e2e "github.com/vmware-tanzu/velero/test/e2e"
 )
 
 type AWSStorage string
@@ -62,10 +62,7 @@ func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix
 	if bslPrefix != "" {
 		objectsInput.Prefix = aws.String(bslPrefix)
 	}
-	s3Config := &aws.Config{
-		Region:      aws.String(region),
-		Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""),
-	}
+	var s3Config *aws.Config
 	if region == "minio" {
 		s3url = config.Data()["s3Url"]
 		s3Config = &aws.Config{
@@ -75,8 +72,12 @@ func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix
 			DisableSSL:       aws.Bool(true),
 			S3ForcePathStyle: aws.Bool(true),
 		}
+	} else {
+		s3Config = &aws.Config{
+			Region:      aws.String(region),
+			Credentials: credentials.NewSharedCredentials(cloudCredentialsFile, ""),
+		}
 	}
-
 	sess, err := session.NewSession(s3Config)
 
 	if err != nil {
@@ -97,7 +98,7 @@ func (s AWSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix
 		fmt.Println("item:")
 		fmt.Println(item)
 		backupNameInStorage = strings.TrimPrefix(*item.Prefix, strings.Trim(bslPrefix, "/")+"/")
-		fmt.Println("backupNameInStorage:" + backupNameInStorage)
+		fmt.Println("backupNameInStorage:" + backupNameInStorage + " backupObject:" + backupObject)
 		if strings.Contains(backupNameInStorage, backupObject) {
 			fmt.Printf("Backup %s was found under prefix %s \n", backupObject, bslPrefix)
 			return true, nil
@@ -144,7 +145,7 @@ func (s AWSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPr
 	return nil
 }
 
-func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
+func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck e2e.SnapshotCheckPoint) error {
 
 	config := flag.NewMap()
 	config.Set(bslConfig)
@@ -172,10 +173,16 @@ func (s AWSStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix
 			},
 		},
 	}
+
 	result, err := svc.DescribeSnapshots(params)
 	if err != nil {
 		fmt.Println(err)
 	}
+
+	for _, n := range result.Snapshots {
+		fmt.Println(n.SnapshotId)
+		fmt.Println(n.Tags)
+	}
 	if len(result.Snapshots) != snapshotCheck.ExpectCount {
 		return errors.New(fmt.Sprintf("Snapshot count is not as expected %d", snapshotCheck.ExpectCount))
 	} else {

test/e2e/util/providers/azure_utils.go

@@ -317,18 +317,9 @@ func mapLookup(data map[string]string) func(string) string {
 		return data[key]
 	}
 }
-func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
+func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
 
 	ctx := context.Background()
-	config := flag.NewMap()
-	config.Set(bslConfig)
-	if err := validateConfigKeys(config.Data(),
-		resourceGroupConfigKey,
-		subscriptionIDConfigKey,
-		storageAccount,
-	); err != nil {
-		return err
-	}
 
 	if err := loadCredentialsIntoEnv(cloudCredentialsFile); err != nil {
 		return err
@@ -348,13 +339,7 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPref
 
 	// set a different subscriptionId for snapshots if specified
 	snapshotsSubscriptionID := envVars[subscriptionIDEnvVar]
-	if val := config.Data()[subscriptionIDConfigKey]; val != "" {
-		// if subscription was set in config, it is required to also set the resource group
-		if _, err := getRequiredValues(mapLookup(config.Data()), resourceGroupConfigKey); err != nil {
-			return errors.Wrap(err, "resourceGroup not specified, but is a requirement when backing up to a different subscription")
-		}
-		snapshotsSubscriptionID = val
-	}
+
 	// set up clients
 	snapsClient := disk.NewSnapshotsClientWithBaseURI(env.ResourceManagerEndpoint, snapshotsSubscriptionID)
 	snapsClient.PollingDelay = 5 * time.Second

test/e2e/util/providers/common.go

@@ -31,7 +31,7 @@ import (
 type ObjectsInStorage interface {
 	IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
 	DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
-	IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error
+	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error
 }
 
 func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
@@ -109,11 +109,11 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
 	return nil
 }
 
-func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
+func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
 	var snapshotCheckPoint SnapshotCheckPoint
 	snapshotCheckPoint.ExpectCount = 0
-	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix, snapshotCheckPoint)
+	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s is existed in cloud after backup as expected", backupName))
 	}
@@ -121,9 +121,9 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
 	return nil
 }
 
-func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, snapshotCheckPoint SnapshotCheckPoint) error {
+func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
-	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix, snapshotCheckPoint)
+	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s are not existed in cloud after backup as expected", backupName))
 	}
@@ -131,8 +131,8 @@ func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBuc
 	return nil
 }
 
-func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, snapshotCheck SnapshotCheckPoint) error {
-	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
+func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error {
+
 	s, err := getProvider(cloudProvider)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
@@ -140,12 +140,13 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix
 	if cloudProvider == "vsphere" {
 		var retSnapshotIDs []string
 		ctx, _ := context.WithTimeout(context.Background(), time.Minute*2)
-		retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp)
+		retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName)
 		if err != nil {
 			return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName))
 		}
-		bslPrefix = "plugins"
-		subPrefix = "vsphere-astrolabe-repo/ivd/data"
+
+		bslPrefix := "plugins"
+		subPrefix := "vsphere-astrolabe-repo/ivd/data"
 		if snapshotCheck.ExpectCount == 0 {
 			for _, snapshotID := range retSnapshotIDs {
 				err := ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, snapshotID, subPrefix, 5)
@@ -165,7 +166,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix
 			}
 		}
 	} else {
-		err = s.IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, snapshotCheck)
+		err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck)
 		if err != nil {
 			return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup%s", backupName))
 		}

test/e2e/util/providers/gcloud_utils.go

@@ -105,7 +105,7 @@ func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPr
 	}
 }
 
-func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
+func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
 	ctx := context.Background()
 	data, err := ioutil.ReadFile(cloudCredentialsFile)
 	if err != nil {
@@ -137,7 +137,8 @@ func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslBucket, bslPrefix
 	}); err != nil {
 		return errors.Wrapf(err, "Failed listing snapshot pages")
 	}
-	if snapshotCountFound != len(snapshotCheck.SnapshotIDList) {
+
+	if snapshotCountFound != snapshotCheck.ExpectCount {
 		return errors.New(fmt.Sprintf("Snapshot count %d is not as expected %d\n", snapshotCountFound, len(snapshotCheck.SnapshotIDList)))
 	} else {
 		fmt.Printf("Snapshot count %d is as expected %d\n", snapshotCountFound, len(snapshotCheck.SnapshotIDList))

test/e2e/util/velero/velero_utils.go

@@ -40,6 +40,7 @@ import (
 	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	common "github.com/vmware-tanzu/velero/test/e2e/util/common"
 )
 
 const BackupObjectsPrefix = "backups"
@@ -233,14 +234,17 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
 }
 
 // VeleroBackupNamespace uses the veleroCLI to backup a namespace.
-func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string,
-	useVolumeSnapshots bool) error {
+func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, backupName, namespace, backupLocation string,
+	useVolumeSnapshots bool, selector string) error {
 	args := []string{
 		"--namespace", veleroNamespace,
 		"create", "backup", backupName,
 		"--include-namespaces", namespace,
 		"--wait",
 	}
+	if selector != "" {
+		args = append(args, "--selector", selector)
+	}
 
 	if useVolumeSnapshots {
 		args = append(args, "--snapshot-volumes")
@@ -478,7 +482,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
 	return err
 }
 
-func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string) ([]string, error) {
+func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace, podName string) ([]string, error) {
 	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
 		"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'")
 	fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
@@ -498,6 +502,9 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 		if len(curLine) == 0 {
 			continue
 		}
+		if podName != "" && !strings.Contains(curLine, podName) {
+			continue
+		}
 		snapshotID := curLine[strings.LastIndex(curLine, ":")+1:]
 		fmt.Println("snapshotID:" + snapshotID)
 		snapshotIDDec, _ := b64.StdEncoding.DecodeString(snapshotID)
@@ -626,7 +633,7 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 				fmt.Printf("|| EXPECTED || - Backup %s was deleted successfully according to message %s\n", backupName, stderr)
 				return nil
 			}
-			return errors.Wrapf(err, "Fail to get delete backup, stdout=%s, stderr=%s", stdout, stderr)
+			return errors.Wrapf(err, "Fail to perform get backup, stdout=%s, stderr=%s", stdout, stderr)
 		}
 		time.Sleep(1 * time.Minute)
 	}
@@ -640,7 +647,8 @@ func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string
 }
 
 func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
-	if _, outerr, err := GetBackup(ctx, veleroCLI, backupName); err != nil {
+	out, outerr, err := GetBackup(ctx, veleroCLI, backupName)
+	if err != nil {
 		if err != nil {
 			if strings.Contains(outerr, "not found") {
 				return false, nil
@@ -648,6 +656,7 @@ func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bo
 			return false, err
 		}
 	}
+	fmt.Printf("Backup %s exist locally according to output %s", backupName, out)
 	return true, nil
 }
 
@@ -664,3 +673,118 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string,
 		}
 	})
 }
+
+func WaitForBackupCreated(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
+			return false, err
+		} else {
+			if exist {
+				return true, nil
+			} else {
+				return false, nil
+			}
+		}
+	})
+}
+func GetBackupsFromBsl(ctx context.Context, veleroCLI, bslName string) ([]string, error) {
+	args1 := []string{"get", "backups"}
+	if strings.TrimSpace(bslName) != "" {
+		args1 = append(args1, "-l", "velero.io/storage-location="+bslName)
+	}
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  veleroCLI,
+		Args: args1,
+	}
+
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "tail",
+		Args: []string{"-n", "+2"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}
+
+func GetAllBackups(ctx context.Context, veleroCLI string) ([]string, error) {
+	return GetBackupsFromBsl(ctx, veleroCLI, "")
+}
+func DeleteBslResource(ctx context.Context, veleroCLI string, bslName string) error {
+	args := []string{"backup-location", "delete", bslName, "--confirm"}
+
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	fmt.Println("Delete backup location Command:" + cmd.String())
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	if err != nil {
+		return errors.Wrapf(err, "Fail to get delete location, stdout=%s, stderr=%s", stdout, stderr)
+	}
+
+	output := strings.Replace(stdout, "\n", " ", -1)
+	fmt.Println("Backup location delete command output:" + output)
+
+	fmt.Println(stdout)
+	fmt.Println(stderr)
+	return nil
+}
+
+func SnapshotCRsCountShouldBe(ctx context.Context, namespace, backupName string, expectedCount int) error {
+
+	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
+		"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.metadata.labels.velero\\.io\\/backup-name}{\"\\n\"}{end}'")
+	fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
+	stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd)
+	if err != nil {
+		fmt.Print(stdout)
+		fmt.Print(stderr)
+		return errors.Wrap(err, fmt.Sprintf("Failed getting snapshot CR of backup %s in namespace %d", backupName, expectedCount))
+	}
+	count := 0
+	stdout = strings.Replace(stdout, "'", "", -1)
+	arr := strings.Split(stdout, "\n")
+	for _, bn := range arr {
+		fmt.Println("Snapshot CR:" + bn)
+		if strings.Contains(bn, backupName) {
+			count++
+		}
+	}
+	if count == expectedCount {
+		return nil
+	} else {
+		return errors.New(fmt.Sprintf("SnapshotCR count %d of backup %s in namespace %s is not as expected %d", count, backupName, namespace, expectedCount))
+	}
+}
+
+func ResticRepositoriesCountShouldBe(ctx context.Context, veleroNamespace, targetNamespace string, expectedCount int) error {
+	resticArr, err := GetResticRepositories(ctx, veleroNamespace, targetNamespace)
+	if err != nil {
+		return errors.Wrapf(err, "Fail to get GetResticRepositories")
+	}
+	if len(resticArr) == expectedCount {
+		return nil
+	} else {
+		return errors.New(fmt.Sprintf("Resticrepositories count %d in namespace %s is not as expected %d", len(resticArr), targetNamespace, expectedCount))
+	}
+}
+
+func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace string) ([]string, error) {
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", "-n", veleroNamespace, "resticrepositories"},
+	}
+
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{targetNamespace},
+	}
+
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}