Merge pull request #4836 from danfengliu/add-csi-checkpoint

Add snapshot checkpoint for E2E test

test/e2e/backup/backup.go

@@ -39,13 +39,17 @@ func BackupRestoreWithRestic() {
 }
 
 func BackupRestoreTest(useVolumeSnapshots bool) {
+	kibishiiNamespace := "kibishii-workload"
 	var (
 		backupName, restoreName string
+		client                  TestClient
+		err                     error
 	)
-	kibishiiNamespace := "kibishii-workload"
-	client, err := NewTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
 
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
 	BeforeEach(func() {
 		if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")

test/e2e/backups/deletion.go

@@ -25,6 +25,7 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
+	waitutil "k8s.io/apimachinery/pkg/util/wait"
 
 	. "github.com/vmware-tanzu/velero/test/e2e"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
@@ -47,11 +48,14 @@ func BackupDeletionWithRestic() {
 func backup_deletion_test(useVolumeSnapshots bool) {
 	var (
 		backupName string
+		client     TestClient
+		err        error
 	)
 
-	client, err := NewTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests")
-
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
 	BeforeEach(func() {
 		if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
 			Skip("Volume snapshots not supported on kind")
@@ -98,11 +102,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
 	}
-	defer func() {
-		if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
-			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
-		}
-	}()
+	if !VeleroCfg.Debug {
+		defer func() {
+			if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
+				fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
+			}
+		}()
+	}
 
 	if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest,
 		registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil {
@@ -131,28 +137,50 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa
 	if err != nil {
 		return err
 	}
+	var snapshotCheckPoint SnapshotCheckPoint
 	if useVolumeSnapshots {
-		var snapshotCheckPoint SnapshotCheckPoint
-		snapshotCheckPoint.ExpectCount = 2
-		snapshotCheckPoint.NamespaceBackedUp = deletionTest
-		err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint)
+		snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
+		Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
 		if err != nil {
-			return err
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
+		}
+		err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
+			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
+			backupName, snapshotCheckPoint)
+		if err != nil {
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
 	err = DeleteBackupResource(context.Background(), veleroCLI, backupName)
 	if err != nil {
 		return err
 	}
+	if useVolumeSnapshots {
+		err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
+			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
+			backupName, snapshotCheckPoint)
+		if err != nil {
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
+		}
+	}
 
 	err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
 	if err != nil {
 		return err
 	}
 	if useVolumeSnapshots {
-		err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName)
+		err = waitutil.PollImmediate(30*time.Second, 3*time.Minute,
+			func() (bool, error) {
+				err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
+					VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+					bslConfig, backupName, snapshotCheckPoint)
+				if err == nil {
+					return true, nil
+				}
+				return false, err
+			})
 		if err != nil {
-			return err
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
 

test/e2e/backups/sync_backups.go

@@ -52,10 +52,16 @@ func (b *SyncBackups) Init() {
 
 func BackupsSyncTest() {
 	test := new(SyncBackups)
-	client, err := NewTestClient()
-	if err != nil {
-		println(err.Error())
-	}
+	var (
+		client TestClient
+		err    error
+	)
+
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
+
 	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
 
 	BeforeEach(func() {

test/e2e/basic/enable_api_group_versions.go

@@ -45,10 +45,13 @@ func APIGropuVersionsTest() {
 		resource, group string
 		err             error
 		ctx             = context.Background()
+		client          TestClient
 	)
 
-	client, err := NewTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
 
 	BeforeEach(func() {
 		resource = "rockbands"

test/e2e/bsl-mgmt/deletion.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/google/go-cmp/cmp"
@@ -28,6 +29,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	. "github.com/vmware-tanzu/velero/test/e2e"
+	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
 
@@ -52,8 +54,16 @@ func BslDeletionWithRestic() {
 	BslDeletionTest(false)
 }
 func BslDeletionTest(useVolumeSnapshots bool) {
-	client, err := NewTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests")
+	var (
+		client TestClient
+		err    error
+	)
+
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
+
 	less := func(a, b string) bool { return a < b }
 	BeforeEach(func() {
 		if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
@@ -198,19 +208,28 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
 							backupName_2, 1)).To(Succeed())
 					})
+				} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+					By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() {
+						Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
+					})
+
+					By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() {
+						Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.")
+					})
 				}
+
 				var snapshotCheckPoint SnapshotCheckPoint
 				snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
 				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
-					snapshotCheckPoint.ExpectCount = 1
-					snapshotCheckPoint.PodName = podName_1
-					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
-						VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
+					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+					Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
+						VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket,
 						VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
 				})
 				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
-					snapshotCheckPoint.ExpectCount = 1
-					snapshotCheckPoint.PodName = podName_2
+					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
+					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
 					var BSLCredentials, BSLConfig string
 					if VeleroCfg.CloudProvider == "vsphere" {
 						BSLCredentials = VeleroCfg.AdditionalBSLCredentials
@@ -219,7 +238,8 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						BSLCredentials = VeleroCfg.CloudCredentialsFile
 						BSLConfig = VeleroCfg.BSLConfig
 					}
-					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
+
+					Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
 						BSLCredentials, VeleroCfg.AdditionalBSLBucket,
 						BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
 				})
@@ -234,12 +254,12 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				})
 			}
 
-			By(fmt.Sprintf("Verify if backup %s is created or not", backupName_1), func() {
+			By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() {
 				Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI,
 					backupName_1, 10*time.Minute)).To(Succeed())
 			})
 
-			By(fmt.Sprintf("Verify if backup %s is created or not", backupName_2), func() {
+			By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() {
 				Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI,
 					backupName_2, 10*time.Minute)).To(Succeed())
 			})
@@ -298,15 +318,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				var snapshotCheckPoint SnapshotCheckPoint
 				snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
 				By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() {
-					snapshotCheckPoint.ExpectCount = 1
-					snapshotCheckPoint.PodName = podName_1
-					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
+
+					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
+					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+					Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
 						VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 						VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
 				})
 				By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() {
-					snapshotCheckPoint.ExpectCount = 1
-					snapshotCheckPoint.PodName = podName_2
 					var BSLCredentials, BSLConfig string
 					if VeleroCfg.CloudProvider == "vsphere" {
 						BSLCredentials = VeleroCfg.AdditionalBSLCredentials
@@ -315,7 +334,9 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 						BSLCredentials = VeleroCfg.CloudCredentialsFile
 						BSLConfig = VeleroCfg.BSLConfig
 					}
-					Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
+					snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
+					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+					Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
 						BSLCredentials, VeleroCfg.AdditionalBSLBucket,
 						BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
 				})

test/e2e/e2e_suite_test.go

@@ -94,9 +94,11 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on i
 var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
 var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
+
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
+
 var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
 var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
 

test/e2e/privilegesmgmt/ssr.go

@@ -36,11 +36,15 @@ import (
 
 func SSRTest() {
 	testNS := "ssr-test"
-	client, err := NewTestClient()
-	if err != nil {
-		println(err.Error())
-	}
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+	var (
+		client TestClient
+		err    error
+	)
+
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
 
 	BeforeEach(func() {
 		flag.Parse()

test/e2e/test/test.go

@@ -72,9 +72,11 @@ var TestClientInstance TestClient
 
 func TestFunc(test VeleroBackupRestoreTest) func() {
 	return func() {
-		var err error
-		TestClientInstance, err = NewTestClient()
-		Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+		By("Create test client instance", func() {
+			var err error
+			TestClientInstance, err = NewTestClient()
+			Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+		})
 		Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases")
 		BeforeEach(func() {
 			flag.Parse()
@@ -99,8 +101,11 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
 	return func() {
 		var err error
 		var countIt int
-		TestClientInstance, err = NewTestClient()
-		Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+		By("Create test client instance", func() {
+			TestClientInstance, err = NewTestClient()
+			Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+		})
+		//Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
 		for k := range tests {
 			Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
 		}

test/e2e/types.go

@@ -55,7 +55,10 @@ type VerleroConfig struct {
 
 type SnapshotCheckPoint struct {
 	NamespaceBackedUp string
-	SnapshotIDList    []string
-	ExpectCount       int
-	PodName           string
+	// SnapshotIDList is for Azure CSI Verification
+	//  we can get SnapshotID from VolumeSnapshotContent from a certain backup
+	SnapshotIDList []string
+	ExpectCount    int
+	PodName        []string
+	EnableCSI      bool
 }

test/e2e/upgrade/upgrade.go

@@ -27,8 +27,10 @@ import (
 	. "github.com/onsi/gomega"
 
 	. "github.com/vmware-tanzu/velero/test/e2e"
+	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
@@ -74,10 +76,13 @@ func BackupUpgradeRestoreWithRestic() {
 func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero UpgradeFromVelero) {
 	var (
 		backupName, restoreName string
+		client                  TestClient
+		err                     error
 	)
-
-	client, err := NewTestClient()
-	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+	By("Create test client instance", func() {
+		client, err = NewTestClient()
+		Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests")
+	})
 	BeforeEach(func() {
 		if !VeleroCfg.InstallVelero {
 			Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false")
@@ -166,11 +171,27 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 				})
 			})
 
-			if useVolumeSnapshots && VeleroCfg.CloudProvider == "vsphere" {
-				// TODO - remove after upload progress monitoring is implemented
-				By("Waiting for vSphere uploads to complete", func() {
-					Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
-						upgradeNamespace)).To(Succeed())
+			if useVolumeSnapshots {
+				if VeleroCfg.CloudProvider == "vsphere" {
+					// TODO - remove after upload progress monitoring is implemented
+					By("Waiting for vSphere uploads to complete", func() {
+						Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
+							upgradeNamespace)).To(Succeed())
+					})
+				} else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+					By("CSI VolumeSnapshotContent CR should be created", func() {
+						Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content")
+					})
+				}
+				var snapshotCheckPoint SnapshotCheckPoint
+				snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace
+				By("Snapshot should be created in cloud object store", func() {
+					snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2,
+						upgradeNamespace, backupName, KibishiiPodNameList)
+					Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
+					Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
+						VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+						VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
 				})
 			}
 
@@ -179,6 +200,20 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade
 					fmt.Sprintf("failed to delete namespace %s", upgradeNamespace))
 			})
 
+			if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+				// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
+				//   v1.0 have issues to fail upgrade case.
+				By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
+					time.Sleep(5 * time.Minute)
+				})
+				// TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly
+				// err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
+				// 	VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
+				// 	backupName, snapshotCheckPoint)
+				// if err != nil {
+				// 	return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
+				// }
+			}
 			// the snapshots of AWS may be still in pending status when do the restore, wait for a while
 			// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
 			// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed

test/e2e/util/csi/common.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2020 the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+)
+
+func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error) {
+	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	configOverrides := &clientcmd.ConfigOverrides{}
+	kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides)
+	clientConfig, err := kubeConfig.ClientConfig()
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	client, err := kubernetes.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	snapshotterClient, err := snapshotterClientSet.NewForConfig(clientConfig)
+	if err != nil {
+		return nil, nil, errors.WithStack(err)
+	}
+
+	return client, snapshotterClient, nil
+}
+
+func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) {
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return nil, err
+	}
+	vscList, err1 := snapshotClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
+	if err1 != nil {
+		return nil, err
+	}
+	var snapshotHandleList []string
+	for _, i := range vscList.Items {
+		if i.Status.SnapshotHandle == nil {
+			fmt.Println("SnapshotHandle is nil")
+			continue
+		}
+		fmt.Println(*i.Status.SnapshotHandle)
+		if i.Labels["velero.io/backup-name"] == backupName {
+			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
+			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	if len(snapshotHandleList) == 0 {
+		return snapshotHandleList, errors.New(fmt.Sprintf("No VolumeSnapshotContent from backup %s", backupName))
+	}
+	return snapshotHandleList, nil
+}
+func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
+	pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
+	if err != nil {
+		return "", err
+	}
+	if len(pvcList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace))
+	}
+	pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0])
+	if err != nil {
+		return "", err
+	}
+	if len(pvList) != 1 {
+		return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace))
+	}
+	pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0])
+	fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"])
+	if err != nil {
+		return "", err
+	}
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return "", err
+	}
+	vsList, err := snapshotClient.SnapshotV1beta1().VolumeSnapshots(namespace).List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return "", err
+	}
+	for _, i := range vsList.Items {
+		if pvcList[0] == *i.Spec.Source.PersistentVolumeClaimName &&
+			i.Labels["velero.io/backup-name"] == backupName {
+			return *i.Status.BoundVolumeSnapshotContentName, nil
+		}
+	}
+	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
+}
+
+func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error {
+	for _, podName := range pods {
+		if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" {
+			return errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		} else {
+			fmt.Println("Found volumesnapshotcontent: " + snapshotContentName)
+		}
+	}
+	return nil
+}

test/e2e/util/k8s/common.go

@@ -99,7 +99,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string,
 	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
 }
 
-func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) {
+func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) {
 	// Example:
 	// 	  NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                              STORAGECLASS             REASON   AGE
 	//    pvc-3f784366-58db-40b2-8fec-77307807e74b   1Gi        RWO            Delete           Bound    bsl-deletion/kibishii-data-kibishii-deployment-0   kibishii-storage-class            6h41m
@@ -110,7 +110,7 @@ func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) {
 
 	CmdLine2 := &common.OsCommandLine{
 		Cmd:  "grep",
-		Args: []string{pvc},
+		Args: []string{namespace + "/" + pvc},
 	}
 
 	CmdLine3 := &common.OsCommandLine{

test/e2e/util/k8s/persistentvolumes.go

@@ -0,0 +1,28 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func GetPersistentVolume(ctx context.Context, client TestClient, namespace string, persistentVolume string) (*corev1.PersistentVolume, error) {
+	return client.ClientGo.CoreV1().PersistentVolumes().Get(ctx, persistentVolume, metav1.GetOptions{})
+}

test/e2e/util/kibishii/kibishii_utils.go

@@ -28,7 +28,9 @@ import (
 
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 	. "github.com/vmware-tanzu/velero/test/e2e"
+	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
@@ -36,6 +38,8 @@ const (
 	jumpPadPod = "jump-pad"
 )
 
+var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
+
 // RunKibishiiTests runs kibishii tests on the provider.
 func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
 	useVolumeSnapshots bool) error {
@@ -47,7 +51,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
 	veleroFeatures := VeleroCfg.Features
 	kibishiiDirectory := VeleroCfg.KibishiiDirectory
 	if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil {
-		fmt.Printf("Workload namespace %s exists, delete it first.", kibishiiNamespace)
+		fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace)
 		if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
 			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
 		}
@@ -73,19 +77,46 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re
 		RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}
-
-	if providerName == "vsphere" && useVolumeSnapshots {
-		// Wait for uploads started by the Velero Plug-in for vSphere to complete
-		// TODO - remove after upload progress monitoring is implemented
-		fmt.Println("Waiting for vSphere uploads to complete")
-		if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
-			return errors.Wrapf(err, "Error waiting for uploads to complete")
+	var snapshotCheckPoint SnapshotCheckPoint
+	var err error
+	if useVolumeSnapshots {
+		if providerName == "vsphere" {
+			// Wait for uploads started by the Velero Plug-in for vSphere to complete
+			// TODO - remove after upload progress monitoring is implemented
+			fmt.Println("Waiting for vSphere uploads to complete")
+			if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
+				return errors.Wrapf(err, "Error waiting for uploads to complete")
+			}
+		} else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
+			if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil {
+				return errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
+			}
+		}
+		snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList)
+		if err != nil {
+			return errors.Wrap(err, "Fail to get snapshot checkpoint")
+		}
+		err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider,
+			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
+			backupName, snapshotCheckPoint)
+		if err != nil {
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
 		}
 	}
+
 	fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
 	if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}
+	if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") {
+		err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider,
+			VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
+			backupName, snapshotCheckPoint)
+		if err != nil {
+			return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
+		}
+	}
+	time.Sleep(5 * time.Minute)
 
 	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
 	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799

test/e2e/util/providers/azure_utils.go

@@ -226,12 +226,11 @@ func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string
 	_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	return err
 }
-func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) {
+func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName string) (bool, error) {
 	accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig)
 	if err != nil {
 		log.Fatal("Fail to get : accountName and accountKey, " + err.Error())
 	}
-
 	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
 	if err != nil {
 		log.Fatal("Invalid credentials with error: " + err.Error())
@@ -250,7 +249,7 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref
 	_, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
 	handleErrors(err)
 
-	fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName)
+	fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupName, containerName)
 	for marker := (azblob.Marker{}); marker.NotDone(); {
 		listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{})
 		if err != nil {
@@ -259,8 +258,8 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref
 		marker = listBlob.NextMarker
 
 		for _, blobInfo := range listBlob.Segment.BlobItems {
-			if strings.Contains(blobInfo.Name, backupObject) {
-				fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name)
+			if strings.Contains(blobInfo.Name, backupName) {
+				fmt.Printf("Blob name: %s exist in %s\n", backupName, blobInfo.Name)
 				return true, nil
 			}
 		}
@@ -317,7 +316,7 @@ func mapLookup(data map[string]string) func(string) string {
 		return data[key]
 	}
 }
-func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error {
+func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error {
 
 	ctx := context.Background()
 
@@ -351,7 +350,7 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO
 	// // if config["snapsIncrementalConfigKey"] is empty, default to nil; otherwise, parse i
 	snapsClient.Authorizer = authorizer
 	snaps := &snapsClient
-	//return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupObject, snapshotCount)
+	//return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupName, snapshotCount)
 	req, err := snaps.ListByResourceGroupPreparer(ctx, envVars[resourceGroupEnvVar])
 	if err != nil {
 		return autorest.NewErrorWithError(err, "compute.SnapshotsClient", "ListByResourceGroup", nil, "Failure preparing request")
@@ -364,11 +363,29 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO
 	result, err := snaps.ListByResourceGroupResponder(resp)
 	snapshotCountFound := 0
 	backupNameInSnapshot := ""
+	if err != nil {
+		errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar]))
+	}
+	if result.Value == nil {
+		errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar]))
+	}
 	for _, v := range *result.Value {
-		backupNameInSnapshot = *v.Tags["velero.io-backup"]
-		fmt.Println(backupNameInSnapshot)
-		if backupObject == backupNameInSnapshot {
-			snapshotCountFound++
+		if snapshotCheck.EnableCSI {
+			for _, s := range snapshotCheck.SnapshotIDList {
+				fmt.Println("Azure CSI local snapshot CR: " + s)
+				fmt.Println("Azure provider snapshot name: " + *v.Name)
+				if s == *v.Name {
+					fmt.Printf("Azure snapshot %s is created.\n", s)
+					snapshotCountFound++
+				}
+			}
+		} else {
+			fmt.Println(v.Tags)
+			backupNameInSnapshot = *v.Tags["velero.io-backup"]
+			fmt.Println(backupNameInSnapshot)
+			if backupName == backupNameInSnapshot {
+				snapshotCountFound++
+			}
 		}
 	}
 	if err != nil {

test/e2e/util/providers/common.go

@@ -24,6 +24,8 @@ import (
 
 	"github.com/pkg/errors"
 
+	waitutil "k8s.io/apimachinery/pkg/util/wait"
+
 	. "github.com/vmware-tanzu/velero/test/e2e"
 	velero "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
@@ -31,11 +33,11 @@ import (
 type ObjectsInStorage interface {
 	IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error)
 	DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error
-	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error
+	IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error
 }
 
 func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
-	fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix)
+	fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix)
 	exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 	if !exist {
 		return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName))
@@ -109,9 +111,37 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
 	return nil
 }
 
-func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string) error {
+func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
+	err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
+		func() (bool, error) {
+			err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				bslConfig, backupName, snapshotCheckPoint)
+			if err == nil {
+				return true, nil
+			}
+			return false, err
+		})
+	return err
+}
+
+func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
+	err := waitutil.PollImmediate(30*time.Second, 3*time.Minute,
+		func() (bool, error) {
+			snapshotCheckPoint.ExpectCount = 0
+			err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
+				VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				bslConfig, backupName, snapshotCheckPoint)
+			if err == nil {
+				return true, nil
+			}
+			return false, err
+		})
+	return err
+}
+
+func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
-	var snapshotCheckPoint SnapshotCheckPoint
 	snapshotCheckPoint.ExpectCount = 0
 	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {

test/e2e/util/velero/velero_utils.go

@@ -40,7 +40,10 @@ import (
 	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+	. "github.com/vmware-tanzu/velero/test/e2e"
 	common "github.com/vmware-tanzu/velero/test/e2e/util/common"
+	util "github.com/vmware-tanzu/velero/test/e2e/util/csi"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 )
 
 const BackupObjectsPrefix = "backups"
@@ -491,7 +494,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
 	return err
 }
 
-func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace, podName string) ([]string, error) {
+func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, podNameList []string) ([]string, error) {
 	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
 		"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'")
 	fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
@@ -504,14 +507,20 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 	stdout = strings.Replace(stdout, "'", "", -1)
 	lines := strings.Split(stdout, "\n")
 	var result []string
-
 	for _, curLine := range lines {
 		fmt.Println("curLine:" + curLine)
 		curLine = strings.Replace(curLine, "\n", "", -1)
 		if len(curLine) == 0 {
 			continue
 		}
-		if podName != "" && !strings.Contains(curLine, podName) {
+		var Exist bool
+		for _, podName := range podNameList {
+			if podName != "" && strings.Contains(curLine, podName) {
+				Exist = true
+				break
+			}
+		}
+		if !Exist {
 			continue
 		}
 		snapshotID := curLine[strings.LastIndex(curLine, ":")+1:]
@@ -519,6 +528,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace
 		snapshotIDDec, _ := b64.StdEncoding.DecodeString(snapshotID)
 		fmt.Println("snapshotIDDec:" + string(snapshotIDDec))
 		result = append(result, string(snapshotIDDec))
+		fmt.Println(result)
 	}
 	fmt.Println(result)
 	return result, nil
@@ -797,3 +807,28 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
 
 	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
 }
+
+func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
+	var snapshotCheckPoint SnapshotCheckPoint
+
+	snapshotCheckPoint.ExpectCount = expectCount
+	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
+	snapshotCheckPoint.PodName = kibishiiPodNameList
+	if strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
+		snapshotCheckPoint.EnableCSI = true
+		if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil {
+			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
+		}
+		var err error
+		snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName)
+		if err != nil {
+			return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName))
+		}
+		fmt.Println(snapshotCheckPoint)
+		if len(snapshotCheckPoint.SnapshotIDList) != expectCount {
+			return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount))
+		}
+	}
+	fmt.Println(snapshotCheckPoint)
+	return snapshotCheckPoint, nil
+}