Refactor Snapshot Checkpoint in E2E Tests (#9083)

* Refactor backup volume info retrieval and snapshot checkpoint building in e2e tests
Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

log backup volume info retrieval and snapshot checkpoint building
Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

Add error handling for volume info retrieval in backup tests
Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

Add error handling for volume info retrieval in backup tests
Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

* Update snapshot checkpoint building to use DefaultKibishiiWorkerCounts
Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

---------

Signed-off-by: Priyansh Choudhary <im1706@gmail.com>

test/util/kibishii/kibishii_utils.go

@@ -141,6 +141,21 @@ func RunKibishiiTests(
 
 	fmt.Printf("VeleroBackupNamespace done %s\n", time.Now().Format("2006-01-02 15:04:05"))
 
+	fmt.Printf("KibishiiVerifyAfterBackup %s\n", time.Now().Format("2006-01-02 15:04:05"))
+	backupVolumeInfo, err := GetVolumeInfo(
+		veleroCfg.ObjectStoreProvider,
+		veleroCfg.CloudCredentialsFile,
+		veleroCfg.BSLBucket,
+		veleroCfg.BSLPrefix,
+		veleroCfg.BSLConfig,
+		backupName,
+		BackupObjectsPrefix+"/"+backupName,
+	)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to get volume info for backup %s", backupName)
+	}
+	fmt.Printf("backupVolumeInfo %v\n", backupVolumeInfo)
+
 	// Checkpoint for a successful backup
 	if useVolumeSnapshots {
 		if veleroCfg.HasVspherePlugin {
@@ -150,7 +165,8 @@ func RunKibishiiTests(
 				return errors.Wrapf(err, "Error waiting for uploads to complete")
 			}
 		}
-		snapshotCheckPoint, err := GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPVCNameList)
+
+		snapshotCheckPoint, err := BuildSnapshotCheckPointFromVolumeInfo(veleroCfg, backupVolumeInfo, DefaultKibishiiWorkerCounts, kibishiiNamespace, backupName, KibishiiPVCNameList)
 		if err != nil {
 			return errors.Wrap(err, "Fail to get snapshot checkpoint")
 		}
@@ -186,7 +202,7 @@ func RunKibishiiTests(
 			// wait for a period to confirm no snapshots content exist for the backup
 			time.Sleep(1 * time.Minute)
 			if strings.EqualFold(veleroFeatures, FeatureCSI) {
-				_, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0,
+				_, err = BuildSnapshotCheckPointFromVolumeInfo(veleroCfg, backupVolumeInfo, 0,
 					kibishiiNamespace, backupName, KibishiiPVCNameList)
 				if err != nil {
 					return errors.Wrap(err, "failed to get snapshot checkPoint")

test/util/velero/velero_utils.go

@@ -43,13 +43,13 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/vmware-tanzu/velero/internal/volume"
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
 	cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 	. "github.com/vmware-tanzu/velero/test"
 	common "github.com/vmware-tanzu/velero/test/util/common"
-	util "github.com/vmware-tanzu/velero/test/util/csi"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -1277,20 +1277,38 @@ func GetRepositories(ctx context.Context, veleroNamespace, targetNamespace strin
 	return common.GetListByCmdPipes(ctx, cmds)
 }
 
-func GetSnapshotCheckPoint(client TestClient, veleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
-	var err error
+// BuildSnapshotCheckPointFromVolumeInfo pulls snapshot handles directly
+func BuildSnapshotCheckPointFromVolumeInfo(
+	veleroCfg VeleroConfig,
+	backupVolumeInfo []*volume.BackupVolumeInfo,
+	expectCount int,
+	namespaceBackedUp string,
+	backupName string,
+	KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
 	var snapshotCheckPoint SnapshotCheckPoint
 
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
+
 	if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.EqualFold(veleroCfg.Features, FeatureCSI) {
 		snapshotCheckPoint.EnableCSI = true
 
-		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, map[string]string{"backupNameLabel": backupName}, expectCount); err != nil {
-			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
+		var VscCount = 0
+		for _, volumeInfo := range backupVolumeInfo {
+			if *volumeInfo.CSISnapshotInfo.ReadyToUse == true {
+				snapshotCheckPoint.SnapshotIDList = append(snapshotCheckPoint.SnapshotIDList, volumeInfo.CSISnapshotInfo.SnapshotHandle)
+				VscCount++
+			} else {
+				return snapshotCheckPoint, errors.New("CSI snapshot is not ready to use")
+			}
+		}
+
+		if VscCount != expectCount {
+			return snapshotCheckPoint, errors.New(fmt.Sprintf("CSI snapshot count %d is not as expected %d", VscCount, expectCount))
 		}
 	}
+
 	fmt.Printf("snapshotCheckPoint: %v \n", snapshotCheckPoint)
 	return snapshotCheckPoint, nil
 }