From 33bf59a9107cc153212ebb6a5d5bc64a4f8cf94a Mon Sep 17 00:00:00 2001 From: danfengl Date: Sun, 10 Apr 2022 10:20:16 +0000 Subject: [PATCH] Add snapshot checkpoint for CSI E2E test and fix CSI snapshot name issue 1. Add checkpoint in snapshot E2E test to verify snapshot CR should be created and snapshot should be created in cloud side after backup completion; 2. Fix snapshot name issue that CSI snapshot name in cloud side is not the same with other non-CSI cloud snapshots; Signed-off-by: danfengl --- test/e2e/backup/backup.go | 10 +- test/e2e/backups/deletion.go | 58 ++++++--- test/e2e/backups/sync_backups.go | 14 ++- test/e2e/basic/enable_api_group_versions.go | 7 +- test/e2e/bsl-mgmt/deletion.go | 55 +++++--- test/e2e/e2e_suite_test.go | 2 + test/e2e/privilegesmgmt/ssr.go | 14 ++- test/e2e/test/test.go | 15 ++- test/e2e/types.go | 9 +- test/e2e/upgrade/upgrade.go | 51 ++++++-- test/e2e/util/csi/common.go | 132 ++++++++++++++++++++ test/e2e/util/k8s/common.go | 4 +- test/e2e/util/k8s/persistentvolumes.go | 28 +++++ test/e2e/util/kibishii/kibishii_utils.go | 47 +++++-- test/e2e/util/providers/azure_utils.go | 39 ++++-- test/e2e/util/providers/common.go | 38 +++++- test/e2e/util/velero/velero_utils.go | 41 +++++- 17 files changed, 474 insertions(+), 90 deletions(-) create mode 100644 test/e2e/util/csi/common.go create mode 100644 test/e2e/util/k8s/persistentvolumes.go diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 757c2e589..103568e05 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -39,13 +39,17 @@ func BackupRestoreWithRestic() { } func BackupRestoreTest(useVolumeSnapshots bool) { + kibishiiNamespace := "kibishii-workload" var ( backupName, restoreName string + client TestClient + err error ) - kibishiiNamespace := "kibishii-workload" - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 69c8bd355..0c5c2deed 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" @@ -47,11 +48,14 @@ func BackupDeletionWithRestic() { func backup_deletion_test(useVolumeSnapshots bool) { var ( backupName string + client TestClient + err error ) - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") - + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") @@ -98,11 +102,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) } - defer func() { - if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) - } - }() + if !VeleroCfg.Debug { + defer func() { + if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) + } + }() + } if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil { @@ -131,28 +137,50 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if err != nil { return err } + var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - var snapshotCheckPoint SnapshotCheckPoint - snapshotCheckPoint.ExpectCount = 2 - snapshotCheckPoint.NamespaceBackedUp = deletionTest - err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") if err != nil { - return err + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } err = DeleteBackupResource(context.Background(), veleroCLI, backupName) if err != nil { return err } + if useVolumeSnapshots { + err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + } err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) if err != nil { return err } if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName) + err = waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) if err != nil { - return err + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 8d1eb3d16..2fb2df6d7 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -52,10 +52,16 @@ func (b *SyncBackups) Init() { func BackupsSyncTest() { test := new(SyncBackups) - client, err := NewTestClient() - if err != nil { - println(err.Error()) - } + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index 1eac63b02..c30716387 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -45,10 +45,13 @@ func APIGropuVersionsTest() { resource, group string err error ctx = context.Background() + client TestClient ) - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { resource = "rockbands" diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index a034b2da3..f385d7371 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -19,6 +19,7 @@ import ( "context" "flag" "fmt" + "strings" "time" "github.com/google/go-cmp/cmp" @@ -28,6 +29,7 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" @@ -52,8 +54,16 @@ func BslDeletionWithRestic() { BslDeletionTest(false) } func BslDeletionTest(useVolumeSnapshots bool) { - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + less := func(a, b string) bool { return a < b } BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { @@ -198,19 +208,28 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName_2, 1)).To(Succeed()) }) + } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() { + Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") + }) + + By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() { + Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") + }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_1 - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_2 + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") var BSLCredentials, BSLConfig string if VeleroCfg.CloudProvider == "vsphere" { BSLCredentials = VeleroCfg.AdditionalBSLCredentials @@ -219,7 +238,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { BSLCredentials = VeleroCfg.CloudCredentialsFile BSLConfig = VeleroCfg.BSLConfig } - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) @@ -234,12 +254,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) } - By(fmt.Sprintf("Verify if backup %s is created or not", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI, backupName_1, 10*time.Minute)).To(Succeed()) }) - By(fmt.Sprintf("Verify if backup %s is created or not", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI, backupName_2, 10*time.Minute)).To(Succeed()) }) @@ -298,15 +318,14 @@ func BslDeletionTest(useVolumeSnapshots bool) { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_1 - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_2 var BSLCredentials, BSLConfig string if VeleroCfg.CloudProvider == "vsphere" { BSLCredentials = VeleroCfg.AdditionalBSLCredentials @@ -315,7 +334,9 @@ func BslDeletionTest(useVolumeSnapshots bool) { BSLCredentials = VeleroCfg.CloudCredentialsFile BSLConfig = VeleroCfg.BSLConfig } - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a0d4c3d59..a2d98727d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -94,9 +94,11 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on i var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) + var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) + var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 90d7cd7f7..ee1ed5631 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -36,11 +36,15 @@ import ( func SSRTest() { testNS := "ssr-test" - client, err := NewTestClient() - if err != nil { - println(err.Error()) - } - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { flag.Parse() diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 995b4fda7..811b69ac9 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -72,9 +72,11 @@ var TestClientInstance TestClient func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - var err error - TestClientInstance, err = NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + var err error + TestClientInstance, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") BeforeEach(func() { flag.Parse() @@ -99,8 +101,11 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { var err error var countIt int - TestClientInstance, err = NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + TestClientInstance, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + //Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) } diff --git a/test/e2e/types.go b/test/e2e/types.go index 5a0a8ae9b..d24c40635 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -55,7 +55,10 @@ type VerleroConfig struct { type SnapshotCheckPoint struct { NamespaceBackedUp string - SnapshotIDList []string - ExpectCount int - PodName string + // SnapshotIDList is for Azure CSI Verification + // we can get SnapshotID from VolumeSnapshotContent from a certain backup + SnapshotIDList []string + ExpectCount int + PodName []string + EnableCSI bool } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 0d98908d5..aff943aa6 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -27,8 +27,10 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -74,10 +76,13 @@ func BackupUpgradeRestoreWithRestic() { func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero UpgradeFromVelero) { var ( backupName, restoreName string + client TestClient + err error ) - - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if !VeleroCfg.InstallVelero { Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false") @@ -166,11 +171,27 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade }) }) - if useVolumeSnapshots && VeleroCfg.CloudProvider == "vsphere" { - // TODO - remove after upload progress monitoring is implemented - By("Waiting for vSphere uploads to complete", func() { - Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, - upgradeNamespace)).To(Succeed()) + if useVolumeSnapshots { + if VeleroCfg.CloudProvider == "vsphere" { + // TODO - remove after upload progress monitoring is implemented + By("Waiting for vSphere uploads to complete", func() { + Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, + upgradeNamespace)).To(Succeed()) + }) + } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + By("CSI VolumeSnapshotContent CR should be created", func() { + Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content") + }) + } + var snapshotCheckPoint SnapshotCheckPoint + snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace + By("Snapshot should be created in cloud object store", func() { + snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2, + upgradeNamespace, backupName, KibishiiPodNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) } @@ -179,6 +200,20 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade fmt.Sprintf("failed to delete namespace %s", upgradeNamespace)) }) + if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before + // v1.0 have issues to fail upgrade case. + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) + }) + // TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly + // err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + // VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + // backupName, snapshotCheckPoint) + // if err != nil { + // return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + // } + } // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go new file mode 100644 index 000000000..b517b5f54 --- /dev/null +++ b/test/e2e/util/csi/common.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + clientConfig, err := kubeConfig.ClientConfig() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + snapshotterClient, err := snapshotterClientSet.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + return client, snapshotterClient, nil +} + +func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) { + _, snapshotClient, err := GetClients() + if err != nil { + return nil, err + } + vscList, err1 := snapshotClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err1 != nil { + return nil, err + } + var snapshotHandleList []string + for _, i := range vscList.Items { + if i.Status.SnapshotHandle == nil { + fmt.Println("SnapshotHandle is nil") + continue + } + fmt.Println(*i.Status.SnapshotHandle) + if i.Labels["velero.io/backup-name"] == backupName { + tmp := strings.Split(*i.Status.SnapshotHandle, "/") + snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) + } + } + if err != nil { + return nil, err + } + if len(snapshotHandleList) == 0 { + return snapshotHandleList, errors.New(fmt.Sprintf("No VolumeSnapshotContent from backup %s", backupName)) + } + return snapshotHandleList, nil +} +func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) { + pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) + if err != nil { + return "", err + } + if len(pvcList) != 1 { + return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace)) + } + pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0]) + if err != nil { + return "", err + } + if len(pvList) != 1 { + return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace)) + } + pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0]) + fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"]) + if err != nil { + return "", err + } + _, snapshotClient, err := GetClients() + if err != nil { + return "", err + } + vsList, err := snapshotClient.SnapshotV1beta1().VolumeSnapshots(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return "", err + } + for _, i := range vsList.Items { + if pvcList[0] == *i.Spec.Source.PersistentVolumeClaimName && + i.Labels["velero.io/backup-name"] == backupName { + return *i.Status.BoundVolumeSnapshotContentName, nil + } + } + return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) +} + +func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error { + for _, podName := range pods { + if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" { + return errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } else { + fmt.Println("Found volumesnapshotcontent: " + snapshotContentName) + } + } + return nil +} diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 06d7cdaea..43812a74a 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -99,7 +99,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } -func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) { +func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) { // Example: // NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE // pvc-3f784366-58db-40b2-8fec-77307807e74b 1Gi RWO Delete Bound bsl-deletion/kibishii-data-kibishii-deployment-0 kibishii-storage-class 6h41m @@ -110,7 +110,7 @@ func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) { CmdLine2 := &common.OsCommandLine{ Cmd: "grep", - Args: []string{pvc}, + Args: []string{namespace + "/" + pvc}, } CmdLine3 := &common.OsCommandLine{ diff --git a/test/e2e/util/k8s/persistentvolumes.go b/test/e2e/util/k8s/persistentvolumes.go new file mode 100644 index 000000000..2f414cda4 --- /dev/null +++ b/test/e2e/util/k8s/persistentvolumes.go @@ -0,0 +1,28 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetPersistentVolume(ctx context.Context, client TestClient, namespace string, persistentVolume string) (*corev1.PersistentVolume, error) { + return client.ClientGo.CoreV1().PersistentVolumes().Get(ctx, persistentVolume, metav1.GetOptions{}) +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index d99b1b939..005354f28 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -28,7 +28,9 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -36,6 +38,8 @@ const ( jumpPadPod = "jump-pad" ) +var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"} + // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots bool) error { @@ -47,7 +51,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re veleroFeatures := VeleroCfg.Features kibishiiDirectory := VeleroCfg.KibishiiDirectory if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil { - fmt.Printf("Workload namespace %s exists, delete it first.", kibishiiNamespace) + fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace) if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } @@ -73,19 +77,46 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } - - if providerName == "vsphere" && useVolumeSnapshots { - // Wait for uploads started by the Velero Plug-in for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - fmt.Println("Waiting for vSphere uploads to complete") - if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { - return errors.Wrapf(err, "Error waiting for uploads to complete") + var snapshotCheckPoint SnapshotCheckPoint + var err error + if useVolumeSnapshots { + if providerName == "vsphere" { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { + if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil { + return errors.Wrapf(err, "Fail to get Azure CSI snapshot content") + } + } + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) + if err != nil { + return errors.Wrap(err, "Fail to get snapshot checkpoint") + } + err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } + fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } + if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { + err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + } + time.Sleep(5 * time.Minute) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go index a34b05a41..8c579c74f 100644 --- a/test/e2e/util/providers/azure_utils.go +++ b/test/e2e/util/providers/azure_utils.go @@ -226,12 +226,11 @@ func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string _, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) return err } -func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { +func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName string) (bool, error) { accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) if err != nil { log.Fatal("Fail to get : accountName and accountKey, " + err.Error()) } - credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { log.Fatal("Invalid credentials with error: " + err.Error()) @@ -250,7 +249,7 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) handleErrors(err) - fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName) + fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupName, containerName) for marker := (azblob.Marker{}); marker.NotDone(); { listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) if err != nil { @@ -259,8 +258,8 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref marker = listBlob.NextMarker for _, blobInfo := range listBlob.Segment.BlobItems { - if strings.Contains(blobInfo.Name, backupObject) { - fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name) + if strings.Contains(blobInfo.Name, backupName) { + fmt.Printf("Blob name: %s exist in %s\n", backupName, blobInfo.Name) return true, nil } } @@ -317,7 +316,7 @@ func mapLookup(data map[string]string) func(string) string { return data[key] } } -func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error { +func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { ctx := context.Background() @@ -351,7 +350,7 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO // // if config["snapsIncrementalConfigKey"] is empty, default to nil; otherwise, parse i snapsClient.Authorizer = authorizer snaps := &snapsClient - //return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupObject, snapshotCount) + //return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupName, snapshotCount) req, err := snaps.ListByResourceGroupPreparer(ctx, envVars[resourceGroupEnvVar]) if err != nil { return autorest.NewErrorWithError(err, "compute.SnapshotsClient", "ListByResourceGroup", nil, "Failure preparing request") @@ -364,11 +363,29 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO result, err := snaps.ListByResourceGroupResponder(resp) snapshotCountFound := 0 backupNameInSnapshot := "" + if err != nil { + errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar])) + } + if result.Value == nil { + errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar])) + } for _, v := range *result.Value { - backupNameInSnapshot = *v.Tags["velero.io-backup"] - fmt.Println(backupNameInSnapshot) - if backupObject == backupNameInSnapshot { - snapshotCountFound++ + if snapshotCheck.EnableCSI { + for _, s := range snapshotCheck.SnapshotIDList { + fmt.Println("Azure CSI local snapshot CR: " + s) + fmt.Println("Azure provider snapshot name: " + *v.Name) + if s == *v.Name { + fmt.Printf("Azure snapshot %s is created.\n", s) + snapshotCountFound++ + } + } + } else { + fmt.Println(v.Tags) + backupNameInSnapshot = *v.Tags["velero.io-backup"] + fmt.Println(backupNameInSnapshot) + if backupName == backupNameInSnapshot { + snapshotCountFound++ + } } } if err != nil { diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go index 26fc8083c..accfafec1 100644 --- a/test/e2e/util/providers/common.go +++ b/test/e2e/util/providers/common.go @@ -24,6 +24,8 @@ import ( "github.com/pkg/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" + . "github.com/vmware-tanzu/velero/test/e2e" velero "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -31,11 +33,11 @@ import ( type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error } func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { - fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix) + fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix) exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if !exist { return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) @@ -109,9 +111,37 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string) error { +func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { + err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) + return err +} + +func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { + err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + snapshotCheckPoint.ExpectCount = 0 + err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) + return err +} + +func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) - var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 4ca48d415..449f6e0a5 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -40,7 +40,10 @@ import ( cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" common "github.com/vmware-tanzu/velero/test/e2e/util/common" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" ) const BackupObjectsPrefix = "backups" @@ -491,7 +494,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, return err } -func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace, podName string) ([]string, error) { +func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, podNameList []string) ([]string, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'") fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) @@ -504,14 +507,20 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace stdout = strings.Replace(stdout, "'", "", -1) lines := strings.Split(stdout, "\n") var result []string - for _, curLine := range lines { fmt.Println("curLine:" + curLine) curLine = strings.Replace(curLine, "\n", "", -1) if len(curLine) == 0 { continue } - if podName != "" && !strings.Contains(curLine, podName) { + var Exist bool + for _, podName := range podNameList { + if podName != "" && strings.Contains(curLine, podName) { + Exist = true + break + } + } + if !Exist { continue } snapshotID := curLine[strings.LastIndex(curLine, ":")+1:] @@ -519,6 +528,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace snapshotIDDec, _ := b64.StdEncoding.DecodeString(snapshotID) fmt.Println("snapshotIDDec:" + string(snapshotIDDec)) result = append(result, string(snapshotIDDec)) + fmt.Println(result) } fmt.Println(result) return result, nil @@ -797,3 +807,28 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } + +func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) { + var snapshotCheckPoint SnapshotCheckPoint + + snapshotCheckPoint.ExpectCount = expectCount + snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp + snapshotCheckPoint.PodName = kibishiiPodNameList + if strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + snapshotCheckPoint.EnableCSI = true + if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil { + return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") + } + var err error + snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName) + if err != nil { + return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName)) + } + fmt.Println(snapshotCheckPoint) + if len(snapshotCheckPoint.SnapshotIDList) != expectCount { + return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount)) + } + } + fmt.Println(snapshotCheckPoint) + return snapshotCheckPoint, nil +}