diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 757c2e589..103568e05 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -39,13 +39,17 @@ func BackupRestoreWithRestic() { } func BackupRestoreTest(useVolumeSnapshots bool) { + kibishiiNamespace := "kibishii-workload" var ( backupName, restoreName string + client TestClient + err error ) - kibishiiNamespace := "kibishii-workload" - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 69c8bd355..0c5c2deed 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" @@ -47,11 +48,14 @@ func BackupDeletionWithRestic() { func backup_deletion_test(useVolumeSnapshots bool) { var ( backupName string + client TestClient + err error ) - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") - + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") @@ -98,11 +102,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) } - defer func() { - if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) - } - }() + if !VeleroCfg.Debug { + defer func() { + if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { + fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) + } + }() + } if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots); err != nil { @@ -131,28 +137,50 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if err != nil { return err } + var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - var snapshotCheckPoint SnapshotCheckPoint - snapshotCheckPoint.ExpectCount = 2 - snapshotCheckPoint.NamespaceBackedUp = deletionTest - err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") if err != nil { - return err + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } err = DeleteBackupResource(context.Background(), veleroCLI, backupName) if err != nil { return err } + if useVolumeSnapshots { + err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + } err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) if err != nil { return err } if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName) + err = waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) if err != nil { - return err + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 8d1eb3d16..2fb2df6d7 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -52,10 +52,16 @@ func (b *SyncBackups) Init() { func BackupsSyncTest() { test := new(SyncBackups) - client, err := NewTestClient() - if err != nil { - println(err.Error()) - } + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index 1eac63b02..c30716387 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -45,10 +45,13 @@ func APIGropuVersionsTest() { resource, group string err error ctx = context.Background() + client TestClient ) - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { resource = "rockbands" diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index a034b2da3..f385d7371 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -19,6 +19,7 @@ import ( "context" "flag" "fmt" + "strings" "time" "github.com/google/go-cmp/cmp" @@ -28,6 +29,7 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" @@ -52,8 +54,16 @@ func BslDeletionWithRestic() { BslDeletionTest(false) } func BslDeletionTest(useVolumeSnapshots bool) { - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup deletion tests") + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + less := func(a, b string) bool { return a < b } BeforeEach(func() { if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { @@ -198,19 +208,28 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName_2, 1)).To(Succeed()) }) + } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() { + Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") + }) + + By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() { + Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") + }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_1 - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.AdditionalBSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_2 + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") var BSLCredentials, BSLConfig string if VeleroCfg.CloudProvider == "vsphere" { BSLCredentials = VeleroCfg.AdditionalBSLCredentials @@ -219,7 +238,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { BSLCredentials = VeleroCfg.CloudCredentialsFile BSLConfig = VeleroCfg.BSLConfig } - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) @@ -234,12 +254,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) } - By(fmt.Sprintf("Verify if backup %s is created or not", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI, backupName_1, 10*time.Minute)).To(Succeed()) }) - By(fmt.Sprintf("Verify if backup %s is created or not", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { Expect(WaitForBackupCreated(context.Background(), VeleroCfg.VeleroCLI, backupName_2, 10*time.Minute)).To(Succeed()) }) @@ -298,15 +318,14 @@ func BslDeletionTest(useVolumeSnapshots bool) { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_1 - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - snapshotCheckPoint.ExpectCount = 1 - snapshotCheckPoint.PodName = podName_2 var BSLCredentials, BSLConfig string if VeleroCfg.CloudProvider == "vsphere" { BSLCredentials = VeleroCfg.AdditionalBSLCredentials @@ -315,7 +334,9 @@ func BslDeletionTest(useVolumeSnapshots bool) { BSLCredentials = VeleroCfg.CloudCredentialsFile BSLConfig = VeleroCfg.BSLConfig } - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, BSLCredentials, VeleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a0d4c3d59..a2d98727d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -94,9 +94,11 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on i var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) + var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) + var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 90d7cd7f7..ee1ed5631 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -36,11 +36,15 @@ import ( func SSRTest() { testNS := "ssr-test" - client, err := NewTestClient() - if err != nil { - println(err.Error()) - } - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + var ( + client TestClient + err error + ) + + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { flag.Parse() diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 995b4fda7..811b69ac9 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -72,9 +72,11 @@ var TestClientInstance TestClient func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - var err error - TestClientInstance, err = NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + var err error + TestClientInstance, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") BeforeEach(func() { flag.Parse() @@ -99,8 +101,11 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { var err error var countIt int - TestClientInstance, err = NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + TestClientInstance, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) + //Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) } diff --git a/test/e2e/types.go b/test/e2e/types.go index 5a0a8ae9b..d24c40635 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -55,7 +55,10 @@ type VerleroConfig struct { type SnapshotCheckPoint struct { NamespaceBackedUp string - SnapshotIDList []string - ExpectCount int - PodName string + // SnapshotIDList is for Azure CSI Verification + // we can get SnapshotID from VolumeSnapshotContent from a certain backup + SnapshotIDList []string + ExpectCount int + PodName []string + EnableCSI bool } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 0d98908d5..aff943aa6 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -27,8 +27,10 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -74,10 +76,13 @@ func BackupUpgradeRestoreWithRestic() { func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero UpgradeFromVelero) { var ( backupName, restoreName string + client TestClient + err error ) - - client, err := NewTestClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + By("Create test client instance", func() { + client, err = NewTestClient() + Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") + }) BeforeEach(func() { if !VeleroCfg.InstallVelero { Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false") @@ -166,11 +171,27 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade }) }) - if useVolumeSnapshots && VeleroCfg.CloudProvider == "vsphere" { - // TODO - remove after upload progress monitoring is implemented - By("Waiting for vSphere uploads to complete", func() { - Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, - upgradeNamespace)).To(Succeed()) + if useVolumeSnapshots { + if VeleroCfg.CloudProvider == "vsphere" { + // TODO - remove after upload progress monitoring is implemented + By("Waiting for vSphere uploads to complete", func() { + Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, + upgradeNamespace)).To(Succeed()) + }) + } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + By("CSI VolumeSnapshotContent CR should be created", func() { + Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content") + }) + } + var snapshotCheckPoint SnapshotCheckPoint + snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace + By("Snapshot should be created in cloud object store", func() { + snapshotCheckPoint, err := GetSnapshotCheckPoint(client, VeleroCfg, 2, + upgradeNamespace, backupName, KibishiiPodNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) } @@ -179,6 +200,20 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade fmt.Sprintf("failed to delete namespace %s", upgradeNamespace)) }) + if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before + // v1.0 have issues to fail upgrade case. + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) + }) + // TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly + // err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + // VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + // backupName, snapshotCheckPoint) + // if err != nil { + // return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + // } + } // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go new file mode 100644 index 000000000..b517b5f54 --- /dev/null +++ b/test/e2e/util/csi/common.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + + snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" +) + +func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + configOverrides := &clientcmd.ConfigOverrides{} + kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) + clientConfig, err := kubeConfig.ClientConfig() + if err != nil { + return nil, nil, errors.WithStack(err) + } + + client, err := kubernetes.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + snapshotterClient, err := snapshotterClientSet.NewForConfig(clientConfig) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + return client, snapshotterClient, nil +} + +func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) { + _, snapshotClient, err := GetClients() + if err != nil { + return nil, err + } + vscList, err1 := snapshotClient.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + if err1 != nil { + return nil, err + } + var snapshotHandleList []string + for _, i := range vscList.Items { + if i.Status.SnapshotHandle == nil { + fmt.Println("SnapshotHandle is nil") + continue + } + fmt.Println(*i.Status.SnapshotHandle) + if i.Labels["velero.io/backup-name"] == backupName { + tmp := strings.Split(*i.Status.SnapshotHandle, "/") + snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) + } + } + if err != nil { + return nil, err + } + if len(snapshotHandleList) == 0 { + return snapshotHandleList, errors.New(fmt.Sprintf("No VolumeSnapshotContent from backup %s", backupName)) + } + return snapshotHandleList, nil +} +func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) { + pvcList, err := GetPvcByPodName(context.Background(), namespace, podName) + if err != nil { + return "", err + } + if len(pvcList) != 1 { + return "", errors.New(fmt.Sprintf("Only 1 PVC of pod %s should be found under namespace %s", podName, namespace)) + } + pvList, err := GetPvByPvc(context.Background(), namespace, pvcList[0]) + if err != nil { + return "", err + } + if len(pvList) != 1 { + return "", errors.New(fmt.Sprintf("Only 1 PV of PVC %s pod %s should be found under namespace %s", pvcList[0], podName, namespace)) + } + pv_value, err := GetPersistentVolume(context.Background(), client, "", pvList[0]) + fmt.Println(pv_value.Annotations["pv.kubernetes.io/provisioned-by"]) + if err != nil { + return "", err + } + _, snapshotClient, err := GetClients() + if err != nil { + return "", err + } + vsList, err := snapshotClient.SnapshotV1beta1().VolumeSnapshots(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return "", err + } + for _, i := range vsList.Items { + if pvcList[0] == *i.Spec.Source.PersistentVolumeClaimName && + i.Labels["velero.io/backup-name"] == backupName { + return *i.Status.BoundVolumeSnapshotContentName, nil + } + } + return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) +} + +func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error { + for _, podName := range pods { + if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" { + return errors.Wrap(err, "Fail to get Azure CSI snapshot content") + } else { + fmt.Println("Found volumesnapshotcontent: " + snapshotContentName) + } + } + return nil +} diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 06d7cdaea..43812a74a 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -99,7 +99,7 @@ func GetPvcByPodName(ctx context.Context, namespace, podName string) ([]string, return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } -func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) { +func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) { // Example: // NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE // pvc-3f784366-58db-40b2-8fec-77307807e74b 1Gi RWO Delete Bound bsl-deletion/kibishii-data-kibishii-deployment-0 kibishii-storage-class 6h41m @@ -110,7 +110,7 @@ func GetPvByPvc(ctx context.Context, pvc string) ([]string, error) { CmdLine2 := &common.OsCommandLine{ Cmd: "grep", - Args: []string{pvc}, + Args: []string{namespace + "/" + pvc}, } CmdLine3 := &common.OsCommandLine{ diff --git a/test/e2e/util/k8s/persistentvolumes.go b/test/e2e/util/k8s/persistentvolumes.go new file mode 100644 index 000000000..2f414cda4 --- /dev/null +++ b/test/e2e/util/k8s/persistentvolumes.go @@ -0,0 +1,28 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GetPersistentVolume(ctx context.Context, client TestClient, namespace string, persistentVolume string) (*corev1.PersistentVolume, error) { + return client.ClientGo.CoreV1().PersistentVolumes().Get(ctx, persistentVolume, metav1.GetOptions{}) +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index d99b1b939..005354f28 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -28,7 +28,9 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -36,6 +38,8 @@ const ( jumpPadPod = "jump-pad" ) +var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"} + // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots bool) error { @@ -47,7 +51,7 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re veleroFeatures := VeleroCfg.Features kibishiiDirectory := VeleroCfg.KibishiiDirectory if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil { - fmt.Printf("Workload namespace %s exists, delete it first.", kibishiiNamespace) + fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace) if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } @@ -73,19 +77,46 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } - - if providerName == "vsphere" && useVolumeSnapshots { - // Wait for uploads started by the Velero Plug-in for vSphere to complete - // TODO - remove after upload progress monitoring is implemented - fmt.Println("Waiting for vSphere uploads to complete") - if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { - return errors.Wrapf(err, "Error waiting for uploads to complete") + var snapshotCheckPoint SnapshotCheckPoint + var err error + if useVolumeSnapshots { + if providerName == "vsphere" { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + fmt.Println("Waiting for vSphere uploads to complete") + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + return errors.Wrapf(err, "Error waiting for uploads to complete") + } + } else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { + if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil { + return errors.Wrapf(err, "Fail to get Azure CSI snapshot content") + } + } + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) + if err != nil { + return errors.Wrap(err, "Fail to get snapshot checkpoint") + } + err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } + fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } + if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { + err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + } + time.Sleep(5 * time.Minute) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 diff --git a/test/e2e/util/providers/azure_utils.go b/test/e2e/util/providers/azure_utils.go index a34b05a41..8c579c74f 100644 --- a/test/e2e/util/providers/azure_utils.go +++ b/test/e2e/util/providers/azure_utils.go @@ -226,12 +226,11 @@ func deleteBlob(p pipeline.Pipeline, accountName, containerName, blobName string _, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) return err } -func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) { +func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName string) (bool, error) { accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) if err != nil { log.Fatal("Fail to get : accountName and accountKey, " + err.Error()) } - credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { log.Fatal("Invalid credentials with error: " + err.Error()) @@ -250,7 +249,7 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) handleErrors(err) - fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupObject, containerName) + fmt.Printf("Finding backup %s blobs in Azure container/bucket %s\n", backupName, containerName) for marker := (azblob.Marker{}); marker.NotDone(); { listBlob, err := containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{}) if err != nil { @@ -259,8 +258,8 @@ func (s AzureStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPref marker = listBlob.NextMarker for _, blobInfo := range listBlob.Segment.BlobItems { - if strings.Contains(blobInfo.Name, backupObject) { - fmt.Printf("Blob name: %s exist in %s\n", backupObject, blobInfo.Name) + if strings.Contains(blobInfo.Name, backupName) { + fmt.Printf("Blob name: %s exist in %s\n", backupName, blobInfo.Name) return true, nil } } @@ -317,7 +316,7 @@ func mapLookup(data map[string]string) func(string) string { return data[key] } } -func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error { +func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { ctx := context.Background() @@ -351,7 +350,7 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO // // if config["snapsIncrementalConfigKey"] is empty, default to nil; otherwise, parse i snapsClient.Authorizer = authorizer snaps := &snapsClient - //return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupObject, snapshotCount) + //return ListByResourceGroup(ctx, snaps, envVars[resourceGroupEnvVar], backupName, snapshotCount) req, err := snaps.ListByResourceGroupPreparer(ctx, envVars[resourceGroupEnvVar]) if err != nil { return autorest.NewErrorWithError(err, "compute.SnapshotsClient", "ListByResourceGroup", nil, "Failure preparing request") @@ -364,11 +363,29 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupO result, err := snaps.ListByResourceGroupResponder(resp) snapshotCountFound := 0 backupNameInSnapshot := "" + if err != nil { + errors.Wrap(err, fmt.Sprintf("Fail to list snapshots %s\n", envVars[resourceGroupEnvVar])) + } + if result.Value == nil { + errors.New(fmt.Sprintf("No snapshots in Azure resource group %s\n", envVars[resourceGroupEnvVar])) + } for _, v := range *result.Value { - backupNameInSnapshot = *v.Tags["velero.io-backup"] - fmt.Println(backupNameInSnapshot) - if backupObject == backupNameInSnapshot { - snapshotCountFound++ + if snapshotCheck.EnableCSI { + for _, s := range snapshotCheck.SnapshotIDList { + fmt.Println("Azure CSI local snapshot CR: " + s) + fmt.Println("Azure provider snapshot name: " + *v.Name) + if s == *v.Name { + fmt.Printf("Azure snapshot %s is created.\n", s) + snapshotCountFound++ + } + } + } else { + fmt.Println(v.Tags) + backupNameInSnapshot = *v.Tags["velero.io-backup"] + fmt.Println(backupNameInSnapshot) + if backupName == backupNameInSnapshot { + snapshotCountFound++ + } } } if err != nil { diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go index 26fc8083c..accfafec1 100644 --- a/test/e2e/util/providers/common.go +++ b/test/e2e/util/providers/common.go @@ -24,6 +24,8 @@ import ( "github.com/pkg/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" + . "github.com/vmware-tanzu/velero/test/e2e" velero "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) @@ -31,11 +33,11 @@ import ( type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObject string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error } func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { - fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage %s", backupName, bslPrefix) + fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix) exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if !exist { return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) @@ -109,9 +111,37 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string) error { +func WaitUntilSnapshotsExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { + err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + err := SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) + return err +} + +func WaitUntilSnapshotsNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { + err := waitutil.PollImmediate(30*time.Second, 3*time.Minute, + func() (bool, error) { + snapshotCheckPoint.ExpectCount = 0 + err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + bslConfig, backupName, snapshotCheckPoint) + if err == nil { + return true, nil + } + return false, err + }) + return err +} + +func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) - var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 4ca48d415..449f6e0a5 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -40,7 +40,10 @@ import ( cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" common "github.com/vmware-tanzu/velero/test/e2e/util/common" + util "github.com/vmware-tanzu/velero/test/e2e/util/csi" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" ) const BackupObjectsPrefix = "backups" @@ -491,7 +494,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, return err } -func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace, podName string) ([]string, error) { +func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace string, podNameList []string) ([]string, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.snapshotID}{\"\\n\"}{end}'") fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) @@ -504,14 +507,20 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace stdout = strings.Replace(stdout, "'", "", -1) lines := strings.Split(stdout, "\n") var result []string - for _, curLine := range lines { fmt.Println("curLine:" + curLine) curLine = strings.Replace(curLine, "\n", "", -1) if len(curLine) == 0 { continue } - if podName != "" && !strings.Contains(curLine, podName) { + var Exist bool + for _, podName := range podNameList { + if podName != "" && strings.Contains(curLine, podName) { + Exist = true + break + } + } + if !Exist { continue } snapshotID := curLine[strings.LastIndex(curLine, ":")+1:] @@ -519,6 +528,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace snapshotIDDec, _ := b64.StdEncoding.DecodeString(snapshotID) fmt.Println("snapshotIDDec:" + string(snapshotIDDec)) result = append(result, string(snapshotIDDec)) + fmt.Println(result) } fmt.Println(result) return result, nil @@ -797,3 +807,28 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } + +func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) { + var snapshotCheckPoint SnapshotCheckPoint + + snapshotCheckPoint.ExpectCount = expectCount + snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp + snapshotCheckPoint.PodName = kibishiiPodNameList + if strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + snapshotCheckPoint.EnableCSI = true + if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil { + return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") + } + var err error + snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName) + if err != nil { + return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName)) + } + fmt.Println(snapshotCheckPoint) + if len(snapshotCheckPoint.SnapshotIDList) != expectCount { + return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount)) + } + } + fmt.Println(snapshotCheckPoint) + return snapshotCheckPoint, nil +}