diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index f5bba56c2..e4bf69702 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -99,8 +99,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc providerName := veleroCfg.CloudProvider veleroNamespace := veleroCfg.VeleroNamespace registryCredentialFile := veleroCfg.RegistryCredentialFile - bslPrefix := veleroCfg.BSLPrefix - bslConfig := veleroCfg.BSLConfig veleroFeatures := veleroCfg.Features for _, ns := range workloadNamespaceList { if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { @@ -153,7 +151,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc } } } - err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix) if err != nil { return err } @@ -165,9 +163,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc for _, ns := range workloadNamespaceList { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -179,9 +180,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") // Get all snapshots base on backup name, regardless of namespaces - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -207,26 +211,34 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc return err } + // Verify snapshots are deleted after backup deletion. if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + snapshotCheckPoint.ExpectCount = 0 + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + return errors.Wrap(err, "fail to verify snapshots are deleted in provider.") } } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + // Verify backup metadata files are deleted in OSS after backup deletion. + err = ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 5, + ) if err != nil { return err } - if useVolumeSnapshots { - if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint); err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - } - } // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded // Sleep for more than 15 seconds to avoid this issue. @@ -243,13 +255,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) - err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) - if err != nil { + if err := DeleteObjectsInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + ); err != nil { return err } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) - if err != nil { + if err := ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 1, + ); err != nil { return err } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 342eac037..5135c4b0b 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -140,13 +140,11 @@ func TTLTest() { Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect( - SnapshotsShouldBeCreatedInCloud( - veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, - veleroCfg.BSLBucket, - veleroCfg.BSLConfig, + CheckSnapshotsInProvider( + veleroCfg, test.backupName, snapshotCheckPoint, + false, ), ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } @@ -202,9 +200,13 @@ func TTLTest() { By("PersistentVolume snapshots should be deleted", func() { if useVolumeSnapshots { - Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint.ExpectCount = 0 + Expect(CheckSnapshotsInProvider( + veleroCfg, + test.backupName, + snapshotCheckPoint, + false, + )).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } }) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 6c3ccee88..ca9d42add 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -36,8 +36,8 @@ import ( const ( // Please make sure length of this namespace should be shorter, - // otherwise ResticRepositories name verification will be wrong - // when making combination of ResticRepositories name(max length is 63) + // otherwise BackupRepositories name verification will be wrong + // when making combination of BackupRepositories name(max length is 63) bslDeletionTestNs = "bsl-deletion" ) @@ -134,12 +134,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { )).To(Succeed()) }) - backupName_1 := "backup1-" + UUIDgen.String() - backupName_2 := "backup2-" + UUIDgen.String() - backupLocation_1 := "default" - backupLocation_2 := additionalBsl - podName_1 := "kibishii-deployment-0" - podName_2 := "kibishii-deployment-1" + backupName1 := "backup1-" + UUIDgen.String() + backupName2 := "backup2-" + UUIDgen.String() + backupLocation1 := "default" + backupLocation2 := additionalBsl + podName1 := "kibishii-deployment-0" + podName2 := "kibishii-deployment-1" label_1 := "for=1" // TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed @@ -157,17 +157,17 @@ func BslDeletionTest(useVolumeSnapshots bool) { // Restic can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { - Expect(AddLabelToPod(context.Background(), podName_1, bslDeletionTestNs, label_1)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label_1)).To(Succeed()) Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed()) }) By("Get all 2 PVCs of Kibishii and label them separately ", func() { - pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1) + pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName1) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc1 := pvc[0] - pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2) + pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName2) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) @@ -177,13 +177,13 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) var BackupCfg BackupConfig - BackupCfg.BackupName = backupName_1 + BackupCfg.BackupName = backupName1 BackupCfg.Namespace = bslDeletionTestNs - BackupCfg.BackupLocation = backupLocation_1 + BackupCfg.BackupLocation = backupLocation1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = label_1 - By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { + By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, @@ -193,10 +193,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - BackupCfg.BackupName = backupName_2 - BackupCfg.BackupLocation = backupLocation_2 + BackupCfg.BackupName = backupName2 + BackupCfg.BackupLocation = backupLocation2 BackupCfg.Selector = label_2 - By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { + By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") @@ -211,82 +211,94 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, bslDeletionTestNs, 2)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName1), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_2), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } if veleroCfg.CloudProvider != VanillaZFS { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName1, + []string{podName1}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) } - } else { // For Restics - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } - By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName1), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_1, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName2), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_2, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName2, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1) + backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation1) Expect(err).To(Succeed()) - backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2) + backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation2) Expect(err).To(Succeed()) backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...) - By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() { + By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation1), func() { backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) - By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) - By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() { - Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) + By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation1), func() { + Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation1)).To(Succeed()) Expect(WaitForBackupsToBeDeleted(context.Background(), backupsInBSL1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) @@ -298,10 +310,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case @@ -313,45 +325,51 @@ func BslDeletionTest(useVolumeSnapshots bool) { if useVolumeSnapshots { if veleroCfg.CloudProvider == Vsphere { - By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { + By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName1, []string{podName1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) - } else { // For Restic - By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be deleted in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 0)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should still exist in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 51edf3407..915ef8c69 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -288,9 +288,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index ffb8f5f22..f3f63fc54 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -202,9 +202,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, upgradeNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 4ab078394..29652fce3 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -131,9 +131,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -166,9 +169,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrap(err, "failed to get snapshot checkPoint") } } else { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, SnapshotCheckPoint{}) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + SnapshotCheckPoint{}, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 46e42894e..a7c68dc37 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -28,14 +28,14 @@ import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/internal/volume" - . "github.com/vmware-tanzu/velero/test" + velerotest "github.com/vmware-tanzu/velero/test" velero "github.com/vmware-tanzu/velero/test/util/velero" ) type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) } @@ -75,13 +75,13 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case AWS, Vsphere: + case velerotest.AWS, velerotest.Vsphere: aws := AWSStorage("") s = &aws - case GCP: + case velerotest.GCP: gcs := GCSStorage("") s = &gcs - case Azure: + case velerotest.Azure: az := AzureStorage("") s = &az default: @@ -102,7 +102,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl bslPrefix = getFullPrefix(bslPrefix, subPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return false, errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) } @@ -116,66 +116,77 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, } s, err := getProvider(objectStoreProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + return errors.Wrapf(err, "Fail to delete %s", bslPrefix) } return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) +func CheckSnapshotsInProvider( + veleroCfg velerotest.VeleroConfig, + backupName string, + snapshotCheckPoint velerotest.SnapshotCheckPoint, + checkStandbyCluster bool, +) error { + fmt.Printf("|| VERIFICATION || - there should %d snapshots in provider, backup %s\n", + snapshotCheckPoint.ExpectCount, backupName) - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", veleroCfg.CloudProvider) return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) - } - - snapshotCheckPoint.ExpectCount = 0 - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName)) - } - fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName) - return nil -} - -func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) - - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.Vsphere && !veleroCfg.HasVspherePlugin { + fmt.Printf("Skip snapshot check for vSphere environment that doesn't have Velero vSphere plugin.") return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + if veleroCfg.CloudCredentialsFile == "" { + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", veleroCfg.CloudProvider)) } - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName)) + cloudCredentialsFile := veleroCfg.CloudCredentialsFile + bslBucket := veleroCfg.BSLBucket + bslConfig := veleroCfg.BSLConfig + if checkStandbyCluster { + bslBucket = veleroCfg.AdditionalBSLBucket + + // Only vSphere environment's snapshot on standby cluster is related to BSL. + if veleroCfg.CloudProvider == velerotest.Vsphere { + cloudCredentialsFile = veleroCfg.AdditionalBSLCredentials + bslConfig = veleroCfg.AdditionalBSLConfig + } } - fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider) + + err := isSnapshotExisted( + veleroCfg.CloudProvider, + cloudCredentialsFile, + bslBucket, + bslConfig, + backupName, + snapshotCheckPoint, + ) + if err != nil { + return errors.Wrapf(err, "|| UNEXPECTED || - Snapshots count is not as expected after backup %s", backupName) + } + + fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in provider %s\n", backupName, veleroCfg.CloudProvider) return nil } -func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { +func isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error { s, err := getProvider(cloudProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return errors.Wrapf(err, "Cloud provider %s is not valid", cloudProvider) } - if cloudProvider == Vsphere { + if cloudProvider == velerotest.Vsphere { var retSnapshotIDs []string ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2) defer ctxCancel() retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot CRs of backup%s", backupName) } bslPrefix := "plugins" @@ -201,7 +212,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } else { err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot of backup %s", backupName) } } return nil @@ -221,7 +232,7 @@ func GetVolumeInfoMetadataContent( fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider)) + return nil, errors.Wrapf(err, "Cloud provider %s is not valid", objectStoreProvider) } return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName)