diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0c5c2deed..a531865f1 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -118,7 +118,13 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if err != nil { return err } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots, ""); err != nil { + var BackupCfg BackupConfig + BackupCfg.BackupName = backupName + BackupCfg.Namespace = deletionTest + BackupCfg.BackupLocation = backupLocation + BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.Selector = "" + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) @@ -141,9 +147,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa if useVolumeSnapshots { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - if err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - } err = WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint) @@ -185,7 +188,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa } backupName = "backup-1-" + UUIDgen.String() - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, deletionTest, backupLocation, useVolumeSnapshots, ""); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 2fb2df6d7..cc3fefb23 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -89,8 +89,14 @@ func BackupsSyncTest() { Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) }() + var BackupCfg BackupConfig + BackupCfg.BackupName = test.backupName + BackupCfg.Namespace = test.testNS + BackupCfg.BackupLocation = "" + BackupCfg.UseVolumeSnapshots = false + BackupCfg.Selector = "" By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false, ""); err != nil { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") } Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) @@ -116,14 +122,21 @@ func BackupsSyncTest() { Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) }) + if !VeleroCfg.Debug { + defer func() { + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), + fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + } - defer func() { - Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), - fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) - }() - + var BackupCfg BackupConfig + BackupCfg.BackupName = test.backupName + BackupCfg.Namespace = test.testNS + BackupCfg.BackupLocation = "" + BackupCfg.UseVolumeSnapshots = false + BackupCfg.Selector = "" By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false, ""); err != nil { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") } Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go new file mode 100644 index 000000000..6b024808e --- /dev/null +++ b/test/e2e/backups/ttl.go @@ -0,0 +1,200 @@ +/* + * + * Copyright the Velero contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package backups + +import ( + "context" + "flag" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + + . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type TTL struct { + testNS string + backupName string + restoreName string + ctx context.Context + ttl time.Duration +} + +func (b *TTL) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + b.testNS = "backup-ttl-test-" + UUIDgen.String() + b.backupName = "backup-ttl-test-" + UUIDgen.String() + b.restoreName = "restore-ttl-test-" + UUIDgen.String() + b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*30)) + b.ttl = time.Duration(10 * time.Minute) + +} + +func TTLTest() { + useVolumeSnapshots := true + test := new(TTL) + client, err := NewTestClient() + if err != nil { + println(err.Error()) + } + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + // Make sure GCFrequency is shorter than backup TTL + VeleroCfg.GCFrequency = "4m0s" + Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + VeleroCfg.GCFrequency = "" + if !VeleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + } + } + }) + + It("Backups in object storage should be synced to a new Velero successfully", func() { + test.Init() + oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS), func() { + Expect(CreateNamespace(oneHourTimeout, client, test.testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", test.testNS)) + }) + + By("Deploy sample workload of Kibishii", func() { + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider, + test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, + VeleroCfg.KibishiiDirectory, useVolumeSnapshots)).To(Succeed()) + }) + + var BackupCfg BackupConfig + BackupCfg.BackupName = test.backupName + BackupCfg.Namespace = test.testNS + BackupCfg.BackupLocation = "" + BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.Selector = "" + BackupCfg.TTL = test.ttl + + By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { + Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + return "Fail to backup workload" + }) + }) + + var snapshotCheckPoint SnapshotCheckPoint + if useVolumeSnapshots { + if VeleroCfg.CloudProvider == "vsphere" { + // TODO - remove after upload progress monitoring is implemented + By("Waiting for vSphere uploads to complete", func() { + Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, + test.testNS)).To(Succeed()) + }) + } + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + + Expect(WaitUntilSnapshotsExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, + test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + } + + By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() { + Expect(DeleteNamespace(test.ctx, client, BackupCfg.BackupName, true)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) + }) + + if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") + time.Sleep(7 * time.Minute) + } + + By(fmt.Sprintf("Restore %s", test.testNS), func() { + Expect(VeleroRestore(test.ctx, VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, test.restoreName, test.backupName)).To(Succeed(), func() string { + RunDebug(test.ctx, VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, "", test.restoreName) + return "Fail to restore workload" + }) + }) + + By("Associated Restores should be created", func() { + Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName, + RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object") + + }) + + By("Check TTL was set correctly", func() { + ttl, err := GetBackupTTL(test.ctx, VeleroCfg.VeleroNamespace, test.backupName) + Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", "")) + fmt.Println(t.Round(time.Minute).String()) + Expect(t).To(Equal(test.ttl)) + }) + + By(fmt.Sprintf("Waiting %s minutes for removing backup ralated resources by GC", test.ttl.String()), func() { + time.Sleep(test.ttl) + }) + + By("Check if backups are deleted by GC", func() { + Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName)) + }) + + By("Backup file from cloud object storage should be deleted", func() { + Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, + BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + }) + + By("PersistentVolume snapshots should be deleted", func() { + if useVolumeSnapshots { + Expect(WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, + test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + } + }) + + By("Associated Restores should be deleted", func() { + Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, + VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName, + RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object") + + }) + }) +} diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index c30716387..1266a79d4 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -242,7 +242,13 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false, "") + var BackupCfg BackupConfig + BackupCfg.BackupName = backup + BackupCfg.Namespace = namespacesStr + BackupCfg.BackupLocation = "" + BackupCfg.UseVolumeSnapshots = false + BackupCfg.Selector = "" + err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg) if err != nil { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index f385d7371..91307f19e 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -19,7 +19,6 @@ import ( "context" "flag" "fmt" - "strings" "time" "github.com/google/go-cmp/cmp" @@ -29,7 +28,6 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" - util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" @@ -179,18 +177,25 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label_2)).To(Succeed()) }) + var BackupCfg BackupConfig + BackupCfg.BackupName = backupName_1 + BackupCfg.Namespace = bslDeletionTestNs + BackupCfg.BackupLocation = backupLocation_1 + BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.Selector = label_1 By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName_1, bslDeletionTestNs, - backupLocation_1, useVolumeSnapshots, label_1)).To(Succeed()) + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed()) }) + BackupCfg.BackupName = backupName_2 + BackupCfg.BackupLocation = backupLocation_2 + BackupCfg.Selector = label_2 By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName_2, bslDeletionTestNs, - backupLocation_2, useVolumeSnapshots, label_2)).To(Succeed()) + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed()) }) if useVolumeSnapshots { @@ -208,14 +213,6 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName_2, 1)).To(Succeed()) }) - } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { - By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_1), func() { - Expect(util.CheckVolumeSnapshotCR(client, []string{podName_1}, bslDeletionTestNs, backupName_1)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") - }) - - By(fmt.Sprintf("CSI VolumeSnapshotContent CR in backup %s should be created", backupName_2), func() { - Expect(util.CheckVolumeSnapshotCR(client, []string{podName_2}, bslDeletionTestNs, backupName_2)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content CR.") - }) } var snapshotCheckPoint SnapshotCheckPoint diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a2d98727d..35ec77638 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -65,6 +65,7 @@ func init() { flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "Switch to control namespace cleaning.") + flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "Frequency of garbage collection.") } var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) @@ -93,12 +94,14 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) -var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) +var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) +var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) +var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) + var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) diff --git a/test/e2e/types.go b/test/e2e/types.go index d24c40635..9a92ea495 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -17,6 +17,8 @@ limitations under the License. package e2e import ( + "time" + "github.com/google/uuid" ) @@ -51,6 +53,7 @@ type VerleroConfig struct { KibishiiDirectory string Features string Debug bool + GCFrequency string } type SnapshotCheckPoint struct { @@ -62,3 +65,12 @@ type SnapshotCheckPoint struct { PodName []string EnableCSI bool } + +type BackupConfig struct { + BackupName string + Namespace string + BackupLocation string + UseVolumeSnapshots bool + Selector string + TTL time.Duration +} diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index aff943aa6..794feba25 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -27,7 +27,6 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test/e2e" - util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/kibishii" . "github.com/vmware-tanzu/velero/test/e2e/util/providers" @@ -79,6 +78,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade client TestClient err error ) + By("Create test client instance", func() { client, err = NewTestClient() Expect(err).NotTo(HaveOccurred(), "Failed to instantiate cluster client for backup tests") @@ -125,7 +125,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade Expect(err).To(Succeed()) }) } - + VeleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", upgradeFromVelero.UpgradeFromVeleroVersion), func() { //Set VeleroImage and ResticHelperImage to blank @@ -162,9 +162,14 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade }) By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() { + var BackupCfg BackupConfig + BackupCfg.BackupName = backupName + BackupCfg.Namespace = upgradeNamespace + BackupCfg.BackupLocation = "" + BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.Selector = "" Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI, - tmpCfg.VeleroNamespace, backupName, upgradeNamespace, "", - useVolumeSnapshots, "")).ShouldNot(HaveOccurred(), func() string { + tmpCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string { err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, backupName) return "Get backup logs" @@ -178,10 +183,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace)).To(Succeed()) }) - } else if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { - By("CSI VolumeSnapshotContent CR should be created", func() { - Expect(util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, upgradeNamespace, backupName)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot content") - }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace @@ -206,13 +207,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { time.Sleep(5 * time.Minute) }) - // TODO: add WaitUntilSnapshotsNotExistInCloud verification when Upgrade test is enabled in nightly - // err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, - // VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, - // backupName, snapshotCheckPoint) - // if err != nil { - // return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - // } } // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 @@ -223,6 +217,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade } By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() { + tmpCfg.GCFrequency = "" Expect(VeleroInstall(context.Background(), &tmpCfg, useVolumeSnapshots)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, tmpCfg.VeleroVersion)).To(Succeed()) diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go index b517b5f54..e2e9265b2 100644 --- a/test/e2e/util/csi/common.go +++ b/test/e2e/util/csi/common.go @@ -65,11 +65,20 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error } var snapshotHandleList []string for _, i := range vscList.Items { + if i.Status == nil { + fmt.Println("SnapshotHandle Status s nil") + continue + } if i.Status.SnapshotHandle == nil { fmt.Println("SnapshotHandle is nil") continue } - fmt.Println(*i.Status.SnapshotHandle) + + if i.Labels == nil { + fmt.Println("VolumeSnapshotContents label is nil") + continue + } + if i.Labels["velero.io/backup-name"] == backupName { tmp := strings.Split(*i.Status.SnapshotHandle, "/") snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) @@ -120,13 +129,12 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, pods []string, nampespace, backupName string) error { - for _, podName := range pods { - if snapshotContentName, err := GetVolumeSnapshotContentNameByPod(client, podName, nampespace, backupName); err != nil || snapshotContentName == "" { - return errors.Wrap(err, "Fail to get Azure CSI snapshot content") - } else { - fmt.Println("Found volumesnapshotcontent: " + snapshotContentName) - } +func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) error { + var err error + var snapshotContentNameList []string + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || len(snapshotContentNameList) != expectedCount { + return errors.Wrap(err, "Fail to get Azure CSI snapshot content") } + fmt.Println(snapshotContentNameList) return nil } diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 005354f28..45d4d8f8e 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -28,7 +28,6 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" - util "github.com/vmware-tanzu/velero/test/e2e/util/csi" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/providers" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" @@ -72,8 +71,13 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, - kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil { + var BackupCfg BackupConfig + BackupCfg.BackupName = backupName + BackupCfg.Namespace = kibishiiNamespace + BackupCfg.BackupLocation = backupLocation + BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.Selector = "" + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -87,10 +91,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } - } else if providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { - if err := util.CheckVolumeSnapshotCR(client, KibishiiPodNameList, kibishiiNamespace, backupName); err != nil { - return errors.Wrapf(err, "Fail to get Azure CSI snapshot content") - } } snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) if err != nil { @@ -108,14 +108,6 @@ func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, re if err := DeleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } - if useVolumeSnapshots && providerName == "azure" && strings.EqualFold(veleroFeatures, "EnableCSI") { - err = WaitUntilSnapshotsNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - } - } time.Sleep(5 * time.Minute) // the snapshots of AWS may be still in pending status when do the restore, wait for a while @@ -215,6 +207,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie if err := WaitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, 10*time.Minute); err != nil { return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) } + // add the image pull secret to avoid the image pull limit issue of Docker Hub if err := PatchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) diff --git a/test/e2e/util/providers/common.go b/test/e2e/util/providers/common.go index accfafec1..ee5dd47da 100644 --- a/test/e2e/util/providers/common.go +++ b/test/e2e/util/providers/common.go @@ -37,10 +37,10 @@ type ObjectsInStorage interface { } func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { - fmt.Printf("|| VERIFICATION || - Backup %s should exist in storage [%s]", backupName, bslPrefix) + fmt.Printf("|| VERIFICATION || - %s %s should exist in storage [%s]\n", subPrefix, backupName, bslPrefix) exist, _ := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if !exist { - return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected", backupName)) + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName)) } fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket) return nil @@ -48,11 +48,11 @@ func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bsl func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { var err error var exist bool - fmt.Printf("|| VERIFICATION || - Backup %s should not exist in storage %s", backupName, bslPrefix) + fmt.Printf("|| VERIFICATION || - %s %s should not exist in storage %s\n", subPrefix, backupName, bslPrefix) for i := 0; i < retryTimes; i++ { exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if err != nil { - return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store", backupName) + return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store\n", backupName) } if !exist { fmt.Printf("|| EXPECTED || - Backup %s is not in object store\n", backupName) @@ -60,7 +60,7 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, } time.Sleep(1 * time.Minute) } - return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion", backupName)) + return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion\n", backupName)) } func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 30b1b1c42..ed3f6156d 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -90,6 +90,8 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps veleroInstallOptions.UseRestic = !useVolumeSnapshots veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace + GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) + veleroInstallOptions.GarbageCollectionFrequency = GCFrequency err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{ InstallOptions: veleroInstallOptions, @@ -131,6 +133,7 @@ func configvSpherePlugin() error { if err != nil { return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace) } + fmt.Println("configvSpherePlugin: WaitForConfigMapComplete") err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName) if err != nil { return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace)) @@ -209,6 +212,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption } } + if options.GarbageCollectionFrequency > 0 { + args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency)) + } if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil { return err diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 449f6e0a5..9c1cb8709 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -47,6 +47,7 @@ import ( ) const BackupObjectsPrefix = "backups" +const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" var pluginsMatrix = map[string]map[string][]string{ @@ -246,19 +247,18 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st } // VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, backupName, namespace, backupLocation string, - useVolumeSnapshots bool, selector string) error { +func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace string, backupCfg BackupConfig) error { args := []string{ "--namespace", veleroNamespace, - "create", "backup", backupName, - "--include-namespaces", namespace, + "create", "backup", backupCfg.BackupName, + "--include-namespaces", backupCfg.Namespace, "--wait", } - if selector != "" { - args = append(args, "--selector", selector) + if backupCfg.Selector != "" { + args = append(args, "--selector", backupCfg.Selector) } - if useVolumeSnapshots { + if backupCfg.UseVolumeSnapshots { args = append(args, "--snapshot-volumes") } else { args = append(args, "--default-volumes-to-restic") @@ -268,11 +268,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, back // TODO This can be removed if the logic of vSphere plugin bump up to 1.3 args = append(args, "--snapshot-volumes=false") } - if backupLocation != "" { - args = append(args, "--storage-location", backupLocation) + if backupCfg.BackupLocation != "" { + args = append(args, "--storage-location", backupCfg.BackupLocation) } - - return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupName, args) + if backupCfg.TTL != 0 { + args = append(args, "--ttl", backupCfg.TTL.String()) + } + return VeleroBackupExec(ctx, veleroCLI, veleroNamespace, backupCfg.BackupName, args) } // VeleroBackupExcludeNamespaces uses the veleroCLI to backup a namespace. @@ -502,7 +504,7 @@ func GetVsphereSnapshotIDs(ctx context.Context, timeout time.Duration, namespace if err != nil { fmt.Print(stdout) fmt.Print(stderr) - return nil, errors.Wrap(err, "failed to verify") + return nil, errors.Wrap(err, "Failed to get vSphere snapshot ID list from snapshots.backupdriver.cnsdp.vmware.com") } stdout = strings.Replace(stdout, "'", "", -1) lines := strings.Split(stdout, "\n") @@ -814,9 +816,9 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = kibishiiPodNameList - if strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true - if err := util.CheckVolumeSnapshotCR(client, kibishiiPodNameList, namespaceBackedUp, backupName); err != nil { + if err := util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } var err error @@ -832,3 +834,24 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou fmt.Println(snapshotCheckPoint) return snapshotCheckPoint, nil } + +func GetBackupTTL(ctx context.Context, veleroNamespace, backupName string) (string, error) { + + checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", + "get", "backup", "-n", veleroNamespace, backupName, "-o=jsonpath='{.spec.ttl}'") + fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd) + stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd) + if err != nil { + fmt.Print(stdout) + fmt.Print(stderr) + return "", errors.Wrap(err, "failed to verify") + } + // lines := strings.Split(stdout, "\n") + // complete := true + // for _, curLine := range lines { + // fmt.Println(curLine) + + // } + // return complete, nil + return stdout, err +}