From 09240a269bef4b0f1ff84ed83c6da72bdded303b Mon Sep 17 00:00:00 2001 From: danfengl Date: Tue, 13 Sep 2022 09:12:37 +0000 Subject: [PATCH] Add Kopia support for Velero installation in E2E test Signed-off-by: danfengl --- test/e2e/Makefile | 5 +++- test/e2e/backups/deletion.go | 33 +++++++++++++-------- test/e2e/backups/sync_backups.go | 14 +++++---- test/e2e/basic/enable_api_group_versions.go | 20 +++++++------ test/e2e/bsl-mgmt/deletion.go | 10 +++++-- test/e2e/e2e_suite_test.go | 1 + test/e2e/migration/migration.go | 19 ++++++------ test/e2e/types.go | 2 ++ test/e2e/upgrade/upgrade.go | 11 ++++--- test/e2e/util/velero/install.go | 5 ++++ test/e2e/util/velero/velero_utils.go | 7 ++++- 11 files changed, 81 insertions(+), 46 deletions(-) diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 53d52364e..8f703cc00 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -95,6 +95,8 @@ DEBUG_E2E_TEST ?= false DEFAULT_CLUSTER ?= STANDBY_CLUSTER ?= +UPLOADER_TYPE ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin @@ -137,7 +139,8 @@ run: ginkgo -kibishii-directory=$(KIBISHII_DIRECTORY) \ -debug-e2e-test=$(DEBUG_E2E_TEST) \ -default-cluster=$(DEFAULT_CLUSTER) \ - -standby-cluster=$(STANDBY_CLUSTER) + -standby-cluster=$(STANDBY_CLUSTER) \ + -uploader-type=$(UPLOADER_TYPE) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 59944d6aa..274df9d20 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -122,12 +122,14 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa BackupCfg.BackupLocation = backupLocation BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.Selector = "" - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { - // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command - // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case - VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) - } + + By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() { + Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI, + veleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "") + return "Fail to backup workload" + }) + }) if providerName == "vsphere" && useVolumeSnapshots { // Wait for uploads started by the Velero Plug-in for vSphere to complete @@ -176,22 +178,27 @@ func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupNa return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } - backupName = "backup-1-" + UUIDgen.String() - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { - // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command - // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case - VeleroBackupLogs(context.Background(), VeleroCfg.UpgradeFromVeleroCLI, veleroNamespace, backupName) - return errors.Wrapf(err, "Failed to backup kibishii namespace %s", deletionTest) - } + BackupCfg.BackupName = backupName + + By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() { + Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI, + veleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "") + return "Fail to backup workload" + }) + }) + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) if err != nil { return err } + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } + err = DeleteBackupResource(context.Background(), veleroCLI, backupName) if err != nil { return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index fc2c27337..deb9befa6 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -93,10 +93,11 @@ func BackupsSyncTest() { BackupCfg.UseVolumeSnapshots = false BackupCfg.Selector = "" By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil { + Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") - } - Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + return "Fail to backup workload" + }) }) By("Uninstall velero", func() { @@ -132,10 +133,11 @@ func BackupsSyncTest() { BackupCfg.UseVolumeSnapshots = false BackupCfg.Selector = "" By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg); err != nil { + Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") - } - Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + return "Fail to backup workload" + }) }) By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() { diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index 5c4d11c34..5441e075f 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -125,10 +125,10 @@ func APIExtensionsVersionsTest() { BackupCfg.IncludeClusterResources = true BackupCfg.Selector = label Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string { - VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName) - return "Get backup logs" + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, backupName, "") + return "Fail to backup workload" }) }) @@ -367,11 +367,13 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso BackupCfg.BackupLocation = "" BackupCfg.UseVolumeSnapshots = false BackupCfg.Selector = "" - err = VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg) - if err != nil { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "") - return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr) - } + + Expect(VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, backup, "") + return "Fail to backup workload" + }) if err := deleteCRD(ctx, tc.srcCrdYaml); err != nil { return errors.Wrapf(err, "delete music-system CRD from source cluster") diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index d7504342a..20b591d73 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -189,7 +189,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed()) + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") + return "Fail to backup workload" + }) }) BackupCfg.BackupName = backupName_2 @@ -197,7 +200,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { BackupCfg.Selector = label_2 By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed()) + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") + return "Fail to backup workload" + }) }) if useVolumeSnapshots { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0f06f6a19..fc396b84e 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -77,6 +77,7 @@ func init() { flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "Frequency of garbage collection.") flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster", "", "Default cluster context for migration test.") flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "Standby cluster context for migration test.") + flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "Identify persistent volume backup uploader.") } var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 90f96f578..6894aff58 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -126,6 +126,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.VeleroImage = "" OriginVeleroCfg.ResticHelperImage = "" OriginVeleroCfg.Plugins = "" + //TODO: Remove this once origin Velero version is 1.10 and upper + OriginVeleroCfg.UploaderType = "" } fmt.Println(OriginVeleroCfg) Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed()) @@ -156,11 +158,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupStorageClassCfg.BackupName = backupScName BackupStorageClassCfg.IncludeResources = "StorageClass" BackupStorageClassCfg.IncludeClusterResources = true + Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupStorageClassCfg)).ShouldNot(HaveOccurred(), func() string { - err = VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName) - return "Get backup logs" + VeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "") + return "Fail to backup workload" }) var BackupCfg BackupConfig @@ -169,14 +171,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.BackupLocation = "" BackupCfg.Selector = "" - //BackupCfg.ExcludeResources = "tierentitlementbindings,tierentitlements,tiers,capabilities,customresourcedefinitions" Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string { - err = VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName) - return "Get backup logs" + VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") + return "Fail to backup workload" }) - }) if useVolumeSnapshots { diff --git a/test/e2e/types.go b/test/e2e/types.go index bb54562e1..068d957eb 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -63,6 +63,7 @@ type VerleroConfig struct { ClientToInstallVelero *TestClient DefaultClient *TestClient StandbyClient *TestClient + UploaderType string } type SnapshotCheckPoint struct { @@ -86,6 +87,7 @@ type BackupConfig struct { ExcludeResources string IncludeClusterResources bool OrderedResources string + UseRestic bool } type VeleroCLI2Version struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index c86273038..a46231b46 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -112,6 +112,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.VeleroImage = "" tmpCfgForOldVeleroInstall.ResticHelperImage = "" tmpCfgForOldVeleroInstall.Plugins = "" + tmpCfgForOldVeleroInstall.UploaderType = "" Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, useVolumeSnapshots)).To(Succeed()) @@ -143,11 +144,13 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BackupCfg.BackupLocation = "" BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.Selector = "" + //TODO: pay attention to this param + BackupCfg.UseRestic = true Expect(VeleroBackupNamespace(oneHourTimeout, tmpCfg.UpgradeFromVeleroCLI, - tmpCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string { - err = VeleroBackupLogs(context.Background(), tmpCfg.UpgradeFromVeleroCLI, - tmpCfg.VeleroNamespace, backupName) - return "Get backup logs" + tmpCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), tmpCfg.UpgradeFromVeleroCLI, tmpCfg.VeleroNamespace, + BackupCfg.BackupName, "") + return "Fail to backup workload" }) }) diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 0b3e5962b..b40ec1721 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -90,6 +90,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, useVolumeSnaps veleroInstallOptions.UseRestic = !useVolumeSnapshots veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace + veleroInstallOptions.UploaderType = veleroCfg.UploaderType GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) veleroInstallOptions.GarbageCollectionFrequency = GCFrequency @@ -213,6 +214,10 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption args = append(args, fmt.Sprintf("--garbage-collection-frequency=%v", options.GarbageCollectionFrequency)) } + if len(options.UploaderType) > 0 { + args = append(args, fmt.Sprintf("--uploader-type=%v", options.UploaderType)) + } + if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil { return err } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index ef50610a5..c67598ad1 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -323,7 +323,12 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin if backupCfg.UseVolumeSnapshots { args = append(args, "--snapshot-volumes") } else { - args = append(args, "--default-volumes-to-fs-backup") + if backupCfg.UseRestic { + args = append(args, "--default-volumes-to-restic") + } else { + args = append(args, "--default-volumes-to-fs-backup") + } + // To workaround https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/347 for vsphere plugin v1.1.1 // if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots // for the volumes even though the "--default-volumes-to-fs-backup" is specified