From 028e784eb67d71d8c083a3c0659fc1b08a81a518 Mon Sep 17 00:00:00 2001 From: danfengl Date: Thu, 1 Jun 2023 03:42:22 +0000 Subject: [PATCH] Prepare E2E test of data movement feature 1. Bumpup velero version to the latest 2 versions in upgrade script; 2. Bumpup velero verioin to the latest 1 vesion in migration script; 3. Bring B/R with restic test back in vSphere pipeline since vSphere plugin issue fix was included in v1.5; 4. Disable nodeport test in AWS pipeline since AWS k8s version bumpup; 5. Prepare for data mover test, allow object store provider diffrent from cloud provider. Signed-off-by: danfengl --- test/e2e/Makefile | 4 +- test/e2e/backup/backup.go | 7 -- test/e2e/test/test.go | 4 +- test/e2e/util/velero/install.go | 11 ++- test/e2e/util/velero/velero_utils.go | 102 +++++++++------------------ 5 files changed, 48 insertions(+), 80 deletions(-) diff --git a/test/e2e/Makefile b/test/e2e/Makefile index c36bac932..f2d7594e1 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -55,7 +55,7 @@ VELERO_VERSION ?= $(VERSION) PLUGINS ?= RESTORE_HELPER_IMAGE ?= #Released version only -UPGRADE_FROM_VELERO_VERSION ?= v1.9.6,v1.10.2 +UPGRADE_FROM_VELERO_VERSION ?= v1.10.2,v1.11.0 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -63,7 +63,7 @@ UPGRADE_FROM_VELERO_VERSION ?= v1.9.6,v1.10.2 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION UPGRADE_FROM_VELERO_CLI ?= -MIGRATE_FROM_VELERO_VERSION ?= v1.10.2,self +MIGRATE_FROM_VELERO_VERSION ?= v1.11.0,self MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 68e0c55cd..9bf694c00 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -76,13 +76,6 @@ func BackupRestoreTest(useVolumeSnapshots bool) { When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { - // TODO[High] - remove code block below when vSphere plugin PR #500 is included in release version. - // because restore will be partiallyFailed when DefaultVolumesToFsBackup is set to true during - // Velero installation with default BSL. - if veleroCfg.CloudProvider == "vsphere" && !useVolumeSnapshots { - Skip("vSphere plugin PR #500 is not included in latest version 1.4.2") - } - if veleroCfg.InstallVelero { if useVolumeSnapshots { //Install node agent also diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 9148030eb..d37b9138c 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -86,7 +86,9 @@ func TestFunc(test VeleroBackupRestoreTest) func() { // Using the global velero config which covered the installation for most common cases veleroCfg := e2e.VeleroCfg // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { + // TODO: Although this issue is closed, but it's not fixed. + // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. + if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") } if veleroCfg.InstallVelero { diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 5a37a1ba1..be50c95b2 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -59,14 +59,17 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) if veleroCfg.CloudProvider != "kind" { fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") - veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider + // If ObjectStoreProvider is not provided, then using the value same as CloudProvider + if veleroCfg.ObjectStoreProvider == "" { + veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider + } } else { if veleroCfg.ObjectStoreProvider == "" { return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider } } - providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins, veleroCfg.Features) + providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.CloudProvider, veleroCfg.Plugins, veleroCfg.Features) if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } @@ -80,7 +83,9 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified - veleroCfg.ObjectStoreProvider = "aws" + if veleroCfg.ObjectStoreProvider == "" { + veleroCfg.ObjectStoreProvider = "aws" + } if err := configvSpherePlugin(*veleroCfg.ClientToInstallVelero); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index abffafa75..7da65f945 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -55,68 +55,26 @@ const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" var pluginsMatrix = map[string]map[string][]string{ - "v1.4": { - "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.1 "}, - }, - "v1.5": { - "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.2 "}, - }, - "v1.6": { - "aws": {"velero/velero-plugin-for-aws:v1.2.1"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.2.1"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.1.2 "}, - }, - "v1.7": { - "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.2.0"}, - }, - "v1.8": { - "aws": {"velero/velero-plugin-for-aws:v1.4.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.4.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.4.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"}, - }, - "v1.9": { - "aws": {"velero/velero-plugin-for-aws:v1.5.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.5.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.5.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.5.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.5.0", "velero/velero-plugin-for-csi:v0.3.0"}, - }, "v1.10": { - "aws": {"velero/velero-plugin-for-aws:v1.6.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.6.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.6.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.1"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.6.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.6.0", "velero/velero-plugin-for-csi:v0.4.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.6.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.6.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.6.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.4.0"}, }, "v1.11": { - "aws": {"velero/velero-plugin-for-aws:v1.7.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.7.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.7.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.7.0"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.7.0", "velero/velero-plugin-for-csi:v0.5.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.7.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.7.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.7.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.5.0"}, }, "main": { - "aws": {"velero/velero-plugin-for-aws:main"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"}, - "gcp": {"velero/velero-plugin-for-gcp:main"}, - "azure-csi": {"velero/velero-plugin-for-microsoft-azure:main", "velero/velero-plugin-for-csi:main"}, + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + "csi": {"velero/velero-plugin-for-csi:main"}, }, } @@ -132,14 +90,19 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin return nil, errors.Errorf("fail to get plugins by version: main") } } - if strings.EqualFold(providerName, "azure") && strings.EqualFold(feature, "EnableCSI") { - providerName = "azure-csi" - } + var pluginsForFeature []string + plugins, ok := cloudMap[providerName] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) } - return plugins, nil + if strings.EqualFold(feature, "EnableCSI") { + pluginsForFeature, ok = cloudMap["csi"] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + } + } + return append(plugins, pluginsForFeature...), nil } // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. @@ -452,8 +415,7 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace } func VeleroSchedulePause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error { - var args []string - args = []string{ + args := []string{ "--namespace", veleroNamespace, "schedule", "pause", scheduleName, } if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { @@ -463,8 +425,7 @@ func VeleroSchedulePause(ctx context.Context, veleroCLI string, veleroNamespace } func VeleroScheduleUnpause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error { - var args []string - args = []string{ + args := []string{ "--namespace", veleroNamespace, "schedule", "unpause", scheduleName, } if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { @@ -555,7 +516,7 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error return nil } -func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) { +func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, cloudProvider, providerPlugins, feature string) ([]string, error) { // Fetch the plugins for the provider before checking for the object store provider below. var plugins []string if len(providerPlugins) > 0 { @@ -569,6 +530,13 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) } + if objectStoreProvider != "" && cloudProvider != "kind" && objectStoreProvider != cloudProvider { + pluginsTmp, err := GetProviderPluginsByVersion(version, cloudProvider, feature) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + plugins = append(plugins, pluginsTmp...) + } } return plugins, nil } @@ -576,7 +544,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins, feature) + plugins, err := getProviderPlugins(ctx, veleroCLI, provider, provider, addPlugins, feature) fmt.Printf("addPlugins cmd =%v\n", addPlugins) fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins)