diff --git a/changelogs/unreleased/4141-danfengliu b/changelogs/unreleased/4141-danfengliu new file mode 100644 index 000000000..1cce90f13 --- /dev/null +++ b/changelogs/unreleased/4141-danfengliu @@ -0,0 +1 @@ +Fix plugins incompatible issue in upgrade test \ No newline at end of file diff --git a/test/e2e/Makefile b/test/e2e/Makefile index b6f6aaf78..8f7916292 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -49,6 +49,7 @@ GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main VELERO_VERSION ?= $(VERSION) +PLUGINS ?= RESTIC_HELPER_IMAGE ?= #Released version only UPGRADE_FROM_VELERO_CLI ?= @@ -66,6 +67,7 @@ INSTALL_VELERO ?= true REGISTRY_CREDENTIAL_FILE ?= # Flags to create an additional BSL for multiple credentials tests +ADDITIONAL_BSL_PLUGINS ?= ADDITIONAL_OBJECT_STORE_PROVIDER ?= ADDITIONAL_CREDS_FILE ?= ADDITIONAL_BSL_BUCKET ?= @@ -86,6 +88,7 @@ run: ginkgo (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ + -plugins=$(PLUGINS) \ -velero-version=$(VELERO_VERSION) \ -restic-helper-image=$(RESTIC_HELPER_IMAGE) \ -upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ @@ -99,6 +102,7 @@ run: ginkgo -vsl-config=$(VSL_CONFIG) \ -cloud-provider=$(CLOUD_PROVIDER) \ -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ + -additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \ -additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ -additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 6e53abc17..9d5f92baf 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -59,7 +59,7 @@ func backup_restore_test(useVolumeSnapshots bool) { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -94,7 +94,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) + Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 81d41f790..37e111703 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -28,26 +28,28 @@ import ( var ( veleroCLI, veleroImage, veleroVersion, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage string - upgradeFromVeleroVersion, upgradeFromVeleroCLI string + upgradeFromVeleroVersion, upgradeFromVeleroCLI, plugins, addBSLPlugins string installVelero bool ) func init() { - flag.StringVar(&cloudProvider, "cloud-provider", "", "Cloud that Velero will be installed into. Required.") - flag.StringVar(&objectStoreProvider, "object-store-provider", "", "Provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") + flag.StringVar(&cloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") + flag.StringVar(&objectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") - flag.StringVar(&veleroVersion, "velero-version", "main", "image for the velero server to be tested.") + flag.StringVar(&plugins, "plugins", "", "provider plugins to be tested.") + flag.StringVar(&addBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.") + flag.StringVar(&veleroVersion, "velero-version", "main", "image version for the velero server to be tested with.") flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.") flag.StringVar(&upgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.") flag.StringVar(&upgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.") flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") - flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into") - flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.") + flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") + flag.BoolVar(&installVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") flag.StringVar(®istryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") // Flags to create an additional BSL for multiple credentials test diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index a3891a10f..68014f184 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -61,6 +61,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", veleroCLI, veleroImage, resticHelperImage, + plugins, veleroNamespace, cloudProvider, objectStoreProvider, diff --git a/test/e2e/install.go b/test/e2e/install.go index 7dc7a63a8..80c344c44 100644 --- a/test/e2e/install.go +++ b/test/e2e/install.go @@ -45,7 +45,7 @@ type installOptions struct { } // TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items -func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, +func veleroInstall(ctx context.Context, cli, veleroImage, resticHelperImage, providerPlugins, veleroNamespace, cloudProvider, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, crdsVersion string, features string, registryCredentialFile string) error { @@ -60,9 +60,10 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperIma } } - // Fetch the plugins for the provider before checking for the object store provider below. - providerPlugins := getProviderPlugins(objectStoreProvider) - + providerPluginsTmp, err := getProviderPlugins(ctx, cli, objectStoreProvider, providerPlugins) + if err != nil { + return errors.WithMessage(err, "Failed to get provider plugins") + } // TODO - handle this better if cloudProvider == "vsphere" { // We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the @@ -70,13 +71,13 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperIma // Snapshot location specified objectStoreProvider = "aws" } - err := ensureClusterExists(ctx) + err = ensureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, - bslPrefix, bslConfig, vslConfig, providerPlugins, features) + bslPrefix, bslConfig, vslConfig, providerPluginsTmp, features) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) } diff --git a/test/e2e/multiple_namespaces_test.go b/test/e2e/multiple_namespaces_test.go index 9f87bcc6c..bb6b71466 100644 --- a/test/e2e/multiple_namespaces_test.go +++ b/test/e2e/multiple_namespaces_test.go @@ -27,7 +27,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -62,7 +62,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 2a9ae9d22..c647e7b31 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -74,7 +74,7 @@ func backup_upgrade_restore_test(useVolumeSnapshots bool) { if installVelero { //Set veleroImage and resticHelperImage to blank //veleroImage and resticHelperImage should be the default value in originalCli - Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed()) Expect(checkVeleroVersion(context.Background(), upgradeFromVeleroCLI, upgradeFromVeleroVersion)).To(Succeed()) } else { @@ -143,7 +143,7 @@ func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVer time.Sleep(5 * time.Minute) } - if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil { return errors.Wrapf(err, "Failed to install velero from image %s", upgradeToVeleroImage) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index cd2b29d4c..760eb4eb1 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -41,20 +41,56 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) -func getProviderPlugins(providerName string) []string { - // TODO: make plugin images configurable - switch providerName { - case "aws": - return []string{"velero/velero-plugin-for-aws:v1.2.1"} - case "azure": - return []string{"velero/velero-plugin-for-microsoft-azure:v1.2.0"} - case "vsphere": - return []string{"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"} - case "gcp": - return []string{"velero/velero-plugin-for-gcp:v1.2.1"} - default: - panic(fmt.Errorf("unknown provider name: %s", providerName)) +var pluginsMatrix = map[string]map[string][]string{ + "v1.4": { + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + }, + "v1.5": { + "aws": {"velero/velero-plugin-for-aws:v1.1.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.1.2"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.1.0"}, + }, + "v1.6": { + "aws": {"velero/velero-plugin-for-aws:v1.2.1"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.2.1"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.2.1"}, + }, + "v1.7": { + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + }, + "main": { + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + }, +} + +func getProviderPluginsByVersion(version, providerName string) ([]string, error) { + var cloudMap map[string][]string + arr := strings.Split(version, ".") + if len(arr) >= 3 { + cloudMap = pluginsMatrix[arr[0]+"."+arr[1]] } + if len(cloudMap) == 0 { + cloudMap = pluginsMatrix["main"] + if len(cloudMap) == 0 { + return nil, errors.Errorf("fail to get plugins by version: main") + } + } + plugins, ok := cloudMap[providerName] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + } + return plugins, nil } // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. @@ -326,10 +362,32 @@ func veleroCreateBackupLocation(ctx context.Context, return bslCreateCmd.Run() } +func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) { + // Fetch the plugins for the provider before checking for the object store provider below. + var plugins []string + if len(providerPlugins) > 0 { + plugins = strings.Split(providerPlugins, ",") + } else { + version, err := getVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + plugins, err = getProviderPluginsByVersion(version, objectStoreProvider) + if err != nil { + return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) + } + } + return plugins, nil +} + // veleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { - for _, plugin := range getProviderPlugins(provider) { +func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error { + plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins) + if err != nil { + return errors.WithMessage(err, "Failed to get plugins") + } + for _, plugin := range plugins { stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) @@ -400,8 +458,12 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, return err } -func getVeleroVersion(ctx context.Context, veleroCLI string) (string, error) { - cmd := exec.CommandContext(ctx, veleroCLI, "version", "--timeout", "60s") +func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) { + args := []string{"version", "--timeout", "60s"} + if clientOnly { + args = append(args, "--client-only") + } + cmd := exec.CommandContext(ctx, veleroCLI, args...) fmt.Println("Get Version Command:" + cmd.String()) stdout, stderr, err := veleroexec.RunCommand(cmd) if err != nil { @@ -410,25 +472,33 @@ func getVeleroVersion(ctx context.Context, veleroCLI string) (string, error) { output := strings.Replace(stdout, "\n", " ", -1) fmt.Println("Version:" + output) - regCompiler := regexp.MustCompile(`(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)`) - versionMatches := regCompiler.FindStringSubmatch(output) - if len(versionMatches) < 3 { - return "", errors.New("Velero version command returned null version") + resultCount := 3 + regexpRule := `(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)` + if clientOnly { + resultCount = 2 + regexpRule = `(?i)client\s*:\s*version\s*:\s*(\S+)` } - if versionMatches[1] != versionMatches[2] { - return "", errors.New("Velero server and client version are not matched") + regCompiler := regexp.MustCompile(regexpRule) + versionMatches := regCompiler.FindStringSubmatch(output) + if len(versionMatches) != resultCount { + return "", errors.New("failed to parse velero version from output") + } + if !clientOnly { + if versionMatches[1] != versionMatches[2] { + return "", errors.New("velero server and client version are not matched") + } } return versionMatches[1], nil } func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error { tag := expectedVer - tagInstalled, err := getVeleroVersion(ctx, veleroCLI) + tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false) if err != nil { - return errors.WithMessagef(err, "Failed to get Velero version") + return errors.WithMessagef(err, "failed to get Velero version") } if strings.Trim(tag, " ") != strings.Trim(tagInstalled, " ") { - return errors.New(fmt.Sprintf("Velero version %s is not as expected %s", tagInstalled, tag)) + return errors.New(fmt.Sprintf("velero version %s is not as expected %s", tagInstalled, tag)) } fmt.Printf("Velero version %s is as expected %s\n", tagInstalled, tag) return nil @@ -440,18 +510,18 @@ func installVeleroCLI(version string) (string, error) { tarball := name + postfix tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball) if err != nil { - return "", errors.WithMessagef(err, "Failed to get Velero CLI tarball") + return "", errors.WithMessagef(err, "failed to get Velero CLI tarball") } tempVeleroCliDir, err := ioutil.TempDir("", "velero-test") if err != nil { - return "", errors.WithMessagef(err, "Failed to create temp dir for tarball extraction") + return "", errors.WithMessagef(err, "failed to create temp dir for tarball extraction") } cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir) defer os.Remove(tempFile.Name()) if _, err := cmd.Output(); err != nil { - return "", errors.WithMessagef(err, "Failed to extract file from velero CLI tarball") + return "", errors.WithMessagef(err, "failed to extract file from velero CLI tarball") } return tempVeleroCliDir + "/" + name + "/velero", nil } @@ -462,21 +532,21 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { resp, err := http.Get(cliTarballUrl) if err != nil { - return nil, errors.WithMessagef(err, "Failed to access Velero CLI tarball") + return nil, errors.WithMessagef(err, "failed to access Velero CLI tarball") } defer resp.Body.Close() tarballBuf, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, errors.WithMessagef(err, "Failed to read buffer for tarball %s.", tarball) + return nil, errors.WithMessagef(err, "failed to read buffer for tarball %s.", tarball) } tmpfile, err := ioutil.TempFile("", tarball) if err != nil { - return nil, errors.WithMessagef(err, "Failed to create temp file for tarball %s locally.", tarball) + return nil, errors.WithMessagef(err, "failed to create temp file for tarball %s locally.", tarball) } if _, err := tmpfile.Write(tarballBuf); err != nil { - return nil, errors.WithMessagef(err, "Failed to write tarball file %s locally.", tarball) + return nil, errors.WithMessagef(err, "failed to write tarball file %s locally.", tarball) } return tmpfile, nil