Fix plugins incompatible issue in upgrade test (#4141)

In upgrade test, both original and to-be-upgrading velero installation should use the compatible plugins, but currently, plugin value is determined by provider.

Signed-off-by: danfengl <danfengl@vmware.com>

changelogs/unreleased/4141-danfengliu

@@ -0,0 +1 @@
+Fix plugins incompatible issue in upgrade test

test/e2e/Makefile

@@ -49,6 +49,7 @@ GINKGO_FOCUS ?=
 VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
 VELERO_IMAGE ?= velero/velero:main
 VELERO_VERSION ?= $(VERSION)
+PLUGINS ?=
 RESTIC_HELPER_IMAGE ?=
 #Released version only
 UPGRADE_FROM_VELERO_CLI ?=
@@ -66,6 +67,7 @@ INSTALL_VELERO ?= true
 REGISTRY_CREDENTIAL_FILE ?=
 
 # Flags to create an additional BSL for multiple credentials tests
+ADDITIONAL_BSL_PLUGINS ?=
 ADDITIONAL_OBJECT_STORE_PROVIDER ?=
 ADDITIONAL_CREDS_FILE ?=
 ADDITIONAL_BSL_BUCKET ?=
@@ -86,6 +88,7 @@ run: ginkgo
 			(echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
 	@$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \
 		-velero-image=$(VELERO_IMAGE) \
+		-plugins=$(PLUGINS) \
 		-velero-version=$(VELERO_VERSION) \
 		-restic-helper-image=$(RESTIC_HELPER_IMAGE) \
 		-upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \
@@ -99,6 +102,7 @@ run: ginkgo
 		-vsl-config=$(VSL_CONFIG) \
 		-cloud-provider=$(CLOUD_PROVIDER) \
 		-object-store-provider="$(OBJECT_STORE_PROVIDER)" \
+		-additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \
 		-additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
 		-additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
 		-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \

test/e2e/backup_test.go

@@ -59,7 +59,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
@@ -94,7 +94,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 				Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
 			}
 
-			Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed())
+			Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed())
 
 			// Create Secret for additional BSL
 			secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)

test/e2e/e2e_suite_test.go

@@ -28,26 +28,28 @@ import (
 var (
 	veleroCLI, veleroImage, veleroVersion, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string
 	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage                 string
-	upgradeFromVeleroVersion, upgradeFromVeleroCLI                                                                                                                            string
+	upgradeFromVeleroVersion, upgradeFromVeleroCLI, plugins, addBSLPlugins                                                                                                    string
 	installVelero                                                                                                                                                             bool
 )
 
 func init() {
-	flag.StringVar(&cloudProvider, "cloud-provider", "", "Cloud that Velero will be installed into.  Required.")
-	flag.StringVar(&objectStoreProvider, "object-store-provider", "", "Provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
+	flag.StringVar(&cloudProvider, "cloud-provider", "", "cloud that Velero will be installed into.  Required.")
+	flag.StringVar(&objectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.")
 	flag.StringVar(&bslBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.")
 	flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
 	flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.")
 	flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
-	flag.StringVar(&veleroVersion, "velero-version", "main", "image for the velero server to be tested.")
+	flag.StringVar(&plugins, "plugins", "", "provider plugins to be tested.")
+	flag.StringVar(&addBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.")
+	flag.StringVar(&veleroVersion, "velero-version", "main", "image version for the velero server to be tested with.")
 	flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
 	flag.StringVar(&upgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "path to the pre-upgrade velero application to use.")
 	flag.StringVar(&upgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.6.3", "image for the pre-upgrade velero server to be tested.")
 	flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
 	flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
 	flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
-	flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
-	flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test.  Optional.")
+	flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "namespace to install Velero into")
+	flag.BoolVar(&installVelero, "install-velero", true, "install/uninstall velero during the test.  Optional.")
 	flag.StringVar(&registryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
 
 	// Flags to create an additional BSL for multiple credentials test

test/e2e/enable_api_group_versions_test.go

@@ -61,6 +61,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
 				veleroCLI,
 				veleroImage,
 				resticHelperImage,
+				plugins,
 				veleroNamespace,
 				cloudProvider,
 				objectStoreProvider,

test/e2e/install.go

@@ -45,7 +45,7 @@ type installOptions struct {
 }
 
 // TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items
-func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool,
+func veleroInstall(ctx context.Context, cli, veleroImage, resticHelperImage, providerPlugins, veleroNamespace, cloudProvider, objectStoreProvider string, useVolumeSnapshots bool,
 	cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
 	crdsVersion string, features string, registryCredentialFile string) error {
 
@@ -60,9 +60,10 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperIma
 		}
 	}
 
-	// Fetch the plugins for the provider before checking for the object store provider below.
-	providerPlugins := getProviderPlugins(objectStoreProvider)
-
+	providerPluginsTmp, err := getProviderPlugins(ctx, cli, objectStoreProvider, providerPlugins)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to get provider plugins")
+	}
 	// TODO - handle this better
 	if cloudProvider == "vsphere" {
 		// We overrider the objectStoreProvider here for vSphere because we want to use the aws plugin for the
@@ -70,13 +71,13 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperIma
 		// Snapshot location specified
 		objectStoreProvider = "aws"
 	}
-	err := ensureClusterExists(ctx)
+	err = ensureClusterExists(ctx)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
 	}
 
 	veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket,
-		bslPrefix, bslConfig, vslConfig, providerPlugins, features)
+		bslPrefix, bslConfig, vslConfig, providerPluginsTmp, features)
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider)
 	}

test/e2e/multiple_namespaces_test.go

@@ -27,7 +27,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
@@ -62,7 +62,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, false,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})

test/e2e/upgrade_test.go

@@ -74,7 +74,7 @@ func backup_upgrade_restore_test(useVolumeSnapshots bool) {
 		if installVelero {
 			//Set veleroImage and resticHelperImage to blank
 			//veleroImage and resticHelperImage should be the default value in originalCli
-			Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
+			Expect(veleroInstall(context.Background(), upgradeFromVeleroCLI, "", "", "", veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "", "", registryCredentialFile)).To(Succeed())
 			Expect(checkVeleroVersion(context.Background(), upgradeFromVeleroCLI, upgradeFromVeleroVersion)).To(Succeed())
 		} else {
@@ -143,7 +143,7 @@ func runUpgradeTests(client testClient, upgradeToVeleroImage, upgradeToVeleroVer
 		time.Sleep(5 * time.Minute)
 	}
 
-	if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
+	if err := veleroInstall(context.Background(), veleroCLI, upgradeToVeleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
 		cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile); err != nil {
 		return errors.Wrapf(err, "Failed to install velero from image %s", upgradeToVeleroImage)
 	}

test/e2e/velero_utils.go

@@ -41,20 +41,56 @@ import (
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 )
 
-func getProviderPlugins(providerName string) []string {
-	// TODO: make plugin images configurable
-	switch providerName {
-	case "aws":
-		return []string{"velero/velero-plugin-for-aws:v1.2.1"}
-	case "azure":
-		return []string{"velero/velero-plugin-for-microsoft-azure:v1.2.0"}
-	case "vsphere":
-		return []string{"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}
-	case "gcp":
-		return []string{"velero/velero-plugin-for-gcp:v1.2.1"}
-	default:
-		panic(fmt.Errorf("unknown provider name: %s", providerName))
+var pluginsMatrix = map[string]map[string][]string{
+	"v1.4": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.1.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.1.0"},
+	},
+	"v1.5": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.1.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.1.0"},
+	},
+	"v1.6": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.2.1"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.2.1"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.2.1"},
+	},
+	"v1.7": {
+		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
+	},
+	"main": {
+		"aws":     {"velero/velero-plugin-for-aws:main"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:main"},
+		"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:main"},
+	},
+}
+
+func getProviderPluginsByVersion(version, providerName string) ([]string, error) {
+	var cloudMap map[string][]string
+	arr := strings.Split(version, ".")
+	if len(arr) >= 3 {
+		cloudMap = pluginsMatrix[arr[0]+"."+arr[1]]
 	}
+	if len(cloudMap) == 0 {
+		cloudMap = pluginsMatrix["main"]
+		if len(cloudMap) == 0 {
+			return nil, errors.Errorf("fail to get plugins by version: main")
+		}
+	}
+	plugins, ok := cloudMap[providerName]
+	if !ok {
+		return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
+	}
+	return plugins, nil
 }
 
 // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
@@ -326,10 +362,32 @@ func veleroCreateBackupLocation(ctx context.Context,
 	return bslCreateCmd.Run()
 }
 
+func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins string) ([]string, error) {
+	// Fetch the plugins for the provider before checking for the object store provider below.
+	var plugins []string
+	if len(providerPlugins) > 0 {
+		plugins = strings.Split(providerPlugins, ",")
+	} else {
+		version, err := getVeleroVersion(ctx, veleroCLI, true)
+		if err != nil {
+			return nil, errors.WithMessage(err, "failed to get velero version")
+		}
+		plugins, err = getProviderPluginsByVersion(version, objectStoreProvider)
+		if err != nil {
+			return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
+		}
+	}
+	return plugins, nil
+}
+
 // veleroAddPluginsForProvider determines which plugins need to be installed for a provider and
 // installs them in the current Velero installation, skipping over those that are already installed.
-func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error {
-	for _, plugin := range getProviderPlugins(provider) {
+func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins string) error {
+	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to get plugins")
+	}
+	for _, plugin := range plugins {
 		stdoutBuf := new(bytes.Buffer)
 		stderrBuf := new(bytes.Buffer)
 
@@ -400,8 +458,12 @@ func waitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
 	return err
 }
 
-func getVeleroVersion(ctx context.Context, veleroCLI string) (string, error) {
-	cmd := exec.CommandContext(ctx, veleroCLI, "version", "--timeout", "60s")
+func getVeleroVersion(ctx context.Context, veleroCLI string, clientOnly bool) (string, error) {
+	args := []string{"version", "--timeout", "60s"}
+	if clientOnly {
+		args = append(args, "--client-only")
+	}
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	fmt.Println("Get Version Command:" + cmd.String())
 	stdout, stderr, err := veleroexec.RunCommand(cmd)
 	if err != nil {
@@ -410,25 +472,33 @@ func getVeleroVersion(ctx context.Context, veleroCLI string) (string, error) {
 
 	output := strings.Replace(stdout, "\n", " ", -1)
 	fmt.Println("Version:" + output)
-	regCompiler := regexp.MustCompile(`(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)`)
-	versionMatches := regCompiler.FindStringSubmatch(output)
-	if len(versionMatches) < 3 {
-		return "", errors.New("Velero version command returned null version")
+	resultCount := 3
+	regexpRule := `(?i)client\s*:\s*version\s*:\s*(\S+).+server\s*:\s*version\s*:\s*(\S+)`
+	if clientOnly {
+		resultCount = 2
+		regexpRule = `(?i)client\s*:\s*version\s*:\s*(\S+)`
 	}
-	if versionMatches[1] != versionMatches[2] {
-		return "", errors.New("Velero server and client version are not matched")
+	regCompiler := regexp.MustCompile(regexpRule)
+	versionMatches := regCompiler.FindStringSubmatch(output)
+	if len(versionMatches) != resultCount {
+		return "", errors.New("failed to parse velero version from output")
+	}
+	if !clientOnly {
+		if versionMatches[1] != versionMatches[2] {
+			return "", errors.New("velero server and client version are not matched")
+		}
 	}
 	return versionMatches[1], nil
 }
 
 func checkVeleroVersion(ctx context.Context, veleroCLI string, expectedVer string) error {
 	tag := expectedVer
-	tagInstalled, err := getVeleroVersion(ctx, veleroCLI)
+	tagInstalled, err := getVeleroVersion(ctx, veleroCLI, false)
 	if err != nil {
-		return errors.WithMessagef(err, "Failed to get Velero version")
+		return errors.WithMessagef(err, "failed to get Velero version")
 	}
 	if strings.Trim(tag, " ") != strings.Trim(tagInstalled, " ") {
-		return errors.New(fmt.Sprintf("Velero version %s is not as expected %s", tagInstalled, tag))
+		return errors.New(fmt.Sprintf("velero version %s is not as expected %s", tagInstalled, tag))
 	}
 	fmt.Printf("Velero version %s is as expected %s\n", tagInstalled, tag)
 	return nil
@@ -440,18 +510,18 @@ func installVeleroCLI(version string) (string, error) {
 	tarball := name + postfix
 	tempFile, err := getVeleroCliTarball("https://github.com/vmware-tanzu/velero/releases/download/" + version + "/" + tarball)
 	if err != nil {
-		return "", errors.WithMessagef(err, "Failed to get Velero CLI tarball")
+		return "", errors.WithMessagef(err, "failed to get Velero CLI tarball")
 	}
 	tempVeleroCliDir, err := ioutil.TempDir("", "velero-test")
 	if err != nil {
-		return "", errors.WithMessagef(err, "Failed to create temp dir for tarball extraction")
+		return "", errors.WithMessagef(err, "failed to create temp dir for tarball extraction")
 	}
 
 	cmd := exec.Command("tar", "-xvf", tempFile.Name(), "-C", tempVeleroCliDir)
 	defer os.Remove(tempFile.Name())
 
 	if _, err := cmd.Output(); err != nil {
-		return "", errors.WithMessagef(err, "Failed to extract file from velero CLI tarball")
+		return "", errors.WithMessagef(err, "failed to extract file from velero CLI tarball")
 	}
 	return tempVeleroCliDir + "/" + name + "/velero", nil
 }
@@ -462,21 +532,21 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) {
 
 	resp, err := http.Get(cliTarballUrl)
 	if err != nil {
-		return nil, errors.WithMessagef(err, "Failed to access Velero CLI tarball")
+		return nil, errors.WithMessagef(err, "failed to access Velero CLI tarball")
 	}
 	defer resp.Body.Close()
 
 	tarballBuf, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return nil, errors.WithMessagef(err, "Failed to read buffer for tarball %s.", tarball)
+		return nil, errors.WithMessagef(err, "failed to read buffer for tarball %s.", tarball)
 	}
 	tmpfile, err := ioutil.TempFile("", tarball)
 	if err != nil {
-		return nil, errors.WithMessagef(err, "Failed to create temp file for tarball %s locally.", tarball)
+		return nil, errors.WithMessagef(err, "failed to create temp file for tarball %s locally.", tarball)
 	}
 
 	if _, err := tmpfile.Write(tarballBuf); err != nil {
-		return nil, errors.WithMessagef(err, "Failed to write tarball file %s locally.", tarball)
+		return nil, errors.WithMessagef(err, "failed to write tarball file %s locally.", tarball)
 	}
 
 	return tmpfile, nil