Fix E2E provider plugin images and add Velero v1.18 (#10774)

* Fix E2E provider plugin images and add Velero v1.18
* Remove obsolete E2E data-mover-plugin parameter
* Bump the UPGRADE_FROM_VELERO_VERSION from 1.15-1.17 to 1.16-1.18

Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>

test/Makefile

@@ -77,7 +77,7 @@ HAS_VSPHERE_PLUGIN ?= false
 RESTORE_HELPER_IMAGE ?=
 
 #Released version only
-UPGRADE_FROM_VELERO_VERSION ?= v1.15.2,v1.16.2,v1.17.2
+UPGRADE_FROM_VELERO_VERSION ?= v1.16.2,v1.17.2,v1.18.4
 
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
@@ -137,7 +137,6 @@ UPLOADER_TYPE ?=
 
 SNAPSHOT_MOVE_DATA ?= false
 DATA_MOVER ?=
-DATA_MOVER_PLUGIN ?=
 DISABLE_INFORMER_CACHE ?= false
 
 DEFAULT_CLUSTER_NAME ?=
@@ -236,7 +235,6 @@ run-e2e: ginkgo
 		--standby-cluster-context=$(STANDBY_CLUSTER) \
 		--snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
 		--data-mover=$(DATA_MOVER) \
-		--data-mover-plugin=$(DATA_MOVER_PLUGIN) \
 		--standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \
 		--standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \
 		--standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \

test/e2e/README.md

@@ -73,7 +73,6 @@ These configuration parameters are expected as values to the following command l
 1. `--uploader-type`: Type of uploader for persistent volume backup.
 1. `--snapshot-move-data`: Enable snapshot data movement for backups.
 1. `--data-mover`: Data mover name for snapshot data movement. Empty or `velero` selects the built-in data mover. Custom movers must be installed separately.
-1. `--data-mover-plugin`: Customized plugin for data mover.
 1. `--standby-cluster-cloud-provider`: Cloud provider for standby cluster.
 1. `--standby-cluster-plugins`: Plugins provider for standby cluster.
 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster.
@@ -127,7 +126,6 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `UPLOADER_TYPE`: `-uploader-type`. Optional.
 1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional.
 1. `DATA_MOVER`: `--data-mover`. Optional; used by snapshot data movement backups and does not enable `SNAPSHOT_MOVE_DATA` automatically.
-1. `DATA_MOVER_PLUGIN`: `-data-mover-plugin`. Optional.
 1. `STANDBY_CLUSTER_CLOUD_PROVIDER`: `-standby-cluster-cloud-provider`. Optional.
 1. `STANDBY_CLUSTER_PLUGINS`: `-standby-cluster-plugins`. Optional.
 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional.

test/e2e/e2e_suite_test.go

@@ -275,12 +275,6 @@ func init() {
 		"",
 		"data mover name for snapshot data movement; empty or velero uses the built-in data mover.",
 	)
-	flag.StringVar(
-		&test.VeleroCfg.DataMoverPlugin,
-		"data-mover-plugin",
-		"",
-		"customized plugin for data mover.",
-	)
 	flag.StringVar(
 		&test.VeleroCfg.StandbyClusterCloudProvider,
 		"standby-cluster-cloud-provider",

test/types.go

@@ -155,7 +155,6 @@ type VeleroConfig struct {
 	VeleroServerDebugMode             bool
 	SnapshotMoveData                  bool
 	DataMover                         string
-	DataMoverPlugin                   string
 	StandbyClusterCloudProvider       string
 	StandbyClusterPlugins             string
 	StandbyClusterObjectStoreProvider string

test/util/velero/velero_utils.go

@@ -65,55 +65,57 @@ var ImagesMatrix = map[string]map[string][]string{
 	"v1.13": {
 		"aws":                   {"velero/velero-plugin-for-aws:v1.9.2"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.9.2"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.9.2"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:v1.9.2"},
 		"csi":                   {"velero/velero-plugin-for-csi:v0.7.1"},
-		"datamover":             {"velero/velero-plugin-for-aws:v1.9.2"},
 		"velero":                {"velero/velero:v1.13.2"},
 		"velero-restore-helper": {"velero/velero-restore-helper:v1.13.2"},
 	},
 	"v1.14": {
 		"aws":                   {"velero/velero-plugin-for-aws:v1.10.1"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.10.1"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.10.1"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:v1.10.1"},
-		"datamover":             {"velero/velero-plugin-for-aws:v1.10.1"},
 		"velero":                {"velero/velero:v1.14.1"},
 		"velero-restore-helper": {"velero/velero-restore-helper:v1.14.1"},
 	},
 	"v1.15": {
 		"aws":                   {"velero/velero-plugin-for-aws:v1.11.0"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.11.0"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.11.0"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:v1.11.0"},
-		"datamover":             {"velero/velero-plugin-for-aws:v1.11.0"},
 		"velero":                {"velero/velero:v1.15.2"},
 		"velero-restore-helper": {"velero/velero-restore-helper:v1.15.2"},
 	},
 	"v1.16": {
 		"aws":                   {"velero/velero-plugin-for-aws:v1.12.2"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.12.2"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.12.2"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:v1.12.2"},
-		"datamover":             {"velero/velero-plugin-for-aws:v1.12.2"},
 		"velero":                {"velero/velero:v1.16.2"},
 		"velero-restore-helper": {"velero/velero:v1.16.2"},
 	},
 	"v1.17": {
 		"aws":                   {"velero/velero-plugin-for-aws:v1.13.2"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.13.2"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.13.2"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:v1.13.2"},
-		"datamover":             {"velero/velero-plugin-for-aws:v1.13.2"},
 		"velero":                {"velero/velero:v1.17.2"},
 		"velero-restore-helper": {"velero/velero:v1.17.2"},
 	},
+	"v1.18": {
+		"aws":                   {"velero/velero-plugin-for-aws:v1.14.4"},
+		"azure":                 {"velero/velero-plugin-for-microsoft-azure:v1.14.4"},
+		"vsphere":               {"velero/velero-plugin-for-aws:v1.14.4"},
+		"gcp":                   {"velero/velero-plugin-for-gcp:v1.14.4"},
+		"velero":                {"velero/velero:v1.18.4"},
+		"velero-restore-helper": {"velero/velero:v1.18.4"},
+	},
 	"main": {
 		"aws":                   {"velero/velero-plugin-for-aws:main"},
 		"azure":                 {"velero/velero-plugin-for-microsoft-azure:main"},
-		"vsphere":               {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"},
+		"vsphere":               {"velero/velero-plugin-for-aws:main"},
 		"gcp":                   {"velero/velero-plugin-for-gcp:main"},
-		"datamover":             {"velero/velero-plugin-for-aws:main"},
 		"velero":                {"velero/velero:main"},
 		"velero-restore-helper": {"velero/velero-restore-helper:main"},
 	},
@@ -155,12 +157,9 @@ func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig
 	if ret.CloudProvider == AWS {
 		ret.Plugins = images[AWS][0]
 	}
-	// If HasVspherePlugin is false, only install the AWS plugin.
-	// If do nothing here, the default behavior is
-	// installing both AWS and vSphere plugins.
-	if ret.CloudProvider == Vsphere &&
-		!ret.HasVspherePlugin {
-		ret.Plugins = images[AWS][0]
+	// vSphere tests use the AWS plugin for S3-compatible backup storage.
+	if ret.CloudProvider == Vsphere {
+		ret.Plugins = images[Vsphere][0]
 	}
 
 	if ret.SnapshotMoveData && ret.CloudProvider == Azure {
@@ -191,7 +190,7 @@ func getVersionWithoutPatch(version string) string {
 	return versionWithoutPatch
 }
 
-func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) {
+func getPluginsByVersion(version string, cloudProvider string) ([]string, error) {
 	var cloudMap map[string][]string
 	arr := strings.Split(version, ".")
 	if len(arr) >= 3 {
@@ -218,17 +217,6 @@ func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlug
 		}
 	}
 
-	if needDataMoverPlugin {
-		pluginsForDatamover, ok := cloudMap["datamover"]
-		if !ok {
-			return nil, errors.Errorf("fail to get plugins by for datamover")
-		}
-		for _, p := range pluginsForDatamover {
-			if !slices.Contains(plugins, p) {
-				plugins = append(plugins, p)
-			}
-		}
-	}
 	return plugins, nil
 }
 
@@ -695,13 +683,12 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error
 
 // GetPlugins will collect all kinds plugins for VeleroInstall, such as provider
 // plugins(cloud provider/object store provider, if object store provider is not
-// provided, it should be set to value as cloud provider's), feature plugins (CSI/Datamover)
+// provided, it should be set to value as cloud provider's), feature plugins
 func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([]string, error) {
 	veleroCLI := veleroCfg.VeleroCLI
 	cloudProvider := veleroCfg.CloudProvider
 	objectStoreProvider := veleroCfg.ObjectStoreProvider
 	providerPlugins := veleroCfg.Plugins
-	needDataMoverPlugin := false
 	var plugins []string
 
 	version, err := GetVeleroVersion(ctx, veleroCLI, true)
@@ -719,7 +706,7 @@ func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([
 				return []string{}, errors.New("AdditionalBSLProvider should be provided.")
 			}
 
-			plugins, err = getPluginsByVersion(version, veleroCfg.AdditionalBSLProvider, false)
+			plugins, err = getPluginsByVersion(version, veleroCfg.AdditionalBSLProvider)
 			if err != nil {
 				return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", veleroCfg.AdditionalBSLProvider, version)
 			}
@@ -745,11 +732,7 @@ func GetPlugins(ctx context.Context, veleroCfg VeleroConfig, defaultBSL bool) ([
 			version = veleroCfg.VeleroVersion
 		}
 
-		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest {
-			needDataMoverPlugin = true
-		}
-
-		plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin)
+		plugins, err = getPluginsByVersion(version, cloudProvider)
 		if err != nil {
 			return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
 		}

test/util/velero/velero_utils_test.go

@@ -20,6 +20,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/velero-io/velero/test"
 )
 
 func Test_getVersionWithoutPatch(t *testing.T) {
@@ -52,3 +54,40 @@ func Test_getVersionWithoutPatch(t *testing.T) {
 		})
 	}
 }
+
+func TestVSpherePluginSelection(t *testing.T) {
+	for _, version := range []string{"v1.13.2", "v1.17.2", "v1.18.4", "main"} {
+		t.Run(version, func(t *testing.T) {
+			images := ImagesMatrix[getVersionWithoutPatch(version)]
+			plugins, err := getPluginsByVersion(version, test.Vsphere)
+			require.NoError(t, err)
+			require.Equal(t, images[test.AWS], plugins)
+
+			for _, hasVSpherePlugin := range []bool{false, true} {
+				for _, moveData := range []bool{false, true} {
+					config, err := SetImagesToDefaultValues(test.VeleroConfig{
+						CloudProvider:    test.Vsphere,
+						HasVspherePlugin: hasVSpherePlugin,
+						SnapshotMoveData: moveData,
+					}, version)
+					require.NoError(t, err)
+					require.Equal(t, images[test.AWS][0], config.Plugins)
+				}
+			}
+		})
+	}
+}
+
+func TestV118PluginSelection(t *testing.T) {
+	for provider, image := range map[string]string{
+		test.AWS:   "velero/velero-plugin-for-aws:v1.14.4",
+		test.Azure: "velero/velero-plugin-for-microsoft-azure:v1.14.4",
+		test.GCP:   "velero/velero-plugin-for-gcp:v1.14.4",
+	} {
+		t.Run(provider, func(t *testing.T) {
+			plugins, err := getPluginsByVersion("v1.18.4", provider)
+			require.NoError(t, err)
+			require.Equal(t, []string{image}, plugins)
+		})
+	}
+}