Merge pull request #7297 from danfengliu/add-irsa-for-eks-pipeline

Support IRSA as credential in one of  nightly EKS pipelines

test/e2e/Makefile

@@ -67,6 +67,8 @@ MIGRATE_FROM_VELERO_CLI ?=
 
 VELERO_NAMESPACE ?= velero
 CREDS_FILE ?=
+DEFAULT_CLS_SERVICE_ACCOUNT_NAME ?=
+STANDBY_CLS_SERVICE_ACCOUNT_NAME ?=
 BSL_BUCKET ?=
 BSL_PREFIX ?=
 BSL_CONFIG ?=
@@ -106,6 +108,10 @@ SNAPSHOT_MOVE_DATA ?= false
 DATA_MOVER_PLUGIN ?=
 DISABLE_INFORMER_CACHE ?= false
 
+DEFAULT_CLUSTER_NAME ?=
+STANDBY_CLUSTER_NAME ?=
+EKS_POLICY_ARN ?=
+
 
 .PHONY:ginkgo
 ginkgo: # Make sure ginkgo is in $GOPATH/bin
@@ -148,8 +154,8 @@ run: ginkgo
 		-kibishii-directory=$(KIBISHII_DIRECTORY) \
 		-debug-e2e-test=$(DEBUG_E2E_TEST) \
 		-velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
-		-default-cluster=$(DEFAULT_CLUSTER) \
-		-standby-cluster=$(STANDBY_CLUSTER) \
+		-default-cluster-context=$(DEFAULT_CLUSTER) \
+		-standby-cluster-context=$(STANDBY_CLUSTER) \
 		-uploader-type=$(UPLOADER_TYPE) \
 		-snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
 		-data-mover-plugin=$(DATA_MOVER_PLUGIN) \
@@ -157,8 +163,12 @@ run: ginkgo
 		-standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \
 		-standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \
 		-debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \
-		-disable-informer-cache=$(DISABLE_INFORMER_CACHE)
-		
+		-disable-informer-cache=$(DISABLE_INFORMER_CACHE) \
+		-default-cluster-name=$(DEFAULT_CLUSTER_NAME) \
+		-standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
+		-eks-policy-arn=$(EKS_POLICY_ARN) \
+		-default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \
+		-standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME)
 
 build: ginkgo
 	mkdir -p $(OUTPUT_DIR)

test/e2e/README.md

@@ -69,8 +69,8 @@ the object-store-provider to be specified.
 1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action.
 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional.
 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server.
-1. `-default-cluster`: Default (source) cluster's kube config context, it's for migration test.
-1. `-standby-cluster`: Standby (destination) cluster's kube config context, it's for migration test.
+1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test.
+1. `-standby-cluster-context`: Standby (destination) cluster's kube config context, it's for migration test.
 1. `-uploader-type`: Type of uploader for persistent volume backup.
 1. `-snapshot-move-data`: A Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin.
 1. `-data-mover-plugin`: Customized plugin for data mover.
@@ -118,8 +118,8 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional.
 1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional.
 1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional.
-1. `DEFAULT_CLUSTER`: `-default-cluster`. Optional.
-1. `STANDBY_CLUSTER`: `-standby-cluster`. Optional.
+1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional.
+1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional.
 1. `UPLOADER_TYPE`: `-uploader-type`. Optional.
 1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional.
 1. `DATA_MOVER_plugin`: `-data-mover-plugin`. Optional.

test/e2e/basic/api-group/enable_api_group_extentions.go

@@ -39,11 +39,11 @@ func APIExtensionsVersionsTest() {
 	label := "for=backup"
 	srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
 	BeforeEach(func() {
-		if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
+		if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
 			Skip("CRD with apiextension versions migration test needs 2 clusters")
 		}
 		veleroCfg = VeleroCfg
-		Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+		Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 		srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName)
 		Expect(err).ShouldNot(HaveOccurred())
 		dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName)
@@ -75,19 +75,19 @@ func APIExtensionsVersionsTest() {
 				By("Uninstall Velero and delete CRD ", func() {
 					ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 					defer ctxCancel()
-					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
 
-					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
+					Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
 					Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 						veleroCfg.VeleroNamespace)).To(Succeed())
 					Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
 				})
 			}
-			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
 			})
 		}
@@ -98,14 +98,14 @@ func APIExtensionsVersionsTest() {
 			backupName = "backup-" + UUIDgen.String()
 			restoreName = "restore-" + UUIDgen.String()
 
-			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 				veleroCfg.Features = "EnableAPIGroupVersions"
 				veleroCfg.UseVolumeSnapshots = false
 				Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
 			})
 
-			By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
+			By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultClusterContext), func() {
 				Expect(InstallCRD(context.Background(), srcCrdYaml)).To(Succeed())
 				Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
 				Expect(WaitForCRDEstablished(crdName)).To(Succeed())
@@ -128,17 +128,17 @@ func APIExtensionsVersionsTest() {
 				})
 			})
 
-			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
+			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
 				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
 				Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
 			})
 
-			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
+			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
 				Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
 			})
 
-			By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() {
+			By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyClusterContext), func() {
 				Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
 			})
 

test/e2e/bsl-mgmt/deletion.go

@@ -229,7 +229,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				})
 				By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
 					snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
-					Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+					Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
 					var BSLCredentials, BSLConfig string
 					if veleroCfg.CloudProvider == "vsphere" {
 						BSLCredentials = veleroCfg.AdditionalBSLCredentials

test/e2e/e2e_suite_test.go

@@ -84,8 +84,8 @@ func init() {
 	flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.")
 	flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.")
 	flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.")
-	flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster", "", "default cluster's kube config context, it's for migration test.")
-	flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.")
+	flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.")
+	flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.")
 	flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.")
 	flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.")
 	flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin")
@@ -95,6 +95,12 @@ func init() {
 	flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
 	flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.")
 	flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.")
+	flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.")
+	flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.")
+	flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.")
+	flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.")
+	flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.")
+
 }
 
 var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest)
@@ -164,20 +170,22 @@ var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume clai
 func GetKubeconfigContext() error {
 	var err error
 	var tcDefault, tcStandby TestClient
-	tcDefault, err = NewTestClient(VeleroCfg.DefaultCluster)
+	tcDefault, err = NewTestClient(VeleroCfg.DefaultClusterContext)
 	VeleroCfg.DefaultClient = &tcDefault
 	VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
+	VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName
+	VeleroCfg.ServiceAccountNameToInstall = VeleroCfg.DefaultCLSServiceAccountName
 	if err != nil {
 		return err
 	}
 
-	if VeleroCfg.DefaultCluster != "" {
-		err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)
+	if VeleroCfg.DefaultClusterContext != "" {
+		err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultClusterContext)
 		if err != nil {
 			return err
 		}
-		if VeleroCfg.StandbyCluster != "" {
-			tcStandby, err = NewTestClient(VeleroCfg.StandbyCluster)
+		if VeleroCfg.StandbyClusterContext != "" {
+			tcStandby, err = NewTestClient(VeleroCfg.StandbyClusterContext)
 			VeleroCfg.StandbyClient = &tcStandby
 			if err != nil {
 				return err

test/e2e/migration/migration.go

@@ -64,7 +64,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			Skip("Volume snapshots not supported on kind")
 		}
 
-		if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
+		if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
 			Skip("Migration test needs 2 clusters")
 		}
 		// need to uninstall Velero first in case of the affection of the existing global velero installation
@@ -79,19 +79,19 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 	})
 	AfterEach(func() {
 		if !veleroCfg.Debug {
-			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultCluster), func() {
+			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 					veleroCfg.VeleroNamespace)).To(Succeed())
 				DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true)
 			})
 
-			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyCluster), func() {
+			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
 				Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
 					veleroCfg.VeleroNamespace)).To(Succeed())
 				DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
@@ -102,9 +102,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
 				})
 			}
-			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+			By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
+				veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
 			})
 		}
 
@@ -142,11 +143,13 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				})
 			}
 			OriginVeleroCfg := veleroCfg
-			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
+			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
 				OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
 				OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
 				OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
+				OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
+				OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName
 				OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 				OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
 
@@ -272,20 +275,22 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				time.Sleep(5 * time.Minute)
 			}
 
-			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
+			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
 				//Ensure workload of "migrationNamespace" existed in cluster-A
 				ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
 				Expect(ns.Name).To(Equal(migrationNamespace))
 				Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err))
 
 				//Ensure cluster-B is the target cluster
-				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
+				Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
 				_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
 				Expect(err).To(HaveOccurred())
 				strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
 				fmt.Println(err)
 
 				veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
+				veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
+				veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName
 				veleroCfg.UseNodeAgent = !useVolumeSnapshots
 				veleroCfg.UseRestic = false
 				if veleroCfg.SnapshotMoveData {
@@ -299,7 +304,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 				Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
 			})
 
-			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
+			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
 				Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
 				Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed())
 			})

test/perf/e2e_suite_test.go

@@ -69,7 +69,7 @@ func init() {
 	flag.DurationVar(&VeleroCfg.PodVolumeOperationTimeout, "pod-volume-operation-timeout", 360*time.Minute, "Timeout for pod volume operations. Optional.")
 	//vmware-tanzu-experiments
 	flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
-	flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster-context", "", "Default cluster context for migration test.")
+	flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.")
 	flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", true, "Switch to control namespace cleaning.")
 	flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.")
 	flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.")

test/types.go

@@ -71,11 +71,14 @@ type VeleroConfig struct {
 	KibishiiDirectory                 string
 	Debug                             bool
 	GCFrequency                       string
-	DefaultCluster                    string
-	StandbyCluster                    string
+	DefaultClusterContext             string
+	StandbyClusterContext             string
 	ClientToInstallVelero             *TestClient
 	DefaultClient                     *TestClient
 	StandbyClient                     *TestClient
+	ClusterToInstallVelero            string
+	DefaultClusterName                string
+	StandbyClusterName                string
 	ProvideSnapshotsVolumeParam       bool
 	VeleroServerDebugMode             bool
 	SnapshotMoveData                  bool
@@ -87,6 +90,11 @@ type VeleroConfig struct {
 	IsUpgradeTest                     bool
 	WithoutDisableInformerCacheParam  bool
 	DisableInformerCache              bool
+	CreateClusterRoleBinding          bool
+	DefaultCLSServiceAccountName      string
+	StandbyCLSServiceAccountName      string
+	ServiceAccountNameToInstall       string
+	EKSPolicyARN                      string
 }
 
 type VeleroCfgInPerf struct {

test/util/eks/eks.go

@@ -0,0 +1,63 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"golang.org/x/net/context"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+)
+
+func KubectlDeleteIAMServiceAcount(ctx context.Context, name, namespace, cluster string) error {
+	args := []string{"delete", "iamserviceaccount", name,
+		"--namespace", namespace, "--cluster", cluster, "--wait"}
+	fmt.Println(args)
+
+	cmd := exec.CommandContext(ctx, "eksctl", args...)
+	fmt.Println(cmd)
+	stdout, stderr, err := veleroexec.RunCommand(cmd)
+	fmt.Printf("Output: %v\n", stdout)
+	if strings.Contains(stderr, "NotFound") {
+		err = nil
+	}
+	return err
+}
+
+func EksctlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error {
+	args := []string{"create", "iamserviceaccount", name,
+		"--namespace", namespace, "--cluster", cluster, "--attach-policy-arn", policyARN,
+		"--approve", "--override-existing-serviceaccounts"}
+
+	PollInterval := 1 * time.Minute
+	PollTimeout := 10 * time.Minute
+	return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
+		cmd := exec.CommandContext(ctx, "eksctl", args...)
+		fmt.Println(cmd)
+		stdout, stderr, err := veleroexec.RunCommand(cmd)
+		if err != nil {
+			fmt.Printf("eksctl return stdout: %v, stderr: %v, err: %v\n", stdout, stderr, err)
+			return false, nil
+		}
+		return true, nil
+	})
+}

test/util/k8s/clusterrolebinding.go

@@ -0,0 +1,46 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8s
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"golang.org/x/net/context"
+
+	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
+)
+
+func KubectlDeleteClusterRoleBinding(ctx context.Context, name string) error {
+	args := []string{"delete", "clusterrolebinding", name}
+	fmt.Println(args)
+	cmd := exec.CommandContext(ctx, "kubectl", args...)
+	fmt.Println(cmd)
+	_, stderr, err := veleroexec.RunCommand(cmd)
+	if strings.Contains(stderr, "NotFound") {
+		fmt.Printf("Ignore error: %v\n", stderr)
+		err = nil
+	}
+	return err
+}
+
+func KubectlCreateClusterRoleBinding(ctx context.Context, name, clusterrole, namespace, serviceaccount string) error {
+	args := []string{"create", "clusterrolebinding", name, fmt.Sprintf("--clusterrole=%s", clusterrole), fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceaccount)}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/util/k8s/namespace.go

@@ -19,12 +19,12 @@ package k8s
 import (
 	"context"
 	"fmt"
+	"os/exec"
 	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
 	corev1api "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -233,3 +233,9 @@ func GetMappingNamespaces(ctx context.Context, client TestClient, excludeNS []st
 	}
 	return joinedNsMapping, nil
 }
+
+func KubectlCreateNamespace(ctx context.Context, name string) error {
+	args := []string{"create", "namespace", name}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}

test/util/providers/common.go

@@ -47,6 +47,9 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
 	var err error
 	var exist bool
 	fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
+	if cloudCredentialsFile == "" {
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+	}
 	for i := 0; i < retryTimes; i++ {
 		exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
 		if err != nil {
@@ -98,6 +101,9 @@ func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix
 func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
 	bslPrefix = getFullPrefix(bslPrefix, subPrefix)
 	fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
+	if cloudCredentialsFile == "" {
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+	}
 	s, err := getProvider(cloudProvider)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
@@ -111,6 +117,9 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
 
 func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
+	if cloudCredentialsFile == "" {
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+	}
 	snapshotCheckPoint.ExpectCount = 0
 	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
@@ -122,6 +131,9 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
 
 func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
 	fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
+	if cloudCredentialsFile == "" {
+		return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
+	}
 	err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
 	if err != nil {
 		return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName))

test/util/velero/install.go

@@ -39,6 +39,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 	. "github.com/vmware-tanzu/velero/test"
+	. "github.com/vmware-tanzu/velero/test/util/eks"
 	. "github.com/vmware-tanzu/velero/test/util/k8s"
 )
 
@@ -75,7 +76,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 		veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
 	}
 	if veleroCfg.CloudProvider != "kind" {
-		fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider")
+		fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
 		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
 		if veleroCfg.ObjectStoreProvider == "" {
 			veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
@@ -112,27 +113,37 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
 	}
-	veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
-	if !veleroCfg.UseRestic {
-		veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
-	}
-	veleroInstallOptions.UseRestic = veleroCfg.UseRestic
-	veleroInstallOptions.Image = veleroCfg.VeleroImage
-	veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
-	veleroInstallOptions.UploaderType = veleroCfg.UploaderType
-	GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
-	veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
-	veleroInstallOptions.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout
-	veleroInstallOptions.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit
-	veleroInstallOptions.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest
-	veleroInstallOptions.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit
-	veleroInstallOptions.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest
-	veleroInstallOptions.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit
-	veleroInstallOptions.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest
-	veleroInstallOptions.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit
-	veleroInstallOptions.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest
-	veleroInstallOptions.DisableInformerCache = veleroCfg.DisableInformerCache
 
+	// For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN
+	// are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part.
+	if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" {
+		if veleroCfg.EKSPolicyARN == "" {
+			return errors.New("Please provide EKSPolicyARN for IRSA test.")
+		}
+		_, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace)
+		// We should uninstall Velero for a new service account creation.
+		if !apierrors.IsNotFound(err) {
+			if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil {
+				return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace)
+			}
+		}
+		// If velero namespace does not exist, we should create it for service account creation
+		if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil {
+			return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace)
+		}
+		if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil {
+			return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
+		}
+		if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil {
+			return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
+		}
+		if err := KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
+			return errors.Wrapf(err, "Failed to delete service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
+		}
+		if err := EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
+			return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
+		}
+	}
 	err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{
 		Options:                          veleroInstallOptions,
 		RegistryCredentialFile:           veleroCfg.RegistryCredentialFile,
@@ -239,8 +250,16 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 	if len(options.Prefix) > 0 {
 		args = append(args, "--prefix", options.Prefix)
 	}
-	if len(options.SecretFile) > 0 {
-		args = append(args, "--secret-file", options.SecretFile)
+	//Treat ServiceAccountName priority higher than SecretFile
+	if len(options.ServiceAccountName) > 0 {
+		args = append(args, "--service-account-name", options.ServiceAccountName)
+	} else {
+		if len(options.SecretFile) > 0 {
+			args = append(args, "--secret-file", options.SecretFile)
+		}
+	}
+	if options.NoSecret {
+		args = append(args, "--no-secret")
 	}
 	if len(options.VolumeSnapshotConfig.Data()) > 0 {
 		args = append(args, "--snapshot-location-config", options.VolumeSnapshotConfig.String())
@@ -257,11 +276,11 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 		}
 	}
 
-	fmt.Println("Start to install Azure VolumeSnapshotClass ...")
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
 		if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
 			if strings.EqualFold(cloudProvider, "azure") {
+				fmt.Println("Start to install Azure VolumeSnapshotClass ...")
 				if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
 					return err
 				}

test/util/velero/velero_utils.go

@@ -174,20 +174,27 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st
 func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	plugins []string) (*cliinstall.Options, error) {
 
-	if veleroCfg.CloudCredentialsFile == "" {
+	if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountNameToInstall == "" {
 		return nil, errors.Errorf("No credentials were supplied to use for E2E tests")
 	}
 
-	realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
-	if err != nil {
-		return nil, err
-	}
-
 	io := cliinstall.NewInstallOptions()
 	// always wait for velero and restic pods to be running.
 	io.Wait = true
 	io.ProviderName = veleroCfg.ObjectStoreProvider
-	io.SecretFile = veleroCfg.CloudCredentialsFile
+
+	if veleroCfg.CloudCredentialsFile != "" {
+		realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
+		if err != nil {
+			return nil, err
+		}
+		io.SecretFile = realPath
+	}
+
+	if veleroCfg.ServiceAccountNameToInstall != "" {
+		io.ServiceAccountName = veleroCfg.ServiceAccountNameToInstall
+		io.NoSecret = true
+	}
 
 	io.BucketName = veleroCfg.BSLBucket
 	io.Prefix = veleroCfg.BSLPrefix
@@ -197,11 +204,31 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
 	io.VolumeSnapshotConfig = flag.NewMap()
 	io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig)
 
-	io.SecretFile = realPath
 	io.Plugins = flag.NewStringArray(plugins...)
 	io.Features = veleroCfg.Features
 	io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup
 	io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
+
+	if !veleroCfg.UseRestic {
+		io.UseNodeAgent = veleroCfg.UseNodeAgent
+	}
+	io.UseRestic = veleroCfg.UseRestic
+	io.Image = veleroCfg.VeleroImage
+	io.Namespace = veleroCfg.VeleroNamespace
+	io.UploaderType = veleroCfg.UploaderType
+	GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
+	io.GarbageCollectionFrequency = GCFrequency
+	io.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout
+	io.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit
+	io.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest
+	io.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit
+	io.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest
+	io.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit
+	io.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest
+	io.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit
+	io.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest
+	io.DisableInformerCache = veleroCfg.DisableInformerCache
+
 	return io, nil
 }