From 72438b7319c9a8d7052e5791d8946022d14fcb06 Mon Sep 17 00:00:00 2001 From: danfengl Date: Wed, 24 Jan 2024 06:21:04 +0000 Subject: [PATCH] Support IRSA for data mover pipeline Signed-off-by: danfengl --- test/e2e/Makefile | 10 ++++++---- test/e2e/README.md | 8 ++++---- test/e2e/bsl-mgmt/deletion.go | 2 +- test/e2e/e2e_suite_test.go | 8 +++++--- test/e2e/migration/migration.go | 2 ++ test/types.go | 4 +++- test/util/eks/eks.go | 6 ++---- test/util/k8s/clusterrolebinding.go | 1 - test/util/providers/common.go | 12 ++++-------- test/util/velero/install.go | 26 ++++++++++++++------------ test/util/velero/velero_utils.go | 6 +++--- 11 files changed, 44 insertions(+), 41 deletions(-) diff --git a/test/e2e/Makefile b/test/e2e/Makefile index e00e29432..3bc3cbc7a 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -67,7 +67,8 @@ MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero CREDS_FILE ?= -SERVICE_ACCOUNT_NAME ?= +DEFAULT_CLS_SERVICE_ACCOUNT_NAME ?= +STANDBY_CLS_SERVICE_ACCOUNT_NAME ?= BSL_BUCKET ?= BSL_PREFIX ?= BSL_CONFIG ?= @@ -153,8 +154,8 @@ run: ginkgo -kibishii-directory=$(KIBISHII_DIRECTORY) \ -debug-e2e-test=$(DEBUG_E2E_TEST) \ -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ - -default-cluster=$(DEFAULT_CLUSTER) \ - -standby-cluster=$(STANDBY_CLUSTER) \ + -default-cluster-context=$(DEFAULT_CLUSTER) \ + -standby-cluster-context=$(STANDBY_CLUSTER) \ -uploader-type=$(UPLOADER_TYPE) \ -snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ -data-mover-plugin=$(DATA_MOVER_PLUGIN) \ @@ -166,7 +167,8 @@ run: ginkgo -default-cluster-name=$(DEFAULT_CLUSTER_NAME) \ -standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ -eks-policy-arn=$(EKS_POLICY_ARN) \ - -service-account-name=$(SERVICE_ACCOUNT_NAME) + -default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \ + -standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/README.md b/test/e2e/README.md index 9207e2da8..cb1d001fd 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -69,8 +69,8 @@ the object-store-provider to be specified. 1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action. 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional. 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server. -1. `-default-cluster`: Default (source) cluster's kube config context, it's for migration test. -1. `-standby-cluster`: Standby (destination) cluster's kube config context, it's for migration test. +1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test. +1. `-standby-cluster-context`: Standby (destination) cluster's kube config context, it's for migration test. 1. `-uploader-type`: Type of uploader for persistent volume backup. 1. `-snapshot-move-data`: A Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin. 1. `-data-mover-plugin`: Customized plugin for data mover. @@ -118,8 +118,8 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional. 1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional. 1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional. -1. `DEFAULT_CLUSTER`: `-default-cluster`. Optional. -1. `STANDBY_CLUSTER`: `-standby-cluster`. Optional. +1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional. +1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional. 1. `UPLOADER_TYPE`: `-uploader-type`. Optional. 1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional. 1. `DATA_MOVER_plugin`: `-data-mover-plugin`. Optional. diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 909458e03..4dc7a0561 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -229,7 +229,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) - Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") var BSLCredentials, BSLConfig string if veleroCfg.CloudProvider == "vsphere" { BSLCredentials = veleroCfg.AdditionalBSLCredentials diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d285a8324..99dd7b16d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -84,8 +84,8 @@ func init() { flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.") flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.") flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.") - flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster", "", "default cluster's kube config context, it's for migration test.") - flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.") + flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.") + flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.") flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin") @@ -98,7 +98,8 @@ func init() { flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.") flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.") flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.") - flag.StringVar(&VeleroCfg.ServiceAccountName, "service-account-name", "", "service account name.") + flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") + flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") } @@ -173,6 +174,7 @@ func GetKubeconfigContext() error { VeleroCfg.DefaultClient = &tcDefault VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName + VeleroCfg.ServiceAccountNameToInstall = VeleroCfg.DefaultCLSServiceAccountName if err != nil { return err } diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 1bfb11e29..7395970a5 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -149,6 +149,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName + OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots @@ -289,6 +290,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName + veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName veleroCfg.UseNodeAgent = !useVolumeSnapshots veleroCfg.UseRestic = false if veleroCfg.SnapshotMoveData { diff --git a/test/types.go b/test/types.go index 20b3ca14c..0d27e96be 100644 --- a/test/types.go +++ b/test/types.go @@ -91,7 +91,9 @@ type VeleroConfig struct { WithoutDisableInformerCacheParam bool DisableInformerCache bool CreateClusterRoleBinding bool - ServiceAccountName string + DefaultCLSServiceAccountName string + StandbyCLSServiceAccountName string + ServiceAccountNameToInstall string EKSPolicyARN string } diff --git a/test/util/eks/eks.go b/test/util/eks/eks.go index 6b83e88df..ff3df3c31 100644 --- a/test/util/eks/eks.go +++ b/test/util/eks/eks.go @@ -40,11 +40,10 @@ func KubectlDeleteIAMServiceAcount(ctx context.Context, name, namespace, cluster if strings.Contains(stderr, "NotFound") { err = nil } - fmt.Printf("err: %v\n", err) return err } -func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error { +func EksctlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error { args := []string{"create", "iamserviceaccount", name, "--namespace", namespace, "--cluster", cluster, "--attach-policy-arn", policyARN, "--approve", "--override-existing-serviceaccounts"} @@ -55,9 +54,8 @@ func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyA cmd := exec.CommandContext(ctx, "eksctl", args...) fmt.Println(cmd) stdout, stderr, err := veleroexec.RunCommand(cmd) - fmt.Printf("Output: %v|%v|%v\n", stdout, stderr, err) if err != nil { - fmt.Printf("err: %v|%v|%v\n", stdout, stderr, err) + fmt.Printf("eksctl return stdout: %v, stderr: %v, err: %v\n", stdout, stderr, err) return false, nil } return true, nil diff --git a/test/util/k8s/clusterrolebinding.go b/test/util/k8s/clusterrolebinding.go index a9fbb7962..c7ce32e31 100644 --- a/test/util/k8s/clusterrolebinding.go +++ b/test/util/k8s/clusterrolebinding.go @@ -32,7 +32,6 @@ func KubectlDeleteClusterRoleBinding(ctx context.Context, name string) error { cmd := exec.CommandContext(ctx, "kubectl", args...) fmt.Println(cmd) _, stderr, err := veleroexec.RunCommand(cmd) - fmt.Printf("Ignore error: %v\n", stderr) if strings.Contains(stderr, "NotFound") { fmt.Printf("Ignore error: %v\n", stderr) err = nil diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 8ef943e0f..2576d5357 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -48,8 +48,7 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, var exist bool fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix) if cloudCredentialsFile == "" { - fmt.Printf("|| SKIPPED || - Skipping object storebackup checkpoint %s\n", backupName) - return nil + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } for i := 0; i < retryTimes; i++ { exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) @@ -103,8 +102,7 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr bslPrefix = getFullPrefix(bslPrefix, subPrefix) fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix) if cloudCredentialsFile == "" { - fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) - return nil + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } s, err := getProvider(cloudProvider) if err != nil { @@ -120,8 +118,7 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) if cloudCredentialsFile == "" { - fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) - return nil + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) @@ -135,8 +132,7 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) if cloudCredentialsFile == "" { - fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) - return nil + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) } err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 7eab157ea..cdb61e740 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -76,7 +76,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider } if veleroCfg.CloudProvider != "kind" { - fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider\n") + fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider if veleroCfg.ObjectStoreProvider == "" { veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider @@ -114,32 +114,34 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } + // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN + // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.EKSPolicyARN == "" { + return errors.New("Please provide EKSPolicyARN for IRSA test.") + } _, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace) + // We should uninstall Velero for a new service account creation. if !apierrors.IsNotFound(err) { if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace) } } + // If velero namespace does not exist, we should create it for service account creation if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace) } if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil { - fmt.Println(err) - return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace) + return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace) } if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil { - fmt.Println(err) - return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace) + return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace) } - if err := KubectlDeleteIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil { - fmt.Println(err) - return errors.Wrapf(err, "Failed to delete service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace) + if err := KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil { + return errors.Wrapf(err, "Failed to delete service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace) } - time.Sleep(10 * time.Second) - if err := KubectlCreateIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil { - fmt.Println(err) - return errors.Wrapf(err, "Failed to create service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace) + if err := EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil { + return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace) } } err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{ diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 6eb924ac8..91f3f8b04 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -174,7 +174,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, plugins []string) (*cliinstall.Options, error) { - if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountName == "" { + if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountNameToInstall == "" { return nil, errors.Errorf("No credentials were supplied to use for E2E tests") } @@ -191,8 +191,8 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.SecretFile = realPath } - if veleroCfg.ServiceAccountName != "" { - io.ServiceAccountName = veleroCfg.ServiceAccountName + if veleroCfg.ServiceAccountNameToInstall != "" { + io.ServiceAccountName = veleroCfg.ServiceAccountNameToInstall io.NoSecret = true }