diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 7bf80c351..e00e29432 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -67,6 +67,7 @@ MIGRATE_FROM_VELERO_CLI ?= VELERO_NAMESPACE ?= velero CREDS_FILE ?= +SERVICE_ACCOUNT_NAME ?= BSL_BUCKET ?= BSL_PREFIX ?= BSL_CONFIG ?= @@ -106,6 +107,10 @@ SNAPSHOT_MOVE_DATA ?= false DATA_MOVER_PLUGIN ?= DISABLE_INFORMER_CACHE ?= false +DEFAULT_CLUSTER_NAME ?= +STANDBY_CLUSTER_NAME ?= +EKS_POLICY_ARN ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin @@ -157,8 +162,11 @@ run: ginkgo -standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \ -standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \ -debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - -disable-informer-cache=$(DISABLE_INFORMER_CACHE) - + -disable-informer-cache=$(DISABLE_INFORMER_CACHE) \ + -default-cluster-name=$(DEFAULT_CLUSTER_NAME) \ + -standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ + -eks-policy-arn=$(EKS_POLICY_ARN) \ + -service-account-name=$(SERVICE_ACCOUNT_NAME) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index 5c77438fc..af6d32304 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -39,11 +39,11 @@ func APIExtensionsVersionsTest() { label := "for=backup" srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml" BeforeEach(func() { - if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { + if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" { Skip("CRD with apiextension versions migration test needs 2 clusters") } veleroCfg = VeleroCfg - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) Expect(err).ShouldNot(HaveOccurred()) dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName) @@ -75,19 +75,19 @@ func APIExtensionsVersionsTest() { By("Uninstall Velero and delete CRD ", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient }) } @@ -98,14 +98,14 @@ func APIExtensionsVersionsTest() { backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) - By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() { + By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultClusterContext), func() { Expect(InstallCRD(context.Background(), srcCrdYaml)).To(Succeed()) Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed()) Expect(WaitForCRDEstablished(crdName)).To(Succeed()) @@ -128,17 +128,17 @@ func APIExtensionsVersionsTest() { }) }) - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) - By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() { Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) }) - By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyClusterContext), func() { Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test") }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9464dfd10..d285a8324 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -84,8 +84,8 @@ func init() { flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.") flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.") flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.") - flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster", "", "default cluster's kube config context, it's for migration test.") - flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.") + flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster", "", "default cluster's kube config context, it's for migration test.") + flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.") flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin") @@ -95,6 +95,11 @@ func init() { flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.") flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.") flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.") + flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.") + flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.") + flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.") + flag.StringVar(&VeleroCfg.ServiceAccountName, "service-account-name", "", "service account name.") + } var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest) @@ -164,20 +169,21 @@ var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume clai func GetKubeconfigContext() error { var err error var tcDefault, tcStandby TestClient - tcDefault, err = NewTestClient(VeleroCfg.DefaultCluster) + tcDefault, err = NewTestClient(VeleroCfg.DefaultClusterContext) VeleroCfg.DefaultClient = &tcDefault VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient + VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName if err != nil { return err } - if VeleroCfg.DefaultCluster != "" { - err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster) + if VeleroCfg.DefaultClusterContext != "" { + err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultClusterContext) if err != nil { return err } - if VeleroCfg.StandbyCluster != "" { - tcStandby, err = NewTestClient(VeleroCfg.StandbyCluster) + if VeleroCfg.StandbyClusterContext != "" { + tcStandby, err = NewTestClient(VeleroCfg.StandbyClusterContext) VeleroCfg.StandbyClient = &tcStandby if err != nil { return err diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 202cad701..1bfb11e29 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -64,7 +64,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Skip("Volume snapshots not supported on kind") } - if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { + if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" { Skip("Migration test needs 2 clusters") } // need to uninstall Velero first in case of the affection of the existing global velero installation @@ -79,19 +79,19 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) AfterEach(func() { if !veleroCfg.Debug { - By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultCluster), func() { + By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true) }) - By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) @@ -102,9 +102,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName }) } @@ -142,11 +143,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) } OriginVeleroCfg := veleroCfg - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient + OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots @@ -272,20 +274,21 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) time.Sleep(5 * time.Minute) } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { //Ensure workload of "migrationNamespace" existed in cluster-A ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) Expect(ns.Name).To(Equal(migrationNamespace)) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err)) //Ensure cluster-B is the target cluster - Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) Expect(err).To(HaveOccurred()) strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found") fmt.Println(err) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient + veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName veleroCfg.UseNodeAgent = !useVolumeSnapshots veleroCfg.UseRestic = false if veleroCfg.SnapshotMoveData { @@ -299,7 +302,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) }) - By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() { Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed()) }) diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index 7d1cae16f..f05f7ffe0 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -69,7 +69,7 @@ func init() { flag.DurationVar(&VeleroCfg.PodVolumeOperationTimeout, "pod-volume-operation-timeout", 360*time.Minute, "Timeout for pod volume operations. Optional.") //vmware-tanzu-experiments flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") - flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster-context", "", "Default cluster context for migration test.") + flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.") flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", true, "Switch to control namespace cleaning.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") diff --git a/test/types.go b/test/types.go index ed00cada6..20b3ca14c 100644 --- a/test/types.go +++ b/test/types.go @@ -71,11 +71,14 @@ type VeleroConfig struct { KibishiiDirectory string Debug bool GCFrequency string - DefaultCluster string - StandbyCluster string + DefaultClusterContext string + StandbyClusterContext string ClientToInstallVelero *TestClient DefaultClient *TestClient StandbyClient *TestClient + ClusterToInstallVelero string + DefaultClusterName string + StandbyClusterName string ProvideSnapshotsVolumeParam bool VeleroServerDebugMode bool SnapshotMoveData bool @@ -87,6 +90,9 @@ type VeleroConfig struct { IsUpgradeTest bool WithoutDisableInformerCacheParam bool DisableInformerCache bool + CreateClusterRoleBinding bool + ServiceAccountName string + EKSPolicyARN string } type VeleroCfgInPerf struct { diff --git a/test/util/eks/eks.go b/test/util/eks/eks.go new file mode 100644 index 000000000..6b83e88df --- /dev/null +++ b/test/util/eks/eks.go @@ -0,0 +1,65 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "os/exec" + "strings" + "time" + + "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/wait" + + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" +) + +func KubectlDeleteIAMServiceAcount(ctx context.Context, name, namespace, cluster string) error { + args := []string{"delete", "iamserviceaccount", name, + "--namespace", namespace, "--cluster", cluster, "--wait"} + fmt.Println(args) + + cmd := exec.CommandContext(ctx, "eksctl", args...) + fmt.Println(cmd) + stdout, stderr, err := veleroexec.RunCommand(cmd) + fmt.Printf("Output: %v\n", stdout) + if strings.Contains(stderr, "NotFound") { + err = nil + } + fmt.Printf("err: %v\n", err) + return err +} + +func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error { + args := []string{"create", "iamserviceaccount", name, + "--namespace", namespace, "--cluster", cluster, "--attach-policy-arn", policyARN, + "--approve", "--override-existing-serviceaccounts"} + + PollInterval := 1 * time.Minute + PollTimeout := 10 * time.Minute + return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { + cmd := exec.CommandContext(ctx, "eksctl", args...) + fmt.Println(cmd) + stdout, stderr, err := veleroexec.RunCommand(cmd) + fmt.Printf("Output: %v|%v|%v\n", stdout, stderr, err) + if err != nil { + fmt.Printf("err: %v|%v|%v\n", stdout, stderr, err) + return false, nil + } + return true, nil + }) +} diff --git a/test/util/k8s/clusterrolebinding.go b/test/util/k8s/clusterrolebinding.go new file mode 100644 index 000000000..a9fbb7962 --- /dev/null +++ b/test/util/k8s/clusterrolebinding.go @@ -0,0 +1,47 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "os/exec" + "strings" + + "golang.org/x/net/context" + + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" +) + +func KubectlDeleteClusterRoleBinding(ctx context.Context, name string) error { + args := []string{"delete", "clusterrolebinding", name} + fmt.Println(args) + cmd := exec.CommandContext(ctx, "kubectl", args...) + fmt.Println(cmd) + _, stderr, err := veleroexec.RunCommand(cmd) + fmt.Printf("Ignore error: %v\n", stderr) + if strings.Contains(stderr, "NotFound") { + fmt.Printf("Ignore error: %v\n", stderr) + err = nil + } + return err +} + +func KubectlCreateClusterRoleBinding(ctx context.Context, name, clusterrole, namespace, serviceaccount string) error { + args := []string{"create", "clusterrolebinding", name, fmt.Sprintf("--clusterrole=%s", clusterrole), fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceaccount)} + fmt.Println(args) + return exec.CommandContext(ctx, "kubectl", args...).Run() +} diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index 3c7686756..479ac1b8a 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -19,12 +19,12 @@ package k8s import ( "context" "fmt" + "os/exec" "strings" "time" "github.com/pkg/errors" "github.com/sirupsen/logrus" - corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -233,3 +233,9 @@ func GetMappingNamespaces(ctx context.Context, client TestClient, excludeNS []st } return joinedNsMapping, nil } + +func KubectlCreateNamespace(ctx context.Context, name string) error { + args := []string{"create", "namespace", name} + fmt.Println(args) + return exec.CommandContext(ctx, "kubectl", args...).Run() +} diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 0764e70de..8ef943e0f 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -47,6 +47,10 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, var err error var exist bool fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix) + if cloudCredentialsFile == "" { + fmt.Printf("|| SKIPPED || - Skipping object storebackup checkpoint %s\n", backupName) + return nil + } for i := 0; i < retryTimes; i++ { exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if err != nil { @@ -98,6 +102,10 @@ func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { bslPrefix = getFullPrefix(bslPrefix, subPrefix) fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix) + if cloudCredentialsFile == "" { + fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) + return nil + } s, err := getProvider(cloudProvider) if err != nil { return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) @@ -111,6 +119,10 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) + if cloudCredentialsFile == "" { + fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) + return nil + } snapshotCheckPoint.ExpectCount = 0 err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { @@ -122,6 +134,10 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) + if cloudCredentialsFile == "" { + fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName) + return nil + } err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName)) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 9c621259c..7eab157ea 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -39,6 +39,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/util/eks" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -75,7 +76,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider } if veleroCfg.CloudProvider != "kind" { - fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") + fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider\n") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider if veleroCfg.ObjectStoreProvider == "" { veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider @@ -112,27 +113,35 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } - veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - if !veleroCfg.UseRestic { - veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent - } - veleroInstallOptions.UseRestic = veleroCfg.UseRestic - veleroInstallOptions.Image = veleroCfg.VeleroImage - veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace - veleroInstallOptions.UploaderType = veleroCfg.UploaderType - GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) - veleroInstallOptions.GarbageCollectionFrequency = GCFrequency - veleroInstallOptions.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout - veleroInstallOptions.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit - veleroInstallOptions.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest - veleroInstallOptions.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit - veleroInstallOptions.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest - veleroInstallOptions.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit - veleroInstallOptions.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest - veleroInstallOptions.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit - veleroInstallOptions.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest - veleroInstallOptions.DisableInformerCache = veleroCfg.DisableInformerCache + if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" { + _, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace) + if !apierrors.IsNotFound(err) { + if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil { + return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace) + } + } + if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace) + } + if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil { + fmt.Println(err) + return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace) + } + if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil { + fmt.Println(err) + return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace) + } + if err := KubectlDeleteIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil { + fmt.Println(err) + return errors.Wrapf(err, "Failed to delete service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace) + } + time.Sleep(10 * time.Second) + if err := KubectlCreateIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil { + fmt.Println(err) + return errors.Wrapf(err, "Failed to create service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace) + } + } err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{ Options: veleroInstallOptions, RegistryCredentialFile: veleroCfg.RegistryCredentialFile, @@ -239,8 +248,16 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Prefix) > 0 { args = append(args, "--prefix", options.Prefix) } - if len(options.SecretFile) > 0 { - args = append(args, "--secret-file", options.SecretFile) + //Treat ServiceAccountName priority higher than SecretFile + if len(options.ServiceAccountName) > 0 { + args = append(args, "--service-account-name", options.ServiceAccountName) + } else { + if len(options.SecretFile) > 0 { + args = append(args, "--secret-file", options.SecretFile) + } + } + if options.NoSecret { + args = append(args, "--no-secret") } if len(options.VolumeSnapshotConfig.Data()) > 0 { args = append(args, "--snapshot-location-config", options.VolumeSnapshotConfig.String()) @@ -257,11 +274,11 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options } } - fmt.Println("Start to install Azure VolumeSnapshotClass ...") if len(options.Features) > 0 { args = append(args, "--features", options.Features) if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { if strings.EqualFold(cloudProvider, "azure") { + fmt.Println("Start to install Azure VolumeSnapshotClass ...") if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { return err } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 61e1db22a..6eb924ac8 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -174,20 +174,27 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, plugins []string) (*cliinstall.Options, error) { - if veleroCfg.CloudCredentialsFile == "" { + if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountName == "" { return nil, errors.Errorf("No credentials were supplied to use for E2E tests") } - realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile) - if err != nil { - return nil, err - } - io := cliinstall.NewInstallOptions() // always wait for velero and restic pods to be running. io.Wait = true io.ProviderName = veleroCfg.ObjectStoreProvider - io.SecretFile = veleroCfg.CloudCredentialsFile + + if veleroCfg.CloudCredentialsFile != "" { + realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile) + if err != nil { + return nil, err + } + io.SecretFile = realPath + } + + if veleroCfg.ServiceAccountName != "" { + io.ServiceAccountName = veleroCfg.ServiceAccountName + io.NoSecret = true + } io.BucketName = veleroCfg.BSLBucket io.Prefix = veleroCfg.BSLPrefix @@ -197,11 +204,31 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.VolumeSnapshotConfig = flag.NewMap() io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig) - io.SecretFile = realPath io.Plugins = flag.NewStringArray(plugins...) io.Features = veleroCfg.Features io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots + + if !veleroCfg.UseRestic { + io.UseNodeAgent = veleroCfg.UseNodeAgent + } + io.UseRestic = veleroCfg.UseRestic + io.Image = veleroCfg.VeleroImage + io.Namespace = veleroCfg.VeleroNamespace + io.UploaderType = veleroCfg.UploaderType + GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) + io.GarbageCollectionFrequency = GCFrequency + io.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout + io.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit + io.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest + io.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit + io.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest + io.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit + io.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest + io.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit + io.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest + io.DisableInformerCache = veleroCfg.DisableInformerCache + return io, nil }