mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-05 04:55:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add a new EKS pipeline with IRSA as credential

Browse Source 
Signed-off-by: danfengl <danfengl@vmware.com>
This commit is contained in:
danfengl
2024-01-04 02:46:06 +00:00
parent 1034d6aee0
commit df585053e7
12 changed files with 271 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
test/e2e/Makefile
View File

@@ -67,6 +67,7 @@ MIGRATE_FROM_VELERO_CLI ?=
VELERO_NAMESPACE ?= velero
CREDS_FILE ?=
SERVICE_ACCOUNT_NAME ?=
BSL_BUCKET ?=
BSL_PREFIX ?=
BSL_CONFIG ?=
@@ -106,6 +107,10 @@ SNAPSHOT_MOVE_DATA ?= false
DATA_MOVER_PLUGIN ?=
DISABLE_INFORMER_CACHE ?= false
DEFAULT_CLUSTER_NAME ?=
STANDBY_CLUSTER_NAME ?=
EKS_POLICY_ARN ?=
.PHONY:ginkgo
ginkgo: # Make sure ginkgo is in $GOPATH/bin
@@ -157,8 +162,11 @@ run: ginkgo
-standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \
-standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \
-debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \
-disable-informer-cache=$(DISABLE_INFORMER_CACHE)
-disable-informer-cache=$(DISABLE_INFORMER_CACHE) \
-default-cluster-name=$(DEFAULT_CLUSTER_NAME) \
-standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
-eks-policy-arn=$(EKS_POLICY_ARN) \
-service-account-name=$(SERVICE_ACCOUNT_NAME)
build: ginkgo
mkdir -p $(OUTPUT_DIR)

26
test/e2e/basic/api-group/enable_api_group_extentions.go
View File

@@ -39,11 +39,11 @@ func APIExtensionsVersionsTest() {
label := "for=backup"
srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
BeforeEach(func() {
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
Skip("CRD with apiextension versions migration test needs 2 clusters")
}
veleroCfg = VeleroCfg
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName)
Expect(err).ShouldNot(HaveOccurred())
dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName)
@@ -75,19 +75,19 @@ func APIExtensionsVersionsTest() {
By("Uninstall Velero and delete CRD ", func() {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
defer ctxCancel()
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
})
}
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
})
}
@@ -98,14 +98,14 @@ func APIExtensionsVersionsTest() {
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
veleroCfg.Features = "EnableAPIGroupVersions"
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
})
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultClusterContext), func() {
Expect(InstallCRD(context.Background(), srcCrdYaml)).To(Succeed())
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
Expect(WaitForCRDEstablished(crdName)).To(Succeed())
@@ -128,17 +128,17 @@ func APIExtensionsVersionsTest() {
})
})
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
})
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyClusterContext), func() {
Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
})

20
test/e2e/e2e_suite_test.go
View File

@@ -84,8 +84,8 @@ func init() {
flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.")
flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.")
flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.")
flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster", "", "default cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster", "", "default cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.")
flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.")
flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin")
@@ -95,6 +95,11 @@ func init() {
flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.")
flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.")
flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.")
flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.")
flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.")
flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.")
flag.StringVar(&VeleroCfg.ServiceAccountName, "service-account-name", "", "service account name.")
}
var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest)
@@ -164,20 +169,21 @@ var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume clai
func GetKubeconfigContext() error {
var err error
var tcDefault, tcStandby TestClient
tcDefault, err = NewTestClient(VeleroCfg.DefaultCluster)
tcDefault, err = NewTestClient(VeleroCfg.DefaultClusterContext)
VeleroCfg.DefaultClient = &tcDefault
VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName
if err != nil {
return err
}
if VeleroCfg.DefaultCluster != "" {
err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)
if VeleroCfg.DefaultClusterContext != "" {
err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultClusterContext)
if err != nil {
return err
}
if VeleroCfg.StandbyCluster != "" {
tcStandby, err = NewTestClient(VeleroCfg.StandbyCluster)
if VeleroCfg.StandbyClusterContext != "" {
tcStandby, err = NewTestClient(VeleroCfg.StandbyClusterContext)
VeleroCfg.StandbyClient = &tcStandby
if err != nil {
return err

27
test/e2e/migration/migration.go
View File

@@ -64,7 +64,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
Skip("Volume snapshots not supported on kind")
}
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
Skip("Migration test needs 2 clusters")
}
// need to uninstall Velero first in case of the affection of the existing global velero installation
@@ -79,19 +79,19 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
AfterEach(func() {
if !veleroCfg.Debug {
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultCluster), func() {
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
defer ctxCancel()
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true)
})
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() {
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
defer ctxCancel()
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
@@ -102,9 +102,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
})
}
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
})
}
@@ -142,11 +143,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
}
OriginVeleroCfg := veleroCfg
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
@@ -272,20 +274,21 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
time.Sleep(5 * time.Minute)
}
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
//Ensure workload of "migrationNamespace" existed in cluster-A
ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
Expect(ns.Name).To(Equal(migrationNamespace))
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err))
//Ensure cluster-B is the target cluster
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
Expect(err).To(HaveOccurred())
strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
fmt.Println(err)
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
veleroCfg.UseNodeAgent = !useVolumeSnapshots
veleroCfg.UseRestic = false
if veleroCfg.SnapshotMoveData {
@@ -299,7 +302,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed())
})

2
test/perf/e2e_suite_test.go
View File

@@ -69,7 +69,7 @@ func init() {
flag.DurationVar(&VeleroCfg.PodVolumeOperationTimeout, "pod-volume-operation-timeout", 360*time.Minute, "Timeout for pod volume operations. Optional.")
//vmware-tanzu-experiments
flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.")
flag.StringVar(&VeleroCfg.DefaultCluster, "default-cluster-context", "", "Default cluster context for migration test.")
flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.")
flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", true, "Switch to control namespace cleaning.")
flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.")
flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.")

10
test/types.go
View File

@@ -71,11 +71,14 @@ type VeleroConfig struct {
KibishiiDirectory string
Debug bool
GCFrequency string
DefaultCluster string
StandbyCluster string
DefaultClusterContext string
StandbyClusterContext string
ClientToInstallVelero *TestClient
DefaultClient *TestClient
StandbyClient *TestClient
ClusterToInstallVelero string
DefaultClusterName string
StandbyClusterName string
ProvideSnapshotsVolumeParam bool
VeleroServerDebugMode bool
SnapshotMoveData bool
@@ -87,6 +90,9 @@ type VeleroConfig struct {
IsUpgradeTest bool
WithoutDisableInformerCacheParam bool
DisableInformerCache bool
CreateClusterRoleBinding bool
ServiceAccountName string
EKSPolicyARN string
}
type VeleroCfgInPerf struct {

65
test/util/eks/eks.go Normal file
View File

@@ -0,0 +1,65 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8s
import (
"fmt"
"os/exec"
"strings"
"time"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/util/wait"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
)
func KubectlDeleteIAMServiceAcount(ctx context.Context, name, namespace, cluster string) error {
args := []string{"delete", "iamserviceaccount", name,
"--namespace", namespace, "--cluster", cluster, "--wait"}
fmt.Println(args)
cmd := exec.CommandContext(ctx, "eksctl", args...)
fmt.Println(cmd)
stdout, stderr, err := veleroexec.RunCommand(cmd)
fmt.Printf("Output: %v\n", stdout)
if strings.Contains(stderr, "NotFound") {
err = nil
}
fmt.Printf("err: %v\n", err)
return err
}
func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error {
args := []string{"create", "iamserviceaccount", name,
"--namespace", namespace, "--cluster", cluster, "--attach-policy-arn", policyARN,
"--approve", "--override-existing-serviceaccounts"}
PollInterval := 1 * time.Minute
PollTimeout := 10 * time.Minute
return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
cmd := exec.CommandContext(ctx, "eksctl", args...)
fmt.Println(cmd)
stdout, stderr, err := veleroexec.RunCommand(cmd)
fmt.Printf("Output: %v|%v|%v\n", stdout, stderr, err)
if err != nil {
fmt.Printf("err: %v|%v|%v\n", stdout, stderr, err)
return false, nil
}
return true, nil
})
}

47
test/util/k8s/clusterrolebinding.go Normal file
View File

@@ -0,0 +1,47 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8s
import (
"fmt"
"os/exec"
"strings"
"golang.org/x/net/context"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
)
func KubectlDeleteClusterRoleBinding(ctx context.Context, name string) error {
args := []string{"delete", "clusterrolebinding", name}
fmt.Println(args)
cmd := exec.CommandContext(ctx, "kubectl", args...)
fmt.Println(cmd)
_, stderr, err := veleroexec.RunCommand(cmd)
fmt.Printf("Ignore error: %v\n", stderr)
if strings.Contains(stderr, "NotFound") {
fmt.Printf("Ignore error: %v\n", stderr)
err = nil
}
return err
}
func KubectlCreateClusterRoleBinding(ctx context.Context, name, clusterrole, namespace, serviceaccount string) error {
args := []string{"create", "clusterrolebinding", name, fmt.Sprintf("--clusterrole=%s", clusterrole), fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceaccount)}
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}

8
test/util/k8s/namespace.go
View File

@@ -19,12 +19,12 @@ package k8s
import (
"context"
"fmt"
"os/exec"
"strings"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -233,3 +233,9 @@ func GetMappingNamespaces(ctx context.Context, client TestClient, excludeNS []st
}
return joinedNsMapping, nil
}
func KubectlCreateNamespace(ctx context.Context, name string) error {
args := []string{"create", "namespace", name}
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}

16
test/util/providers/common.go
View File

@@ -47,6 +47,10 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
var err error
var exist bool
fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping object storebackup checkpoint %s\n", backupName)
return nil
}
for i := 0; i < retryTimes; i++ {
exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
if err != nil {
@@ -98,6 +102,10 @@ func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix
func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error {
bslPrefix = getFullPrefix(bslPrefix, subPrefix)
fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
}
s, err := getProvider(cloudProvider)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider))
@@ -111,6 +119,10 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
}
snapshotCheckPoint.ExpectCount = 0
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
@@ -122,6 +134,10 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
}
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s do not exist in cloud after backup as expected", backupName))

65
test/util/velero/install.go
View File

@@ -39,6 +39,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test"
. "github.com/vmware-tanzu/velero/test/util/eks"
. "github.com/vmware-tanzu/velero/test/util/k8s"
)
@@ -75,7 +76,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
}
if veleroCfg.CloudProvider != "kind" {
fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider")
fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider\n")
// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
if veleroCfg.ObjectStoreProvider == "" {
veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
@@ -112,27 +113,35 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
if err != nil {
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
if !veleroCfg.UseRestic {
veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
}
veleroInstallOptions.UseRestic = veleroCfg.UseRestic
veleroInstallOptions.Image = veleroCfg.VeleroImage
veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
veleroInstallOptions.UploaderType = veleroCfg.UploaderType
GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
veleroInstallOptions.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout
veleroInstallOptions.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit
veleroInstallOptions.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest
veleroInstallOptions.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit
veleroInstallOptions.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest
veleroInstallOptions.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit
veleroInstallOptions.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest
veleroInstallOptions.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit
veleroInstallOptions.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest
veleroInstallOptions.DisableInformerCache = veleroCfg.DisableInformerCache
if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" {
_, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace)
if !apierrors.IsNotFound(err) {
if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace)
}
}
if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace)
}
if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace)
}
if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace)
}
if err := KubectlDeleteIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to delete service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace)
}
time.Sleep(10 * time.Second)
if err := KubectlCreateIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to create service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace)
}
}
err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{
Options: veleroInstallOptions,
RegistryCredentialFile: veleroCfg.RegistryCredentialFile,
@@ -239,8 +248,16 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
if len(options.Prefix) > 0 {
args = append(args, "--prefix", options.Prefix)
}
if len(options.SecretFile) > 0 {
args = append(args, "--secret-file", options.SecretFile)
//Treat ServiceAccountName priority higher than SecretFile
if len(options.ServiceAccountName) > 0 {
args = append(args, "--service-account-name", options.ServiceAccountName)
} else {
if len(options.SecretFile) > 0 {
args = append(args, "--secret-file", options.SecretFile)
}
}
if options.NoSecret {
args = append(args, "--no-secret")
}
if len(options.VolumeSnapshotConfig.Data()) > 0 {
args = append(args, "--snapshot-location-config", options.VolumeSnapshotConfig.String())
@@ -257,11 +274,11 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
}
}
fmt.Println("Start to install Azure VolumeSnapshotClass ...")
if len(options.Features) > 0 {
args = append(args, "--features", options.Features)
if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots {
if strings.EqualFold(cloudProvider, "azure") {
fmt.Println("Start to install Azure VolumeSnapshotClass ...")
if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
return err
}

43
test/util/velero/velero_utils.go
View File

@@ -174,20 +174,27 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st
func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
plugins []string) (*cliinstall.Options, error) {
if veleroCfg.CloudCredentialsFile == "" {
if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountName == "" {
return nil, errors.Errorf("No credentials were supplied to use for E2E tests")
}
realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
if err != nil {
return nil, err
}
io := cliinstall.NewInstallOptions()
// always wait for velero and restic pods to be running.
io.Wait = true
io.ProviderName = veleroCfg.ObjectStoreProvider
io.SecretFile = veleroCfg.CloudCredentialsFile
if veleroCfg.CloudCredentialsFile != "" {
realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
if err != nil {
return nil, err
}
io.SecretFile = realPath
}
if veleroCfg.ServiceAccountName != "" {
io.ServiceAccountName = veleroCfg.ServiceAccountName
io.NoSecret = true
}
io.BucketName = veleroCfg.BSLBucket
io.Prefix = veleroCfg.BSLPrefix
@@ -197,11 +204,31 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
io.VolumeSnapshotConfig = flag.NewMap()
io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig)
io.SecretFile = realPath
io.Plugins = flag.NewStringArray(plugins...)
io.Features = veleroCfg.Features
io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup
io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
if !veleroCfg.UseRestic {
io.UseNodeAgent = veleroCfg.UseNodeAgent
}
io.UseRestic = veleroCfg.UseRestic
io.Image = veleroCfg.VeleroImage
io.Namespace = veleroCfg.VeleroNamespace
io.UploaderType = veleroCfg.UploaderType
GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
io.GarbageCollectionFrequency = GCFrequency
io.PodVolumeOperationTimeout = veleroCfg.PodVolumeOperationTimeout
io.NodeAgentPodCPULimit = veleroCfg.NodeAgentPodCPULimit
io.NodeAgentPodCPURequest = veleroCfg.NodeAgentPodCPURequest
io.NodeAgentPodMemLimit = veleroCfg.NodeAgentPodMemLimit
io.NodeAgentPodMemRequest = veleroCfg.NodeAgentPodMemRequest
io.VeleroPodCPULimit = veleroCfg.VeleroPodCPULimit
io.VeleroPodCPURequest = veleroCfg.VeleroPodCPURequest
io.VeleroPodMemLimit = veleroCfg.VeleroPodMemLimit
io.VeleroPodMemRequest = veleroCfg.VeleroPodMemRequest
io.DisableInformerCache = veleroCfg.DisableInformerCache
return io, nil
}