From 9ad1e898d67180f6296f79d8e48b36ef2b15753a Mon Sep 17 00:00:00 2001 From: Bridget McErlean Date: Thu, 4 Mar 2021 13:19:42 -0500 Subject: [PATCH] Add E2E test for multiple credentials This change adds an E2E test which exercises the mulitple credentials feature added in #3489. The test creates a secret from the given credentials and creates a BackupStorageLocation which uses those credentials. A backup and restore is then performed to the default BSL and to the newly created BSL. This change adds new flags to the E2E test suite to configure the BSL created and used in the test. Signed-off-by: Bridget McErlean --- go.mod | 2 +- go.sum | 6 ++- test/e2e/Makefile | 14 +++++- test/e2e/README.md | 17 +++++++ test/e2e/backup_test.go | 54 ++++++++++++++++++++-- test/e2e/common.go | 19 ++++++++ test/e2e/e2e_suite_test.go | 8 ++++ test/e2e/enable_api_group_versions_test.go | 2 +- test/e2e/kibishii_tests.go | 4 +- test/e2e/velero_utils.go | 52 +++++++++++++++++++-- 10 files changed, 165 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 22536bf57..a0125034f 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26 github.com/joho/godotenv v1.3.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0 - github.com/onsi/ginkgo v1.15.0 + github.com/onsi/ginkgo v1.15.1 github.com/onsi/gomega v1.10.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.7.1 diff --git a/go.sum b/go.sum index b42b8b609..b99400f65 100644 --- a/go.sum +++ b/go.sum @@ -419,6 +419,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -429,8 +431,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= -github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/ginkgo v1.15.1 h1:DsXNrKujDlkMS9Rsxmd+Fg7S6Kc5lhE+qX8tY6laOxc= +github.com/onsi/ginkgo v1.15.1/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 931c55ce6..2d43d85ae 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -57,6 +57,13 @@ VSL_CONFIG ?= CLOUD_PROVIDER ?= OBJECT_STORE_PROVIDER ?= +# Flags to create an additional BSL for multiple credentials tests +ADDITIONAL_OBJECT_STORE_PROVIDER ?= +ADDITIONAL_CREDS_FILE ?= +ADDITIONAL_BSL_BUCKET ?= +ADDITIONAL_BSL_PREFIX ?= +ADDITIONAL_BSL_CONFIG ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin go get github.com/onsi/ginkgo/ginkgo @@ -78,7 +85,12 @@ run: ginkgo -bsl-config=$(BSL_CONFIG) \ -vsl-config=$(VSL_CONFIG) \ -cloud-provider=$(CLOUD_PROVIDER) \ - -object-store-provider="$(OBJECT_STORE_PROVIDER)" + -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ + -additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ + -additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ + -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ + -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ + -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/README.md b/test/e2e/README.md index b6a4e9b3b..08eeb40c3 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -41,6 +41,11 @@ the object-store-provider to be specified. 1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional. 1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero". 1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true". +1. `-additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support. +1. `-additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support. +1. `-additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional. +1. `-additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional. +1. `-additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -62,6 +67,11 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `BSL_PREFIX`: `-prefix`. Optional. 1. `BSL_CONFIG`: `-bsl-config`. Optional. 1. `VSL_CONFIG`: `-vsl-config`. Optional. +1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional. +1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional. +1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional. +1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional. +1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional. For example, E2E tests can be run from Velero repository roots using the commands below: @@ -82,6 +92,13 @@ For example, E2E tests can be run from Velero repository roots using the command ```bash BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=http://192.168.1.124:9000" BSL_PREFIX=veldat BSL_BUCKET=velero CREDS_FILE=~/go/src/github.com/vmware-tanzu/velero/frankie-secrets/credentials-minio PLUGIN_PROVIDER=aws VELERO_IMAGE=projects.registry.vmware.com/tanzu_migrator/velero-pr3133:0.0.5 GINKGO_FOCUS="API group versions" make test-e2e ``` +1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location: + ```bash + make test-e2e \ + CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws BSL_BUCKET= BSL_PREFIX= CREDS_FILE=/path/to/aws-creds \ + ADDITIONAL_OBJECT_STORE_PROVIDER=azure ADDITIONAL_BSL_BUCKET= ADDITIONAL_BSL_PREFIX= ADDITIONAL_BSL_CONFIG= ADDITIONAL_CREDS_FILE=/path/to/azure-creds + ``` + Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container) ## Filtering tests diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index fe6359974..c44a491d0 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -3,6 +3,7 @@ package e2e import ( "context" "flag" + "fmt" "time" "github.com/google/uuid" @@ -49,14 +50,61 @@ var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for }) - Context("When kibishii is the sample workload", func() { - It("should be successfully backed up and restored", func() { + When("kibishii is the sample workload", func() { + It("should be successfully backed up and restored to the default BackupStorageLocation", func() { backupName = "backup-" + uuidgen.String() restoreName = "restore-" + uuidgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(), + Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "")).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) + + It("should successfully back up and restore to multiple BackupStorageLocations with unique credentials", func() { + if additionalBslProvider == "" { + Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") + } + + if additionalBslBucket == "" { + Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests") + } + + if additionalBslCredentials == "" { + Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") + } + + // Create Secret for additional BSL + secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) + secretKey := fmt.Sprintf("creds-%s", additionalBslProvider) + files := map[string]string{ + secretKey: additionalBslCredentials, + } + + Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + + // Create additional BSL using credential + additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) + Expect(VeleroCreateBackupLocation(context.TODO(), + veleroCLI, + veleroNamespace, + additionalBsl, + additionalBslProvider, + additionalBslBucket, + additionalBslPrefix, + additionalBslConfig, + secretName, + secretKey, + )).To(Succeed()) + + bsls := []string{"default", additionalBsl} + + for _, bsl := range bsls { + backupName = fmt.Sprintf("backup-%s-%s", bsl, uuidgen) + restoreName = fmt.Sprintf("restore-%s-%s", bsl, uuidgen) + + Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl)).To(Succeed(), + "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) + } + }) }) }) diff --git a/test/e2e/common.go b/test/e2e/common.go index 41e4f5002..961eea24b 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -2,9 +2,11 @@ package e2e import ( "fmt" + "io/ioutil" "os/exec" "time" + "github.com/pkg/errors" "golang.org/x/net/context" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,3 +46,20 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete }) return err } + +func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { + data := make(map[string][]byte) + + for key, filePath := range files { + contents, err := ioutil.ReadFile(filePath) + if err != nil { + return errors.WithMessagef(err, "Failed to read secret file %q", filePath) + } + + data[key] = contents + } + + secret := builder.ForSecret(namespace, name).Data(data).Result() + _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + return err +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index f6ec08630..e53bd3f92 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -10,6 +10,7 @@ import ( var ( veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string + additionalBslProvider, additionalBslBucket, additionalBslPrefix, additionalBslConfig, additionalBslCredentials string installVelero, useVolumeSnapshots bool ) @@ -26,6 +27,13 @@ func init() { flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into") flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.") flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup. Optional") + + // Flags to create an additional BSL for multiple credentials test + flag.StringVar(&additionalBslProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&additionalBslBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") + flag.StringVar(&additionalBslPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") + flag.StringVar(&additionalBslConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") + flag.StringVar(&additionalBslCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") } func TestE2e(t *testing.T) { diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 29e463e48..dc5b69712 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -230,7 +230,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr) + err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "") if err != nil { VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr) diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 868322f0f..416cb8b83 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -85,7 +85,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error { +func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) timeout := 10 * time.Minute @@ -103,7 +103,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to generate data") } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation); err != nil { VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index b120b6b81..9a00be799 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -210,11 +210,20 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st } // VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error { - backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName, +func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string) error { + args := []string{ + "--namespace", veleroNamespace, + "create", "backup", backupName, "--include-namespaces", namespace, - "--default-volumes-to-restic", "--wait") + "--default-volumes-to-restic", + "--wait", + } + if backupLocation != "" { + args = append(args, "--storage-location", backupLocation) + } + + backupCmd := exec.CommandContext(ctx, veleroCLI, args...) backupCmd.Stdout = os.Stdout backupCmd.Stderr = os.Stderr fmt.Printf("backup cmd =%v\n", backupCmd) @@ -314,3 +323,40 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st } return nil } + +func VeleroCreateBackupLocation(ctx context.Context, + veleroCLI string, + veleroNamespace string, + name string, + objectStoreProvider string, + bucket string, + prefix string, + config string, + secretName string, + secretKey string, +) error { + args := []string{ + "--namespace", veleroNamespace, + "create", "backup-location", name, + "--provider", objectStoreProvider, + "--bucket", bucket, + } + + if prefix != "" { + args = append(args, "--prefix", prefix) + } + + if config != "" { + args = append(args, "--config", config) + } + + if secretName != "" && secretKey != "" { + args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey)) + } + + bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...) + bslCreateCmd.Stdout = os.Stdout + bslCreateCmd.Stderr = os.Stderr + + return bslCreateCmd.Run() +}