mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-08-16 04:06:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #3559 from zubron/add-e2e-test-for-multi-creds

Browse Source 
Add E2E test for multiple credentials
This commit is contained in:
David L. Smith-Uchida
2021-03-10 15:00:26 -08:00
committed by GitHub
parent 55a9b65c17 26c14933cc
commit 574bc16aa1
10 changed files with 165 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -20,7 +20,7 @@ require (
github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
github.com/joho/godotenv v1.3.0
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
github.com/onsi/ginkgo v1.15.0
github.com/onsi/ginkgo v1.15.1
github.com/onsi/gomega v1.10.2
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.7.1
go.sum
+4 -2
View File
@@ -419,6 +419,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -429,8 +431,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
github.com/onsi/ginkgo v1.15.1 h1:DsXNrKujDlkMS9Rsxmd+Fg7S6Kc5lhE+qX8tY6laOxc=
github.com/onsi/ginkgo v1.15.1/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
test/e2e/Makefile
+13 -1
View File
@@ -57,6 +57,13 @@ VSL_CONFIG ?=
CLOUD_PROVIDER ?=
OBJECT_STORE_PROVIDER ?=
# Flags to create an additional BSL for multiple credentials tests
ADDITIONAL_OBJECT_STORE_PROVIDER ?=
ADDITIONAL_CREDS_FILE ?=
ADDITIONAL_BSL_BUCKET ?=
ADDITIONAL_BSL_PREFIX ?=
ADDITIONAL_BSL_CONFIG ?=
.PHONY:ginkgo
ginkgo: # Make sure ginkgo is in $GOPATH/bin
go get github.com/onsi/ginkgo/ginkgo
@@ -78,7 +85,12 @@ run: ginkgo
-bsl-config=$(BSL_CONFIG) \
-vsl-config=$(VSL_CONFIG) \
-cloud-provider=$(CLOUD_PROVIDER) \
-object-store-provider="$(OBJECT_STORE_PROVIDER)"
-object-store-provider="$(OBJECT_STORE_PROVIDER)" \
-additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
-additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG)
build: ginkgo
mkdir -p $(OUTPUT_DIR)
test/e2e/README.md
+17
View File
@@ -41,6 +41,11 @@ the object-store-provider to be specified.
1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional.
1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero".
1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true".
1. `-additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.
1. `-additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.
1. `-additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional.
1. `-additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional.
1. `-additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support.
These configurations or parameters are used to generate install options for Velero for each test suite.
@@ -62,6 +67,11 @@ Below is a mapping between `make` variables to E2E configuration flags.
1. `BSL_PREFIX`: `-prefix`. Optional.
1. `BSL_CONFIG`: `-bsl-config`. Optional.
1. `VSL_CONFIG`: `-vsl-config`. Optional.
1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional.
1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional.
1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional.
1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional.
1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional.
For example, E2E tests can be run from Velero repository roots using the commands below:
@@ -82,6 +92,13 @@ For example, E2E tests can be run from Velero repository roots using the command
```bash
BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=http://192.168.1.124:9000" BSL_PREFIX=veldat BSL_BUCKET=velero CREDS_FILE=~/go/src/github.com/vmware-tanzu/velero/frankie-secrets/credentials-minio PLUGIN_PROVIDER=aws VELERO_IMAGE=projects.registry.vmware.com/tanzu_migrator/velero-pr3133:0.0.5 GINKGO_FOCUS="API group versions" make test-e2e
```
1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location:
```bash
make test-e2e \
CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> BSL_PREFIX=<PREFIX_UNDER_BUCKET> CREDS_FILE=/path/to/aws-creds \
ADDITIONAL_OBJECT_STORE_PROVIDER=azure ADDITIONAL_BSL_BUCKET=<BUCKET_FOR_AZURE_BSL> ADDITIONAL_BSL_PREFIX=<PREFIX_UNDER_BUCKET> ADDITIONAL_BSL_CONFIG=<CONFIG_FOR_AZURE_BUCKET> ADDITIONAL_CREDS_FILE=/path/to/azure-creds
```
Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container)
## Filtering tests
test/e2e/backup_test.go
+51 -3
View File
@@ -3,6 +3,7 @@ package e2e
import (
"context"
"flag"
"fmt"
"time"
"github.com/google/uuid"
@@ -49,14 +50,61 @@ var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for
})
Context("When kibishii is the sample workload", func() {
It("should be successfully backed up and restored", func() {
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + uuidgen.String()
restoreName = "restore-" + uuidgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(),
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "")).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
if additionalBSLProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}
if additionalBSLBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}
if additionalBSLCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}
// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider)
files := map[string]string{
secretKey: additionalBSLCredentials,
}
Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())
// Create additional BSL using credential
additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
Expect(VeleroCreateBackupLocation(context.TODO(),
veleroCLI,
veleroNamespace,
additionalBsl,
additionalBSLProvider,
additionalBSLBucket,
additionalBSLPrefix,
additionalBSLConfig,
secretName,
secretKey,
)).To(Succeed())
bsls := []string{"default", additionalBsl}
for _, bsl := range bsls {
backupName = fmt.Sprintf("backup-%s-%s", bsl, uuidgen)
restoreName = fmt.Sprintf("restore-%s-%s", bsl, uuidgen)
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
}
})
})
})
test/e2e/common.go
+19
View File
@@ -2,9 +2,11 @@ package e2e
import (
"fmt"
"io/ioutil"
"os/exec"
"time"
"github.com/pkg/errors"
"golang.org/x/net/context"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,3 +46,20 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete
})
return err
}
func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error {
data := make(map[string][]byte)
for key, filePath := range files {
contents, err := ioutil.ReadFile(filePath)
if err != nil {
return errors.WithMessagef(err, "Failed to read secret file %q", filePath)
}
data[key] = contents
}
secret := builder.ForSecret(namespace, name).Data(data).Result()
_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
return err
}
test/e2e/e2e_suite_test.go
+8
View File
@@ -10,6 +10,7 @@ import (
var (
veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string
additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials string
installVelero, useVolumeSnapshots bool
)
@@ -26,6 +27,13 @@ func init() {
flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.")
flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup. Optional")
// Flags to create an additional BSL for multiple credentials test
flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
flag.StringVar(&additionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&additionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
}
func TestE2e(t *testing.T) {
test/e2e/enable_api_group_versions_test.go
+1 -1
View File
@@ -230,7 +230,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
namespacesStr := strings.Join(tc.namespaces, ",")
err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr)
err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "")
if err != nil {
VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup)
return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr)
test/e2e/kibishii_tests.go
+2 -2
View File
@@ -85,7 +85,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
}
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error {
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string) error {
fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
timeout := 10 * time.Minute
@@ -103,7 +103,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel
return errors.Wrap(err, "Failed to generate data")
}
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil {
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation); err != nil {
VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName)
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
test/e2e/velero_utils.go
+49 -3
View File
@@ -210,11 +210,20 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
}
// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error {
backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string) error {
args := []string{
"--namespace", veleroNamespace,
"create", "backup", backupName,
"--include-namespaces", namespace,
"--default-volumes-to-restic", "--wait")
"--default-volumes-to-restic",
"--wait",
}
if backupLocation != "" {
args = append(args, "--storage-location", backupLocation)
}
backupCmd := exec.CommandContext(ctx, veleroCLI, args...)
backupCmd.Stdout = os.Stdout
backupCmd.Stderr = os.Stderr
fmt.Printf("backup cmd =%v\n", backupCmd)
@@ -314,3 +323,40 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st
}
return nil
}
func VeleroCreateBackupLocation(ctx context.Context,
veleroCLI string,
veleroNamespace string,
name string,
objectStoreProvider string,
bucket string,
prefix string,
config string,
secretName string,
secretKey string,
) error {
args := []string{
"--namespace", veleroNamespace,
"create", "backup-location", name,
"--provider", objectStoreProvider,
"--bucket", bucket,
}
if prefix != "" {
args = append(args, "--prefix", prefix)
}
if config != "" {
args = append(args, "--config", config)
}
if secretName != "" && secretKey != "" {
args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
}
bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...)
bslCreateCmd.Stdout = os.Stdout
bslCreateCmd.Stderr = os.Stderr
return bslCreateCmd.Run()
}