diff --git a/pkg/install/install.go b/pkg/install/install.go index 85257f4ad..ebe4c3751 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -50,6 +50,7 @@ var kindToResource = map[string]string{ "Deployment": "deployments", "DaemonSet": "daemonsets", "Secret": "secrets", + "ConfigMap": "configmaps", "BackupStorageLocation": "backupstoragelocations", "VolumeSnapshotLocation": "volumesnapshotlocations", } diff --git a/test/e2e/Makefile b/test/e2e/Makefile index ed47614b2..6f7140463 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main +RESTIC_HELPER_IMAGE ?= CRDS_VERSION ?= v1 VELERO_NAMESPACE ?= velero CREDS_FILE ?= @@ -81,6 +82,7 @@ run: ginkgo (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER="; exit 1) @$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \ -velero-image=$(VELERO_IMAGE) \ + -restic-helper-image=$(RESTIC_HELPER_IMAGE) \ -velero-namespace=$(VELERO_NAMESPACE) \ -crds-version=$(CRDS_VERSION) \ -credentials-file=$(CREDS_FILE) \ diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index b906759b3..6e53abc17 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -59,7 +59,7 @@ func backup_restore_test(useVolumeSnapshots bool) { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7f0712e31..8cc3dfe96 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -27,7 +27,7 @@ import ( var ( veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string - additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile string + additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage string installVelero bool ) @@ -38,6 +38,7 @@ func init() { flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.") flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") + flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.") flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 42f9bf23f..9a1eebcf2 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -60,6 +60,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", context.Background(), veleroCLI, veleroImage, + resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, diff --git a/test/e2e/install.go b/test/e2e/install.go index 0d31f8d82..7a57acf24 100644 --- a/test/e2e/install.go +++ b/test/e2e/install.go @@ -41,10 +41,11 @@ import ( type installOptions struct { *install.InstallOptions RegistryCredentialFile string + ResticHelperImage string } // TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items -func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, +func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, crdsVersion string, features string, registryCredentialFile string) error { @@ -88,6 +89,7 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace err = installVeleroServer(ctx, cli, &installOptions{ InstallOptions: veleroInstallOptions, RegistryCredentialFile: registryCredentialFile, + ResticHelperImage: resticHelperImage, }) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") @@ -140,14 +142,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption args = append(args, "--features", options.Features) } - if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile); err != nil { + if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil { return err } return waitVeleroReady(ctx, namespace, options.UseRestic) } -func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile string) error { +func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile, resticHelperImage string) error { args = append(args, "--dry-run", "--output", "json", "--crds-only") // get the CRD definitions @@ -192,7 +194,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout)) } - if err = patchResources(ctx, resources, namespace, registryCredentialFile); err != nil { + if err = patchResources(ctx, resources, namespace, registryCredentialFile, resticHelperImage); err != nil { return errors.Wrapf(err, "failed to patch resources") } @@ -214,7 +216,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st } // patch the velero resources -func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile string) error { +func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile, resticHelperImage string) error { // apply the image pull secret to avoid the image pull limit of Docker Hub if len(registryCredentialFile) > 0 { credential, err := ioutil.ReadFile(registryCredentialFile) @@ -257,6 +259,34 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis resources.Items = append(resources.Items, un) } + // customize the restic restore helper image + if len(resticHelperImage) > 0 { + restoreActionConfig := corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "restic-restore-action-config", + Namespace: namespace, + Labels: map[string]string{ + "velero.io/plugin-config": "", + "velero.io/restic": "RestoreItemAction", + }, + }, + Data: map[string]string{ + "image": resticHelperImage, + }, + } + + un, err := toUnstructured(restoreActionConfig) + if err != nil { + return errors.Wrapf(err, "failed to convert restore action config to unstructure") + } + resources.Items = append(resources.Items, un) + fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config") + } + return nil } diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 27c32b8e9..d15893d72 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -81,9 +81,9 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes)) fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd) - _, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) + stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) if err != nil { - return errors.Wrapf(err, "failed to verify, stderr=%s", stderr) + return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout) } return nil } @@ -97,7 +97,8 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { - if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil { + // if other functions runs timeout, the defer has no change to run, so use a separated context rather than the "oneHourTimeout" to avoid this + if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) } }() @@ -144,6 +145,13 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace) } + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if providerName == "aws" && useVolumeSnapshots { + fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") + time.Sleep(5 * time.Minute) + } if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) diff --git a/test/e2e/multiple_namespaces_test.go b/test/e2e/multiple_namespaces_test.go index 8a1673b80..b9a73a9ba 100644 --- a/test/e2e/multiple_namespaces_test.go +++ b/test/e2e/multiple_namespaces_test.go @@ -27,7 +27,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -62,7 +62,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -87,7 +87,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { }) func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error { - defer cleanupNamespaces(ctx, client, nsBaseName) // Run at exit for final cleanup + defer cleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup var excludeNamespaces []string // Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead