Several fixes to improve the stability of E2E testing (#4056)

1. Support to customize the restic restore helper image 2. Use a seperated context when doing the clean up works 3. Wait a while before doing the the restore for aws to avoid #1799 Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
2026-01-04 04:04:02 +00:00 · 2021-09-01 00:50:38 +08:00
parent 7c75cd6cf8
commit 8d57215ded
8 changed files with 56 additions and 13 deletions
--- a/pkg/install/install.go
+++ b/pkg/install/install.go
@@ -50,6 +50,7 @@ var kindToResource = map[string]string{
 	"Deployment":               "deployments",
 	"DaemonSet":                "daemonsets",
 	"Secret":                   "secrets",
+	"ConfigMap":                "configmaps",
 	"BackupStorageLocation":    "backupstoragelocations",
 	"VolumeSnapshotLocation":   "volumesnapshotlocations",
 }
--- a/test/e2e/Makefile
+++ b/test/e2e/Makefile
@@ -48,6 +48,7 @@ OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
 VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero
 VELERO_IMAGE ?= velero/velero:main
+RESTIC_HELPER_IMAGE ?=
 CRDS_VERSION ?= v1
 VELERO_NAMESPACE ?= velero
 CREDS_FILE ?=
@@ -81,6 +82,7 @@ run: ginkgo
 		    (echo "Cloud provider for target cloud/plug-in provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
 	@$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=$(VELERO_CLI) \
 		-velero-image=$(VELERO_IMAGE) \
+		-restic-helper-image=$(RESTIC_HELPER_IMAGE) \
 		-velero-namespace=$(VELERO_NAMESPACE) \
 		-crds-version=$(CRDS_VERSION) \
 		-credentials-file=$(CREDS_FILE) \
--- a/test/e2e/backup_test.go
+++ b/test/e2e/backup_test.go
@@ -59,7 +59,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
--- a/test/e2e/e2e_suite_test.go
+++ b/test/e2e/e2e_suite_test.go
@@ -27,7 +27,7 @@ import (

 var (
 	veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string
-	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile                     string
+	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile, resticHelperImage  string
 	installVelero                                                                                                                                              bool
 )

@@ -38,6 +38,7 @@ func init() {
 	flag.StringVar(&cloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.")
 	flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use.")
 	flag.StringVar(&veleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.")
+	flag.StringVar(&resticHelperImage, "restic-helper-image", "", "image for the velero restic restore helper to be tested.")
 	flag.StringVar(&bslConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2")
 	flag.StringVar(&bslPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.")
 	flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
--- a/test/e2e/enable_api_group_versions_test.go
+++ b/test/e2e/enable_api_group_versions_test.go
@@ -60,6 +60,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
 				context.Background(),
 				veleroCLI,
 				veleroImage,
+				resticHelperImage,
 				veleroNamespace,
 				cloudProvider,
 				objectStoreProvider,
--- a/test/e2e/install.go
+++ b/test/e2e/install.go
@@ -41,10 +41,11 @@ import (
 type installOptions struct {
 	*install.InstallOptions
 	RegistryCredentialFile string
+	ResticHelperImage      string
 }

 // TODO too many parameters for this function, better to make it a structure, we can introduces a structure `config` for the E2E to hold all configuration items
-func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool,
+func veleroInstall(ctx context.Context, cli, veleroImage string, resticHelperImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool,
 	cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
 	crdsVersion string, features string, registryCredentialFile string) error {

@@ -88,6 +89,7 @@ func veleroInstall(ctx context.Context, cli, veleroImage string, veleroNamespace
 	err = installVeleroServer(ctx, cli, &installOptions{
 		InstallOptions:         veleroInstallOptions,
 		RegistryCredentialFile: registryCredentialFile,
+		ResticHelperImage:      resticHelperImage,
 	})
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
@@ -140,14 +142,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
 		args = append(args, "--features", options.Features)
 	}

-	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile); err != nil {
+	if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {
 		return err
 	}

 	return waitVeleroReady(ctx, namespace, options.UseRestic)
 }

-func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile string) error {
+func createVelereResources(ctx context.Context, cli, namespace string, args []string, registryCredentialFile, resticHelperImage string) error {
 	args = append(args, "--dry-run", "--output", "json", "--crds-only")

 	// get the CRD definitions
@@ -192,7 +194,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
 		return errors.Wrapf(err, "failed to unmarshal the resources: %s", string(stdout))
 	}

-	if err = patchResources(ctx, resources, namespace, registryCredentialFile); err != nil {
+	if err = patchResources(ctx, resources, namespace, registryCredentialFile, resticHelperImage); err != nil {
 		return errors.Wrapf(err, "failed to patch resources")
 	}

@@ -214,7 +216,7 @@ func createVelereResources(ctx context.Context, cli, namespace string, args []st
 }

 // patch the velero resources
-func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile string) error {
+func patchResources(ctx context.Context, resources *unstructured.UnstructuredList, namespace, registryCredentialFile, resticHelperImage string) error {
 	// apply the image pull secret to avoid the image pull limit of Docker Hub
 	if len(registryCredentialFile) > 0 {
 		credential, err := ioutil.ReadFile(registryCredentialFile)
@@ -257,6 +259,34 @@ func patchResources(ctx context.Context, resources *unstructured.UnstructuredLis
 		resources.Items = append(resources.Items, un)
 	}

+	// customize the restic restore helper image
+	if len(resticHelperImage) > 0 {
+		restoreActionConfig := corev1.ConfigMap{
+			TypeMeta: metav1.TypeMeta{
+				Kind:       "ConfigMap",
+				APIVersion: corev1.SchemeGroupVersion.String(),
+			},
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "restic-restore-action-config",
+				Namespace: namespace,
+				Labels: map[string]string{
+					"velero.io/plugin-config": "",
+					"velero.io/restic":        "RestoreItemAction",
+				},
+			},
+			Data: map[string]string{
+				"image": resticHelperImage,
+			},
+		}
+
+		un, err := toUnstructured(restoreActionConfig)
+		if err != nil {
+			return errors.Wrapf(err, "failed to convert restore action config to unstructure")
+		}
+		resources.Items = append(resources.Items, un)
+		fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config")
+	}
+
 	return nil
 }

--- a/test/e2e/kibishii_tests.go
+++ b/test/e2e/kibishii_tests.go
@@ -81,9 +81,9 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
 		strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
 	fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)

-	_, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
+	stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
 	if err != nil {
-		return errors.Wrapf(err, "failed to verify, stderr=%s", stderr)
+		return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout)
 	}
 	return nil
 }
@@ -97,7 +97,8 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
 	}
 	defer func() {
-		if err := deleteNamespace(oneHourTimeout, client, kibishiiNamespace, true); err != nil {
+		// if other functions runs timeout, the defer has no change to run, so use a separated context rather than the "oneHourTimeout" to avoid this
+		if err := deleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
 			fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace))
 		}
 	}()
@@ -144,6 +145,13 @@ func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac
 		return errors.Wrapf(err, "failed to delete namespace %s", kibishiiNamespace)
 	}

+	// the snapshots of AWS may be still in pending status when do the restore, wait for a while
+	// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
+	// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
+	if providerName == "aws" && useVolumeSnapshots {
+		fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
+		time.Sleep(5 * time.Minute)
+	}
 	if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
 		veleroRestoreLogs(oneHourTimeout, veleroCLI, veleroNamespace, restoreName)
 		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
--- a/test/e2e/multiple_namespaces_test.go
+++ b/test/e2e/multiple_namespaces_test.go
@@ -27,7 +27,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
@@ -62,7 +62,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
 		uuidgen, err = uuid.NewRandom()
 		Expect(err).To(Succeed())
 		if installVelero {
-			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
+			Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
 				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
@@ -87,7 +87,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
 })

 func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName string, numberOfNamespaces int, backupName string, restoreName string) error {
-	defer cleanupNamespaces(ctx, client, nsBaseName) // Run at exit for final cleanup
+	defer cleanupNamespaces(context.Background(), client, nsBaseName) // Run at exit for final cleanup
 	var excludeNamespaces []string

 	// Currently it's hard to build a large list of namespaces to include and wildcards do not work so instead