From b84ce9b6aa7ebd60f90f27f191f602d4e24aa1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Tue, 27 Jul 2021 06:20:47 +0800 Subject: [PATCH] Patch the resources of velero and kibishii when running E2E testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the image pull secret to the service account when deploying velero and kibishii to avoid the image pull limit issue of Docker Hub Fixes #3966 Signed-off-by: Wenkai Yin(尹文开) --- test/e2e/Makefile | 4 +- test/e2e/backup_test.go | 6 +- test/e2e/e2e_suite_test.go | 3 +- test/e2e/enable_api_group_versions_test.go | 2 +- test/e2e/kibishii_tests.go | 12 +++- test/e2e/multiple_namespaces_test.go | 8 +-- test/e2e/serviceaccount.go | 66 ++++++++++++++++++++ test/e2e/velero_utils.go | 72 +++++++++++++++++++++- 8 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 test/e2e/serviceaccount.go diff --git a/test/e2e/Makefile b/test/e2e/Makefile index ba58f9476..ed47614b2 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -58,6 +58,7 @@ VSL_CONFIG ?= CLOUD_PROVIDER ?= OBJECT_STORE_PROVIDER ?= INSTALL_VELERO ?= true +REGISTRY_CREDENTIAL_FILE ?= # Flags to create an additional BSL for multiple credentials tests ADDITIONAL_OBJECT_STORE_PROVIDER ?= @@ -94,7 +95,8 @@ run: ginkgo -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ - -install-velero=$(INSTALL_VELERO) + -install-velero=$(INSTALL_VELERO) \ + -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 1832ec60f..5840c0771 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -60,7 +60,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Expect(err).To(Succeed()) if installVelero { Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed()) + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -77,7 +77,7 @@ func backup_restore_test(useVolumeSnapshots bool) { restoreName = "restore-" + uuidgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(), + Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) @@ -131,7 +131,7 @@ func backup_restore_test(useVolumeSnapshots bool) { restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen) } - Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots)).To(Succeed(), + Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 88ffb6b8c..7f0712e31 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -27,7 +27,7 @@ import ( var ( veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string - additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials string + additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile string installVelero bool ) @@ -43,6 +43,7 @@ func init() { flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into") flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.") + flag.StringVar(®istryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") // Flags to create an additional BSL for multiple credentials test flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 442d911cd..b0cffe172 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -70,7 +70,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", vslConfig, crdsVersion, "EnableAPIGroupVersions", // TODO: remove when feature flag is removed - ) + registryCredentialFile) Expect(err).NotTo(HaveOccurred()) } }) diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 4f5899cc3..5ab5f04c9 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -92,16 +92,26 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel // runKibishiiTests runs kibishii tests on the provider. func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool) error { + useVolumeSnapshots bool, registryCredentialFile string) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) timeout := 10 * time.Minute interval := 5 * time.Second + serviceAccountName := "default" if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } + // wait until the service account is created before patch the image pull secret + if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, timeout); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) + } + if err := installKibishii(fiveMinTimeout, kibishiiNamespace, providerName); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } diff --git a/test/e2e/multiple_namespaces_test.go b/test/e2e/multiple_namespaces_test.go index 5e809b698..7c5c1b6ca 100644 --- a/test/e2e/multiple_namespaces_test.go +++ b/test/e2e/multiple_namespaces_test.go @@ -28,8 +28,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() { Expect(err).To(Succeed()) if installVelero { Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed()) - + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -65,8 +64,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() { Expect(err).To(Succeed()) if installVelero { Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, - cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed()) - + cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed()) } }) @@ -112,7 +110,7 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName } } if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil { - veleroBackupLogs(ctx, veleroCLI, "", backupName) + veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName) } diff --git a/test/e2e/serviceaccount.go b/test/e2e/serviceaccount.go new file mode 100644 index 000000000..30e46a85c --- /dev/null +++ b/test/e2e/serviceaccount.go @@ -0,0 +1,66 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "io/ioutil" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/vmware-tanzu/velero/pkg/builder" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error { + return wait.PollImmediate(5*time.Second, timeout, + func() (bool, error) { + if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil { + if !apierrors.IsNotFound(err) { + return false, err + } + return false, nil + } + return true, nil + }) +} + +func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error { + credential, err := ioutil.ReadFile(dockerCredentialFile) + if err != nil { + return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile) + } + secretName := "image-pull-secret" + secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result() + secret.Type = corev1.SecretTypeDockerConfigJson + if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace) + } + + if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType, + []byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace) + } + return nil +} diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index e01791a54..e8e753b8c 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -22,6 +22,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "os" "os/exec" "path/filepath" @@ -29,6 +30,9 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -96,7 +100,7 @@ func getProviderVeleroInstallOptions( } // installVeleroServer installs velero in the cluster. -func installVeleroServer(io *cliinstall.InstallOptions) error { +func installVeleroServer(io *cliinstall.InstallOptions, registryCredentialFile string) error { vo, err := io.AsVeleroOptions() if err != nil { return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero") @@ -109,6 +113,14 @@ func installVeleroServer(io *cliinstall.InstallOptions) error { errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" resources := install.AllResources(vo) + + // apply the image pull secret to avoid the image pull limit of Docker Hub + if len(registryCredentialFile) > 0 { + if err = patchResources(io.Namespace, registryCredentialFile, resources); err != nil { + return err + } + } + err = install.Install(client.dynamicFactory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) @@ -131,6 +143,60 @@ func installVeleroServer(io *cliinstall.InstallOptions) error { return nil } +// patch the velero resources for E2E testing +func patchResources(namespace, registryCredentialFile string, resources *unstructured.UnstructuredList) error { + credential, err := ioutil.ReadFile(registryCredentialFile) + if err != nil { + return errors.Wrapf(err, "failed to read the registry credential file %s", registryCredentialFile) + } + + imagePullSecret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "image-pull-secret", + Namespace: namespace, + }, + Type: corev1.SecretTypeDockerConfigJson, + Data: map[string][]byte{ + ".dockerconfigjson": credential, + }, + } + + for resourceIndex, resource := range resources.Items { + if resource.GetKind() == "ServiceAccount" && resource.GetName() == "velero" { + resource.Object["imagePullSecrets"] = []map[string]interface{}{ + { + "name": "image-pull-secret", + }, + } + resources.Items[resourceIndex] = resource + fmt.Printf("image pull secret %q set for velero serviceaccount \n", "image-pull-secret") + continue + } + } + + un, err := toUnstructured(imagePullSecret) + if err != nil { + return errors.Wrapf(err, "failed to convert pull secret to unstructure") + } + resources.Items = append(resources.Items, un) + + return nil +} + +func toUnstructured(res interface{}) (unstructured.Unstructured, error) { + un := unstructured.Unstructured{} + data, err := json.Marshal(res) + if err != nil { + return un, err + } + err = json.Unmarshal(data, &un) + return un, err +} + // checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { @@ -286,7 +352,7 @@ func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, - crdsVersion string, features string) error { + crdsVersion string, features string, registryCredentialFile string) error { if cloudProvider != "kind" { if objectStoreProvider != "" { @@ -333,7 +399,7 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri veleroInstallOptions.CRDsVersion = crdsVersion veleroInstallOptions.Namespace = veleroNamespace - err = installVeleroServer(veleroInstallOptions) + err = installVeleroServer(veleroInstallOptions, registryCredentialFile) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") }