Merge pull request #3992 from ywk253100/210727_image_pull_secret

Patch the resources of velero and kibishii when running E2E testing

test/e2e/Makefile

@@ -58,6 +58,7 @@ VSL_CONFIG ?=
 CLOUD_PROVIDER ?=
 OBJECT_STORE_PROVIDER ?=
 INSTALL_VELERO ?= true
+REGISTRY_CREDENTIAL_FILE ?=
 
 # Flags to create an additional BSL for multiple credentials tests
 ADDITIONAL_OBJECT_STORE_PROVIDER ?=
@@ -94,7 +95,8 @@ run: ginkgo
 		-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
 		-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
 		-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \
-		-install-velero=$(INSTALL_VELERO)
+		-install-velero=$(INSTALL_VELERO) \
+		-registry-credential-file=$(REGISTRY_CREDENTIAL_FILE)
 
 build: ginkgo
 	mkdir -p $(OUTPUT_DIR)

test/e2e/backup_test.go

@@ -60,7 +60,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 		Expect(err).To(Succeed())
 		if installVelero {
 			Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed())
+				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
@@ -77,7 +77,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 			restoreName = "restore-" + uuidgen.String()
 			// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
 			// KinD. So use the kind installation for Kibishii.
-			Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots)).To(Succeed(),
+			Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
 				"Failed to successfully backup and restore Kibishii namespace")
 		})
 
@@ -131,7 +131,7 @@ func backup_restore_test(useVolumeSnapshots bool) {
 					restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen)
 				}
 
-				Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots)).To(Succeed(),
+				Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
 					"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
 			}
 		})

test/e2e/e2e_suite_test.go

@@ -27,7 +27,7 @@ import (
 
 var (
 	veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace, crdsVersion string
-	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials                                             string
+	additionalBSLProvider, additionalBSLBucket, additionalBSLPrefix, additionalBSLConfig, additionalBSLCredentials, registryCredentialFile                     string
 	installVelero                                                                                                                                              bool
 )
 
@@ -43,6 +43,7 @@ func init() {
 	flag.StringVar(&vslConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2")
 	flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
 	flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test.  Optional.")
+	flag.StringVar(&registryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.")
 
 	// Flags to create an additional BSL for multiple credentials test
 	flag.StringVar(&additionalBSLProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")

test/e2e/enable_api_group_versions_test.go

@@ -70,7 +70,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
 				vslConfig,
 				crdsVersion,
 				"EnableAPIGroupVersions", // TODO: remove when feature flag is removed
-			)
+				registryCredentialFile)
 			Expect(err).NotTo(HaveOccurred())
 		}
 	})

test/e2e/kibishii_tests.go

@@ -92,16 +92,26 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
 
 // runKibishiiTests runs kibishii tests on the provider.
 func runKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string,
-	useVolumeSnapshots bool) error {
+	useVolumeSnapshots bool, registryCredentialFile string) error {
 	fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	timeout := 10 * time.Minute
 	interval := 5 * time.Second
+	serviceAccountName := "default"
 
 	if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
 	}
 
+	// wait until the service account is created before patch the image pull secret
+	if err := waitUntilServiceAccountCreated(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, timeout); err != nil {
+		return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+	// add the image pull secret to avoid the image pull limit issue of Docker Hub
+	if err := patchServiceAccountWithImagePullSecret(oneHourTimeout, client, kibishiiNamespace, serviceAccountName, registryCredentialFile); err != nil {
+		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace)
+	}
+
 	if err := installKibishii(fiveMinTimeout, kibishiiNamespace, providerName); err != nil {
 		return errors.Wrap(err, "Failed to install Kibishii workload")
 	}

test/e2e/multiple_namespaces_test.go

@@ -28,8 +28,7 @@ var _ = Describe("[Basic] Backup/restore of 2 namespaces", func() {
 		Expect(err).To(Succeed())
 		if installVelero {
 			Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed())
-
+				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
@@ -65,8 +64,7 @@ var _ = Describe("[Scale] Backup/restore of 2500 namespaces", func() {
 		Expect(err).To(Succeed())
 		if installVelero {
 			Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false,
-				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "")).To(Succeed())
-
+				cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
 		}
 	})
 
@@ -112,7 +110,7 @@ func RunMultipleNamespaceTest(ctx context.Context, client testClient, nsBaseName
 		}
 	}
 	if err := veleroBackupExcludeNamespaces(ctx, veleroCLI, veleroNamespace, backupName, excludeNamespaces); err != nil {
-		veleroBackupLogs(ctx, veleroCLI, "", backupName)
+		veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backupName)
 		return errors.Wrapf(err, "Failed to backup backup namespaces %s-*", nsBaseName)
 	}
 

test/e2e/serviceaccount.go

@@ -0,0 +1,66 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"time"
+
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/vmware-tanzu/velero/pkg/builder"
+
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func waitUntilServiceAccountCreated(ctx context.Context, client testClient, namespace, serviceAccount string, timeout time.Duration) error {
+	return wait.PollImmediate(5*time.Second, timeout,
+		func() (bool, error) {
+			if _, err := client.clientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}); err != nil {
+				if !apierrors.IsNotFound(err) {
+					return false, err
+				}
+				return false, nil
+			}
+			return true, nil
+		})
+}
+
+func patchServiceAccountWithImagePullSecret(ctx context.Context, client testClient, namespace, serviceAccount, dockerCredentialFile string) error {
+	credential, err := ioutil.ReadFile(dockerCredentialFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile)
+	}
+	secretName := "image-pull-secret"
+	secret := builder.ForSecret(namespace, secretName).Data(map[string][]byte{".dockerconfigjson": credential}).Result()
+	secret.Type = corev1.SecretTypeDockerConfigJson
+	if _, err = client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
+		return errors.Wrapf(err, "failed to create secret %q under namespace %q", secretName, namespace)
+	}
+
+	if _, err = client.clientGo.CoreV1().ServiceAccounts(namespace).Patch(ctx, serviceAccount, types.StrategicMergePatchType,
+		[]byte(fmt.Sprintf(`{"imagePullSecrets": [{"name": "%s"}]}`, secretName)), metav1.PatchOptions{}); err != nil {
+		return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccount, namespace)
+	}
+	return nil
+}

test/e2e/velero_utils.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -29,6 +30,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
 	kbclient "sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -96,7 +100,7 @@ func getProviderVeleroInstallOptions(
 }
 
 // installVeleroServer installs velero in the cluster.
-func installVeleroServer(io *cliinstall.InstallOptions) error {
+func installVeleroServer(io *cliinstall.InstallOptions, registryCredentialFile string) error {
 	vo, err := io.AsVeleroOptions()
 	if err != nil {
 		return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero")
@@ -109,6 +113,14 @@ func installVeleroServer(io *cliinstall.InstallOptions) error {
 
 	errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs"
 	resources := install.AllResources(vo)
+
+	// apply the image pull secret to avoid the image pull limit of Docker Hub
+	if len(registryCredentialFile) > 0 {
+		if err = patchResources(io.Namespace, registryCredentialFile, resources); err != nil {
+			return err
+		}
+	}
+
 	err = install.Install(client.dynamicFactory, resources, os.Stdout)
 	if err != nil {
 		return errors.Wrap(err, errorMsg)
@@ -131,6 +143,60 @@ func installVeleroServer(io *cliinstall.InstallOptions) error {
 	return nil
 }
 
+// patch the velero resources for E2E testing
+func patchResources(namespace, registryCredentialFile string, resources *unstructured.UnstructuredList) error {
+	credential, err := ioutil.ReadFile(registryCredentialFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read the registry credential file %s", registryCredentialFile)
+	}
+
+	imagePullSecret := corev1.Secret{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Secret",
+			APIVersion: corev1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "image-pull-secret",
+			Namespace: namespace,
+		},
+		Type: corev1.SecretTypeDockerConfigJson,
+		Data: map[string][]byte{
+			".dockerconfigjson": credential,
+		},
+	}
+
+	for resourceIndex, resource := range resources.Items {
+		if resource.GetKind() == "ServiceAccount" && resource.GetName() == "velero" {
+			resource.Object["imagePullSecrets"] = []map[string]interface{}{
+				{
+					"name": "image-pull-secret",
+				},
+			}
+			resources.Items[resourceIndex] = resource
+			fmt.Printf("image pull secret %q set for velero serviceaccount \n", "image-pull-secret")
+			continue
+		}
+	}
+
+	un, err := toUnstructured(imagePullSecret)
+	if err != nil {
+		return errors.Wrapf(err, "failed to convert pull secret to unstructure")
+	}
+	resources.Items = append(resources.Items, un)
+
+	return nil
+}
+
+func toUnstructured(res interface{}) (unstructured.Unstructured, error) {
+	un := unstructured.Unstructured{}
+	data, err := json.Marshal(res)
+	if err != nil {
+		return un, err
+	}
+	err = json.Unmarshal(data, &un)
+	return un, err
+}
+
 // checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup.
 func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string,
 	expectedPhase velerov1api.BackupPhase) error {
@@ -286,7 +352,7 @@ func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string
 
 func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool,
 	cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string,
-	crdsVersion string, features string) error {
+	crdsVersion string, features string, registryCredentialFile string) error {
 
 	if cloudProvider != "kind" {
 		if objectStoreProvider != "" {
@@ -333,7 +399,7 @@ func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri
 	veleroInstallOptions.CRDsVersion = crdsVersion
 	veleroInstallOptions.Namespace = veleroNamespace
 
-	err = installVeleroServer(veleroInstallOptions)
+	err = installVeleroServer(veleroInstallOptions, registryCredentialFile)
 	if err != nil {
 		return errors.WithMessagef(err, "Failed to install Velero in the cluster")
 	}