Merge pull request #9219 from blackpiglet/9157_e2e

Add E2E auto case for node-agent-config validation.

test/e2e/e2e_suite_test.go

@@ -39,6 +39,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
 	. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
 	. "github.com/vmware-tanzu/velero/test/e2e/migration"
+	. "github.com/vmware-tanzu/velero/test/e2e/nodeagentconfig"
 	. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesdownload"
 	. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload"
 	. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
@@ -673,6 +674,12 @@ var _ = Describe(
 	SpecificRepoMaintenanceTest,
 )
 
+var _ = Describe(
+	"Test node agent config's LoadAffinity part",
+	Label("NodeAgentConfig", "LoadAffinity"),
+	LoadAffinities,
+)
+
 func GetKubeConfigContext() error {
 	var err error
 	var tcDefault, tcStandby k8s.TestClient
@@ -753,7 +760,7 @@ var _ = BeforeSuite(func() {
 		).To(Succeed())
 	}
 
-	// Create the needed PriorityClasses
+	By("Install PriorityClasses for E2E.")
 	Expect(veleroutil.CreatePriorityClasses(
 		context.Background(),
 		test.VeleroCfg.ClientToInstallVelero.Kubebuilder,
@@ -783,6 +790,8 @@ var _ = AfterSuite(func() {
 			test.StorageClassName,
 		),
 	).To(Succeed())
+
+	By("Delete PriorityClasses created by E2E")
 	Expect(
 		k8s.DeleteStorageClass(
 			ctx,

test/e2e/migration/migration.go

@@ -342,7 +342,7 @@ func (m *migrationE2E) Restore() error {
 		Expect(veleroutil.InstallStorageClasses(
 			m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed())
 
-		// Create the needed PriorityClasses
+		By("Install PriorityClass for E2E.")
 		Expect(veleroutil.CreatePriorityClasses(
 			context.Background(),
 			test.VeleroCfg.StandbyClient.Kubebuilder,
@@ -453,6 +453,7 @@ func (m *migrationE2E) Clean() error {
 
 		Expect(k8sutil.KubectlConfigUseContext(
 			m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed())
+
 		m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient
 		m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName
 
@@ -465,7 +466,6 @@ func (m *migrationE2E) Clean() error {
 			fmt.Println("Fail to delete StorageClass1: ", err)
 			return
 		}
-
 		if err := k8sutil.DeleteStorageClass(
 			m.Ctx,
 			*m.VeleroCfg.ClientToInstallVelero,
@@ -475,6 +475,12 @@ func (m *migrationE2E) Clean() error {
 			return
 		}
 
+		By("Delete PriorityClasses created by E2E")
+		Expect(veleroutil.DeletePriorityClasses(
+			m.Ctx,
+			m.VeleroCfg.ClientToInstallVelero.Kubebuilder,
+		)).To(Succeed())
+
 		if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
 			m.VeleroCfg.UseVolumeSnapshots {
 			By("Delete VolumeSnapshotClass created by E2E")

test/e2e/nodeagentconfig/node-agent-config.go

@@ -0,0 +1,347 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nodeagentconfig
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"github.com/pkg/errors"
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1"
+	"github.com/vmware-tanzu/velero/pkg/builder"
+	velerotypes "github.com/vmware-tanzu/velero/pkg/types"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+	velerokubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
+	"github.com/vmware-tanzu/velero/test"
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	k8sutil "github.com/vmware-tanzu/velero/test/util/k8s"
+	veleroutil "github.com/vmware-tanzu/velero/test/util/velero"
+)
+
+type NodeAgentConfigTestCase struct {
+	TestCase
+	nodeAgentConfigs       velerotypes.NodeAgentConfigs
+	nodeAgentConfigMapName string
+}
+
+var LoadAffinities func() = TestFunc(&NodeAgentConfigTestCase{
+	nodeAgentConfigs: velerotypes.NodeAgentConfigs{
+		LoadAffinity: []*kube.LoadAffinity{
+			{
+				NodeSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"beta.kubernetes.io/arch": "amd64",
+					},
+				},
+				StorageClass: test.StorageClassName,
+			},
+			{
+				NodeSelector: metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"kubernetes.io/arch": "amd64",
+					},
+				},
+				StorageClass: test.StorageClassName2,
+			},
+		},
+		BackupPVCConfig: map[string]velerotypes.BackupPVC{
+			test.StorageClassName: {
+				StorageClass: test.StorageClassName2,
+			},
+		},
+		RestorePVCConfig: &velerotypes.RestorePVC{
+			IgnoreDelayBinding: true,
+		},
+		PriorityClassName: test.PriorityClassNameForDataMover,
+	},
+	nodeAgentConfigMapName: "node-agent-config",
+})
+
+func (n *NodeAgentConfigTestCase) Init() error {
+	// generate random number as UUIDgen and set one default timeout duration
+	n.TestCase.Init()
+
+	// generate variable names based on CaseBaseName + UUIDgen
+	n.CaseBaseName = "node-agent-config-" + n.UUIDgen
+	n.BackupName = "backup-" + n.CaseBaseName
+	n.RestoreName = "restore-" + n.CaseBaseName
+
+	// generate namespaces by NamespacesTotal
+	n.NamespacesTotal = 1
+	n.NSIncluded = &[]string{}
+	for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ {
+		createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum)
+		*n.NSIncluded = append(*n.NSIncluded, createNSName)
+	}
+
+	// assign values to the inner variable for specific case
+	n.VeleroCfg.UseNodeAgent = true
+	n.VeleroCfg.UseNodeAgentWindows = true
+
+	// Need to verify the data mover pod content, so don't wait until backup completion.
+	n.BackupArgs = []string{
+		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName,
+		"--include-namespaces", strings.Join(*n.NSIncluded, ","),
+		"--snapshot-volumes=true", "--snapshot-move-data",
+	}
+
+	// Need to verify the data mover pod content, so don't wait until restore completion.
+	n.RestoreArgs = []string{
+		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
+		"--from-backup", n.BackupName,
+	}
+
+	// Message output by ginkgo
+	n.TestMsg = &TestMSG{
+		Desc:      "Validate Node Agent ConfigMap configuration",
+		FailedMSG: "Failed to apply and / or validate configuration in VGDP pod.",
+		Text:      "Should be able to apply and validate configuration in VGDP pod.",
+	}
+	return nil
+}
+
+func (n *NodeAgentConfigTestCase) InstallVelero() error {
+	// Because this test needs to use customized Node Agent ConfigMap,
+	// need to uninstall and reinstall Velero.
+
+	fmt.Println("Start to uninstall Velero")
+	if err := veleroutil.VeleroUninstall(n.Ctx, n.VeleroCfg); err != nil {
+		fmt.Printf("Fail to uninstall Velero: %s\n", err.Error())
+		return err
+	}
+
+	result, err := json.Marshal(n.nodeAgentConfigs)
+	if err != nil {
+		return err
+	}
+
+	repoMaintenanceConfig := builder.ForConfigMap(n.VeleroCfg.VeleroNamespace, n.nodeAgentConfigMapName).
+		Data("node-agent-config", string(result)).Result()
+
+	n.VeleroCfg.NodeAgentConfigMap = n.nodeAgentConfigMapName
+
+	return veleroutil.PrepareVelero(
+		n.Ctx,
+		n.CaseBaseName,
+		n.VeleroCfg,
+		repoMaintenanceConfig,
+	)
+}
+
+func (n *NodeAgentConfigTestCase) CreateResources() error {
+	for _, ns := range *n.NSIncluded {
+		if err := k8sutil.CreateNamespace(n.Ctx, n.Client, ns); err != nil {
+			fmt.Printf("Fail to create ns %s: %s\n", ns, err.Error())
+			return err
+		}
+
+		pvc, err := k8sutil.CreatePVC(n.Client, ns, "volume-1", test.StorageClassName, nil)
+		if err != nil {
+			fmt.Printf("Fail to create PVC %s: %s\n", "volume-1", err.Error())
+			return err
+		}
+
+		vols := k8sutil.CreateVolumes(pvc.Name, []string{"volume-1"})
+
+		deployment := k8sutil.NewDeployment(
+			n.CaseBaseName,
+			(*n.NSIncluded)[0],
+			1,
+			map[string]string{"app": "test"},
+			n.VeleroCfg.ImageRegistryProxy,
+			n.VeleroCfg.WorkerOS,
+		).WithVolume(vols).Result()
+
+		deployment, err = k8sutil.CreateDeployment(n.Client.ClientGo, ns, deployment)
+		if err != nil {
+			fmt.Printf("Fail to create deployment %s: %s \n", deployment.Name, err.Error())
+			return errors.Wrap(err, fmt.Sprintf("failed to create deployment: %s", err.Error()))
+		}
+
+		if err := k8sutil.WaitForReadyDeployment(n.Client.ClientGo, deployment.Namespace, deployment.Name); err != nil {
+			fmt.Printf("Fail to create deployment %s: %s\n", n.CaseBaseName, err.Error())
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (n *NodeAgentConfigTestCase) Backup() error {
+	if err := veleroutil.VeleroCmdExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.BackupArgs); err != nil {
+		return err
+	}
+
+	backupPodList := new(corev1api.PodList)
+
+	wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+		duList := new(velerov2alpha1api.DataUploadList)
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(
+			n.Ctx,
+			duList,
+			&client.ListOptions{Namespace: n.VeleroCfg.VeleroNamespace},
+		); err != nil {
+			fmt.Printf("Fail to list DataUpload: %s\n", err.Error())
+			return false, fmt.Errorf("Fail to list DataUpload: %w", err)
+		} else {
+			if len(duList.Items) <= 0 {
+				fmt.Println("No DataUpload found yet. Continue polling.")
+				return false, nil
+			}
+		}
+
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(
+			n.Ctx,
+			backupPodList,
+			&client.ListOptions{
+				LabelSelector: labels.SelectorFromSet(map[string]string{
+					velerov1api.DataUploadLabel: duList.Items[0].Name,
+				}),
+			}); err != nil {
+			fmt.Printf("Fail to list backupPod %s\n", err.Error())
+			return false, errors.Wrapf(err, "error to list backup pods")
+		} else {
+			if len(backupPodList.Items) <= 0 {
+				fmt.Println("No backupPod found yet. Continue polling.")
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+
+	fmt.Println("Start to verify backupPod content.")
+
+	Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName))
+
+	// In backup, only the second element of LoadAffinity array should be used.
+	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:])
+
+	Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity))
+
+	fmt.Println("backupPod content verification completed successfully.")
+
+	wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+		backup := new(velerov1api.Backup)
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.Get(
+			n.Ctx,
+			client.ObjectKey{Namespace: n.VeleroCfg.VeleroNamespace, Name: n.BackupName},
+			backup,
+		); err != nil {
+			return false, err
+		}
+
+		if backup.Status.Phase != velerov1api.BackupPhaseCompleted &&
+			backup.Status.Phase != velerov1api.BackupPhaseFailed &&
+			backup.Status.Phase != velerov1api.BackupPhasePartiallyFailed {
+			fmt.Printf("backup status is %s. Continue polling until backup reach to a final state.\n", backup.Status.Phase)
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	return nil
+}
+
+func (n *NodeAgentConfigTestCase) Restore() error {
+	if err := veleroutil.VeleroCmdExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.RestoreArgs); err != nil {
+		return err
+	}
+
+	restorePodList := new(corev1api.PodList)
+
+	wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+		ddList := new(velerov2alpha1api.DataDownloadList)
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(
+			n.Ctx,
+			ddList,
+			&client.ListOptions{Namespace: n.VeleroCfg.VeleroNamespace},
+		); err != nil {
+			fmt.Printf("Fail to list DataDownload: %s\n", err.Error())
+			return false, fmt.Errorf("Fail to list DataDownload %w", err)
+		} else {
+			if len(ddList.Items) <= 0 {
+				fmt.Println("No DataDownload found yet. Continue polling.")
+				return false, nil
+			}
+		}
+
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List(
+			n.Ctx,
+			restorePodList,
+			&client.ListOptions{
+				LabelSelector: labels.SelectorFromSet(map[string]string{
+					velerov1api.DataDownloadLabel: ddList.Items[0].Name,
+				}),
+			}); err != nil {
+			fmt.Printf("Fail to list restorePod %s\n", err.Error())
+			return false, errors.Wrapf(err, "error to list restore pods")
+		} else {
+			if len(restorePodList.Items) <= 0 {
+				fmt.Println("No restorePod found yet. Continue polling.")
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+
+	fmt.Println("Start to verify restorePod content.")
+
+	Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName))
+
+	// In restore, only the first element of LoadAffinity array should be used.
+	expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1])
+
+	Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity))
+
+	fmt.Println("restorePod content verification completed successfully.")
+
+	wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) {
+		restore := new(velerov1api.Restore)
+		if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.Get(
+			n.Ctx,
+			client.ObjectKey{Namespace: n.VeleroCfg.VeleroNamespace, Name: n.RestoreName},
+			restore,
+		); err != nil {
+			return false, err
+		}
+
+		if restore.Status.Phase != velerov1api.RestorePhaseCompleted &&
+			restore.Status.Phase != velerov1api.RestorePhaseFailed &&
+			restore.Status.Phase != velerov1api.RestorePhasePartiallyFailed {
+			fmt.Printf("restore status is %s. Continue polling until restore reach to a final state.\n", restore.Status.Phase)
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	return nil
+}

test/util/velero/install.go

@@ -163,6 +163,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC
 			veleroCfg.VeleroNamespace,
 		)
 	}
+	veleroCfg.BackupRepoConfigMap = test.BackupRepositoryConfigName
 
 	// Install the passed-in objects in Velero installed namespace
 	for _, obj := range objects {
@@ -654,7 +655,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string,
 				APIVersion: corev1api.SchemeGroupVersion.String(),
 			},
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      "restic-restore-action-config",
+				Name:      "fs-restore-action-config",
 				Namespace: namespace,
 				Labels: map[string]string{
 					"velero.io/plugin-config":      "",
@@ -671,7 +672,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string,
 			return errors.Wrapf(err, "failed to convert restore action config to unstructure")
 		}
 		resources.Items = append(resources.Items, un)
-		fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config")
+		fmt.Printf("the restic restore helper image is set by the configmap %q \n", "fs-restore-action-config")
 	}
 
 	return nil