diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0e632dcc4..71d501e60 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" . "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt" . "github.com/vmware-tanzu/velero/test/e2e/migration" + . "github.com/vmware-tanzu/velero/test/e2e/nodeagentconfig" . "github.com/vmware-tanzu/velero/test/e2e/parallelfilesdownload" . "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload" . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" @@ -673,6 +674,12 @@ var _ = Describe( SpecificRepoMaintenanceTest, ) +var _ = Describe( + "Test node agent config's LoadAffinity part", + Label("NodeAgentConfig", "LoadAffinity"), + LoadAffinities, +) + func GetKubeConfigContext() error { var err error var tcDefault, tcStandby k8s.TestClient @@ -753,7 +760,7 @@ var _ = BeforeSuite(func() { ).To(Succeed()) } - // Create the needed PriorityClasses + By("Install PriorityClasses for E2E.") Expect(veleroutil.CreatePriorityClasses( context.Background(), test.VeleroCfg.ClientToInstallVelero.Kubebuilder, @@ -783,6 +790,8 @@ var _ = AfterSuite(func() { test.StorageClassName, ), ).To(Succeed()) + + By("Delete PriorityClasses created by E2E") Expect( k8s.DeleteStorageClass( ctx, diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index b26f87f38..f91bcffff 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -342,7 +342,7 @@ func (m *migrationE2E) Restore() error { Expect(veleroutil.InstallStorageClasses( m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed()) - // Create the needed PriorityClasses + By("Install PriorityClass for E2E.") Expect(veleroutil.CreatePriorityClasses( context.Background(), test.VeleroCfg.StandbyClient.Kubebuilder, @@ -453,6 +453,7 @@ func (m *migrationE2E) Clean() error { Expect(k8sutil.KubectlConfigUseContext( m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed()) + m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName @@ -465,7 +466,6 @@ func (m *migrationE2E) Clean() error { fmt.Println("Fail to delete StorageClass1: ", err) return } - if err := k8sutil.DeleteStorageClass( m.Ctx, *m.VeleroCfg.ClientToInstallVelero, @@ -475,6 +475,12 @@ func (m *migrationE2E) Clean() error { return } + By("Delete PriorityClasses created by E2E") + Expect(veleroutil.DeletePriorityClasses( + m.Ctx, + m.VeleroCfg.ClientToInstallVelero.Kubebuilder, + )).To(Succeed()) + if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && m.VeleroCfg.UseVolumeSnapshots { By("Delete VolumeSnapshotClass created by E2E") diff --git a/test/e2e/nodeagentconfig/node-agent-config.go b/test/e2e/nodeagentconfig/node-agent-config.go new file mode 100644 index 000000000..1b46eed65 --- /dev/null +++ b/test/e2e/nodeagentconfig/node-agent-config.go @@ -0,0 +1,347 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeagentconfig + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + . "github.com/onsi/gomega" + "github.com/pkg/errors" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov2alpha1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v2alpha1" + "github.com/vmware-tanzu/velero/pkg/builder" + velerotypes "github.com/vmware-tanzu/velero/pkg/types" + "github.com/vmware-tanzu/velero/pkg/util/kube" + velerokubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" + "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" +) + +type NodeAgentConfigTestCase struct { + TestCase + nodeAgentConfigs velerotypes.NodeAgentConfigs + nodeAgentConfigMapName string +} + +var LoadAffinities func() = TestFunc(&NodeAgentConfigTestCase{ + nodeAgentConfigs: velerotypes.NodeAgentConfigs{ + LoadAffinity: []*kube.LoadAffinity{ + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "beta.kubernetes.io/arch": "amd64", + }, + }, + StorageClass: test.StorageClassName, + }, + { + NodeSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "kubernetes.io/arch": "amd64", + }, + }, + StorageClass: test.StorageClassName2, + }, + }, + BackupPVCConfig: map[string]velerotypes.BackupPVC{ + test.StorageClassName: { + StorageClass: test.StorageClassName2, + }, + }, + RestorePVCConfig: &velerotypes.RestorePVC{ + IgnoreDelayBinding: true, + }, + PriorityClassName: test.PriorityClassNameForDataMover, + }, + nodeAgentConfigMapName: "node-agent-config", +}) + +func (n *NodeAgentConfigTestCase) Init() error { + // generate random number as UUIDgen and set one default timeout duration + n.TestCase.Init() + + // generate variable names based on CaseBaseName + UUIDgen + n.CaseBaseName = "node-agent-config-" + n.UUIDgen + n.BackupName = "backup-" + n.CaseBaseName + n.RestoreName = "restore-" + n.CaseBaseName + + // generate namespaces by NamespacesTotal + n.NamespacesTotal = 1 + n.NSIncluded = &[]string{} + for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { + createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) + *n.NSIncluded = append(*n.NSIncluded, createNSName) + } + + // assign values to the inner variable for specific case + n.VeleroCfg.UseNodeAgent = true + n.VeleroCfg.UseNodeAgentWindows = true + + // Need to verify the data mover pod content, so don't wait until backup completion. + n.BackupArgs = []string{ + "create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName, + "--include-namespaces", strings.Join(*n.NSIncluded, ","), + "--snapshot-volumes=true", "--snapshot-move-data", + } + + // Need to verify the data mover pod content, so don't wait until restore completion. + n.RestoreArgs = []string{ + "create", "--namespace", n.VeleroCfg.VeleroNamespace, "restore", n.RestoreName, + "--from-backup", n.BackupName, + } + + // Message output by ginkgo + n.TestMsg = &TestMSG{ + Desc: "Validate Node Agent ConfigMap configuration", + FailedMSG: "Failed to apply and / or validate configuration in VGDP pod.", + Text: "Should be able to apply and validate configuration in VGDP pod.", + } + return nil +} + +func (n *NodeAgentConfigTestCase) InstallVelero() error { + // Because this test needs to use customized Node Agent ConfigMap, + // need to uninstall and reinstall Velero. + + fmt.Println("Start to uninstall Velero") + if err := veleroutil.VeleroUninstall(n.Ctx, n.VeleroCfg); err != nil { + fmt.Printf("Fail to uninstall Velero: %s\n", err.Error()) + return err + } + + result, err := json.Marshal(n.nodeAgentConfigs) + if err != nil { + return err + } + + repoMaintenanceConfig := builder.ForConfigMap(n.VeleroCfg.VeleroNamespace, n.nodeAgentConfigMapName). + Data("node-agent-config", string(result)).Result() + + n.VeleroCfg.NodeAgentConfigMap = n.nodeAgentConfigMapName + + return veleroutil.PrepareVelero( + n.Ctx, + n.CaseBaseName, + n.VeleroCfg, + repoMaintenanceConfig, + ) +} + +func (n *NodeAgentConfigTestCase) CreateResources() error { + for _, ns := range *n.NSIncluded { + if err := k8sutil.CreateNamespace(n.Ctx, n.Client, ns); err != nil { + fmt.Printf("Fail to create ns %s: %s\n", ns, err.Error()) + return err + } + + pvc, err := k8sutil.CreatePVC(n.Client, ns, "volume-1", test.StorageClassName, nil) + if err != nil { + fmt.Printf("Fail to create PVC %s: %s\n", "volume-1", err.Error()) + return err + } + + vols := k8sutil.CreateVolumes(pvc.Name, []string{"volume-1"}) + + deployment := k8sutil.NewDeployment( + n.CaseBaseName, + (*n.NSIncluded)[0], + 1, + map[string]string{"app": "test"}, + n.VeleroCfg.ImageRegistryProxy, + n.VeleroCfg.WorkerOS, + ).WithVolume(vols).Result() + + deployment, err = k8sutil.CreateDeployment(n.Client.ClientGo, ns, deployment) + if err != nil { + fmt.Printf("Fail to create deployment %s: %s \n", deployment.Name, err.Error()) + return errors.Wrap(err, fmt.Sprintf("failed to create deployment: %s", err.Error())) + } + + if err := k8sutil.WaitForReadyDeployment(n.Client.ClientGo, deployment.Namespace, deployment.Name); err != nil { + fmt.Printf("Fail to create deployment %s: %s\n", n.CaseBaseName, err.Error()) + return err + } + } + + return nil +} + +func (n *NodeAgentConfigTestCase) Backup() error { + if err := veleroutil.VeleroCmdExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.BackupArgs); err != nil { + return err + } + + backupPodList := new(corev1api.PodList) + + wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + duList := new(velerov2alpha1api.DataUploadList) + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List( + n.Ctx, + duList, + &client.ListOptions{Namespace: n.VeleroCfg.VeleroNamespace}, + ); err != nil { + fmt.Printf("Fail to list DataUpload: %s\n", err.Error()) + return false, fmt.Errorf("Fail to list DataUpload: %w", err) + } else { + if len(duList.Items) <= 0 { + fmt.Println("No DataUpload found yet. Continue polling.") + return false, nil + } + } + + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List( + n.Ctx, + backupPodList, + &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + velerov1api.DataUploadLabel: duList.Items[0].Name, + }), + }); err != nil { + fmt.Printf("Fail to list backupPod %s\n", err.Error()) + return false, errors.Wrapf(err, "error to list backup pods") + } else { + if len(backupPodList.Items) <= 0 { + fmt.Println("No backupPod found yet. Continue polling.") + return false, nil + } + } + + return true, nil + }) + + fmt.Println("Start to verify backupPod content.") + + Expect(backupPodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) + + // In backup, only the second element of LoadAffinity array should be used. + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[1:]) + + Expect(backupPodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) + + fmt.Println("backupPod content verification completed successfully.") + + wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + backup := new(velerov1api.Backup) + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.Get( + n.Ctx, + client.ObjectKey{Namespace: n.VeleroCfg.VeleroNamespace, Name: n.BackupName}, + backup, + ); err != nil { + return false, err + } + + if backup.Status.Phase != velerov1api.BackupPhaseCompleted && + backup.Status.Phase != velerov1api.BackupPhaseFailed && + backup.Status.Phase != velerov1api.BackupPhasePartiallyFailed { + fmt.Printf("backup status is %s. Continue polling until backup reach to a final state.\n", backup.Status.Phase) + return false, nil + } + + return true, nil + }) + + return nil +} + +func (n *NodeAgentConfigTestCase) Restore() error { + if err := veleroutil.VeleroCmdExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.RestoreArgs); err != nil { + return err + } + + restorePodList := new(corev1api.PodList) + + wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + ddList := new(velerov2alpha1api.DataDownloadList) + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List( + n.Ctx, + ddList, + &client.ListOptions{Namespace: n.VeleroCfg.VeleroNamespace}, + ); err != nil { + fmt.Printf("Fail to list DataDownload: %s\n", err.Error()) + return false, fmt.Errorf("Fail to list DataDownload %w", err) + } else { + if len(ddList.Items) <= 0 { + fmt.Println("No DataDownload found yet. Continue polling.") + return false, nil + } + } + + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.List( + n.Ctx, + restorePodList, + &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{ + velerov1api.DataDownloadLabel: ddList.Items[0].Name, + }), + }); err != nil { + fmt.Printf("Fail to list restorePod %s\n", err.Error()) + return false, errors.Wrapf(err, "error to list restore pods") + } else { + if len(restorePodList.Items) <= 0 { + fmt.Println("No restorePod found yet. Continue polling.") + return false, nil + } + } + + return true, nil + }) + + fmt.Println("Start to verify restorePod content.") + + Expect(restorePodList.Items[0].Spec.PriorityClassName).To(Equal(n.nodeAgentConfigs.PriorityClassName)) + + // In restore, only the first element of LoadAffinity array should be used. + expectedAffinity := velerokubeutil.ToSystemAffinity(n.nodeAgentConfigs.LoadAffinity[:1]) + + Expect(restorePodList.Items[0].Spec.Affinity).To(Equal(expectedAffinity)) + + fmt.Println("restorePod content verification completed successfully.") + + wait.PollUntilContextTimeout(n.Ctx, 5*time.Second, 5*time.Minute, true, func(ctx context.Context) (bool, error) { + restore := new(velerov1api.Restore) + if err := n.VeleroCfg.ClientToInstallVelero.Kubebuilder.Get( + n.Ctx, + client.ObjectKey{Namespace: n.VeleroCfg.VeleroNamespace, Name: n.RestoreName}, + restore, + ); err != nil { + return false, err + } + + if restore.Status.Phase != velerov1api.RestorePhaseCompleted && + restore.Status.Phase != velerov1api.RestorePhaseFailed && + restore.Status.Phase != velerov1api.RestorePhasePartiallyFailed { + fmt.Printf("restore status is %s. Continue polling until restore reach to a final state.\n", restore.Status.Phase) + return false, nil + } + + return true, nil + }) + + return nil +} diff --git a/test/util/velero/install.go b/test/util/velero/install.go index ca6e207d1..6ffa8812d 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -163,6 +163,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC veleroCfg.VeleroNamespace, ) } + veleroCfg.BackupRepoConfigMap = test.BackupRepositoryConfigName // Install the passed-in objects in Velero installed namespace for _, obj := range objects { @@ -654,7 +655,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string, APIVersion: corev1api.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: "restic-restore-action-config", + Name: "fs-restore-action-config", Namespace: namespace, Labels: map[string]string{ "velero.io/plugin-config": "", @@ -671,7 +672,7 @@ func patchResources(resources *unstructured.UnstructuredList, namespace string, return errors.Wrapf(err, "failed to convert restore action config to unstructure") } resources.Items = append(resources.Items, un) - fmt.Printf("the restic restore helper image is set by the configmap %q \n", "restic-restore-action-config") + fmt.Printf("the restic restore helper image is set by the configmap %q \n", "fs-restore-action-config") } return nil