From 6b9b13790eed55ac7ce5b29e5c694feee9e5b4dd Mon Sep 17 00:00:00 2001 From: Ming Date: Sun, 15 May 2022 05:20:43 +0000 Subject: [PATCH] Add schedule ordered resources test Signed-off-by: Ming --- changelogs/unreleased/4913-qiuming-best | 1 + test/e2e/e2e_suite_test.go | 3 + .../e2e/orderedresources/ordered_resources.go | 191 ++++++++++++++++++ test/e2e/util/velero/velero_utils.go | 159 ++++++++++----- 4 files changed, 306 insertions(+), 48 deletions(-) create mode 100644 changelogs/unreleased/4913-qiuming-best create mode 100644 test/e2e/orderedresources/ordered_resources.go diff --git a/changelogs/unreleased/4913-qiuming-best b/changelogs/unreleased/4913-qiuming-best new file mode 100644 index 000000000..406319cc5 --- /dev/null +++ b/changelogs/unreleased/4913-qiuming-best @@ -0,0 +1 @@ +Add schedule ordered resources E2E test diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index a2d98727d..a8be501ea 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -30,6 +30,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/basic" . "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" . "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt" + . "github.com/vmware-tanzu/velero/test/e2e/orderedresources" . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" . "github.com/vmware-tanzu/velero/test/e2e/scale" @@ -102,6 +103,8 @@ var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) +var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) + func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: // 1. E2E tests are long running tests involving installation of Velero and performing backup and restore operations. diff --git a/test/e2e/orderedresources/ordered_resources.go b/test/e2e/orderedresources/ordered_resources.go new file mode 100644 index 000000000..ff141470d --- /dev/null +++ b/test/e2e/orderedresources/ordered_resources.go @@ -0,0 +1,191 @@ +package orderedresources + +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +//the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 +import ( + "context" + "flag" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + waitutil "k8s.io/apimachinery/pkg/util/wait" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type OrderedResources struct { + Namespace string + ScheduleName string + OrderMap map[string]string + ScheduleArgs []string + TestCase +} + +func ScheduleOrderedResources() { + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero && !VeleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } + }) + + It("Create a schedule to backup resources in a specific order should be successful", func() { + test := &OrderedResources{} + err := test.Init() + Expect(err).To(Succeed(), err) + defer func() { + Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace)) + err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err)) + }() + + By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() { + err = test.CreateResources() + Expect(err).To(Succeed(), fmt.Sprintf("Failed to create resources to backup with err %v", err)) + }) + + By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() { + err = VeleroScheduleCreate(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err)) + }) + + By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() { + err = CheckScheduleWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err)) + }) + + By("Checking resource order in backup cr", func() { + backupList := new(velerov1api.BackupList) + err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) { + if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + } + + for _, backup := range backupList.Items { + if err = CheckBackupWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil { + return true, nil + } + } + fmt.Printf("still finding backup created by schedule %s ...\n", test.ScheduleName) + return false, nil + }) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", test.ScheduleName, err)) + }) + + }) +} + +func (o *OrderedResources) Init() error { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + client, err := NewTestClient() + if err != nil { + return fmt.Errorf("failed to init ordered resources test with err %v", err) + } + o.Client = client + o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String() + o.NSBaseName = "schedule-ordered-resources" + o.Namespace = o.NSBaseName + "-" + UUIDgen.String() + o.OrderMap = map[string]string{ + "deployments": fmt.Sprintf("deploy-%s", o.NSBaseName), + "secrets": fmt.Sprintf("secret-%s", o.NSBaseName), + "configmaps": fmt.Sprintf("configmap-%s", o.NSBaseName), + } + + o.ScheduleArgs = []string{"--schedule", "@every 1m", + "--include-namespaces", o.Namespace, "--default-volumes-to-restic", "--ordered-resources"} + var orderStr string + for kind, resource := range o.OrderMap { + orderStr += fmt.Sprintf("%s=%s;", kind, resource) + } + o.ScheduleArgs = append(o.ScheduleArgs, strings.TrimRight(orderStr, ";")) + + return nil +} + +func (o *OrderedResources) CreateResources() error { + o.Ctx, _ = context.WithTimeout(context.Background(), 5*time.Minute) + label := map[string]string{ + "orderedresources": "true", + } + fmt.Printf("Creating resources in %s namespace ...\n", o.Namespace) + if err := CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil { + return errors.Wrapf(err, "failed to create namespace %s", o.Namespace) + } + serviceAccountName := "default" + // wait until the service account is created before patch the image pull secret + if err := WaitUntilServiceAccountCreated(o.Ctx, o.Client, o.Namespace, serviceAccountName, 10*time.Minute); err != nil { + return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace) + } + // add the image pull secret to avoid the image pull limit issue of Docker Hub + if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace) + } + //Create deployment + deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName) + fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) + deployment := NewDeployment(deploymentName, o.Namespace, 1, label) + deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err)) + } + err = WaitForReadyDeployment(o.Client.ClientGo, o.Namespace, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", o.Namespace)) + } + //Create Secret + secretName := fmt.Sprintf("secret-%s", o.NSBaseName) + fmt.Printf("Creating secret %s in %s namespaces ...\n", secretName, o.Namespace) + _, err = CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", o.Namespace)) + } + err = WaitForSecretsComplete(o.Client.ClientGo, o.Namespace, secretName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace)) + } + //Create Configmap + configmapName := fmt.Sprintf("configmap-%s", o.NSBaseName) + fmt.Printf("Creating configmap %s in %s namespaces ...\n", configmapName, o.Namespace) + _, err = CreateConfigMap(o.Client.ClientGo, o.Namespace, configmapName, label) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", o.Namespace)) + } + err = WaitForConfigMapComplete(o.Client.ClientGo, o.Namespace, configmapName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace)) + } + return nil +} diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 449f6e0a5..fc14335c8 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -28,6 +28,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "regexp" "runtime" "strings" @@ -157,6 +158,36 @@ func getProviderVeleroInstallOptions( return io, nil } +func CMDExecWithOutput(checkCMD *exec.Cmd) (*[]byte, error) { + stdoutPipe, err := checkCMD.StdoutPipe() + if err != nil { + return nil, err + } + + jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening + + err = checkCMD.Start() + if err != nil { + return nil, err + } + + bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) + + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + if bytesRead == len(jsonBuf) { + return nil, errors.New("yaml returned bigger than max allowed") + } + + jsonBuf = jsonBuf[0:bytesRead] + err = checkCMD.Wait() + if err != nil { + return nil, err + } + return &jsonBuf, err +} + // checkBackupPhase uses VeleroCLI to inspect the phase of a Velero backup. func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { @@ -164,34 +195,12 @@ func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str backupName) fmt.Printf("get backup cmd =%v\n", checkCMD) - stdoutPipe, err := checkCMD.StdoutPipe() - if err != nil { - return err - } - - jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening - - err = checkCMD.Start() - if err != nil { - return err - } - - bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) - - if err != nil && err != io.ErrUnexpectedEOF { - return err - } - if bytesRead == len(jsonBuf) { - return errors.New("yaml returned bigger than max allowed") - } - - jsonBuf = jsonBuf[0:bytesRead] - err = checkCMD.Wait() + jsonBuf, err := CMDExecWithOutput(checkCMD) if err != nil { return err } backup := velerov1api.Backup{} - err = json.Unmarshal(jsonBuf, &backup) + err = json.Unmarshal(*jsonBuf, &backup) if err != nil { return err } @@ -208,34 +217,12 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st restoreName) fmt.Printf("get restore cmd =%v\n", checkCMD) - stdoutPipe, err := checkCMD.StdoutPipe() - if err != nil { - return err - } - - jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening - - err = checkCMD.Start() - if err != nil { - return err - } - - bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf) - - if err != nil && err != io.ErrUnexpectedEOF { - return err - } - if bytesRead == len(jsonBuf) { - return errors.New("yaml returned bigger than max allowed") - } - - jsonBuf = jsonBuf[0:bytesRead] - err = checkCMD.Wait() + jsonBuf, err := CMDExecWithOutput(checkCMD) if err != nil { return err } restore := velerov1api.Restore{} - err = json.Unmarshal(jsonBuf, &restore) + err = json.Unmarshal(*jsonBuf, &restore) if err != nil { return err } @@ -245,6 +232,67 @@ func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st return nil } +func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") + jsonBuf, err := CMDExecWithOutput(checkCMD) + if err != nil { + return err + } + schedule := velerov1api.Schedule{} + err = json.Unmarshal(*jsonBuf, &schedule) + if err != nil { + return err + } + + if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled { + return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled) + } + return nil +} + +func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson") + jsonBuf, err := CMDExecWithOutput(checkCMD) + if err != nil { + return err + } + schedule := velerov1api.Schedule{} + err = json.Unmarshal(*jsonBuf, &schedule) + if err != nil { + return err + } + + if schedule.Status.Phase != velerov1api.SchedulePhaseEnabled { + return errors.Errorf("Unexpected restore phase got %s, expecting %s", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled) + } + if reflect.DeepEqual(schedule.Spec.Template.OrderedResources, order) { + return nil + } else { + return fmt.Errorf("resource order %v set in schedule command is not equal with order %v stored in schedule cr", order, schedule.Spec.Template.OrderedResources) + } +} + +func CheckBackupWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, backupName string, order map[string]string) error { + checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "get", "backup", backupName, "-ojson") + jsonBuf, err := CMDExecWithOutput(checkCMD) + if err != nil { + return err + } + backup := velerov1api.Backup{} + err = json.Unmarshal(*jsonBuf, &backup) + if err != nil { + return err + } + if backup.Status.Phase != velerov1api.BackupPhaseCompleted { + return errors.Errorf("Unexpected restore phase got %s, expecting %s", backup.Status.Phase, velerov1api.BackupPhaseCompleted) + } + if reflect.DeepEqual(backup.Spec.OrderedResources, order) { + return nil + } else { + return fmt.Errorf("resource order %v set in backup command is not equal with order %v stored in backup cr", order, backup.Spec.OrderedResources) + } +} + // VeleroBackupNamespace uses the veleroCLI to backup a namespace. func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace, backupName, namespace, backupLocation string, useVolumeSnapshots bool, selector string) error { @@ -320,6 +368,21 @@ func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace str return checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) } +func VeleroScheduleDelete(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error { + args := []string{"--namespace", veleroNamespace, "delete", "schedule", scheduleName, "--confirm"} + return VeleroCmdExec(ctx, veleroCLI, args) +} + +func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string, args []string) error { + args = append([]string{ + "--namespace", veleroNamespace, "create", "schedule", scheduleName, + }, args...) + if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { + return err + } + return checkSchedulePhase(ctx, veleroCLI, veleroNamespace, scheduleName) +} + func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error { cmd := exec.CommandContext(ctx, veleroCLI, args...) cmd.Stdout = os.Stdout