From a628cb525f2234b7388f1cb5b5b2046ae44f03cd Mon Sep 17 00:00:00 2001 From: Ming Qiu Date: Wed, 17 Apr 2024 03:32:03 +0000 Subject: [PATCH] Add E2E test for parallel files upload and download Signed-off-by: Ming Qiu --- test/e2e/e2e_suite_test.go | 5 + .../parallel_files_download.go | 147 ++++++++++++++++++ .../parallel_files_upload.go | 112 +++++++++++++ test/util/k8s/common.go | 24 +++ test/util/velero/install.go | 24 ++- 5 files changed, 305 insertions(+), 7 deletions(-) create mode 100644 test/e2e/parallelfilesdownload/parallel_files_download.go create mode 100644 test/e2e/parallelfilesupload/parallel_files_upload.go diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7fca51363..aacd2c0b4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -39,6 +39,8 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" . "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt" . "github.com/vmware-tanzu/velero/test/e2e/migration" + . "github.com/vmware-tanzu/velero/test/e2e/parallelfilesdownload" + . "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload" . "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt" . "github.com/vmware-tanzu/velero/test/e2e/pv-backup" . "github.com/vmware-tanzu/velero/test/e2e/resource-filtering" @@ -182,6 +184,9 @@ var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest) var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest) +var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest) +var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest) + func GetKubeconfigContext() error { var err error var tcDefault, tcStandby TestClient diff --git a/test/e2e/parallelfilesdownload/parallel_files_download.go b/test/e2e/parallelfilesdownload/parallel_files_download.go new file mode 100644 index 000000000..f607ba449 --- /dev/null +++ b/test/e2e/parallelfilesdownload/parallel_files_download.go @@ -0,0 +1,147 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parallelfilesdownload + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/k8s" +) + +type ParallelFilesDownload struct { + TestCase + parallel string + namespace string + pod string + pvc string + volume string + fileName string + fileNum int + fileSize int64 + hash []string +} + +var ParallelFilesDownloadTest func() = TestFunc(&ParallelFilesDownload{}) + +func (p *ParallelFilesDownload) Init() error { + // generate random number as UUIDgen and set one default timeout duration + p.TestCase.Init() + + // generate variable names based on CaseBaseName + UUIDgen + p.CaseBaseName = "parallel-files-download" + p.UUIDgen + p.BackupName = p.CaseBaseName + "-backup" + p.RestoreName = p.CaseBaseName + "-restore" + p.pod = p.CaseBaseName + "-pod" + p.pvc = p.CaseBaseName + "-pvc" + p.fileName = p.CaseBaseName + "-file" + p.parallel = "3" + p.fileNum = 10 + p.fileSize = 1 * 1024 * 1024 // 1MB + p.volume = p.CaseBaseName + "-vol" + + // generate namespace + p.VeleroCfg.UseVolumeSnapshots = false + p.VeleroCfg.UseNodeAgent = true + p.namespace = p.CaseBaseName + "-ns" + + p.BackupArgs = []string{ + "create", "--namespace", p.VeleroCfg.VeleroNamespace, + "backup", p.BackupName, + "--include-namespaces", p.namespace, + "--default-volumes-to-fs-backup", + "--snapshot-volumes=false", + "--wait", + } + + p.RestoreArgs = []string{ + "create", "--namespace", p.VeleroCfg.VeleroNamespace, + "restore", p.RestoreName, + "--parallel-files-download", p.parallel, + "--from-backup", p.BackupName, "--wait", + } + + // Message output by ginkgo + p.TestMsg = &TestMSG{ + Desc: "Test parallel files download", + FailedMSG: "Failed to test parallel files download", + Text: "Test parallel files download with parallel download " + p.parallel + " files", + } + return nil +} + +func (p *ParallelFilesDownload) CreateResources() error { + err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) + if err != nil { + return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") + } + + By(fmt.Sprintf("Create namespace %s", p.namespace), func() { + Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s", p.namespace)) + }) + + By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() { + _, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil) + Expect(err).To(Succeed()) + err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod}) + Expect(err).To(Succeed()) + }) + + podList, err := ListPods(p.Ctx, p.Client, p.namespace) + Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err)) + + for _, pod := range podList.Items { + for i := 0; i < p.fileNum; i++ { + fileName := fmt.Sprintf("%s-%d", p.fileName, i) + // Write random data to file in pod + Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume, + fileName, p.fileSize)).To(Succeed()) + // Calculate hash of the file + hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName)) + Expect(err).To(Succeed()) + p.hash = append(p.hash, hash) + } + } + + return nil +} + +func (p *ParallelFilesDownload) Verify() error { + podList, err := ListPods(p.Ctx, p.Client, p.namespace) + Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err)) + + for _, pod := range podList.Items { + err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{pod.Name}) + Expect(err).To(Succeed()) + + for i := 0; i < p.fileNum; i++ { + fileName := fmt.Sprintf("%s-%d", p.fileName, i) + // Calculate hash of the file + hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName)) + Expect(err).To(Succeed()) + Expect(hash).To(Equal(p.hash[i])) + } + } + + return nil +} diff --git a/test/e2e/parallelfilesupload/parallel_files_upload.go b/test/e2e/parallelfilesupload/parallel_files_upload.go new file mode 100644 index 000000000..313edac11 --- /dev/null +++ b/test/e2e/parallelfilesupload/parallel_files_upload.go @@ -0,0 +1,112 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package parallelfilesupload + +import ( + "fmt" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/k8s" +) + +type ParallelFilesUpload struct { + TestCase + parallel string + namespace string + pod string + pvc string + volume string + fileName string + fileNum int + fileSize int64 +} + +var ParallelFilesUploadTest func() = TestFunc(&ParallelFilesUpload{}) + +func (p *ParallelFilesUpload) Init() error { + // generate random number as UUIDgen and set one default timeout duration + p.TestCase.Init() + + // generate variable names based on CaseBaseName + UUIDgen + p.CaseBaseName = "parallel-files-upload" + p.UUIDgen + p.BackupName = p.CaseBaseName + "-backup" + p.pod = p.CaseBaseName + "-pod" + p.pvc = p.CaseBaseName + "-pvc" + p.fileName = p.CaseBaseName + "-file" + p.parallel = "3" + p.fileNum = 10 + p.fileSize = 1 * 1024 * 1024 // 1MB + p.volume = p.CaseBaseName + "-vol" + // generate namespace + p.VeleroCfg.UseVolumeSnapshots = false + p.VeleroCfg.UseNodeAgent = true + p.namespace = p.CaseBaseName + "-ns" + + p.BackupArgs = []string{ + "create", "--namespace", p.VeleroCfg.VeleroNamespace, + "backup", p.BackupName, + "--include-namespaces", p.namespace, + "--parallel-files-upload", p.parallel, + "--default-volumes-to-fs-backup", + "--snapshot-volumes=false", + "--wait", + } + + // Message output by ginkgo + p.TestMsg = &TestMSG{ + Desc: "Test parallel files upload", + FailedMSG: "Failed to test parallel files upload", + Text: "Test parallel files upload with parallel upload " + p.parallel + " files", + } + return nil +} + +func (p *ParallelFilesUpload) CreateResources() error { + err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) + if err != nil { + return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") + } + + By(fmt.Sprintf("Create namespace %s", p.namespace), func() { + Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s", p.namespace)) + }) + + By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() { + _, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil) + Expect(err).To(Succeed()) + err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod}) + Expect(err).To(Succeed()) + }) + + podList, err := ListPods(p.Ctx, p.Client, p.namespace) + Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err)) + + for _, pod := range podList.Items { + for i := 0; i < p.fileNum; i++ { + Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume, + fmt.Sprintf("%s-%d", p.fileName, i), p.fileSize)).To(Succeed()) + } + } + + return nil +} diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 933581562..95f685811 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -299,6 +299,29 @@ func PrepareVolumeList(volumeNameList []string) (vols []*corev1.Volume) { return } +func CalFileHashInPod(ctx context.Context, namespace, podName, containerName, filePath string) (string, error) { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, + "--", "/bin/sh", "-c", fmt.Sprintf("sha256sum %s | awk '{ print $1 }'", filePath)} + cmd := exec.CommandContext(ctx, "kubectl", arg...) + output, err := cmd.Output() + if err != nil { + return "", err + } + // Trim any leading or trailing whitespace characters from the output + hash := string(output) + hash = strings.TrimSpace(hash) + + return hash, nil +} + +func WriteRandomDataToFileInPod(ctx context.Context, namespace, podName, containerName, volume, filename string, fileSize int64) error { + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, + "--", "/bin/sh", "-c", fmt.Sprintf("dd if=/dev/urandom of=/%s/%s bs=%d count=1", volume, filename, fileSize)} + cmd := exec.CommandContext(ctx, "kubectl", arg...) + fmt.Printf("Kubectl exec cmd =%v\n", cmd) + return cmd.Run() +} + func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error { arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)} @@ -306,6 +329,7 @@ func CreateFileToPod(ctx context.Context, namespace, podName, containerName, vol fmt.Printf("Kubectl exec cmd =%v\n", cmd) return cmd.Run() } + func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) { stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 48e2faf1d..e067ea879 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -598,22 +598,32 @@ func IsVeleroReady(ctx context.Context, veleroCfg *VeleroConfig) (bool, error) { } } - // Check BSL - stdout, stderr, err = velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default", + // Check BSL with poll + err = wait.PollUntilContextTimeout(ctx, PollInterval, time.Minute, true, func(ctx context.Context) (bool, error) { + return checkBSL(ctx, veleroCfg) == nil, nil + }) + if err != nil { + return false, errors.Wrap(err, "failed to check the bsl") + } + return true, nil +} + +func checkBSL(ctx context.Context, veleroCfg *VeleroConfig) error { + namespace := veleroCfg.VeleroNamespace + stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "bsl", "default", "-o", "json", "-n", namespace)) if err != nil { - return false, errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", veleroCfg.BSLBucket, stdout, stderr) + return errors.Wrapf(err, "failed to get bsl %s stdout=%s, stderr=%s", veleroCfg.BSLBucket, stdout, stderr) } else { bsl := &velerov1api.BackupStorageLocation{} if err = json.Unmarshal([]byte(stdout), bsl); err != nil { - return false, errors.Wrapf(err, "failed to unmarshal the velero bsl") + return errors.Wrapf(err, "failed to unmarshal the velero bsl") } if bsl.Status.Phase != velerov1api.BackupStorageLocationPhaseAvailable { - return false, fmt.Errorf("current bsl %s is not available", veleroCfg.BSLBucket) + return fmt.Errorf("current bsl %s is not available", veleroCfg.BSLBucket) } } - - return true, nil + return nil } func PrepareVelero(ctx context.Context, caseName string, veleroCfg VeleroConfig) error {