mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-10-01 12:15:36 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #7691 from qiuming-best/e2e-parallel-upload-download

Browse Source 
Add E2E test for parallel files upload and download
This commit is contained in:
qiuming
2024-05-09 15:27:06 +08:00
committed by GitHub
parent 0d85a647b5 a628cb525f
commit b8a48c0ef8
5 changed files with 305 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/e2e_suite_test.go
+5
View File
@@ -39,6 +39,8 @@ import (
. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
. "github.com/vmware-tanzu/velero/test/e2e/migration"
. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesdownload"
. "github.com/vmware-tanzu/velero/test/e2e/parallelfilesupload"
. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
@@ -182,6 +184,9 @@ var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore
var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest)
var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)
var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest)
var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest)
func GetKubeconfigContext() error {
var err error
var tcDefault, tcStandby TestClient
test/e2e/parallelfilesdownload/parallel_files_download.go
+147
View File
@@ -0,0 +1,147 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package parallelfilesdownload
import (
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
. "github.com/vmware-tanzu/velero/test"
. "github.com/vmware-tanzu/velero/test/e2e/test"
. "github.com/vmware-tanzu/velero/test/util/k8s"
)
type ParallelFilesDownload struct {
TestCase
parallel string
namespace string
pod string
pvc string
volume string
fileName string
fileNum int
fileSize int64
hash []string
}
var ParallelFilesDownloadTest func() = TestFunc(&ParallelFilesDownload{})
func (p *ParallelFilesDownload) Init() error {
// generate random number as UUIDgen and set one default timeout duration
p.TestCase.Init()
// generate variable names based on CaseBaseName + UUIDgen
p.CaseBaseName = "parallel-files-download" + p.UUIDgen
p.BackupName = p.CaseBaseName + "-backup"
p.RestoreName = p.CaseBaseName + "-restore"
p.pod = p.CaseBaseName + "-pod"
p.pvc = p.CaseBaseName + "-pvc"
p.fileName = p.CaseBaseName + "-file"
p.parallel = "3"
p.fileNum = 10
p.fileSize = 1 * 1024 * 1024 // 1MB
p.volume = p.CaseBaseName + "-vol"
// generate namespace
p.VeleroCfg.UseVolumeSnapshots = false
p.VeleroCfg.UseNodeAgent = true
p.namespace = p.CaseBaseName + "-ns"
p.BackupArgs = []string{
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
"backup", p.BackupName,
"--include-namespaces", p.namespace,
"--default-volumes-to-fs-backup",
"--snapshot-volumes=false",
"--wait",
}
p.RestoreArgs = []string{
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
"restore", p.RestoreName,
"--parallel-files-download", p.parallel,
"--from-backup", p.BackupName, "--wait",
}
// Message output by ginkgo
p.TestMsg = &TestMSG{
Desc: "Test parallel files download",
FailedMSG: "Failed to test parallel files download",
Text: "Test parallel files download with parallel download " + p.parallel + " files",
}
return nil
}
func (p *ParallelFilesDownload) CreateResources() error {
err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))
if err != nil {
return errors.Wrapf(err, "failed to install storage class for pv backup filtering test")
}
By(fmt.Sprintf("Create namespace %s", p.namespace), func() {
Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(),
fmt.Sprintf("Failed to create namespace %s", p.namespace))
})
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
Expect(err).To(Succeed())
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
Expect(err).To(Succeed())
})
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
for _, pod := range podList.Items {
for i := 0; i < p.fileNum; i++ {
fileName := fmt.Sprintf("%s-%d", p.fileName, i)
// Write random data to file in pod
Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume,
fileName, p.fileSize)).To(Succeed())
// Calculate hash of the file
hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName))
Expect(err).To(Succeed())
p.hash = append(p.hash, hash)
}
}
return nil
}
func (p *ParallelFilesDownload) Verify() error {
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
for _, pod := range podList.Items {
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{pod.Name})
Expect(err).To(Succeed())
for i := 0; i < p.fileNum; i++ {
fileName := fmt.Sprintf("%s-%d", p.fileName, i)
// Calculate hash of the file
hash, err := CalFileHashInPod(p.Ctx, p.namespace, pod.Name, pod.Name, fmt.Sprintf("%s/%s", p.volume, fileName))
Expect(err).To(Succeed())
Expect(hash).To(Equal(p.hash[i]))
}
}
return nil
}
test/e2e/parallelfilesupload/parallel_files_upload.go
+112
View File
@@ -0,0 +1,112 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package parallelfilesupload
import (
"fmt"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pkg/errors"
. "github.com/vmware-tanzu/velero/test"
. "github.com/vmware-tanzu/velero/test/e2e/test"
. "github.com/vmware-tanzu/velero/test/util/k8s"
)
type ParallelFilesUpload struct {
TestCase
parallel string
namespace string
pod string
pvc string
volume string
fileName string
fileNum int
fileSize int64
}
var ParallelFilesUploadTest func() = TestFunc(&ParallelFilesUpload{})
func (p *ParallelFilesUpload) Init() error {
// generate random number as UUIDgen and set one default timeout duration
p.TestCase.Init()
// generate variable names based on CaseBaseName + UUIDgen
p.CaseBaseName = "parallel-files-upload" + p.UUIDgen
p.BackupName = p.CaseBaseName + "-backup"
p.pod = p.CaseBaseName + "-pod"
p.pvc = p.CaseBaseName + "-pvc"
p.fileName = p.CaseBaseName + "-file"
p.parallel = "3"
p.fileNum = 10
p.fileSize = 1 * 1024 * 1024 // 1MB
p.volume = p.CaseBaseName + "-vol"
// generate namespace
p.VeleroCfg.UseVolumeSnapshots = false
p.VeleroCfg.UseNodeAgent = true
p.namespace = p.CaseBaseName + "-ns"
p.BackupArgs = []string{
"create", "--namespace", p.VeleroCfg.VeleroNamespace,
"backup", p.BackupName,
"--include-namespaces", p.namespace,
"--parallel-files-upload", p.parallel,
"--default-volumes-to-fs-backup",
"--snapshot-volumes=false",
"--wait",
}
// Message output by ginkgo
p.TestMsg = &TestMSG{
Desc: "Test parallel files upload",
FailedMSG: "Failed to test parallel files upload",
Text: "Test parallel files upload with parallel upload " + p.parallel + " files",
}
return nil
}
func (p *ParallelFilesUpload) CreateResources() error {
err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))
if err != nil {
return errors.Wrapf(err, "failed to install storage class for pv backup filtering test")
}
By(fmt.Sprintf("Create namespace %s", p.namespace), func() {
Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(),
fmt.Sprintf("Failed to create namespace %s", p.namespace))
})
By(fmt.Sprintf("Create pod %s in namespace %s", p.pod, p.namespace), func() {
_, err := CreatePod(p.Client, p.namespace, p.pod, StorageClassName, p.pvc, []string{p.volume}, nil, nil)
Expect(err).To(Succeed())
err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.pod})
Expect(err).To(Succeed())
})
podList, err := ListPods(p.Ctx, p.Client, p.namespace)
Expect(err).To(Succeed(), fmt.Sprintf("failed to list pods in namespace: %q with error %v", p.namespace, err))
for _, pod := range podList.Items {
for i := 0; i < p.fileNum; i++ {
Expect(WriteRandomDataToFileInPod(p.Ctx, p.namespace, pod.Name, pod.Name, p.volume,
fmt.Sprintf("%s-%d", p.fileName, i), p.fileSize)).To(Succeed())
}
}
return nil
}