mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-09-30 19:55:36 +00:00
Add pause/unpause schedule e2e test (#5609)
Signed-off-by: danfengl <danfengl@vmware.com>
This commit is contained in:
danfengliu
@@ -0,0 +1,204 @@
|
||||
package schedule
|
||||
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
//the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
waitutil "k8s.io/apimachinery/pkg/util/wait"
|
||||
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
type OrderedResources struct {
|
||||
Namespace string
|
||||
ScheduleName string
|
||||
OrderMap map[string]string
|
||||
ScheduleArgs []string
|
||||
TestCase
|
||||
}
|
||||
|
||||
func ScheduleOrderedResources() {
|
||||
BeforeEach(func() {
|
||||
flag.Parse()
|
||||
if VeleroCfg.InstallVelero {
|
||||
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
|
||||
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
It("Create a schedule to backup resources in a specific order should be successful", func() {
|
||||
test := &OrderedResources{}
|
||||
err := test.Init()
|
||||
Expect(err).To(Succeed(), err)
|
||||
defer func() {
|
||||
Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace))
|
||||
err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err))
|
||||
err = test.DeleteBackups()
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err))
|
||||
}()
|
||||
|
||||
By(fmt.Sprintf("Prepare workload as target to backup in base namespace %s", test.Namespace), func() {
|
||||
err = test.CreateResources()
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to create resources to backup with err %v", err))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() {
|
||||
err = VeleroScheduleCreate(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() {
|
||||
err = CheckScheduleWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
By("Checking resource order in backup cr", func() {
|
||||
backupList := new(velerov1api.BackupList)
|
||||
err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) {
|
||||
if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
|
||||
return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
|
||||
}
|
||||
|
||||
for _, backup := range backupList.Items {
|
||||
if err = CheckBackupWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
fmt.Printf("still finding backup created by schedule %s ...\n", test.ScheduleName)
|
||||
return false, nil
|
||||
})
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s created backup with err %v", test.ScheduleName, err))
|
||||
})
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
func (o *OrderedResources) Init() error {
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
UUIDgen, _ = uuid.NewRandom()
|
||||
client := *VeleroCfg.ClientToInstallVelero
|
||||
|
||||
o.Client = client
|
||||
o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String()
|
||||
o.NSBaseName = "schedule-ordered-resources"
|
||||
o.Namespace = o.NSBaseName + "-" + UUIDgen.String()
|
||||
o.OrderMap = map[string]string{
|
||||
"deployments": fmt.Sprintf("deploy-%s", o.NSBaseName),
|
||||
"secrets": fmt.Sprintf("secret-%s", o.NSBaseName),
|
||||
"configmaps": fmt.Sprintf("configmap-%s", o.NSBaseName),
|
||||
}
|
||||
|
||||
o.ScheduleArgs = []string{"--schedule", "@every 1m",
|
||||
"--include-namespaces", o.Namespace, "--default-volumes-to-fs-backup", "--ordered-resources"}
|
||||
var orderStr string
|
||||
for kind, resource := range o.OrderMap {
|
||||
orderStr += fmt.Sprintf("%s=%s;", kind, resource)
|
||||
}
|
||||
o.ScheduleArgs = append(o.ScheduleArgs, strings.TrimRight(orderStr, ";"))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OrderedResources) CreateResources() error {
|
||||
o.Ctx, _ = context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
label := map[string]string{
|
||||
"orderedresources": "true",
|
||||
}
|
||||
fmt.Printf("Creating resources in %s namespace ...\n", o.Namespace)
|
||||
if err := CreateNamespace(o.Ctx, o.Client, o.Namespace); err != nil {
|
||||
return errors.Wrapf(err, "failed to create namespace %s", o.Namespace)
|
||||
}
|
||||
serviceAccountName := "default"
|
||||
// wait until the service account is created before patch the image pull secret
|
||||
if err := WaitUntilServiceAccountCreated(o.Ctx, o.Client, o.Namespace, serviceAccountName, 10*time.Minute); err != nil {
|
||||
return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace)
|
||||
}
|
||||
// add the image pull secret to avoid the image pull limit issue of Docker Hub
|
||||
if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil {
|
||||
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace)
|
||||
}
|
||||
//Create deployment
|
||||
deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
|
||||
deployment := NewDeployment(deploymentName, o.Namespace, 1, label)
|
||||
deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))
|
||||
}
|
||||
err = WaitForReadyDeployment(o.Client.ClientGo, o.Namespace, deployment.Name)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
//Create Secret
|
||||
secretName := fmt.Sprintf("secret-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating secret %s in %s namespaces ...\n", secretName, o.Namespace)
|
||||
_, err = CreateSecret(o.Client.ClientGo, o.Namespace, secretName, label)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create secret in the namespace %q", o.Namespace))
|
||||
}
|
||||
err = WaitForSecretsComplete(o.Client.ClientGo, o.Namespace, secretName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
//Create Configmap
|
||||
configmapName := fmt.Sprintf("configmap-%s", o.NSBaseName)
|
||||
fmt.Printf("Creating configmap %s in %s namespaces ...\n", configmapName, o.Namespace)
|
||||
_, err = CreateConfigMap(o.Client.ClientGo, o.Namespace, configmapName, label)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to create configmap in the namespace %q", o.Namespace))
|
||||
}
|
||||
err = WaitForConfigMapComplete(o.Client.ClientGo, o.Namespace, configmapName)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, fmt.Sprintf("failed to ensure secret completion in namespace: %q", o.Namespace))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (o *OrderedResources) DeleteBackups() error {
|
||||
backupList := new(velerov1api.BackupList)
|
||||
if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
|
||||
return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
|
||||
}
|
||||
for _, backup := range backupList.Items {
|
||||
if err := VeleroBackupDelete(o.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
package schedule
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
type ScheduleBackup struct {
|
||||
TestCase
|
||||
ScheduleName string
|
||||
ScheduleArgs []string
|
||||
Period int //Limitation: The unit is minitue only and 60 is divisible by it
|
||||
randBackupName string
|
||||
verifyTimes int
|
||||
}
|
||||
|
||||
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test"}})
|
||||
|
||||
func (n *ScheduleBackup) Init() error {
|
||||
n.Client = TestClientInstance
|
||||
n.Period = 3 // Unit is minute
|
||||
n.verifyTimes = 5 // More verify times more confidence
|
||||
n.TestMsg = &TestMSG{
|
||||
Desc: "Set up a scheduled backup defined by a Cron expression",
|
||||
FailedMSG: "Failed to schedule a backup",
|
||||
Text: "should backup periodly according to the schedule",
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *ScheduleBackup) StartRun() error {
|
||||
n.NSIncluded = &[]string{fmt.Sprintf("%s-%s", n.NSBaseName, "ns")}
|
||||
n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String()
|
||||
n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
|
||||
|
||||
n.ScheduleArgs = []string{
|
||||
"--include-namespaces", strings.Join(*n.NSIncluded, ","),
|
||||
"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
|
||||
}
|
||||
Expect(n.Period < 30).To(Equal(true))
|
||||
return nil
|
||||
}
|
||||
func (n *ScheduleBackup) CreateResources() error {
|
||||
n.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
|
||||
for _, ns := range *n.NSIncluded {
|
||||
By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() {
|
||||
Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
|
||||
})
|
||||
configmaptName := n.NSBaseName
|
||||
fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns)
|
||||
_, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns))
|
||||
Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(),
|
||||
fmt.Sprintf("ailed to ensure secret completion in namespace: %q", ns))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *ScheduleBackup) Backup() error {
|
||||
// Wait until the beginning of the given period to create schedule, it will give us
|
||||
// a predictable period to wait for the first scheduled backup, and verify no immediate
|
||||
// scheduled backup was created between schedule creation and first scheduled backup.
|
||||
By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() {
|
||||
for i := 0; i < n.Period*60/30; i++ {
|
||||
time.Sleep(30 * time.Second)
|
||||
now := time.Now().Minute()
|
||||
triggerNow := now % n.Period
|
||||
if triggerNow == 0 {
|
||||
Expect(VeleroScheduleCreate(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
return nil
|
||||
}
|
||||
func (n *ScheduleBackup) Destroy() error {
|
||||
By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() {
|
||||
creationTimestamp, err := GetSchedule(context.Background(), VeleroCfg.VeleroNamespace, n.ScheduleName)
|
||||
Expect(err).To(Succeed())
|
||||
|
||||
creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1))
|
||||
Expect(err).To(Succeed())
|
||||
fmt.Printf("Schedule %s created at %s\n", n.ScheduleName, creationTime)
|
||||
now := time.Now()
|
||||
diff := creationTime.Sub(now)
|
||||
Expect(diff.Minutes() < 1).To(Equal(true))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("No immediate backup is created by schedule %s\n", n.ScheduleName), func() {
|
||||
for i := 0; i < n.Period; i++ {
|
||||
time.Sleep(1 * time.Minute)
|
||||
now := time.Now()
|
||||
fmt.Printf("Get backup for #%d time at %v\n", i, now)
|
||||
//Ignore the last minute in the period avoiding met the 1st backup by schedule
|
||||
if i != n.Period-1 {
|
||||
backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
|
||||
Expect(err).To(Succeed())
|
||||
Expect(len(backupsInfo) == 0).To(Equal(true))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
By("Delay one more minute to make sure the new backup was created in the given period", func() {
|
||||
time.Sleep(1 * time.Minute)
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() {
|
||||
for i := 0; i < n.verifyTimes; i++ {
|
||||
fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1)
|
||||
time.Sleep(time.Duration(n.Period) * time.Minute)
|
||||
bMap := make(map[string]string)
|
||||
backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
|
||||
Expect(err).To(Succeed())
|
||||
Expect(len(backupsInfo) == i+2).To(Equal(true))
|
||||
for index, bi := range backupsInfo {
|
||||
bList := strings.Split(bi, ",")
|
||||
fmt.Printf("Backup %d: %v\n", index, bList)
|
||||
bMap[bList[0]] = bList[1]
|
||||
_, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1])
|
||||
Expect(err).To(Succeed())
|
||||
}
|
||||
if i == n.verifyTimes-1 {
|
||||
backupInfo := backupsInfo[rand.Intn(len(backupsInfo))]
|
||||
n.randBackupName = strings.Split(backupInfo, ",")[0]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
n.BackupName = strings.Replace(n.randBackupName, " ", "", -1)
|
||||
|
||||
By("Delete all namespaces", func() {
|
||||
Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces")
|
||||
})
|
||||
|
||||
n.RestoreArgs = []string{
|
||||
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
|
||||
"--from-backup", n.BackupName,
|
||||
"--wait",
|
||||
}
|
||||
|
||||
backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
|
||||
fmt.Println(backupsInfo)
|
||||
backupCount := len(backupsInfo)
|
||||
|
||||
By(fmt.Sprintf("Pause schedule %s ......\n", n.ScheduleName), func() {
|
||||
Expect(VeleroSchedulePause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
})
|
||||
|
||||
periodCount := 3
|
||||
sleepDuration := time.Duration(n.Period*periodCount) * time.Minute
|
||||
By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
|
||||
time.Sleep(sleepDuration)
|
||||
})
|
||||
|
||||
backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
|
||||
|
||||
backupCountPostPause := len(backupsInfo)
|
||||
fmt.Printf("After pause, backkups count is %d\n", backupCountPostPause)
|
||||
|
||||
By(fmt.Sprintf("Verify no new backups from %s ......\n", n.ScheduleName), func() {
|
||||
Expect(backupCountPostPause == backupCount).To(Equal(true))
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Unpause schedule %s ......\n", n.ScheduleName), func() {
|
||||
Expect(VeleroScheduleUnpause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
|
||||
return "Fail to unpause schedule"
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
|
||||
time.Sleep(sleepDuration)
|
||||
})
|
||||
|
||||
backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
|
||||
fmt.Println(backupsInfo)
|
||||
backupCountPostUnpause := len(backupsInfo)
|
||||
fmt.Printf("After unpause, backkups count is %d\n", backupCountPostUnpause)
|
||||
By(fmt.Sprintf("Verify no new backups by schedule %s ......\n", n.ScheduleName), func() {
|
||||
Expect(backupCountPostUnpause-backupCount >= periodCount-1).To(Equal(true))
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *ScheduleBackup) Verify() error {
|
||||
By("Namespaces were restored", func() {
|
||||
for _, ns := range *n.NSIncluded {
|
||||
configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.NSBaseName)
|
||||
fmt.Printf("Restored configmap is %v\n", configmap)
|
||||
Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns))
|
||||
}
|
||||
|
||||
})
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user