Add pause/unpause schedule e2e test (#5609)

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/e2e_suite_test.go

@@ -33,15 +33,13 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/basic"
 	. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
 	. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
-	. "github.com/vmware-tanzu/velero/test/e2e/orderedresources"
+	. "github.com/vmware-tanzu/velero/test/e2e/migration"
 	. "github.com/vmware-tanzu/velero/test/e2e/privilegesmgmt"
 	. "github.com/vmware-tanzu/velero/test/e2e/pv-backup"
 	. "github.com/vmware-tanzu/velero/test/e2e/resource-filtering"
-
 	. "github.com/vmware-tanzu/velero/test/e2e/scale"
+	. "github.com/vmware-tanzu/velero/test/e2e/schedule"
 	. "github.com/vmware-tanzu/velero/test/e2e/upgrade"
-
-	. "github.com/vmware-tanzu/velero/test/e2e/migration"
 	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
 )
 
@@ -114,7 +112,9 @@ var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup dele
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
 var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
-var _ = Describe("[Backups][Schedule] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
+
+var _ = Describe("[Schedule][BR][Pause] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
+var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
 
 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
 
@@ -125,8 +125,6 @@ var _ = Describe("[Migration][Restic]", MigrationWithRestic)
 
 var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots)
 
-var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
-
 var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
 var _ = Describe("[NamespaceMapping][Multiple][Restic]  Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
 var _ = Describe("[NamespaceMapping][Single][Snapshot]  Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)

test/e2e/schedule/ordered_resources.go

@@ -1,4 +1,4 @@
-package orderedresources
+package schedule
 
 /*
 Copyright the Velero contributors.

test/e2e/schedule/schedule.go

@@ -1,4 +1,4 @@
-package backups
+package schedule
 
 import (
 	"context"
@@ -25,11 +25,11 @@ type ScheduleBackup struct {
 	verifyTimes    int
 }
 
-var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test-ns", NSIncluded: &[]string{"ns1"}}})
+var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test"}})
 
 func (n *ScheduleBackup) Init() error {
 	n.Client = TestClientInstance
-	n.Period = 3
+	n.Period = 3      // Unit is minute
 	n.verifyTimes = 5 // More verify times more confidence
 	n.TestMsg = &TestMSG{
 		Desc:      "Set up a scheduled backup defined by a Cron expression",
@@ -40,7 +40,7 @@ func (n *ScheduleBackup) Init() error {
 }
 
 func (n *ScheduleBackup) StartRun() error {
-
+	n.NSIncluded = &[]string{fmt.Sprintf("%s-%s", n.NSBaseName, "ns")}
 	n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String()
 	n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
 
@@ -48,7 +48,7 @@ func (n *ScheduleBackup) StartRun() error {
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","),
 		"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
 	}
-
+	Expect(n.Period < 30).To(Equal(true))
 	return nil
 }
 func (n *ScheduleBackup) CreateResources() error {
@@ -152,6 +152,53 @@ func (n *ScheduleBackup) Destroy() error {
 		"--wait",
 	}
 
+	backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
+	fmt.Println(backupsInfo)
+	backupCount := len(backupsInfo)
+
+	By(fmt.Sprintf("Pause schedule %s ......\n", n.ScheduleName), func() {
+		Expect(VeleroSchedulePause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
+			RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
+			return "Fail to restore workload"
+		})
+	})
+
+	periodCount := 3
+	sleepDuration := time.Duration(n.Period*periodCount) * time.Minute
+	By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
+		time.Sleep(sleepDuration)
+	})
+
+	backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
+
+	backupCountPostPause := len(backupsInfo)
+	fmt.Printf("After pause, backkups count is %d\n", backupCountPostPause)
+
+	By(fmt.Sprintf("Verify no new backups from %s ......\n", n.ScheduleName), func() {
+		Expect(backupCountPostPause == backupCount).To(Equal(true))
+	})
+
+	By(fmt.Sprintf("Unpause schedule %s ......\n", n.ScheduleName), func() {
+		Expect(VeleroScheduleUnpause(n.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
+			RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", "")
+			return "Fail to unpause schedule"
+		})
+	})
+
+	By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
+		time.Sleep(sleepDuration)
+	})
+
+	backupsInfo, err = GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
+	fmt.Println(backupsInfo)
+	backupCountPostUnpause := len(backupsInfo)
+	fmt.Printf("After unpause, backkups count is %d\n", backupCountPostUnpause)
+	By(fmt.Sprintf("Verify no new backups by schedule %s ......\n", n.ScheduleName), func() {
+		Expect(backupCountPostUnpause-backupCount >= periodCount-1).To(Equal(true))
+	})
 	return nil
 }
 

test/e2e/util/velero/velero_utils.go

@@ -270,6 +270,24 @@ func checkSchedulePhase(ctx context.Context, veleroCLI, veleroNamespace, schedul
 	})
 }
 
+func checkSchedulePause(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, pause bool) error {
+	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson")
+	jsonBuf, err := CMDExecWithOutput(checkCMD)
+	if err != nil {
+		return err
+	}
+	schedule := velerov1api.Schedule{}
+	err = json.Unmarshal(*jsonBuf, &schedule)
+	if err != nil {
+		return err
+	}
+
+	if schedule.Spec.Paused != pause {
+		fmt.Printf("Unexpected schedule phase got %s, expecting %s, still waiting...", schedule.Status.Phase, velerov1api.SchedulePhaseEnabled)
+		return nil
+	}
+	return nil
+}
 func CheckScheduleWithResourceOrder(ctx context.Context, veleroCLI, veleroNamespace, scheduleName string, order map[string]string) error {
 	checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "schedule", "get", scheduleName, "-ojson")
 	jsonBuf, err := CMDExecWithOutput(checkCMD)
@@ -442,6 +460,28 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace
 	return checkSchedulePhase(ctx, veleroCLI, veleroNamespace, scheduleName)
 }
 
+func VeleroSchedulePause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
+	var args []string
+	args = append([]string{
+		"--namespace", veleroNamespace, "schedule", "pause", scheduleName,
+	})
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	return checkSchedulePause(ctx, veleroCLI, veleroNamespace, scheduleName, true)
+}
+
+func VeleroScheduleUnpause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
+	var args []string
+	args = append([]string{
+		"--namespace", veleroNamespace, "schedule", "unpause", scheduleName,
+	})
+	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
+		return err
+	}
+	return checkSchedulePause(ctx, veleroCLI, veleroNamespace, scheduleName, false)
+}
+
 func VeleroCmdExec(ctx context.Context, veleroCLI string, args []string) error {
 	cmd := exec.CommandContext(ctx, veleroCLI, args...)
 	cmd.Stdout = os.Stdout