velero/test/e2e/schedule/schedule.go

package schedule

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	. "github.com/vmware-tanzu/velero/test/e2e/test"
	. "github.com/vmware-tanzu/velero/test/util/k8s"
	. "github.com/vmware-tanzu/velero/test/util/velero"
)

type ScheduleBackup struct {
	TestCase
	ScheduleName   string
	ScheduleArgs   []string
	Period         int //Limitation: The unit is minitue only and 60 is divisible by it
	randBackupName string
	verifyTimes    int
}

var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{})

func (n *ScheduleBackup) Init() error {
	n.TestCase.Init()
	n.CaseBaseName = "schedule-backup-" + n.UUIDgen
	n.NSIncluded = &[]string{n.CaseBaseName}
	n.ScheduleName = "schedule-" + n.CaseBaseName
	n.RestoreName = "restore-" + n.CaseBaseName
	n.Period = 3      // Unit is minute
	n.verifyTimes = 5 // More larger verify times more confidence we have
	n.TestMsg = &TestMSG{
		Desc:      "Set up a scheduled backup defined by a Cron expression",
		FailedMSG: "Failed to schedule a backup",
		Text:      "should backup periodly according to the schedule",
	}
	n.ScheduleArgs = []string{
		"--include-namespaces", strings.Join(*n.NSIncluded, ","),
		"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
	}

	Expect(n.Period).To(BeNumerically("<", 30))
	return nil
}
func (n *ScheduleBackup) CreateResources() error {
	for _, ns := range *n.NSIncluded {
		By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() {
			Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
		})
		configmaptName := n.CaseBaseName
		fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns)
		_, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil, nil)
		Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns))
		Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(),
			fmt.Sprintf("failed to ensure secret completion in namespace: %q", ns))
	}
	return nil
}

func (n *ScheduleBackup) Backup() error {
	// Wait until the beginning of the given period to create schedule, it will give us
	//   a predictable period to wait for the first scheduled backup, and verify no immediate
	//   scheduled backup was created between schedule creation and first scheduled backup.
	By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() {
		for i := 0; i < n.Period*60/30; i++ {
			time.Sleep(30 * time.Second)
			now := time.Now().Minute()
			triggerNow := now % n.Period
			if triggerNow == 0 {
				Expect(VeleroScheduleCreate(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string {
					RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "")
					return "Fail to create schedule"
				})
				break
			}
		}
	})

	By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() {
		creationTimestamp, err := GetSchedule(n.Ctx, n.VeleroCfg.VeleroNamespace, n.ScheduleName)
		Expect(err).To(Succeed())

		creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1))
		Expect(err).To(Succeed())
		fmt.Printf("Schedule %s created at %s\n", n.ScheduleName, creationTime)
		now := time.Now()
		diff := creationTime.Sub(now)
		Expect(diff.Minutes()).To(BeNumerically("<", 1))
	})

	By(fmt.Sprintf("No immediate backup is created by schedule %s\n", n.ScheduleName), func() {
		for i := 0; i < n.Period; i++ {
			time.Sleep(1 * time.Minute)
			now := time.Now()
			fmt.Printf("Get backup for #%d time at %v\n", i, now)
			//Ignore the last minute in the period avoiding met the 1st backup by schedule
			if i != n.Period-1 {
				backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
				Expect(err).To(Succeed())
				Expect(backupsInfo).To(BeEmpty())
			}
		}
	})

	By("Delay one more minute to make sure the new backup was created in the given period", func() {
		time.Sleep(time.Minute)
	})

	By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() {
		for i := 0; i < n.verifyTimes; i++ {
			fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1)
			time.Sleep(time.Duration(n.Period) * time.Minute)
			bMap := make(map[string]string)
			backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
			Expect(err).To(Succeed())
			Expect(backupsInfo).To(HaveLen(i + 2))
			for index, bi := range backupsInfo {
				bList := strings.Split(bi, ",")
				fmt.Printf("Backup %d: %v\n", index, bList)
				bMap[bList[0]] = bList[1]
				_, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1])
				Expect(err).To(Succeed())
			}
			if i == n.verifyTimes-1 {
				backupInfo := backupsInfo[rand.Intn(len(backupsInfo))]
				n.randBackupName = strings.Split(backupInfo, ",")[0]
			}
		}
	})

	n.BackupName = strings.Replace(n.randBackupName, " ", "", -1)

	n.RestoreArgs = []string{
		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
		"--from-backup", n.BackupName,
		"--wait",
	}

	backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
	fmt.Println(backupsInfo)
	backupCount := len(backupsInfo)

	By(fmt.Sprintf("Pause schedule %s ......\n", n.ScheduleName), func() {
		Expect(VeleroSchedulePause(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
			RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "")
			return "Fail to pause schedule"
		})
	})

	periodCount := 3
	sleepDuration := time.Duration(n.Period*periodCount) * time.Minute
	By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
		time.Sleep(sleepDuration)
	})

	backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))

	backupCountPostPause := len(backupsInfo)
	fmt.Printf("After pause, backkups count is %d\n", backupCountPostPause)

	By(fmt.Sprintf("Verify no new backups from %s ......\n", n.ScheduleName), func() {
		Expect(backupCountPostPause).To(Equal(backupCount))
	})

	By(fmt.Sprintf("Unpause schedule %s ......\n", n.ScheduleName), func() {
		Expect(VeleroScheduleUnpause(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed(), func() string {
			RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "")
			return "Fail to unpause schedule"
		})
	})

	By(fmt.Sprintf("Sleep for %s ......\n", sleepDuration), func() {
		time.Sleep(sleepDuration)
	})

	backupsInfo, err = GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
	Expect(err).To(Succeed(), fmt.Sprintf("Fail to get backups from schedule %s", n.ScheduleName))
	fmt.Println(backupsInfo)
	backupCountPostUnpause := len(backupsInfo)
	fmt.Printf("After unpause, backkups count is %d\n", backupCountPostUnpause)
	By(fmt.Sprintf("Verify no new backups by schedule %s ......\n", n.ScheduleName), func() {
		Expect(backupCountPostUnpause - backupCount).To(BeNumerically(">=", periodCount-1))
	})
	return nil
}

func (n *ScheduleBackup) Verify() error {
	By("Namespaces were restored", func() {
		for _, ns := range *n.NSIncluded {
			configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.CaseBaseName)
			fmt.Printf("Restored configmap is %v\n", configmap)
			Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns))
		}
	})
	return nil
}

func (n *ScheduleBackup) Clean() error {
	if !n.VeleroCfg.Debug {
		Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed())
		Expect(n.TestCase.Clean()).To(Succeed())
	}
	return nil
}