mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-01-07 13:55:20 +00:00
Refactor the migration cases.
Signed-off-by: Xun Jiang <xun.jiang@broadcom.com>
This commit is contained in:
Xun Jiang
@@ -575,9 +575,9 @@ var _ = Describe(
|
||||
)
|
||||
|
||||
var _ = Describe(
|
||||
"Migrate resources between clusters by Restic",
|
||||
Label("Migration", "Restic"),
|
||||
MigrationWithRestic,
|
||||
"Migrate resources between clusters by FileSystem backup",
|
||||
Label("Migration", "FSB"),
|
||||
MigrationWithFS,
|
||||
)
|
||||
var _ = Describe(
|
||||
"Migrate resources between clusters by snapshot",
|
||||
|
||||
@@ -17,388 +17,455 @@ package migration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
. "github.com/vmware-tanzu/velero/test"
|
||||
"github.com/vmware-tanzu/velero/test"
|
||||
framework "github.com/vmware-tanzu/velero/test/e2e/test"
|
||||
util "github.com/vmware-tanzu/velero/test/util/csi"
|
||||
. "github.com/vmware-tanzu/velero/test/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/util/kibishii"
|
||||
. "github.com/vmware-tanzu/velero/test/util/providers"
|
||||
. "github.com/vmware-tanzu/velero/test/util/velero"
|
||||
k8sutil "github.com/vmware-tanzu/velero/test/util/k8s"
|
||||
"github.com/vmware-tanzu/velero/test/util/kibishii"
|
||||
"github.com/vmware-tanzu/velero/test/util/providers"
|
||||
veleroutil "github.com/vmware-tanzu/velero/test/util/velero"
|
||||
)
|
||||
|
||||
var migrationNamespace string
|
||||
var veleroCfg VeleroConfig
|
||||
type migrationE2E struct {
|
||||
framework.TestCase
|
||||
useVolumeSnapshots bool
|
||||
veleroCLI2Version test.VeleroCLI2Version
|
||||
kibishiiData kibishii.KibishiiData
|
||||
}
|
||||
|
||||
func MigrationWithSnapshots() {
|
||||
veleroCfg = VeleroCfg
|
||||
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
|
||||
MigrationTest(true, veleroCLI2Version)
|
||||
for _, veleroCLI2Version := range veleroutil.GetVersionList(
|
||||
test.VeleroCfg.MigrateFromVeleroCLI,
|
||||
test.VeleroCfg.MigrateFromVeleroVersion) {
|
||||
framework.TestFunc(
|
||||
&migrationE2E{
|
||||
useVolumeSnapshots: true,
|
||||
veleroCLI2Version: veleroCLI2Version,
|
||||
},
|
||||
)()
|
||||
}
|
||||
}
|
||||
|
||||
func MigrationWithRestic() {
|
||||
veleroCfg = VeleroCfg
|
||||
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
|
||||
MigrationTest(false, veleroCLI2Version)
|
||||
func MigrationWithFS() {
|
||||
for _, veleroCLI2Version := range veleroutil.GetVersionList(
|
||||
test.VeleroCfg.MigrateFromVeleroCLI,
|
||||
test.VeleroCfg.MigrateFromVeleroVersion) {
|
||||
framework.TestFunc(
|
||||
&migrationE2E{
|
||||
useVolumeSnapshots: false,
|
||||
veleroCLI2Version: veleroCLI2Version,
|
||||
},
|
||||
)()
|
||||
}
|
||||
}
|
||||
|
||||
func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) {
|
||||
var (
|
||||
backupName, restoreName string
|
||||
backupScName, restoreScName string
|
||||
kibishiiWorkerCount int
|
||||
err error
|
||||
)
|
||||
BeforeEach(func() {
|
||||
kibishiiWorkerCount = 3
|
||||
veleroCfg = VeleroCfg
|
||||
UUIDgen, err = uuid.NewRandom()
|
||||
migrationNamespace = "migration-" + UUIDgen.String()
|
||||
if useVolumeSnapshots && veleroCfg.CloudProvider == Kind {
|
||||
Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind))
|
||||
}
|
||||
|
||||
if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
|
||||
func (m *migrationE2E) Init() error {
|
||||
By("Skip check", func() {
|
||||
if m.VeleroCfg.DefaultClusterContext == "" && m.VeleroCfg.StandbyClusterContext == "" {
|
||||
Skip("Migration test needs 2 clusters")
|
||||
}
|
||||
// need to uninstall Velero first in case of the affection of the existing global velero installation
|
||||
if InstallVelero {
|
||||
By("Uninstall Velero", func() {
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
||||
defer ctxCancel()
|
||||
Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
if m.useVolumeSnapshots && m.VeleroCfg.CloudProvider == test.Kind {
|
||||
Skip(fmt.Sprintf("Volume snapshots not supported on %s", test.Kind))
|
||||
}
|
||||
|
||||
if m.VeleroCfg.SnapshotMoveData && !m.useVolumeSnapshots {
|
||||
Skip("FSB migration test is not needed in data mover scenario")
|
||||
}
|
||||
})
|
||||
AfterEach(func() {
|
||||
if CurrentSpecReport().Failed() && veleroCfg.FailFast {
|
||||
fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
|
||||
|
||||
By("Call the base E2E init", func() {
|
||||
Expect(m.TestCase.Init()).To(Succeed())
|
||||
})
|
||||
|
||||
m.kibishiiData = *kibishii.DefaultKibishiiData
|
||||
m.kibishiiData.ExpectedNodes = 3
|
||||
m.CaseBaseName = "migration-" + m.UUIDgen
|
||||
m.BackupName = m.CaseBaseName + "-backup"
|
||||
m.RestoreName = m.CaseBaseName + "-restore"
|
||||
m.NSIncluded = &[]string{m.CaseBaseName}
|
||||
|
||||
m.RestoreArgs = []string{
|
||||
"create", "--namespace", m.VeleroCfg.VeleroNamespace,
|
||||
"restore", m.RestoreName,
|
||||
"--from-backup", m.BackupName, "--wait",
|
||||
}
|
||||
|
||||
// Message output by ginkgo
|
||||
m.TestMsg = &framework.TestMSG{
|
||||
Desc: "Test migration workload on two clusters",
|
||||
FailedMSG: "Fail to test migrate between two clusters",
|
||||
Text: "Test back up on default cluster, restore on standby cluster",
|
||||
}
|
||||
|
||||
// Need to uninstall Velero on the default cluster.
|
||||
if test.InstallVelero {
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
||||
defer ctxCancel()
|
||||
Expect(veleroutil.VeleroUninstall(ctx, m.VeleroCfg)).To(Succeed())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrationE2E) Backup() error {
|
||||
OriginVeleroCfg := m.VeleroCfg
|
||||
var err error
|
||||
|
||||
if m.veleroCLI2Version.VeleroCLI == "" {
|
||||
//Assume tag of velero server image is identical to velero CLI version
|
||||
//Download velero CLI if it's empty according to velero CLI version
|
||||
By(
|
||||
fmt.Sprintf("Install the expected version Velero CLI %s",
|
||||
m.veleroCLI2Version.VeleroVersion),
|
||||
func() {
|
||||
// "self" represents 1.14.x and future versions
|
||||
if m.veleroCLI2Version.VeleroVersion == "self" {
|
||||
m.veleroCLI2Version.VeleroCLI = m.VeleroCfg.VeleroCLI
|
||||
} else {
|
||||
OriginVeleroCfg, err = veleroutil.SetImagesToDefaultValues(
|
||||
OriginVeleroCfg,
|
||||
m.veleroCLI2Version.VeleroVersion,
|
||||
)
|
||||
Expect(err).To(Succeed(),
|
||||
"Fail to set images for the migrate-from Velero installation.")
|
||||
|
||||
m.veleroCLI2Version.VeleroCLI, err = veleroutil.InstallVeleroCLI(
|
||||
m.veleroCLI2Version.VeleroVersion)
|
||||
Expect(err).To(Succeed())
|
||||
}
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Install Velero on default cluster (%s)", m.VeleroCfg.DefaultClusterContext),
|
||||
func() {
|
||||
Expect(k8sutil.KubectlConfigUseContext(
|
||||
m.Ctx, m.VeleroCfg.DefaultClusterContext)).To(Succeed())
|
||||
OriginVeleroCfg.MigrateFromVeleroVersion = m.veleroCLI2Version.VeleroVersion
|
||||
OriginVeleroCfg.VeleroCLI = m.veleroCLI2Version.VeleroCLI
|
||||
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
|
||||
OriginVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.DefaultClusterName
|
||||
OriginVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.DefaultCLSServiceAccountName
|
||||
OriginVeleroCfg.UseVolumeSnapshots = m.useVolumeSnapshots
|
||||
OriginVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots
|
||||
|
||||
version, err := veleroutil.GetVeleroVersion(m.Ctx, OriginVeleroCfg.VeleroCLI, true)
|
||||
Expect(err).To(Succeed(), "Fail to get Velero version")
|
||||
OriginVeleroCfg.VeleroVersion = version
|
||||
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
OriginVeleroCfg.UseNodeAgent = true
|
||||
}
|
||||
|
||||
Expect(veleroutil.VeleroInstall(m.Ctx, &OriginVeleroCfg, false)).To(Succeed())
|
||||
if m.veleroCLI2Version.VeleroVersion != "self" {
|
||||
Expect(veleroutil.CheckVeleroVersion(
|
||||
m.Ctx,
|
||||
OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.MigrateFromVeleroVersion,
|
||||
)).To(Succeed())
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
By("Create namespace for sample workload", func() {
|
||||
Expect(k8sutil.CreateNamespace(
|
||||
m.Ctx,
|
||||
*m.VeleroCfg.DefaultClient,
|
||||
m.CaseBaseName,
|
||||
)).To(Succeed(),
|
||||
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload",
|
||||
m.CaseBaseName))
|
||||
})
|
||||
|
||||
By("Deploy sample workload of Kibishii", func() {
|
||||
Expect(kibishii.KibishiiPrepareBeforeBackup(
|
||||
m.Ctx,
|
||||
*OriginVeleroCfg.DefaultClient,
|
||||
OriginVeleroCfg.CloudProvider,
|
||||
m.CaseBaseName,
|
||||
OriginVeleroCfg.RegistryCredentialFile,
|
||||
OriginVeleroCfg.Features,
|
||||
OriginVeleroCfg.KibishiiDirectory,
|
||||
OriginVeleroCfg.UseVolumeSnapshots,
|
||||
&m.kibishiiData,
|
||||
)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Backup namespace %s", m.CaseBaseName), func() {
|
||||
m.BackupArgs = []string{
|
||||
"create", "--namespace", m.VeleroCfg.VeleroNamespace,
|
||||
"backup", m.BackupName,
|
||||
"--include-namespaces", strings.Join(*m.NSIncluded, ","),
|
||||
"--wait",
|
||||
}
|
||||
|
||||
if m.useVolumeSnapshots {
|
||||
m.BackupArgs = append(m.BackupArgs, "--snapshot-volumes=true")
|
||||
} else {
|
||||
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
||||
defer ctxCancel()
|
||||
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
||||
veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
|
||||
Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
|
||||
|
||||
By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
|
||||
Expect(
|
||||
DeleteNamespace(
|
||||
context.Background(),
|
||||
*veleroCfg.DefaultClient,
|
||||
migrationNamespace,
|
||||
true),
|
||||
).To(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() {
|
||||
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
||||
defer ctxCancel()
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
|
||||
veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
|
||||
|
||||
By("Delete StorageClasses created by E2E")
|
||||
Expect(
|
||||
DeleteStorageClass(
|
||||
ctx,
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
StorageClassName,
|
||||
),
|
||||
).To(Succeed())
|
||||
Expect(
|
||||
DeleteStorageClass(
|
||||
ctx,
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
StorageClassName2,
|
||||
),
|
||||
).To(Succeed())
|
||||
|
||||
if strings.EqualFold(veleroCfg.Features, FeatureCSI) &&
|
||||
veleroCfg.UseVolumeSnapshots {
|
||||
By("Delete VolumeSnapshotClass created by E2E")
|
||||
Expect(
|
||||
KubectlDeleteByFile(
|
||||
ctx,
|
||||
fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.CloudProvider),
|
||||
),
|
||||
).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed())
|
||||
|
||||
By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
|
||||
Expect(
|
||||
DeleteNamespace(
|
||||
context.Background(),
|
||||
*veleroCfg.StandbyClient,
|
||||
migrationNamespace,
|
||||
true,
|
||||
),
|
||||
).To(Succeed())
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Switch to default KubeConfig context %s", veleroCfg.DefaultClusterContext), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
||||
veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
|
||||
})
|
||||
m.BackupArgs = append(m.BackupArgs, "--default-volumes-to-fs-backup")
|
||||
}
|
||||
})
|
||||
When("kibishii is the sample workload", func() {
|
||||
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
|
||||
var backupNames []string
|
||||
if veleroCfg.SnapshotMoveData {
|
||||
if !useVolumeSnapshots {
|
||||
Skip("FSB migration test is not needed in data mover scenario")
|
||||
}
|
||||
}
|
||||
oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
|
||||
defer ctxCancel()
|
||||
flag.Parse()
|
||||
UUIDgen, err = uuid.NewRandom()
|
||||
Expect(err).To(Succeed())
|
||||
supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion)
|
||||
Expect(err).To(Succeed())
|
||||
|
||||
OriginVeleroCfg := veleroCfg
|
||||
if veleroCLI2Version.VeleroCLI == "" {
|
||||
//Assume tag of velero server image is identical to velero CLI version
|
||||
//Download velero CLI if it's empty according to velero CLI version
|
||||
By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero",
|
||||
veleroCLI2Version.VeleroVersion), func() {
|
||||
//"self" represents 1.14.x and future versions
|
||||
if veleroCLI2Version.VeleroVersion == "self" {
|
||||
veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI
|
||||
} else {
|
||||
OriginVeleroCfg, err = SetImagesToDefaultValues(
|
||||
OriginVeleroCfg,
|
||||
veleroCLI2Version.VeleroVersion,
|
||||
)
|
||||
Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.")
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
m.BackupArgs = append(m.BackupArgs, "--snapshot-move-data")
|
||||
}
|
||||
|
||||
veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)
|
||||
Expect(err).To(Succeed())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
||||
OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
|
||||
OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
|
||||
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
|
||||
OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
|
||||
OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName
|
||||
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
|
||||
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
|
||||
|
||||
version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true)
|
||||
Expect(err).To(Succeed(), "Fail to get Velero version")
|
||||
OriginVeleroCfg.VeleroVersion = version
|
||||
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
OriginVeleroCfg.UseNodeAgent = true
|
||||
}
|
||||
|
||||
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed())
|
||||
if veleroCLI2Version.VeleroVersion != "self" {
|
||||
Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed())
|
||||
}
|
||||
})
|
||||
|
||||
backupName = "backup-" + UUIDgen.String()
|
||||
backupScName = backupName + "-sc"
|
||||
restoreName = "restore-" + UUIDgen.String()
|
||||
restoreScName = restoreName + "-sc"
|
||||
|
||||
By("Create namespace for sample workload", func() {
|
||||
Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(),
|
||||
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace))
|
||||
})
|
||||
|
||||
KibishiiData := *DefaultKibishiiData
|
||||
By("Deploy sample workload of Kibishii", func() {
|
||||
KibishiiData.ExpectedNodes = kibishiiWorkerCount
|
||||
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider,
|
||||
migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
|
||||
veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() {
|
||||
var BackupStorageClassCfg BackupConfig
|
||||
BackupStorageClassCfg.BackupName = backupScName
|
||||
BackupStorageClassCfg.IncludeResources = "StorageClass"
|
||||
BackupStorageClassCfg.IncludeClusterResources = true
|
||||
|
||||
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
|
||||
BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType
|
||||
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
backupNames = append(backupNames, BackupStorageClassCfg.BackupName)
|
||||
|
||||
var BackupCfg BackupConfig
|
||||
BackupCfg.BackupName = backupName
|
||||
BackupCfg.Namespace = migrationNamespace
|
||||
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
|
||||
BackupCfg.BackupLocation = ""
|
||||
BackupCfg.Selector = ""
|
||||
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
|
||||
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
|
||||
BackupCfg.UseResticIfFSBackup = !supportUploaderType
|
||||
BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData
|
||||
|
||||
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
backupNames = append(backupNames, BackupCfg.BackupName)
|
||||
})
|
||||
|
||||
if useVolumeSnapshots {
|
||||
// Only wait for the snapshots.backupdriver.cnsdp.vmware.com
|
||||
// when the vSphere plugin is used.
|
||||
if veleroCfg.HasVspherePlugin {
|
||||
By("Waiting for vSphere uploads to complete", func() {
|
||||
Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour,
|
||||
migrationNamespace, kibishiiWorkerCount)).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
var snapshotCheckPoint SnapshotCheckPoint
|
||||
snapshotCheckPoint.NamespaceBackedUp = migrationNamespace
|
||||
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
//VolumeSnapshotContent should be deleted after data movement
|
||||
_, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0)
|
||||
Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0")
|
||||
} else {
|
||||
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
|
||||
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
|
||||
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
|
||||
if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS {
|
||||
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
|
||||
time.Sleep(5 * time.Minute)
|
||||
})
|
||||
}
|
||||
|
||||
By("Snapshot should be created in cloud object store with retain policy", func() {
|
||||
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount,
|
||||
migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount))
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
Expect(CheckSnapshotsInProvider(
|
||||
veleroCfg,
|
||||
backupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
|
||||
//Ensure workload of "migrationNamespace" existed in cluster-A
|
||||
ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
|
||||
Expect(ns.Name).To(Equal(migrationNamespace))
|
||||
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err))
|
||||
|
||||
//Ensure cluster-B is the target cluster
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
|
||||
_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
|
||||
Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace))
|
||||
fmt.Println(err)
|
||||
Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue())
|
||||
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
|
||||
veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
|
||||
veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName
|
||||
veleroCfg.UseNodeAgent = !useVolumeSnapshots
|
||||
veleroCfg.UseRestic = false
|
||||
if veleroCfg.SnapshotMoveData {
|
||||
veleroCfg.UseNodeAgent = true
|
||||
// For SnapshotMoveData pipelines, we should use standby cluster setting for Velero installation
|
||||
// In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData.
|
||||
veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins
|
||||
veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider
|
||||
}
|
||||
|
||||
By("Install StorageClass for E2E.")
|
||||
Expect(InstallStorageClasses(veleroCfg.StandbyClusterCloudProvider)).To(Succeed())
|
||||
|
||||
if strings.EqualFold(veleroCfg.Features, FeatureCSI) &&
|
||||
veleroCfg.UseVolumeSnapshots {
|
||||
By("Install VolumeSnapshotClass for E2E.")
|
||||
Expect(
|
||||
KubectlApplyByFile(
|
||||
context.Background(),
|
||||
fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider),
|
||||
),
|
||||
).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
|
||||
Expect(WaitForBackupToBeCreated(context.Background(), backupName, 5*time.Minute, &veleroCfg)).To(Succeed())
|
||||
Expect(WaitForBackupToBeCreated(context.Background(), backupScName, 5*time.Minute, &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Restore %s", migrationNamespace), func() {
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
cmName := "datamover-storage-class-config"
|
||||
labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
|
||||
"velero.io/plugin-config": ""}
|
||||
data := map[string]string{KibishiiStorageClassName: StorageClassName}
|
||||
|
||||
By(fmt.Sprintf("Create ConfigMap %s in namespace %s", cmName, veleroCfg.VeleroNamespace), func() {
|
||||
_, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, cmName, labels, data)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace))
|
||||
})
|
||||
} else {
|
||||
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, "", restoreName)
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
}
|
||||
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, "", restoreName)
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
|
||||
Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
|
||||
oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
|
||||
})
|
||||
|
||||
// TODO: delete backup created by case self, not all
|
||||
By("Clean backups after test", func() {
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
||||
Expect(DeleteBackups(context.Background(), backupNames, &veleroCfg)).To(Succeed())
|
||||
})
|
||||
Expect(veleroutil.VeleroBackupExec(
|
||||
m.Ctx,
|
||||
OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.VeleroNamespace,
|
||||
m.BackupName,
|
||||
m.BackupArgs,
|
||||
)).To(Succeed(), func() string {
|
||||
veleroutil.RunDebug(
|
||||
context.Background(),
|
||||
OriginVeleroCfg.VeleroCLI,
|
||||
OriginVeleroCfg.VeleroNamespace,
|
||||
m.BackupName,
|
||||
"",
|
||||
)
|
||||
return "Failed to backup resources"
|
||||
})
|
||||
})
|
||||
|
||||
if m.useVolumeSnapshots {
|
||||
// Only wait for the snapshots.backupdriver.cnsdp.vmware.com
|
||||
// when the vSphere plugin is used.
|
||||
if OriginVeleroCfg.HasVspherePlugin {
|
||||
By("Waiting for vSphere uploads to complete", func() {
|
||||
Expect(
|
||||
veleroutil.WaitForVSphereUploadCompletion(
|
||||
context.Background(),
|
||||
time.Hour,
|
||||
m.CaseBaseName,
|
||||
m.kibishiiData.ExpectedNodes,
|
||||
),
|
||||
).To(Succeed())
|
||||
})
|
||||
}
|
||||
|
||||
var snapshotCheckPoint test.SnapshotCheckPoint
|
||||
snapshotCheckPoint.NamespaceBackedUp = m.CaseBaseName
|
||||
|
||||
if OriginVeleroCfg.SnapshotMoveData {
|
||||
//VolumeSnapshotContent should be deleted after data movement
|
||||
_, err := util.CheckVolumeSnapshotCR(
|
||||
*m.VeleroCfg.DefaultClient,
|
||||
map[string]string{"namespace": m.CaseBaseName},
|
||||
0,
|
||||
)
|
||||
By("Check the VSC account", func() {
|
||||
Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0")
|
||||
})
|
||||
} else {
|
||||
// the snapshots of AWS may be still in pending status when do the restore.
|
||||
// wait for a while to avoid this https://github.com/vmware-tanzu/velero/issues/1799
|
||||
if OriginVeleroCfg.CloudProvider == test.Azure &&
|
||||
strings.EqualFold(OriginVeleroCfg.Features, test.FeatureCSI) ||
|
||||
OriginVeleroCfg.CloudProvider == test.AWS {
|
||||
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
|
||||
time.Sleep(5 * time.Minute)
|
||||
})
|
||||
}
|
||||
|
||||
By("Snapshot should be created in cloud object store with retain policy", func() {
|
||||
snapshotCheckPoint, err = veleroutil.GetSnapshotCheckPoint(
|
||||
*OriginVeleroCfg.DefaultClient,
|
||||
OriginVeleroCfg,
|
||||
m.kibishiiData.ExpectedNodes,
|
||||
m.CaseBaseName,
|
||||
m.BackupName,
|
||||
kibishii.GetKibishiiPVCNameList(m.kibishiiData.ExpectedNodes),
|
||||
)
|
||||
|
||||
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
||||
Expect(providers.CheckSnapshotsInProvider(
|
||||
OriginVeleroCfg,
|
||||
m.BackupName,
|
||||
snapshotCheckPoint,
|
||||
false,
|
||||
)).To(Succeed())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrationE2E) Restore() error {
|
||||
StandbyVeleroCfg := m.VeleroCfg
|
||||
|
||||
By("Install Velero in standby cluster.", func() {
|
||||
// Ensure cluster-B is the target cluster
|
||||
Expect(k8sutil.KubectlConfigUseContext(
|
||||
m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed())
|
||||
|
||||
// Check the workload namespace not exist in standby cluster.
|
||||
_, err := k8sutil.GetNamespace(
|
||||
m.Ctx, *m.VeleroCfg.StandbyClient, m.CaseBaseName)
|
||||
Expect(err).To(HaveOccurred(), fmt.Sprintf(
|
||||
"get namespace in dst cluster successfully, it's not as expected: %s", m.CaseBaseName))
|
||||
Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+m.CaseBaseName+"\" not found")).
|
||||
Should(BeTrue())
|
||||
|
||||
By("Install StorageClass for E2E.")
|
||||
Expect(veleroutil.InstallStorageClasses(
|
||||
m.VeleroCfg.StandbyClusterCloudProvider)).To(Succeed())
|
||||
|
||||
if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
|
||||
m.VeleroCfg.UseVolumeSnapshots {
|
||||
By("Install VolumeSnapshotClass for E2E.")
|
||||
Expect(
|
||||
k8sutil.KubectlApplyByFile(
|
||||
m.Ctx,
|
||||
fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml",
|
||||
m.VeleroCfg.StandbyClusterCloudProvider),
|
||||
),
|
||||
).To(Succeed())
|
||||
}
|
||||
|
||||
StandbyVeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient
|
||||
StandbyVeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName
|
||||
StandbyVeleroCfg.ServiceAccountNameToInstall = m.VeleroCfg.StandbyCLSServiceAccountName
|
||||
StandbyVeleroCfg.UseNodeAgent = !m.useVolumeSnapshots
|
||||
if StandbyVeleroCfg.SnapshotMoveData {
|
||||
StandbyVeleroCfg.UseNodeAgent = true
|
||||
// For SnapshotMoveData pipelines, we should use standby cluster setting
|
||||
// for Velero installation.
|
||||
// In nightly CI, StandbyClusterPlugins is set properly
|
||||
// if pipeline is for SnapshotMoveData.
|
||||
StandbyVeleroCfg.Plugins = m.VeleroCfg.StandbyClusterPlugins
|
||||
StandbyVeleroCfg.ObjectStoreProvider = m.VeleroCfg.StandbyClusterObjectStoreProvider
|
||||
}
|
||||
|
||||
Expect(veleroutil.VeleroInstall(
|
||||
context.Background(), &StandbyVeleroCfg, true)).To(Succeed())
|
||||
})
|
||||
|
||||
By("Waiting for backups sync to Velero in standby cluster", func() {
|
||||
Expect(veleroutil.WaitForBackupToBeCreated(
|
||||
m.Ctx, m.BackupName, 5*time.Minute, &StandbyVeleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Restore %s", m.CaseBaseName), func() {
|
||||
if m.VeleroCfg.SnapshotMoveData {
|
||||
cmName := "datamover-storage-class-config"
|
||||
labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
|
||||
"velero.io/plugin-config": ""}
|
||||
data := map[string]string{kibishii.KibishiiStorageClassName: test.StorageClassName}
|
||||
|
||||
By(fmt.Sprintf("Create ConfigMap %s in namespace %s",
|
||||
cmName, StandbyVeleroCfg.VeleroNamespace), func() {
|
||||
_, err := k8sutil.CreateConfigMap(
|
||||
StandbyVeleroCfg.StandbyClient.ClientGo,
|
||||
StandbyVeleroCfg.VeleroNamespace,
|
||||
cmName,
|
||||
labels,
|
||||
data,
|
||||
)
|
||||
Expect(err).To(Succeed(), fmt.Sprintf(
|
||||
"failed to create ConfigMap in the namespace %q",
|
||||
StandbyVeleroCfg.VeleroNamespace))
|
||||
})
|
||||
}
|
||||
|
||||
Expect(veleroutil.VeleroRestore(
|
||||
m.Ctx,
|
||||
StandbyVeleroCfg.VeleroCLI,
|
||||
StandbyVeleroCfg.VeleroNamespace,
|
||||
m.RestoreName,
|
||||
m.BackupName,
|
||||
"",
|
||||
)).To(Succeed(), func() string {
|
||||
veleroutil.RunDebug(
|
||||
m.Ctx, StandbyVeleroCfg.VeleroCLI,
|
||||
StandbyVeleroCfg.VeleroNamespace, "", m.RestoreName)
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrationE2E) Verify() error {
|
||||
By(fmt.Sprintf("Verify workload %s after restore on standby cluster", m.CaseBaseName), func() {
|
||||
Expect(kibishii.KibishiiVerifyAfterRestore(
|
||||
*m.VeleroCfg.StandbyClient,
|
||||
m.CaseBaseName,
|
||||
m.Ctx,
|
||||
&m.kibishiiData,
|
||||
"",
|
||||
)).To(Succeed(), "Fail to verify workload after restore")
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *migrationE2E) Clean() error {
|
||||
By("Clean resource on default cluster.", func() {
|
||||
Expect(m.TestCase.Clean()).To(Succeed())
|
||||
})
|
||||
|
||||
By("Clean resource on standby cluster.", func() {
|
||||
Expect(k8sutil.KubectlConfigUseContext(
|
||||
m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed())
|
||||
m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient
|
||||
m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName
|
||||
|
||||
By("Delete StorageClasses created by E2E")
|
||||
Expect(
|
||||
k8sutil.DeleteStorageClass(
|
||||
m.Ctx,
|
||||
*m.VeleroCfg.ClientToInstallVelero,
|
||||
test.StorageClassName,
|
||||
),
|
||||
).To(Succeed())
|
||||
Expect(
|
||||
k8sutil.DeleteStorageClass(
|
||||
m.Ctx,
|
||||
*m.VeleroCfg.ClientToInstallVelero,
|
||||
test.StorageClassName2,
|
||||
),
|
||||
).To(Succeed())
|
||||
|
||||
if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) &&
|
||||
m.VeleroCfg.UseVolumeSnapshots {
|
||||
By("Delete VolumeSnapshotClass created by E2E")
|
||||
Expect(
|
||||
k8sutil.KubectlDeleteByFile(
|
||||
m.Ctx,
|
||||
fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml",
|
||||
m.VeleroCfg.StandbyClusterCloudProvider),
|
||||
),
|
||||
).To(Succeed())
|
||||
}
|
||||
|
||||
Expect(veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg)).To(Succeed())
|
||||
|
||||
Expect(
|
||||
k8sutil.DeleteNamespace(
|
||||
m.Ctx,
|
||||
*m.VeleroCfg.StandbyClient,
|
||||
m.CaseBaseName,
|
||||
true,
|
||||
),
|
||||
).To(Succeed())
|
||||
})
|
||||
|
||||
By("Switch to default KubeConfig context", func() {
|
||||
Expect(k8sutil.KubectlConfigUseContext(
|
||||
m.Ctx,
|
||||
m.VeleroCfg.DefaultClusterContext,
|
||||
)).To(Succeed())
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -91,8 +91,9 @@ func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
|
||||
}
|
||||
|
||||
func TestIt(test VeleroBackupRestoreTest) error {
|
||||
test.Init()
|
||||
It(test.GetTestMsg().Text, func() {
|
||||
It("Run E2E test case", func() {
|
||||
Expect(test.Init()).To(Succeed())
|
||||
|
||||
Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG)
|
||||
})
|
||||
return nil
|
||||
@@ -213,6 +214,7 @@ func RunTestCase(test VeleroBackupRestoreTest) error {
|
||||
if test == nil {
|
||||
return errors.New("No case should be tested")
|
||||
}
|
||||
fmt.Println("Running case: ", test.GetTestMsg().Text)
|
||||
test.Start()
|
||||
defer test.GetTestCase().CtxCancel()
|
||||
|
||||
|
||||
@@ -138,13 +138,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
|
||||
tmpCfgForOldVeleroInstall.VeleroVersion = version
|
||||
tmpCfgForOldVeleroInstall.UseVolumeSnapshots = useVolumeSnapshots
|
||||
|
||||
if supportUploaderType {
|
||||
tmpCfgForOldVeleroInstall.UseRestic = false
|
||||
tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots
|
||||
} else {
|
||||
tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots
|
||||
tmpCfgForOldVeleroInstall.UseNodeAgent = false
|
||||
}
|
||||
tmpCfgForOldVeleroInstall.UseNodeAgent = !useVolumeSnapshots
|
||||
|
||||
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed())
|
||||
Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI,
|
||||
@@ -230,7 +224,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
|
||||
|
||||
By(fmt.Sprintf("Upgrade Velero by CLI %s", tmpCfg.VeleroCLI), func() {
|
||||
tmpCfg.GCFrequency = ""
|
||||
tmpCfg.UseRestic = false
|
||||
tmpCfg.UseNodeAgent = !useVolumeSnapshots
|
||||
Expect(err).To(Succeed())
|
||||
if supportUploaderType {
|
||||
|
||||
Reference in New Issue
Block a user