mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-02-09 21:40:17 +00:00
82fc557bd1
1. In data movement scenario, volumesnapshotcontent by Velero backup will be deleted instead of retained in CSI scenaito, so add a checkpoint for data movement scenario to verify no volumesnapshotcontent left after Velero backup; 2. Fix global context varaible issue, context varaible is not effective due to it's initialized right after the very beginning of all tests instead of beginning of each test, so if someone script a new E2E test and did not overwrite it in the test body, then it will fail the test if it was triggerd one hour later; 3. Due to CSI plugin is deprecated, it breaked down migration tests, because v1.13 still needs to install CSI plugin for the test. Signed-off-by: danfengl <danfengl@vmware.com>
361 lines
16 KiB
Go
361 lines
16 KiB
Go
/*
|
|
Copyright the Velero contributors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
package migration
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
|
|
. "github.com/vmware-tanzu/velero/test"
|
|
util "github.com/vmware-tanzu/velero/test/util/csi"
|
|
. "github.com/vmware-tanzu/velero/test/util/k8s"
|
|
. "github.com/vmware-tanzu/velero/test/util/kibishii"
|
|
. "github.com/vmware-tanzu/velero/test/util/providers"
|
|
. "github.com/vmware-tanzu/velero/test/util/velero"
|
|
)
|
|
|
|
var migrationNamespace string
|
|
var veleroCfg VeleroConfig
|
|
|
|
func MigrationWithSnapshots() {
|
|
veleroCfg = VeleroCfg
|
|
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
|
|
MigrationTest(true, veleroCLI2Version)
|
|
}
|
|
}
|
|
|
|
func MigrationWithRestic() {
|
|
veleroCfg = VeleroCfg
|
|
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
|
|
MigrationTest(false, veleroCLI2Version)
|
|
}
|
|
}
|
|
|
|
func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) {
|
|
var (
|
|
backupName, restoreName string
|
|
backupScName, restoreScName string
|
|
kibishiiWorkerCount int
|
|
err error
|
|
)
|
|
BeforeEach(func() {
|
|
kibishiiWorkerCount = 3
|
|
veleroCfg = VeleroCfg
|
|
UUIDgen, err = uuid.NewRandom()
|
|
migrationNamespace = "migration-" + UUIDgen.String()
|
|
if useVolumeSnapshots && veleroCfg.CloudProvider == Kind {
|
|
Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind))
|
|
}
|
|
|
|
if veleroCfg.DefaultClusterContext == "" && veleroCfg.StandbyClusterContext == "" {
|
|
Skip("Migration test needs 2 clusters")
|
|
}
|
|
// need to uninstall Velero first in case of the affection of the existing global velero installation
|
|
if InstallVelero {
|
|
By("Uninstall Velero", func() {
|
|
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
|
defer ctxCancel()
|
|
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace)).To(Succeed())
|
|
})
|
|
}
|
|
})
|
|
AfterEach(func() {
|
|
if !veleroCfg.Debug {
|
|
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
|
|
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
|
defer ctxCancel()
|
|
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
|
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace)).To(Succeed())
|
|
DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true)
|
|
})
|
|
|
|
By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() {
|
|
ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
|
|
defer ctxCancel()
|
|
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
|
|
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace)).To(Succeed())
|
|
DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
|
|
})
|
|
|
|
if InstallVelero {
|
|
By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() {
|
|
DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
|
|
})
|
|
}
|
|
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() {
|
|
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
|
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
|
veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
|
|
})
|
|
}
|
|
})
|
|
When("kibishii is the sample workload", func() {
|
|
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
|
|
var backupNames []string
|
|
if veleroCfg.SnapshotMoveData {
|
|
if !useVolumeSnapshots {
|
|
Skip("FSB migration test is not needed in data mover scenario")
|
|
}
|
|
}
|
|
oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
|
|
defer ctxCancel()
|
|
flag.Parse()
|
|
UUIDgen, err = uuid.NewRandom()
|
|
Expect(err).To(Succeed())
|
|
supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion)
|
|
Expect(err).To(Succeed())
|
|
|
|
OriginVeleroCfg := veleroCfg
|
|
if veleroCLI2Version.VeleroCLI == "" {
|
|
//Assume tag of velero server image is identical to velero CLI version
|
|
//Download velero CLI if it's empty according to velero CLI version
|
|
By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero",
|
|
veleroCLI2Version.VeleroVersion), func() {
|
|
//"self" represents 1.14.x and future versions
|
|
if veleroCLI2Version.VeleroVersion == "self" {
|
|
veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI
|
|
} else {
|
|
fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion)
|
|
OriginVeleroCfg.VeleroImage = ""
|
|
OriginVeleroCfg.RestoreHelperImage = ""
|
|
OriginVeleroCfg.Plugins = ""
|
|
|
|
// It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't
|
|
// pass velero-plugin-for-csi to E2E test anymore, and velero installation will not
|
|
// fetch velero-plugin-for-csi automatically, so add it as hardcode below.
|
|
// TODO: remove this section from future version like v1.15, e.g.
|
|
if OriginVeleroCfg.CloudProvider == Azure {
|
|
OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0"
|
|
}
|
|
if OriginVeleroCfg.CloudProvider == AWS {
|
|
OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0"
|
|
}
|
|
if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) {
|
|
OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0"
|
|
}
|
|
if OriginVeleroCfg.SnapshotMoveData {
|
|
if OriginVeleroCfg.CloudProvider == Azure {
|
|
OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0"
|
|
}
|
|
}
|
|
veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)
|
|
Expect(err).To(Succeed())
|
|
}
|
|
})
|
|
}
|
|
|
|
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() {
|
|
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed())
|
|
OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
|
|
OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
|
|
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
|
|
OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
|
|
OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName
|
|
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
|
|
|
|
version, err := GetVeleroVersion(oneHourTimeout, OriginVeleroCfg.VeleroCLI, true)
|
|
Expect(err).To(Succeed(), "Fail to get Velero version")
|
|
OriginVeleroCfg.VeleroVersion = version
|
|
|
|
if OriginVeleroCfg.SnapshotMoveData {
|
|
OriginVeleroCfg.UseNodeAgent = true
|
|
}
|
|
|
|
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed())
|
|
if veleroCLI2Version.VeleroVersion != "self" {
|
|
Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI,
|
|
OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed())
|
|
}
|
|
})
|
|
|
|
backupName = "backup-" + UUIDgen.String()
|
|
backupScName = backupName + "-sc"
|
|
restoreName = "restore-" + UUIDgen.String()
|
|
restoreScName = restoreName + "-sc"
|
|
|
|
By("Create namespace for sample workload", func() {
|
|
Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(),
|
|
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace))
|
|
})
|
|
|
|
KibishiiData := *DefaultKibishiiData
|
|
By("Deploy sample workload of Kibishii", func() {
|
|
KibishiiData.ExpectedNodes = kibishiiWorkerCount
|
|
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider,
|
|
migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
|
|
veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed())
|
|
})
|
|
|
|
By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() {
|
|
var BackupStorageClassCfg BackupConfig
|
|
BackupStorageClassCfg.BackupName = backupScName
|
|
BackupStorageClassCfg.IncludeResources = "StorageClass"
|
|
BackupStorageClassCfg.IncludeClusterResources = true
|
|
|
|
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
|
|
BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType
|
|
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
|
|
OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
|
|
return "Fail to backup workload"
|
|
})
|
|
backupNames = append(backupNames, BackupStorageClassCfg.BackupName)
|
|
|
|
var BackupCfg BackupConfig
|
|
BackupCfg.BackupName = backupName
|
|
BackupCfg.Namespace = migrationNamespace
|
|
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
BackupCfg.BackupLocation = ""
|
|
BackupCfg.Selector = ""
|
|
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
|
|
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
|
|
BackupCfg.UseResticIfFSBackup = !supportUploaderType
|
|
BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData
|
|
|
|
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
|
|
OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
|
|
return "Fail to backup workload"
|
|
})
|
|
backupNames = append(backupNames, BackupCfg.BackupName)
|
|
})
|
|
|
|
if useVolumeSnapshots {
|
|
if veleroCfg.CloudProvider == Vsphere {
|
|
// TODO - remove after upload progress monitoring is implemented
|
|
By("Waiting for vSphere uploads to complete", func() {
|
|
Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour,
|
|
migrationNamespace, kibishiiWorkerCount)).To(Succeed())
|
|
})
|
|
}
|
|
|
|
var snapshotCheckPoint SnapshotCheckPoint
|
|
snapshotCheckPoint.NamespaceBackedUp = migrationNamespace
|
|
|
|
if OriginVeleroCfg.SnapshotMoveData {
|
|
//VolumeSnapshotContent should be deleted after data movement
|
|
_, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0)
|
|
Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0")
|
|
} else {
|
|
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
|
|
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
|
|
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
|
|
if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS {
|
|
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
|
|
time.Sleep(5 * time.Minute)
|
|
})
|
|
}
|
|
|
|
By("Snapshot should be created in cloud object store with retain policy", func() {
|
|
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount,
|
|
migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount))
|
|
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
|
|
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
|
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
|
|
})
|
|
}
|
|
}
|
|
|
|
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() {
|
|
//Ensure workload of "migrationNamespace" existed in cluster-A
|
|
ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
|
|
Expect(ns.Name).To(Equal(migrationNamespace))
|
|
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err))
|
|
|
|
//Ensure cluster-B is the target cluster
|
|
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed())
|
|
_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
|
|
Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace))
|
|
fmt.Println(err)
|
|
Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue())
|
|
|
|
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
|
|
veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
|
|
veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName
|
|
veleroCfg.UseNodeAgent = !useVolumeSnapshots
|
|
veleroCfg.UseRestic = false
|
|
if veleroCfg.SnapshotMoveData {
|
|
veleroCfg.UseNodeAgent = true
|
|
// For SnapshotMoveData pipelines, we should use standby cluster setting for Velero installation
|
|
// In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData.
|
|
veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins
|
|
veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider
|
|
}
|
|
|
|
Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
|
|
})
|
|
|
|
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyClusterContext), func() {
|
|
Expect(WaitForBackupToBeCreated(context.Background(), backupName, 5*time.Minute, &veleroCfg)).To(Succeed())
|
|
Expect(WaitForBackupToBeCreated(context.Background(), backupScName, 5*time.Minute, &veleroCfg)).To(Succeed())
|
|
})
|
|
|
|
By(fmt.Sprintf("Restore %s", migrationNamespace), func() {
|
|
if OriginVeleroCfg.SnapshotMoveData {
|
|
By(fmt.Sprintf("Create a storage class %s for restore PV provisioned by storage class %s on different cloud provider", StorageClassName, KibishiiStorageClassName), func() {
|
|
Expect(InstallStorageClass(context.Background(), fmt.Sprintf("../testdata/storage-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider))).To(Succeed())
|
|
})
|
|
configmaptName := "datamover-storage-class-config"
|
|
labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
|
|
"velero.io/plugin-config": ""}
|
|
data := map[string]string{KibishiiStorageClassName: StorageClassName}
|
|
|
|
By(fmt.Sprintf("Create ConfigMap %s in namespace %s", configmaptName, veleroCfg.VeleroNamespace), func() {
|
|
_, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, configmaptName, labels, data)
|
|
Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace))
|
|
})
|
|
} else {
|
|
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, "", restoreName)
|
|
return "Fail to restore workload"
|
|
})
|
|
}
|
|
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
|
veleroCfg.VeleroNamespace, "", restoreName)
|
|
return "Fail to restore workload"
|
|
})
|
|
})
|
|
|
|
By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
|
|
Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
|
|
oneHourTimeout, &KibishiiData, "")).To(Succeed(), "Fail to verify workload after restore")
|
|
})
|
|
|
|
// TODO: delete backup created by case self, not all
|
|
By("Clean backups after test", func() {
|
|
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
|
DeleteBackups(context.Background(), backupNames, &veleroCfg)
|
|
})
|
|
})
|
|
})
|
|
}
|