mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-07 05:46:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #6447 from danfengliu/add-plugin-for-datamover-e2e-test

Browse Source 
Support datamover test in E2E script
This commit is contained in:
danfengliu
2023-07-10 18:16:37 +08:00
committed by GitHub
parent f6294cc2a3 bc8742566b
commit a07bbb551b
18 changed files with 276 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -35,6 +35,7 @@ require (
github.com/stretchr/testify v1.8.2
github.com/vmware-tanzu/crash-diagnostics v0.3.7
go.uber.org/zap v1.24.0
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f
golang.org/x/mod v0.10.0
golang.org/x/net v0.9.0
golang.org/x/oauth2 v0.7.0
@@ -139,7 +140,6 @@ require (
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/crypto v0.8.0 // indirect
golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect

9
test/e2e/Makefile
View File

@@ -73,6 +73,7 @@ BSL_PREFIX ?=
BSL_CONFIG ?=
VSL_CONFIG ?=
CLOUD_PROVIDER ?=
STANDBY_CLUSTER_CLOUD_PROVIDER ?=
OBJECT_STORE_PROVIDER ?=
INSTALL_VELERO ?= true
REGISTRY_CREDENTIAL_FILE ?=
@@ -99,6 +100,9 @@ STANDBY_CLUSTER ?=
UPLOADER_TYPE ?=
SNAPSHOT_MOVE_DATA ?= false
DATA_MOVER_PLUGIN ?=
.PHONY:ginkgo
ginkgo: # Make sure ginkgo is in $GOPATH/bin
@@ -143,7 +147,10 @@ run: ginkgo
-velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
-default-cluster=$(DEFAULT_CLUSTER) \
-standby-cluster=$(STANDBY_CLUSTER) \
-uploader-type=$(UPLOADER_TYPE)
-uploader-type=$(UPLOADER_TYPE) \
-snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
-data-mover-plugin=$(DATA_MOVER_plugin) \
-standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER)
build: ginkgo
mkdir -p $(OUTPUT_DIR)

9
test/e2e/backup/backup.go
View File

@@ -87,7 +87,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
} else {
veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
}
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
}
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
@@ -125,12 +125,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots
}
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
}
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider,
veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed())
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)

4
test/e2e/backups/sync_backups.go
View File

@@ -59,7 +59,7 @@ func BackupsSyncTest() {
if VeleroCfg.InstallVelero {
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
}
})
@@ -109,7 +109,7 @@ func BackupsSyncTest() {
By("Install velero", func() {
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(ctx, &VeleroCfg)).To(Succeed())
Expect(VeleroInstall(ctx, &VeleroCfg, false)).To(Succeed())
})
By("Check all backups in object storage are synced to Velero", func() {

2
test/e2e/backups/ttl.go
View File

@@ -70,7 +70,7 @@ func TTLTest() {
// Make sure GCFrequency is shorter than backup TTL
veleroCfg.GCFrequency = "4m0s"
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
}
})

4
test/e2e/basic/api-group/enable_api_group_extentions.go
View File

@@ -100,7 +100,7 @@ func APIExtensionsVersionsTest() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
veleroCfg.Features = "EnableAPIGroupVersions"
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
})
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
@@ -129,7 +129,7 @@ func APIExtensionsVersionsTest() {
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {

2
test/e2e/basic/api-group/enable_api_group_versions.go
View File

@@ -75,7 +75,7 @@ func APIGropuVersionsTest() {
if veleroCfg.InstallVelero {
veleroCfg.Features = "EnableAPIGroupVersions"
veleroCfg.UseVolumeSnapshots = false
err = VeleroInstall(context.Background(), &veleroCfg)
err = VeleroInstall(context.Background(), &veleroCfg, false)
Expect(err).NotTo(HaveOccurred())
}
testCaseNum = 4

2
test/e2e/basic/namespace-mapping.go
View File

@@ -115,7 +115,7 @@ func (n *NamespaceMapping) Verify() error {
func (n *NamespaceMapping) Clean() error {
if !n.VeleroCfg.Debug {
if err := DeleteStorageClass(context.Background(), n.Client, "kibishii-storage-class"); err != nil {
if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil {
return err
}
for _, ns := range n.MappedNamespaceList {

4
test/e2e/bsl-mgmt/deletion.go
View File

@@ -104,9 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
}
By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() {
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider,
veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed())
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed())
})
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)

4
test/e2e/e2e_suite_test.go
View File

@@ -84,7 +84,9 @@ func init() {
flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "Standby cluster context for migration test.")
flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "Identify persistent volume backup uploader.")
flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.")
flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "Install default plugin for data mover.")
flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "Install customized plugin for data mover.")
flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "Install customized plugin for data mover.")
}
var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest)

116
test/e2e/migration/migration.go
View File

@@ -59,13 +59,14 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BeforeEach(func() {
veleroCfg = VeleroCfg
UUIDgen, err = uuid.NewRandom()
migrationNamespace = "migration-workload-" + UUIDgen.String()
migrationNamespace = "migration-" + UUIDgen.String()
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" && !veleroCfg.SnapshotMoveData {
Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool")
}
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
Skip("Migration test needs 2 clusters")
}
@@ -79,9 +80,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
AfterEach(func() {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
})
// TODO: delete backup created by case self, not all
// By("Clean backups after test", func() {
// DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
// })
if veleroCfg.InstallVelero {
By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
@@ -104,6 +106,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
if veleroCfg.SnapshotMoveData {
if !useVolumeSnapshots {
Skip("FSB migration test is not needed in data mover scenario")
}
// TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path.
if veleroCLI2Version.VeleroVersion != "self" {
Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion))
}
}
oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
defer ctxCancel()
flag.Parse()
@@ -132,25 +144,20 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
// TODO: self means 1.10 and upper version
if veleroCLI2Version.VeleroVersion != "self" {
// self represents v1.12
if veleroCLI2Version.VeleroVersion == "self" {
if OriginVeleroCfg.SnapshotMoveData {
OriginVeleroCfg.UseNodeAgent = true
}
} else {
Expect(err).To(Succeed())
fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion)
OriginVeleroCfg.VeleroImage = ""
OriginVeleroCfg.RestoreHelperImage = ""
OriginVeleroCfg.Plugins = ""
//TODO: Remove this once origin Velero version is 1.10 and upper
OriginVeleroCfg.UploaderType = ""
if supportUploaderType {
OriginVeleroCfg.UseRestic = false
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
} else {
OriginVeleroCfg.UseRestic = !useVolumeSnapshots
OriginVeleroCfg.UseNodeAgent = false
}
}
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed())
if veleroCLI2Version.VeleroVersion != "self" {
Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed())
@@ -167,10 +174,15 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace))
})
KibishiiData := *DefaultKibishiiData
By("Deploy sample workload of Kibishii", func() {
if OriginVeleroCfg.SnapshotMoveData {
KibishiiData.ExpectedNodes = 6
}
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider,
migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed())
})
By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() {
@@ -178,6 +190,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupStorageClassCfg.BackupName = backupScName
BackupStorageClassCfg.IncludeResources = "StorageClass"
BackupStorageClassCfg.IncludeClusterResources = true
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
@@ -195,6 +208,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupCfg.UseResticIfFSBackup = !supportUploaderType
BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
@@ -211,21 +225,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
migrationNamespace, 2)).To(Succeed())
})
}
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = migrationNamespace
By("Snapshot should be created in cloud object store", func() {
snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2,
migrationNamespace, backupName, KibishiiPVCNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})
if !OriginVeleroCfg.SnapshotMoveData {
By("Snapshot should be created in cloud object store", func() {
snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2,
migrationNamespace, backupName, KibishiiPVCNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})
} else {
//TODO: checkpoint for datamover
}
}
if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") {
// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
// v1.0 have issues to fail upgrade case.
if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" &&
strings.EqualFold(veleroCfg.Features, "EnableCSI") &&
!OriginVeleroCfg.SnapshotMoveData {
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
time.Sleep(5 * time.Minute)
})
@@ -233,7 +253,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
@@ -253,7 +273,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
veleroCfg.UseNodeAgent = !useVolumeSnapshots
veleroCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
if veleroCfg.SnapshotMoveData {
veleroCfg.UseNodeAgent = true
}
Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
@@ -262,12 +285,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
By(fmt.Sprintf("Restore %s", migrationNamespace), func() {
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
if OriginVeleroCfg.SnapshotMoveData {
By(fmt.Sprintf("Create a storage class %s for restore PV provisioned by storage class %s on different cloud provider", StorageClassName, KibishiiStorageClassName), func() {
Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider))).To(Succeed())
})
configmaptName := "datamover-storage-class-config"
labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction",
"velero.io/plugin-config": ""}
data := map[string]string{KibishiiStorageClassName: StorageClassName}
By(fmt.Sprintf("Create ConfigMap %s in namespace %s", configmaptName, veleroCfg.VeleroNamespace), func() {
_, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, configmaptName, labels, data)
Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace))
})
} else {
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
}
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
@@ -278,7 +316,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore")
})
})
})

4
test/e2e/types.go
View File

@@ -72,6 +72,9 @@ type VeleroConfig struct {
DefaultVolumesToFsBackup bool
UseVolumeSnapshots bool
VeleroServerDebugMode bool
SnapshotMoveData bool
DataMoverPlugin string
StandbyClusterCloudProvider string
}
type SnapshotCheckPoint struct {
@@ -98,6 +101,7 @@ type BackupConfig struct {
OrderedResources string
UseResticIfFSBackup bool
DefaultVolumesToFsBackup bool
SnapshotMoveData bool
}
type VeleroCLI2Version struct {

4
test/e2e/upgrade/upgrade.go
View File

@@ -136,7 +136,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.UseNodeAgent = false
}
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall)).To(Succeed())
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI,
tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed())
})
@@ -223,7 +223,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfg.UseNodeAgent = !useVolumeSnapshots
Expect(err).To(Succeed())
if supportUploaderType {
Expect(VeleroInstall(context.Background(), &tmpCfg)).To(Succeed())
Expect(VeleroInstall(context.Background(), &tmpCfg, false)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI,
tmpCfg.VeleroVersion)).To(Succeed())
} else {

1
test/e2e/util/k8s/common.go
View File

@@ -200,6 +200,7 @@ func AddLabelToCRD(ctx context.Context, crd, label string) error {
func KubectlApplyByFile(ctx context.Context, file string) error {
args := []string{"apply", "-f", file, "--force=true"}
fmt.Println(args)
return exec.CommandContext(ctx, "kubectl", args...).Run()
}

39
test/e2e/util/k8s/statefulset.go Normal file
View File

@@ -0,0 +1,39 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8s
import (
"fmt"
"os/exec"
"github.com/pkg/errors"
"golang.org/x/net/context"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
)
func ScaleStatefulSet(ctx context.Context, namespace, name string, replicas int) error {
cmd := exec.CommandContext(ctx, "kubectl", "scale", "statefulsets", name, fmt.Sprintf("--replicas=%d", replicas), "-n", namespace)
fmt.Printf("Scale kibishii stateful set in namespace %s with CMD: %s", name, cmd.Args)
_, stderr, err := veleroexec.RunCommand(cmd)
if err != nil {
return errors.Wrap(err, stderr)
}
return nil
}

19
test/e2e/util/kibishii/kibishii_utils.go
View File

@@ -48,8 +48,11 @@ type KibishiiData struct {
ExpectedNodes int
}
var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
var DefaultKibishiiWorkerCounts = 2
var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, DefaultKibishiiWorkerCounts}
var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"}
var KibishiiStorageClassName = "kibishii-storage-class"
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
@@ -196,11 +199,15 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
}
func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures,
kibishiiDirectory string, useVolumeSnapshots bool) error {
kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error {
if strings.EqualFold(cloudPlatform, "azure") &&
strings.EqualFold(veleroFeatures, "EnableCSI") {
cloudPlatform = "azure-csi"
}
if strings.EqualFold(cloudPlatform, "aws") &&
strings.EqualFold(veleroFeatures, "EnableCSI") {
cloudPlatform = "aws-csi"
}
// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
kibishiiDirectory+cloudPlatform, "--timeout=90s")
@@ -216,6 +223,12 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
if err != nil {
return errors.Wrapf(err, "failed to label namespace with PSA policy, stderr=%s", stderr)
}
if workerReplicas != DefaultKibishiiWorkerCounts {
err = ScaleStatefulSet(ctx, namespace, "kibishii-deployment", workerReplicas)
if err != nil {
return errors.Wrapf(err, "failed to scale statefulset, stderr=%s", err.Error())
}
}
kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment",
"-n", namespace, "-w", "--timeout=30m")
@@ -311,7 +324,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie
}
if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures,
kibishiiDirectory, useVolumeSnapshots); err != nil {
kibishiiDirectory, useVolumeSnapshots, kibishiiData.ExpectedNodes); err != nil {
return errors.Wrap(err, "Failed to install Kibishii workload")
}
// wait for kibishii pod startup

38
test/e2e/util/velero/install.go
View File

@@ -55,8 +55,23 @@ type installOptions struct {
VeleroServerDebugMode bool
}
func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error {
fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05"))
// veleroCfg struct including a set of BSL params and a set of additional BSL params,
// additional BSL set is for additional BSL test only, so only default BSL set is effective
// for VeleroInstall().
//
// veleroCfg struct including 2 sets of cluster setting, but VeleroInstall() only read
// default cluster settings, so if E2E test needs install on the standby cluster, default cluster
// setting should be reset to the value of standby cluster's.
//
// Some other setting might not needed by standby cluster installation like "snapshotMoveData", because in
// standby cluster only restore if performed, so CSI plugin is not needed, but it is installed due to
// the only one veleroCfg setting is provided as current design, since it will not introduce any issues as
// we can predict, so keep it intact for now.
if isStandbyCluster {
veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
}
if veleroCfg.CloudProvider != "kind" {
fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider")
// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
@@ -69,7 +84,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
}
}
providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.CloudProvider, veleroCfg.Plugins, veleroCfg.Features)
pluginsTmp, err := getPlugins(ctx, *veleroCfg)
if err != nil {
return errors.WithMessage(err, "Failed to get provider plugins")
}
@@ -91,22 +106,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
}
}
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, providerPluginsTmp)
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, pluginsTmp)
if err != nil {
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
if !veleroCfg.UseRestic {
veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
}
veleroInstallOptions.UseRestic = veleroCfg.UseRestic
veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
veleroInstallOptions.Image = veleroCfg.VeleroImage
veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace
veleroInstallOptions.UploaderType = veleroCfg.UploaderType
GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency)
veleroInstallOptions.GarbageCollectionFrequency = GCFrequency
err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{
err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{
Options: veleroInstallOptions,
RegistryCredentialFile: veleroCfg.RegistryCredentialFile,
RestoreHelperImage: veleroCfg.RestoreHelperImage,
@@ -176,7 +188,7 @@ func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMap
return nil
}
func installVeleroServer(ctx context.Context, cli string, options *installOptions) error {
func installVeleroServer(ctx context.Context, cli, cloudProvider string, options *installOptions) error {
args := []string{"install"}
namespace := "velero"
if len(options.Namespace) > 0 {
@@ -192,9 +204,6 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
if options.DefaultVolumesToFsBackup {
args = append(args, "--default-volumes-to-fs-backup")
}
if options.UseRestic {
args = append(args, "--use-restic")
}
if options.UseVolumeSnapshots {
args = append(args, "--use-volume-snapshots")
}
@@ -219,10 +228,11 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
if len(options.Plugins) > 0 {
args = append(args, "--plugins", options.Plugins.String())
}
fmt.Println("Start to install Azure VolumeSnapshotClass ...")
if len(options.Features) > 0 {
args = append(args, "--features", options.Features)
if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots {
if strings.EqualFold(options.ProviderName, "Azure") {
if strings.EqualFold(cloudProvider, "azure") {
if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil {
return err
}
@@ -528,7 +538,7 @@ func PrepareVelero(ctx context.Context, caseName string) error {
return nil
}
fmt.Printf("need to install velero for case %s \n", caseName)
return VeleroInstall(context.Background(), &VeleroCfg)
return VeleroInstall(context.Background(), &VeleroCfg, false)
}
func VeleroUninstall(ctx context.Context, cli, namespace string) error {

113
test/e2e/util/velero/velero_utils.go
View File

@@ -34,6 +34,7 @@ import (
"time"
"github.com/pkg/errors"
"golang.org/x/exp/slices"
"k8s.io/apimachinery/pkg/util/wait"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
@@ -70,15 +71,16 @@ var pluginsMatrix = map[string]map[string][]string{
"csi": {"velero/velero-plugin-for-csi:v0.5.0"},
},
"main": {
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
"csi": {"velero/velero-plugin-for-csi:main"},
"aws": {"velero/velero-plugin-for-aws:main"},
"azure": {"velero/velero-plugin-for-microsoft-azure:main"},
"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
"gcp": {"velero/velero-plugin-for-gcp:main"},
"csi": {"velero/velero-plugin-for-csi:main"},
"datamover": {"velero/velero-plugin-for-aws:main"},
},
}
func GetProviderPluginsByVersion(version, providerName, feature string) ([]string, error) {
func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature string, needDataMoverPlugin bool) ([]string, error) {
var cloudMap map[string][]string
arr := strings.Split(version, ".")
if len(arr) >= 3 {
@@ -92,17 +94,47 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin
}
var pluginsForFeature []string
plugins, ok := cloudMap[providerName]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
if cloudProvider == "kind" {
plugins, ok := cloudMap["aws"]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider)
}
return plugins, nil
}
plugins, ok := cloudMap[cloudProvider]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider)
}
if objectStoreProvider != cloudProvider {
pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider)
}
plugins = append(plugins, pluginsForObjectStoreProvider...)
}
if strings.EqualFold(feature, "EnableCSI") {
pluginsForFeature, ok = cloudMap["csi"]
if !ok {
return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
return nil, errors.Errorf("fail to get CSI plugins by version: %s ", version)
}
plugins = append(plugins, pluginsForFeature...)
}
return append(plugins, pluginsForFeature...), nil
if needDataMoverPlugin {
pluginsForDatamover, ok := cloudMap["datamover"]
if !ok {
return nil, errors.Errorf("fail to get plugins by for datamover")
}
for _, p := range pluginsForDatamover {
if !slices.Contains(plugins, p) {
plugins = append(plugins, pluginsForDatamover...)
}
}
}
return plugins, nil
}
// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
@@ -280,6 +312,10 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
args = append(args, "--selector", backupCfg.Selector)
}
if backupCfg.SnapshotMoveData {
args = append(args, "--snapshot-move-data")
}
if backupCfg.UseVolumeSnapshots {
if backupCfg.ProvideSnapshotsVolumeParam {
args = append(args, "--snapshot-volumes")
@@ -516,36 +552,67 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error
return nil
}
func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, cloudProvider, providerPlugins, feature string) ([]string, error) {
// getProviderPlugins only provide plugin for specific cloud provider
func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider string) ([]string, error) {
if cloudProvider == "" {
return []string{}, errors.New("CloudProvider should be provided")
}
version, err := getVeleroVersion(ctx, veleroCLI, true)
if err != nil {
return nil, errors.WithMessage(err, "failed to get velero version")
}
plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, "", false)
if err != nil {
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version)
}
return plugins, nil
}
// getPlugins will collect all kinds plugins for VeleroInstall, such as provider
// plugins(cloud provider/object store provider, if object store provider is not
// provided, it should be set to value as cloud provider's), feature plugins (CSI/Datamover)
func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
veleroCLI := veleroCfg.VeleroCLI
cloudProvider := veleroCfg.CloudProvider
objectStoreProvider := veleroCfg.ObjectStoreProvider
providerPlugins := veleroCfg.Plugins
feature := veleroCfg.Features
needDataMoverPlugin := false
// Fetch the plugins for the provider before checking for the object store provider below.
var plugins []string
if len(providerPlugins) > 0 {
plugins = strings.Split(providerPlugins, ",")
} else {
if cloudProvider == "" {
return []string{}, errors.New("CloudProvider should be provided")
}
if objectStoreProvider == "" {
objectStoreProvider = cloudProvider
}
version, err := getVeleroVersion(ctx, veleroCLI, true)
if err != nil {
return nil, errors.WithMessage(err, "failed to get velero version")
}
plugins, err = GetProviderPluginsByVersion(version, objectStoreProvider, feature)
if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" {
needDataMoverPlugin = true
}
plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature, needDataMoverPlugin)
if err != nil {
return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
}
if objectStoreProvider != "" && cloudProvider != "kind" && objectStoreProvider != cloudProvider {
pluginsTmp, err := GetProviderPluginsByVersion(version, cloudProvider, feature)
if err != nil {
return nil, errors.WithMessage(err, "failed to get velero version")
}
plugins = append(plugins, pluginsTmp...)
}
}
return plugins, nil
}
// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
// installs them in the current Velero installation, skipping over those that are already installed.
func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error {
plugins, err := getProviderPlugins(ctx, veleroCLI, provider, provider, addPlugins, feature)
fmt.Printf("addPlugins cmd =%v\n", addPlugins)
func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error {
plugins, err := getProviderPlugins(ctx, veleroCLI, provider)
fmt.Printf("provider cmd = %v\n", provider)
fmt.Printf("plugins cmd = %v\n", plugins)
if err != nil {