From bc8742566bfcdfc24758114dee1c51bc6e9033cc Mon Sep 17 00:00:00 2001 From: danfengl Date: Tue, 20 Jun 2023 10:24:11 +0000 Subject: [PATCH] Install plugin for datamover pipeline Signed-off-by: danfengl --- go.mod | 2 +- test/e2e/Makefile | 9 +- test/e2e/backup/backup.go | 9 +- test/e2e/backups/sync_backups.go | 4 +- test/e2e/backups/ttl.go | 2 +- .../api-group/enable_api_group_extentions.go | 4 +- .../api-group/enable_api_group_versions.go | 2 +- test/e2e/basic/namespace-mapping.go | 2 +- test/e2e/bsl-mgmt/deletion.go | 4 +- test/e2e/e2e_suite_test.go | 4 +- test/e2e/migration/migration.go | 116 ++++++++++++------ test/e2e/types.go | 4 + test/e2e/upgrade/upgrade.go | 4 +- test/e2e/util/k8s/common.go | 1 + test/e2e/util/k8s/statefulset.go | 39 ++++++ test/e2e/util/kibishii/kibishii_utils.go | 19 ++- test/e2e/util/velero/install.go | 38 +++--- test/e2e/util/velero/velero_utils.go | 113 +++++++++++++---- 18 files changed, 276 insertions(+), 100 deletions(-) create mode 100644 test/e2e/util/k8s/statefulset.go diff --git a/go.mod b/go.mod index 61885fe44..ee43d5fa3 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/stretchr/testify v1.8.2 github.com/vmware-tanzu/crash-diagnostics v0.3.7 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f golang.org/x/mod v0.10.0 golang.org/x/net v0.9.0 golang.org/x/oauth2 v0.7.0 @@ -139,7 +140,6 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.8.0 // indirect - golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.7.0 // indirect diff --git a/test/e2e/Makefile b/test/e2e/Makefile index f2d7594e1..0fbf46e3e 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -73,6 +73,7 @@ BSL_PREFIX ?= BSL_CONFIG ?= VSL_CONFIG ?= CLOUD_PROVIDER ?= +STANDBY_CLUSTER_CLOUD_PROVIDER ?= OBJECT_STORE_PROVIDER ?= INSTALL_VELERO ?= true REGISTRY_CREDENTIAL_FILE ?= @@ -99,6 +100,9 @@ STANDBY_CLUSTER ?= UPLOADER_TYPE ?= +SNAPSHOT_MOVE_DATA ?= false +DATA_MOVER_PLUGIN ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin @@ -143,7 +147,10 @@ run: ginkgo -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ -default-cluster=$(DEFAULT_CLUSTER) \ -standby-cluster=$(STANDBY_CLUSTER) \ - -uploader-type=$(UPLOADER_TYPE) + -uploader-type=$(UPLOADER_TYPE) \ + -snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ + -data-mover-plugin=$(DATA_MOVER_plugin) \ + -standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) build: ginkgo mkdir -p $(OUTPUT_DIR) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 9bf694c00..8a1f2d40e 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -87,7 +87,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { } else { veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots } - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() @@ -125,12 +125,9 @@ func BackupRestoreTest(useVolumeSnapshots bool) { veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots } - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } - - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, - veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 1bfea90b2..a12a270c8 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -59,7 +59,7 @@ func BackupsSyncTest() { if VeleroCfg.InstallVelero { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) } }) @@ -109,7 +109,7 @@ func BackupsSyncTest() { By("Install velero", func() { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(ctx, &VeleroCfg)).To(Succeed()) + Expect(VeleroInstall(ctx, &VeleroCfg, false)).To(Succeed()) }) By("Check all backups in object storage are synced to Velero", func() { diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 09afb23e8..a60d95ea5 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -70,7 +70,7 @@ func TTLTest() { // Make sure GCFrequency is shorter than backup TTL veleroCfg.GCFrequency = "4m0s" veleroCfg.UseVolumeSnapshots = useVolumeSnapshots - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) } }) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index a3d6a3b88..002833d55 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -100,7 +100,7 @@ func APIExtensionsVersionsTest() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() { @@ -129,7 +129,7 @@ func APIExtensionsVersionsTest() { By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg, false)).To(Succeed()) }) By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index ba1d73f6e..61d7ba08e 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -75,7 +75,7 @@ func APIGropuVersionsTest() { if veleroCfg.InstallVelero { veleroCfg.Features = "EnableAPIGroupVersions" veleroCfg.UseVolumeSnapshots = false - err = VeleroInstall(context.Background(), &veleroCfg) + err = VeleroInstall(context.Background(), &veleroCfg, false) Expect(err).NotTo(HaveOccurred()) } testCaseNum = 4 diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 8ea16d365..ae9c0509e 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -115,7 +115,7 @@ func (n *NamespaceMapping) Verify() error { func (n *NamespaceMapping) Clean() error { if !n.VeleroCfg.Debug { - if err := DeleteStorageClass(context.Background(), n.Client, "kibishii-storage-class"); err != nil { + if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil { return err } for _, ns := range n.MappedNamespaceList { diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index e15307369..874cf53c7 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -104,9 +104,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { } By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, - veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 651affb0e..df9d18ea4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -84,7 +84,9 @@ func init() { flag.StringVar(&VeleroCfg.StandbyCluster, "standby-cluster", "", "Standby cluster context for migration test.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") - + flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "Install default plugin for data mover.") + flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "Install customized plugin for data mover.") + flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "Install customized plugin for data mover.") } var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 0a133f23e..6b10ba925 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -59,13 +59,14 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BeforeEach(func() { veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() - migrationNamespace = "migration-workload-" + UUIDgen.String() + migrationNamespace = "migration-" + UUIDgen.String() if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" && !veleroCfg.SnapshotMoveData { Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool") } + if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("Migration test needs 2 clusters") } @@ -79,9 +80,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) AfterEach(func() { if !veleroCfg.Debug { - By("Clean backups after test", func() { - DeleteBackups(context.Background(), *veleroCfg.DefaultClient) - }) + // TODO: delete backup created by case self, not all + // By("Clean backups after test", func() { + // DeleteBackups(context.Background(), *veleroCfg.DefaultClient) + // }) if veleroCfg.InstallVelero { By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) @@ -104,6 +106,16 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { + + if veleroCfg.SnapshotMoveData { + if !useVolumeSnapshots { + Skip("FSB migration test is not needed in data mover scenario") + } + // TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path. + if veleroCLI2Version.VeleroVersion != "self" { + Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion)) + } + } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() flag.Parse() @@ -132,25 +144,20 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots - // TODO: self means 1.10 and upper version - if veleroCLI2Version.VeleroVersion != "self" { + + // self represents v1.12 + if veleroCLI2Version.VeleroVersion == "self" { + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true + } + } else { Expect(err).To(Succeed()) fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) OriginVeleroCfg.VeleroImage = "" OriginVeleroCfg.RestoreHelperImage = "" OriginVeleroCfg.Plugins = "" - //TODO: Remove this once origin Velero version is 1.10 and upper - OriginVeleroCfg.UploaderType = "" - if supportUploaderType { - OriginVeleroCfg.UseRestic = false - OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots - } else { - OriginVeleroCfg.UseRestic = !useVolumeSnapshots - OriginVeleroCfg.UseNodeAgent = false - } } - - Expect(VeleroInstall(context.Background(), &OriginVeleroCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed()) @@ -167,10 +174,15 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace)) }) + KibishiiData := *DefaultKibishiiData By("Deploy sample workload of Kibishii", func() { + if OriginVeleroCfg.SnapshotMoveData { + KibishiiData.ExpectedNodes = 6 + } + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, - veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) + veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) }) By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() { @@ -178,6 +190,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupStorageClassCfg.BackupName = backupScName BackupStorageClassCfg.IncludeResources = "StorageClass" BackupStorageClassCfg.IncludeClusterResources = true + //TODO Remove UseRestic parameter once minor version is 1.10 or upper BackupStorageClassCfg.UseResticIfFSBackup = !supportUploaderType Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -195,6 +208,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots //TODO Remove UseRestic parameter once minor version is 1.10 or upper BackupCfg.UseResticIfFSBackup = !supportUploaderType + BackupCfg.SnapshotMoveData = OriginVeleroCfg.SnapshotMoveData Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { @@ -211,21 +225,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) migrationNamespace, 2)).To(Succeed()) }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace - By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPVCNameList) - Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) - }) + + if !OriginVeleroCfg.SnapshotMoveData { + By("Snapshot should be created in cloud object store", func() { + snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, + migrationNamespace, backupName, KibishiiPVCNameList) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + }) + } else { + //TODO: checkpoint for datamover + } } - if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") { - // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before - // v1.0 have issues to fail upgrade case. + if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && + strings.EqualFold(veleroCfg.Features, "EnableCSI") && + !OriginVeleroCfg.SnapshotMoveData { By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { time.Sleep(5 * time.Minute) }) @@ -233,7 +253,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -253,7 +273,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.UseNodeAgent = !useVolumeSnapshots veleroCfg.UseRestic = false - Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + if veleroCfg.SnapshotMoveData { + veleroCfg.UseNodeAgent = true + } + Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) }) By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { @@ -262,12 +285,27 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) By(fmt.Sprintf("Restore %s", migrationNamespace), func() { - Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { - RunDebug(context.Background(), veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace, "", restoreName) - return "Fail to restore workload" - }) + if OriginVeleroCfg.SnapshotMoveData { + By(fmt.Sprintf("Create a storage class %s for restore PV provisioned by storage class %s on different cloud provider", StorageClassName, KibishiiStorageClassName), func() { + Expect(InstallStorageClass(context.Background(), fmt.Sprintf("testdata/storage-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider))).To(Succeed()) + }) + configmaptName := "datamover-storage-class-config" + labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction", + "velero.io/plugin-config": ""} + data := map[string]string{KibishiiStorageClassName: StorageClassName} + + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", configmaptName, veleroCfg.VeleroNamespace), func() { + _, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, configmaptName, labels, data) + Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace)) + }) + } else { + Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", restoreName) + return "Fail to restore workload" + }) + } Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, @@ -278,7 +316,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() { Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace, - oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore") + oneHourTimeout, &KibishiiData)).To(Succeed(), "Fail to verify workload after restore") }) }) }) diff --git a/test/e2e/types.go b/test/e2e/types.go index 99f4c23cb..4e3dcd1cd 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -72,6 +72,9 @@ type VeleroConfig struct { DefaultVolumesToFsBackup bool UseVolumeSnapshots bool VeleroServerDebugMode bool + SnapshotMoveData bool + DataMoverPlugin string + StandbyClusterCloudProvider string } type SnapshotCheckPoint struct { @@ -98,6 +101,7 @@ type BackupConfig struct { OrderedResources string UseResticIfFSBackup bool DefaultVolumesToFsBackup bool + SnapshotMoveData bool } type VeleroCLI2Version struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 1e661d823..58b892255 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -136,7 +136,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UseNodeAgent = false } - Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed()) }) @@ -223,7 +223,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfg.UseNodeAgent = !useVolumeSnapshots Expect(err).To(Succeed()) if supportUploaderType { - Expect(VeleroInstall(context.Background(), &tmpCfg)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfg, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfg.VeleroCLI, tmpCfg.VeleroVersion)).To(Succeed()) } else { diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index cf18e7f0a..020de0e28 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -200,6 +200,7 @@ func AddLabelToCRD(ctx context.Context, crd, label string) error { func KubectlApplyByFile(ctx context.Context, file string) error { args := []string{"apply", "-f", file, "--force=true"} + fmt.Println(args) return exec.CommandContext(ctx, "kubectl", args...).Run() } diff --git a/test/e2e/util/k8s/statefulset.go b/test/e2e/util/k8s/statefulset.go new file mode 100644 index 000000000..e9a1e564d --- /dev/null +++ b/test/e2e/util/k8s/statefulset.go @@ -0,0 +1,39 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "fmt" + "os/exec" + + "github.com/pkg/errors" + "golang.org/x/net/context" + + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" +) + +func ScaleStatefulSet(ctx context.Context, namespace, name string, replicas int) error { + cmd := exec.CommandContext(ctx, "kubectl", "scale", "statefulsets", name, fmt.Sprintf("--replicas=%d", replicas), "-n", namespace) + fmt.Printf("Scale kibishii stateful set in namespace %s with CMD: %s", name, cmd.Args) + + _, stderr, err := veleroexec.RunCommand(cmd) + if err != nil { + return errors.Wrap(err, stderr) + } + + return nil +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 2edb3acc0..ea4e419a7 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -48,8 +48,11 @@ type KibishiiData struct { ExpectedNodes int } -var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} +var DefaultKibishiiWorkerCounts = 2 +var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, DefaultKibishiiWorkerCounts} + var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} +var KibishiiStorageClassName = "kibishii-storage-class" // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, @@ -196,11 +199,15 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, - kibishiiDirectory string, useVolumeSnapshots bool) error { + kibishiiDirectory string, useVolumeSnapshots bool, workerReplicas int) error { if strings.EqualFold(cloudPlatform, "azure") && strings.EqualFold(veleroFeatures, "EnableCSI") { cloudPlatform = "azure-csi" } + if strings.EqualFold(cloudPlatform, "aws") && + strings.EqualFold(veleroFeatures, "EnableCSI") { + cloudPlatform = "aws-csi" + } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", kibishiiDirectory+cloudPlatform, "--timeout=90s") @@ -216,6 +223,12 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler if err != nil { return errors.Wrapf(err, "failed to label namespace with PSA policy, stderr=%s", stderr) } + if workerReplicas != DefaultKibishiiWorkerCounts { + err = ScaleStatefulSet(ctx, namespace, "kibishii-deployment", workerReplicas) + if err != nil { + return errors.Wrapf(err, "failed to scale statefulset, stderr=%s", err.Error()) + } + } kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment", "-n", namespace, "-w", "--timeout=30m") @@ -311,7 +324,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie } if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, - kibishiiDirectory, useVolumeSnapshots); err != nil { + kibishiiDirectory, useVolumeSnapshots, kibishiiData.ExpectedNodes); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } // wait for kibishii pod startup diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index be50c95b2..1830b15d3 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -55,8 +55,23 @@ type installOptions struct { VeleroServerDebugMode bool } -func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { +func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluster bool) error { fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05")) + // veleroCfg struct including a set of BSL params and a set of additional BSL params, + // additional BSL set is for additional BSL test only, so only default BSL set is effective + // for VeleroInstall(). + // + // veleroCfg struct including 2 sets of cluster setting, but VeleroInstall() only read + // default cluster settings, so if E2E test needs install on the standby cluster, default cluster + // setting should be reset to the value of standby cluster's. + // + // Some other setting might not needed by standby cluster installation like "snapshotMoveData", because in + // standby cluster only restore if performed, so CSI plugin is not needed, but it is installed due to + // the only one veleroCfg setting is provided as current design, since it will not introduce any issues as + // we can predict, so keep it intact for now. + if isStandbyCluster { + veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider + } if veleroCfg.CloudProvider != "kind" { fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider @@ -69,7 +84,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { } } - providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.CloudProvider, veleroCfg.Plugins, veleroCfg.Features) + pluginsTmp, err := getPlugins(ctx, *veleroCfg) if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } @@ -91,22 +106,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { } } - veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, providerPluginsTmp) + veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, pluginsTmp) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots - if !veleroCfg.UseRestic { - veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent - } - veleroInstallOptions.UseRestic = veleroCfg.UseRestic + veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent veleroInstallOptions.Image = veleroCfg.VeleroImage veleroInstallOptions.Namespace = veleroCfg.VeleroNamespace veleroInstallOptions.UploaderType = veleroCfg.UploaderType GCFrequency, _ := time.ParseDuration(veleroCfg.GCFrequency) veleroInstallOptions.GarbageCollectionFrequency = GCFrequency - err = installVeleroServer(ctx, veleroCfg.VeleroCLI, &installOptions{ + err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{ Options: veleroInstallOptions, RegistryCredentialFile: veleroCfg.RegistryCredentialFile, RestoreHelperImage: veleroCfg.RestoreHelperImage, @@ -176,7 +188,7 @@ func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMap return nil } -func installVeleroServer(ctx context.Context, cli string, options *installOptions) error { +func installVeleroServer(ctx context.Context, cli, cloudProvider string, options *installOptions) error { args := []string{"install"} namespace := "velero" if len(options.Namespace) > 0 { @@ -192,9 +204,6 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if options.DefaultVolumesToFsBackup { args = append(args, "--default-volumes-to-fs-backup") } - if options.UseRestic { - args = append(args, "--use-restic") - } if options.UseVolumeSnapshots { args = append(args, "--use-volume-snapshots") } @@ -219,10 +228,11 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if len(options.Plugins) > 0 { args = append(args, "--plugins", options.Plugins.String()) } + fmt.Println("Start to install Azure VolumeSnapshotClass ...") if len(options.Features) > 0 { args = append(args, "--features", options.Features) if strings.EqualFold(options.Features, "EnableCSI") && options.UseVolumeSnapshots { - if strings.EqualFold(options.ProviderName, "Azure") { + if strings.EqualFold(cloudProvider, "azure") { if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { return err } @@ -528,7 +538,7 @@ func PrepareVelero(ctx context.Context, caseName string) error { return nil } fmt.Printf("need to install velero for case %s \n", caseName) - return VeleroInstall(context.Background(), &VeleroCfg) + return VeleroInstall(context.Background(), &VeleroCfg, false) } func VeleroUninstall(ctx context.Context, cli, namespace string) error { diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 7da65f945..c5d1834c1 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -34,6 +34,7 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/exp/slices" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -70,15 +71,16 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.5.0"}, }, "main": { - "aws": {"velero/velero-plugin-for-aws:main"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"velero/velero-plugin-for-gcp:main"}, - "csi": {"velero/velero-plugin-for-csi:main"}, + "aws": {"velero/velero-plugin-for-aws:main"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"velero/velero-plugin-for-gcp:main"}, + "csi": {"velero/velero-plugin-for-csi:main"}, + "datamover": {"velero/velero-plugin-for-aws:main"}, }, } -func GetProviderPluginsByVersion(version, providerName, feature string) ([]string, error) { +func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -92,17 +94,47 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin } var pluginsForFeature []string - plugins, ok := cloudMap[providerName] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + if cloudProvider == "kind" { + plugins, ok := cloudMap["aws"] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) + } + return plugins, nil } + + plugins, ok := cloudMap[cloudProvider] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) + } + + if objectStoreProvider != cloudProvider { + pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] + if !ok { + return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) + } + plugins = append(plugins, pluginsForObjectStoreProvider...) + } + if strings.EqualFold(feature, "EnableCSI") { pluginsForFeature, ok = cloudMap["csi"] if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName) + return nil, errors.Errorf("fail to get CSI plugins by version: %s ", version) } + plugins = append(plugins, pluginsForFeature...) } - return append(plugins, pluginsForFeature...), nil + if needDataMoverPlugin { + pluginsForDatamover, ok := cloudMap["datamover"] + if !ok { + return nil, errors.Errorf("fail to get plugins by for datamover") + } + for _, p := range pluginsForDatamover { + if !slices.Contains(plugins, p) { + plugins = append(plugins, pluginsForDatamover...) + } + } + + } + return plugins, nil } // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. @@ -280,6 +312,10 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin args = append(args, "--selector", backupCfg.Selector) } + if backupCfg.SnapshotMoveData { + args = append(args, "--snapshot-move-data") + } + if backupCfg.UseVolumeSnapshots { if backupCfg.ProvideSnapshotsVolumeParam { args = append(args, "--snapshot-volumes") @@ -516,36 +552,67 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error return nil } -func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, cloudProvider, providerPlugins, feature string) ([]string, error) { +// getProviderPlugins only provide plugin for specific cloud provider +func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider string) ([]string, error) { + if cloudProvider == "" { + return []string{}, errors.New("CloudProvider should be provided") + } + + version, err := getVeleroVersion(ctx, veleroCLI, true) + if err != nil { + return nil, errors.WithMessage(err, "failed to get velero version") + } + + plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, "", false) + if err != nil { + return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) + } + + return plugins, nil +} + +// getPlugins will collect all kinds plugins for VeleroInstall, such as provider +// plugins(cloud provider/object store provider, if object store provider is not +// provided, it should be set to value as cloud provider's), feature plugins (CSI/Datamover) +func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { + veleroCLI := veleroCfg.VeleroCLI + cloudProvider := veleroCfg.CloudProvider + objectStoreProvider := veleroCfg.ObjectStoreProvider + providerPlugins := veleroCfg.Plugins + feature := veleroCfg.Features + needDataMoverPlugin := false + // Fetch the plugins for the provider before checking for the object store provider below. var plugins []string if len(providerPlugins) > 0 { plugins = strings.Split(providerPlugins, ",") } else { + if cloudProvider == "" { + return []string{}, errors.New("CloudProvider should be provided") + } + if objectStoreProvider == "" { + objectStoreProvider = cloudProvider + } version, err := getVeleroVersion(ctx, veleroCLI, true) if err != nil { return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err = GetProviderPluginsByVersion(version, objectStoreProvider, feature) + + if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" { + needDataMoverPlugin = true + } + plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) } - if objectStoreProvider != "" && cloudProvider != "kind" && objectStoreProvider != cloudProvider { - pluginsTmp, err := GetProviderPluginsByVersion(version, cloudProvider, feature) - if err != nil { - return nil, errors.WithMessage(err, "failed to get velero version") - } - plugins = append(plugins, pluginsTmp...) - } } return plugins, nil } // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error { - plugins, err := getProviderPlugins(ctx, veleroCLI, provider, provider, addPlugins, feature) - fmt.Printf("addPlugins cmd =%v\n", addPlugins) +func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { + plugins, err := getProviderPlugins(ctx, veleroCLI, provider) fmt.Printf("provider cmd = %v\n", provider) fmt.Printf("plugins cmd = %v\n", plugins) if err != nil {