diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 550773718..635a1315e 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -39,32 +39,36 @@ func BackupRestoreWithRestic() { } func BackupRestoreTest(useVolumeSnapshots bool) { - kibishiiNamespace := "kibishii-workload" + var ( - backupName, restoreName string - err error + backupName, restoreName, kibishiiNamespace string + err error + provideSnapshotVolumesParmInBackup bool + veleroCfg VeleroConfig ) + provideSnapshotVolumesParmInBackup = false BeforeEach(func() { - if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + veleroCfg = VeleroCfg + veleroCfg.UseVolumeSnapshots = useVolumeSnapshots + veleroCfg.UseNodeAgent = !useVolumeSnapshots + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } var err error flag.Parse() UUIDgen, err = uuid.NewRandom() + kibishiiNamespace = "kibishii-workload" + UUIDgen.String() Expect(err).To(Succeed()) - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) - } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if VeleroCfg.InstallVelero { - err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + if veleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } } @@ -72,50 +76,83 @@ func BackupRestoreTest(useVolumeSnapshots bool) { When("kibishii is the sample workload", func() { It("should be successfully backed up and restored to the default BackupStorageLocation", func() { + // TODO[High] - remove code block below when vSphere plugin PR #500 is included in release version. + // because restore will be partiallyFailed when DefaultVolumesToFsBackup is set to true during + // Velero installation with default BSL. + if veleroCfg.CloudProvider == "vsphere" && !useVolumeSnapshots { + Skip("vSphere plugin PR #500 is not included in latest version 1.4.2") + } + + if veleroCfg.InstallVelero { + if useVolumeSnapshots { + //Install node agent also + veleroCfg.UseNodeAgent = useVolumeSnapshots + veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots + } else { + veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots + } + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + } backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), + + // if set ProvideSnapshotsVolumeParam to false here, make sure set it true in other tests of this case + veleroCfg.ProvideSnapshotsVolumeParam = provideSnapshotVolumesParmInBackup + + // Set DefaultVolumesToFsBackup to false since DefaultVolumesToFsBackup was set to true during installation + Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots, false)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() { - if VeleroCfg.AdditionalBSLProvider == "" { + if veleroCfg.AdditionalBSLProvider == "" { Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") } - if VeleroCfg.AdditionalBSLBucket == "" { + if veleroCfg.AdditionalBSLBucket == "" { Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests") } - if VeleroCfg.AdditionalBSLCredentials == "" { + if veleroCfg.AdditionalBSLCredentials == "" { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } + if veleroCfg.InstallVelero { + if useVolumeSnapshots { + veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots + } else { + // Install VolumeSnapshots also + veleroCfg.UseVolumeSnapshots = !useVolumeSnapshots + veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots + } - Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, - VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) + } + + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, + veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) - secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider) + secretKey := fmt.Sprintf("creds-%s", veleroCfg.AdditionalBSLProvider) files := map[string]string{ - secretKey: VeleroCfg.AdditionalBSLCredentials, + secretKey: veleroCfg.AdditionalBSLCredentials, } - Expect(CreateSecretFromFiles(context.TODO(), *VeleroCfg.ClientToInstallVelero, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) + Expect(CreateSecretFromFiles(context.TODO(), *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) Expect(VeleroCreateBackupLocation(context.TODO(), - VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, + veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, additionalBsl, - VeleroCfg.AdditionalBSLProvider, - VeleroCfg.AdditionalBSLBucket, - VeleroCfg.AdditionalBSLPrefix, - VeleroCfg.AdditionalBSLConfig, + veleroCfg.AdditionalBSLProvider, + veleroCfg.AdditionalBSLBucket, + veleroCfg.AdditionalBSLPrefix, + veleroCfg.AdditionalBSLConfig, secretName, secretKey, )).To(Succeed()) @@ -131,7 +168,8 @@ func BackupRestoreTest(useVolumeSnapshots bool) { backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - Expect(RunKibishiiTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), + veleroCfg.ProvideSnapshotsVolumeParam = !provideSnapshotVolumesParmInBackup + Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index db1313c5a..4f871ce05 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -48,38 +48,41 @@ func backup_deletion_test(useVolumeSnapshots bool) { var ( backupName string err error + veleroCfg VeleroConfig ) + veleroCfg = VeleroCfg + veleroCfg.UseVolumeSnapshots = useVolumeSnapshots + veleroCfg.UseNodeAgent = !useVolumeSnapshots BeforeEach(func() { - if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } var err error flag.Parse() UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + if veleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if VeleroCfg.InstallVelero { - err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) + if veleroCfg.InstallVelero { + err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) Expect(err).To(Succeed()) } } - }) When("kibishii is the sample workload", func() { It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { backupName = "backup-" + UUIDgen.String() - Expect(runBackupDeletionTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, backupName, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(), "Failed to run backup deletion test") }) }) @@ -89,18 +92,18 @@ func backup_deletion_test(useVolumeSnapshots bool) { func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string, useVolumeSnapshots bool, kibishiiDirectory string) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - veleroCLI := VeleroCfg.VeleroCLI - providerName := VeleroCfg.CloudProvider - veleroNamespace := VeleroCfg.VeleroNamespace - registryCredentialFile := VeleroCfg.RegistryCredentialFile - bslPrefix := VeleroCfg.BSLPrefix - bslConfig := VeleroCfg.BSLConfig - veleroFeatures := VeleroCfg.Features + veleroCLI := veleroCfg.VeleroCLI + providerName := veleroCfg.CloudProvider + veleroNamespace := veleroCfg.VeleroNamespace + registryCredentialFile := veleroCfg.RegistryCredentialFile + bslPrefix := veleroCfg.BSLPrefix + bslConfig := veleroCfg.BSLConfig + veleroFeatures := veleroCfg.Features if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) } - if !VeleroCfg.Debug { + if !veleroCfg.Debug { defer func() { if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest)) @@ -112,7 +115,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) } - err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) + err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } @@ -121,6 +124,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam BackupCfg.Namespace = deletionTest BackupCfg.BackupLocation = backupLocation BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = "" By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() { @@ -135,20 +139,20 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") - if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil { + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } - err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) if err != nil { return err } var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig, + err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") @@ -159,21 +163,21 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam return err } if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } - err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) if err != nil { return err } if useVolumeSnapshots { - if err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, backupName, snapshotCheckPoint); err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -189,12 +193,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam }) }) - err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) if err != nil { return err } - err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) + err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 1c628ca50..d725e9e20 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -59,7 +59,9 @@ func BackupsSyncTest() { BeforeEach(func() { flag.Parse() if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + veleroCfg := VeleroCfg + veleroCfg.UseVolumeSnapshots = false + Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed()) } }) @@ -105,8 +107,9 @@ func BackupsSyncTest() { }) By("Install velero", func() { - VeleroCfg.ObjectStoreProvider = "" - Expect(VeleroInstall(test.ctx, &VeleroCfg, false)).To(Succeed()) + veleroCfg := VeleroCfg + veleroCfg.UseVolumeSnapshots = false + Expect(VeleroInstall(test.ctx, &VeleroCfg)).To(Succeed()) }) By("Check all backups in object storage are synced to Velero", func() { diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index d10c3fe7f..863217dc7 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -59,29 +59,33 @@ func (b *TTL) Init() { func TTLTest() { var err error + var veleroCfg VeleroConfig useVolumeSnapshots := true test := new(TTL) - client := *VeleroCfg.ClientToInstallVelero + veleroCfg = VeleroCfg + client := *veleroCfg.ClientToInstallVelero //Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { + veleroCfg = VeleroCfg + if veleroCfg.InstallVelero { // Make sure GCFrequency is shorter than backup TTL - VeleroCfg.GCFrequency = "4m0s" - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + veleroCfg.GCFrequency = "4m0s" + veleroCfg.UseVolumeSnapshots = useVolumeSnapshots + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - VeleroCfg.GCFrequency = "" - if !VeleroCfg.Debug { + veleroCfg.GCFrequency = "" + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if VeleroCfg.InstallVelero { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + if veleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) } @@ -95,9 +99,9 @@ func TTLTest() { }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(test.ctx, client, VeleroCfg.CloudProvider, - test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, - VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) + Expect(KibishiiPrepareBeforeBackup(test.ctx, client, veleroCfg.CloudProvider, + test.testNS, veleroCfg.RegistryCredentialFile, veleroCfg.Features, + veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) }) var BackupCfg BackupConfig @@ -109,26 +113,26 @@ func TTLTest() { BackupCfg.TTL = test.ttl By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { - Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + Expect(VeleroBackupNamespace(test.ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.backupName, "") return "Fail to backup workload" }) }) var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - if VeleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == "vsphere" { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(test.ctx, time.Hour, - test.testNS)).To(Succeed()) + test.testNS, 2)).To(Succeed()) }) } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } @@ -137,30 +141,30 @@ func TTLTest() { fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) }) - if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") time.Sleep(7 * time.Minute) } By(fmt.Sprintf("Restore %s", test.testNS), func() { - Expect(VeleroRestore(test.ctx, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string { - RunDebug(test.ctx, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, "", test.restoreName) + Expect(VeleroRestore(test.ctx, veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string { + RunDebug(test.ctx, veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", test.restoreName) return "Fail to restore workload" }) }) By("Associated Restores should be created", func() { - Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName, + Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName, RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object") }) By("Check TTL was set correctly", func() { - ttl, err := GetBackupTTL(test.ctx, VeleroCfg.VeleroNamespace, test.backupName) + ttl, err := GetBackupTTL(test.ctx, veleroCfg.VeleroNamespace, test.backupName) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", "")) fmt.Println(t.Round(time.Minute).String()) @@ -172,28 +176,28 @@ func TTLTest() { }) By("Check if backups are deleted by GC", func() { - Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName)) + Expect(WaitBackupDeleted(test.ctx, veleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName)) }) By("Backup file from cloud object storage should be deleted", func() { - Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, + Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") }) By("PersistentVolume snapshots should be deleted", func() { if useVolumeSnapshots { - Expect(SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig, + Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } }) By("Associated Restores should be deleted", func() { - Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName, + Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName, RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object") }) diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index bd83aafc2..8102765f0 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -40,22 +40,26 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) +var veleroCfg VeleroConfig + func APIExtensionsVersionsTest() { var ( backupName, restoreName string ) + resourceName := "apiextensions.k8s.io" crdName := "rocknrollbands.music.example.io" label := "for=backup" srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml" BeforeEach(func() { - if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" { + if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("CRD with apiextension versions migration test needs 2 clusters") } - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName) + veleroCfg = VeleroCfg + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) Expect(err).ShouldNot(HaveOccurred()) - dstVersions, err := GetAPIVersions(VeleroCfg.StandbyClient, resourceName) + dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName) Expect(err).ShouldNot(HaveOccurred()) Expect(srcVersions).Should(ContainElement("v1"), func() string { @@ -76,26 +80,26 @@ func APIExtensionsVersionsTest() { }) }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + DeleteBackups(context.Background(), *veleroCfg.DefaultClient) }) - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { By("Uninstall Velero and delete CRD ", func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed()) - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed()) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient }) } @@ -105,19 +109,19 @@ func APIExtensionsVersionsTest() { backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - VeleroCfg.ObjectStoreProvider = "" - VeleroCfg.Features = "EnableAPIGroupVersions" - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + veleroCfg.Features = "EnableAPIGroupVersions" + veleroCfg.UseVolumeSnapshots = false + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", VeleroCfg.DefaultCluster), func() { + By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() { Expect(installCRD(context.Background(), srcCrdYaml)).To(Succeed()) Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed()) Expect(WaitForCRDEstablished(crdName)).To(Succeed()) Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed()) - // Discovery helper is refreshed every 5 minutes + // Velero server refresh api version data by discovery helper every 5 minutes time.Sleep(6 * time.Minute) }) @@ -127,34 +131,33 @@ func APIExtensionsVersionsTest() { BackupCfg.IncludeResources = "crd" BackupCfg.IncludeClusterResources = true BackupCfg.Selector = label - Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backupName, "") + Expect(VeleroBackupNamespace(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, backupName, "") return "Fail to backup workload" }) }) - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) - VeleroCfg.ObjectStoreProvider = "" - VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() { - Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) + By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { + Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) }) - By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, VeleroCfg.StandbyCluster), func() { + By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() { Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test") }) By("Restore CRD", func() { - Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, "", restoreName) + Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", restoreName) return "Fail to restore workload" }) }) @@ -173,6 +176,7 @@ func APIGropuVersionsTest() { ) BeforeEach(func() { + veleroCfg = VeleroCfg resource = "rockbands" group = "music.example.io" UUIDgen, err = uuid.NewRandom() @@ -180,15 +184,16 @@ func APIGropuVersionsTest() { flag.Parse() // TODO: install Velero once for the test suite once feature flag is // removed and velero installation becomes the same as other e2e tests. - if VeleroCfg.InstallVelero { - VeleroCfg.Features = "EnableAPIGroupVersions" - err = VeleroInstall(context.Background(), &VeleroCfg, false) + if veleroCfg.InstallVelero { + veleroCfg.Features = "EnableAPIGroupVersions" + veleroCfg.UseVolumeSnapshots = false + err = VeleroInstall(context.Background(), &veleroCfg) Expect(err).NotTo(HaveOccurred()) } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group) cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group) _, stderr, err := veleroexec.RunCommand(cmd) @@ -198,12 +203,12 @@ func APIGropuVersionsTest() { } Expect(err).NotTo(HaveOccurred()) By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { By("Uninstall Velero", func() { - Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) + Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) }) } } @@ -213,7 +218,7 @@ func APIGropuVersionsTest() { It("Should back up API group version and restore by version priority", func() { Expect(runEnableAPIGroupVersionsTests( ctx, - *VeleroCfg.ClientToInstallVelero, + *veleroCfg.ClientToInstallVelero, resource, group, )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") @@ -294,7 +299,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso }, tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", tgtVer: "v2beta1", - cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data( + cm: builder.ForConfigMap(veleroCfg.VeleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", `rockbands.music.example.io=v2beta1,v2beta2,v2`, ).Result(), @@ -354,12 +359,8 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso tc.namespaces = append(tc.namespaces, ns) } - // Restart Velero pods in order to recognize music-system CRD right away - // instead of waiting for discovery helper to refresh. See - // https://github.com/vmware-tanzu/velero/issues/3471. - if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { - return errors.Wrapf(err, "restart Velero pods") - } + // Velero server refresh api version data by discovery helper every 5 minutes + time.Sleep(6 * time.Minute) backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") @@ -371,10 +372,10 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso BackupCfg.UseVolumeSnapshots = false BackupCfg.Selector = "" - Expect(VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, backup, "") + Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, backup, "") return "Fail to backup workload" }) @@ -397,23 +398,21 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso // Apply config map if there is one. if tc.cm != nil { - _, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { return errors.Wrap(err, "create config map with user version priorities") } } - // Reset Velero to recognize music-system CRD. - if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil { - return errors.Wrapf(err, "restart Velero pods") - } + // Velero server refresh api version data by discovery helper every 5 minutes + time.Sleep(6 * time.Minute) // Restore rockbands namespaces. restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup, ""); err != nil { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore) + if err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, ""); err != nil { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restore) return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr) } @@ -451,7 +450,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup, "") + err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, "") if !strings.Contains(err.Error(), "Unexpected restore phase got PartiallyFailed, expecting Completed") { return errors.New("expected error but not none") } @@ -502,20 +501,6 @@ func deleteCRDByName(ctx context.Context, name string) error { return nil } -func restartPods(ctx context.Context, ns string) error { - fmt.Printf("Restart pods in %s namespace.\n", ns) - cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns, "--wait=true") - - _, stderr, err := veleroexec.RunCommand(cmd) - if strings.Contains(stderr, "not found") { - return nil - } - if err != nil { - return errors.Wrap(err, stderr) - } - return nil -} - func installCR(ctx context.Context, crFile, ns string) error { retries := 5 var stderr string diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index e0b61fcf1..973916969 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -29,7 +29,11 @@ var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: true}}) func (n *NamespaceMapping) Init() error { - n.Client = TestClientInstance + //n.Client = TestClientInstance + n.VeleroCfg = VeleroCfg + n.Client = *n.VeleroCfg.ClientToInstallVelero + n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots + n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots n.kibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} backupType := "restic" if n.UseVolumeSnapshots { diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go new file mode 100644 index 000000000..a475b5702 --- /dev/null +++ b/test/e2e/basic/nodeport.go @@ -0,0 +1,182 @@ +package basic + +import ( + "context" + "fmt" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type NodePort struct { + TestCase + replica int32 + labels map[string]string + containers *[]v1.Container + serviceName string + serviceSpec *v1.ServiceSpec + nodePort int32 + namespaceToCollision string + namespace string +} + +const NodeportBaseName string = "nodeport-" + +var NodePortTest func() = TestFunc(&NodePort{namespace: NodeportBaseName + "1", TestCase: TestCase{NSBaseName: NodeportBaseName}}) + +func (n *NodePort) Init() error { + n.VeleroCfg = VeleroCfg + n.Client = *n.VeleroCfg.ClientToInstallVelero + n.NSBaseName = NodeportBaseName + n.NamespacesTotal = 1 + n.TestMsg = &TestMSG{ + Desc: fmt.Sprintf("Nodeport preservation"), + FailedMSG: "Failed to restore with nodeport preservation", + Text: fmt.Sprintf("Nodeport can be preserved or omit during restore"), + } + n.BackupName = "backup-label-selector-" + UUIDgen.String() + n.RestoreName = "restore-" + UUIDgen.String() + n.serviceName = "nginx-service-" + UUIDgen.String() + n.labels = map[string]string{"app": "nginx"} + return nil +} + +func (n *NodePort) StartRun() error { + n.BackupArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName, + "--include-namespaces", n.namespace, "--wait", + } + n.RestoreArgs = []string{ + "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", + "--from-backup", n.BackupName, + "--wait", + } + return nil +} +func (n *NodePort) CreateResources() error { + n.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) + + By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, n.namespace), func() { + Expect(CreateNamespace(n.Ctx, n.Client, n.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespace)) + Expect(createServiceWithNodeport(n.Ctx, n.Client, n.namespace, n.serviceName, n.labels, 0)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) + service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + n.nodePort = service.Spec.Ports[0].NodePort + _, err = GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + return nil +} + +func (n *NodePort) Destroy() error { + By(fmt.Sprintf("Start to destroy namespace %s......", n.NSBaseName), func() { + Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, NodeportBaseName)).To(Succeed(), + fmt.Sprintf("Failed to delete namespace %s", n.NSBaseName)) + Expect(WaitForServiceDelete(n.Client, n.namespace, n.serviceName, false)).To(Succeed(), "fail to delete service") + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + + n.namespaceToCollision = NodeportBaseName + "tmp" + By(fmt.Sprintf("Creating a new service which has the same nodeport as backed up service has in a new namespaces for nodeport collision ...%s\n", n.namespaceToCollision), func() { + Expect(CreateNamespace(n.Ctx, n.Client, n.namespaceToCollision)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespaceToCollision)) + Expect(createServiceWithNodeport(n.Ctx, n.Client, n.namespaceToCollision, n.serviceName, n.labels, n.nodePort)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName)) + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + + return nil +} + +func (n *NodePort) Restore() error { + index := 4 + restoreName1 := n.RestoreName + "-1" + restoreName2 := restoreName1 + "-1" + + args := n.RestoreArgs + args = append(args[:index], append([]string{n.RestoreName}, args[index:]...)...) + args = append(args, "--preserve-nodeports=true") + By(fmt.Sprintf("Start to restore %s with nodeports preservation when port %d is already occupied by other service", n.RestoreName, n.nodePort), func() { + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, n.RestoreName, + args, velerov1api.RestorePhasePartiallyFailed)).To( + Succeed(), + func() string { + RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, + n.VeleroCfg.VeleroNamespace, "", n.RestoreName) + return "Fail to restore workload" + }) + }) + + args = n.RestoreArgs + args = append(args[:index], append([]string{restoreName1}, args[index:]...)...) + args = append(args, "--preserve-nodeports=false") + By(fmt.Sprintf("Start to restore %s without nodeports preservation ......", restoreName1), func() { + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, + restoreName1, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string { + RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName1) + return "Fail to restore workload" + }) + }) + + By(fmt.Sprintf("Delete service %s by deleting namespace %s", n.serviceName, n.namespace), func() { + service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + fmt.Println(service.Spec.Ports) + Expect(DeleteNamespace(n.Ctx, n.Client, n.namespace, true)).To(Succeed()) + }) + + By(fmt.Sprintf("Start to delete service %s in namespace %s ......", n.serviceName, n.namespaceToCollision), func() { + Expect(WaitForServiceDelete(n.Client, n.namespaceToCollision, n.serviceName, true)).To(Succeed(), "fail to delete service") + _, err := GetAllService(n.Ctx) + Expect(err).To(Succeed(), "fail to get service") + }) + + args = n.RestoreArgs + args = append(args[:index], append([]string{restoreName2}, args[index:]...)...) + args = append(args, "--preserve-nodeports=true") + By(fmt.Sprintf("Start to restore %s with nodeports preservation ......", restoreName2), func() { + Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, + restoreName2, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string { + RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName2) + return "Fail to restore workload" + }) + }) + + By(fmt.Sprintf("Verify service %s was restore successfully with the origin nodeport.", n.namespace), func() { + service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName) + Expect(err).To(Succeed()) + Expect(len(service.Spec.Ports)).To(Equal(1)) + Expect(service.Spec.Ports[0].NodePort).To(Equal(n.nodePort)) + }) + + return nil +} + +func createServiceWithNodeport(ctx context.Context, client TestClient, namespace string, + service string, labels map[string]string, nodePort int32) error { + serviceSpec := &v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Port: 80, + TargetPort: intstr.IntOrString{IntVal: 80}, + NodePort: nodePort, + }, + }, + Selector: nil, + Type: v1.ServiceTypeLoadBalancer, + } + return CreateService(ctx, client, namespace, service, labels, serviceSpec) +} diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 67da3481b..87694a9dd 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -45,7 +45,8 @@ func (m *MultiNSBackup) Init() error { m.BackupName = "backup-" + UUIDgen.String() m.RestoreName = "restore-" + UUIDgen.String() m.NSBaseName = "nstest-" + UUIDgen.String() - m.Client = TestClientInstance + m.VeleroCfg = VeleroCfg + m.Client = *m.VeleroCfg.ClientToInstallVelero m.NSExcluded = &[]string{} if m.IsScalTest { @@ -79,13 +80,13 @@ func (m *MultiNSBackup) StartRun() error { } m.BackupArgs = []string{ - "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", m.BackupName, + "create", "--namespace", m.VeleroCfg.VeleroNamespace, "backup", m.BackupName, "--exclude-namespaces", strings.Join(*m.NSExcluded, ","), "--default-volumes-to-fs-backup", "--wait", } m.RestoreArgs = []string{ - "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", m.RestoreName, + "create", "--namespace", m.VeleroCfg.VeleroNamespace, "restore", m.RestoreName, "--from-backup", m.BackupName, "--wait", } return nil diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index 5033dd14e..b21bfb728 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -43,7 +43,8 @@ func (n *NSAnnotationCase) Init() error { n.NSBaseName = "namespace-annotations-" + UUIDgen.String() n.NamespacesTotal = 1 n.NSIncluded = &[]string{} - n.Client = TestClientInstance + n.VeleroCfg = VeleroCfg + n.Client = *n.VeleroCfg.ClientToInstallVelero for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum) *n.NSIncluded = append(*n.NSIncluded, createNSName) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 7657669cf..1b988bc9e 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -78,7 +78,8 @@ func (r *RBACCase) Init() error { "create", "--namespace", VeleroCfg.VeleroNamespace, "restore", r.RestoreName, "--from-backup", r.BackupName, "--wait", } - r.Client = TestClientInstance + r.VeleroCfg = VeleroCfg + r.Client = *r.VeleroCfg.ClientToInstallVelero return nil } diff --git a/test/e2e/basic/resources-check/resources_check.go b/test/e2e/basic/resources-check/resources_check.go index b3ddadcb9..4fc32d568 100644 --- a/test/e2e/basic/resources-check/resources_check.go +++ b/test/e2e/basic/resources-check/resources_check.go @@ -32,13 +32,16 @@ limitations under the License. package basic -import . "github.com/vmware-tanzu/velero/test/e2e/test" +import ( + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/test" +) func GetResourcesCheckTestCases() []VeleroBackupRestoreTest { return []VeleroBackupRestoreTest{ - &NSAnnotationCase{}, - &MultiNSBackup{IsScalTest: false}, - &RBACCase{}, + &NSAnnotationCase{TestCase{VeleroCfg: VeleroCfg}}, + &MultiNSBackup{IsScalTest: false, TestCase: TestCase{VeleroCfg: VeleroCfg}}, + &RBACCase{TestCase{VeleroCfg: VeleroCfg}}, } } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 20b591d73..c5086f89f 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -53,37 +53,41 @@ func BslDeletionWithRestic() { } func BslDeletionTest(useVolumeSnapshots bool) { var ( - err error + err error + veleroCfg VeleroConfig ) - + veleroCfg = VeleroCfg + veleroCfg.UseVolumeSnapshots = useVolumeSnapshots + veleroCfg.UseNodeAgent = !useVolumeSnapshots less := func(a, b string) bool { return a < b } + BeforeEach(func() { - if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } var err error flag.Parse() UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + if veleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + DeleteBackups(context.Background(), *veleroCfg.DefaultClient) }) By(fmt.Sprintf("Delete sample workload namespace %s", bslDeletionTestNs), func() { - Expect(DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, bslDeletionTestNs, + Expect(DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, bslDeletionTestNs, true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", bslDeletionTestNs)) }) - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { By("Uninstall Velero", func() { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) }) } } @@ -92,44 +96,44 @@ func BslDeletionTest(useVolumeSnapshots bool) { When("kibishii is the sample workload", func() { It("Local backups and restic repos (if Velero was installed with Restic) will be deleted once the corresponding backup storage location is deleted", func() { - if VeleroCfg.AdditionalBSLProvider == "" { + if veleroCfg.AdditionalBSLProvider == "" { Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") } - if VeleroCfg.AdditionalBSLBucket == "" { + if veleroCfg.AdditionalBSLBucket == "" { Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests") } - if VeleroCfg.AdditionalBSLCredentials == "" { + if veleroCfg.AdditionalBSLCredentials == "" { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - By(fmt.Sprintf("Add an additional plugin for provider %s", VeleroCfg.AdditionalBSLProvider), func() { - Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, - VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed()) + By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() { + Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider, + veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed()) }) additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen) secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen) - secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider) + secretKey := fmt.Sprintf("creds-%s", veleroCfg.AdditionalBSLProvider) files := map[string]string{ - secretKey: VeleroCfg.AdditionalBSLCredentials, + secretKey: veleroCfg.AdditionalBSLCredentials, } By(fmt.Sprintf("Create Secret for additional BSL %s", additionalBsl), func() { - Expect(CreateSecretFromFiles(context.TODO(), *VeleroCfg.ClientToInstallVelero, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) + Expect(CreateSecretFromFiles(context.TODO(), *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace, secretName, files)).To(Succeed()) }) By(fmt.Sprintf("Create additional BSL using credential %s", secretName), func() { Expect(VeleroCreateBackupLocation(context.TODO(), - VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, + veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, additionalBsl, - VeleroCfg.AdditionalBSLProvider, - VeleroCfg.AdditionalBSLBucket, - VeleroCfg.AdditionalBSLPrefix, - VeleroCfg.AdditionalBSLConfig, + veleroCfg.AdditionalBSLProvider, + veleroCfg.AdditionalBSLBucket, + veleroCfg.AdditionalBSLPrefix, + veleroCfg.AdditionalBSLConfig, secretName, secretKey, )).To(Succeed()) @@ -149,13 +153,13 @@ func BslDeletionTest(useVolumeSnapshots bool) { //label_2 := "for!=1" label_2 := "for=2" By("Create namespace for sample workload", func() { - Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed()) + Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed()) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, VeleroCfg.CloudProvider, - bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, - VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, veleroCfg.CloudProvider, + bslDeletionTestNs, veleroCfg.RegistryCredentialFile, veleroCfg.Features, + veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) }) // Restic can not backup PV only, so pod need to be labeled also @@ -184,13 +188,14 @@ func BslDeletionTest(useVolumeSnapshots bool) { BackupCfg.Namespace = bslDeletionTestNs BackupCfg.BackupLocation = backupLocation_1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = label_1 By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case - Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") + Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") return "Fail to backup workload" }) }) @@ -199,19 +204,19 @@ func BslDeletionTest(useVolumeSnapshots bool) { BackupCfg.BackupLocation = backupLocation_2 BackupCfg.Selector = label_2 By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { - Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "") + Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") return "Fail to backup workload" }) }) if useVolumeSnapshots { - if VeleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == "vsphere" { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, - bslDeletionTestNs)).To(Succeed()) + bslDeletionTestNs, 2)).To(Succeed()) }) By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, @@ -226,73 +231,73 @@ func BslDeletionTest(useVolumeSnapshots bool) { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") var BSLCredentials, BSLConfig string - if VeleroCfg.CloudProvider == "vsphere" { - BSLCredentials = VeleroCfg.AdditionalBSLCredentials - BSLConfig = VeleroCfg.AdditionalBSLConfig + if veleroCfg.CloudProvider == "vsphere" { + BSLCredentials = veleroCfg.AdditionalBSLCredentials + BSLConfig = veleroCfg.AdditionalBSLConfig } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = VeleroCfg.CloudCredentialsFile - BSLConfig = VeleroCfg.BSLConfig + BSLCredentials = veleroCfg.CloudCredentialsFile + BSLConfig = veleroCfg.BSLConfig } - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - BSLCredentials, VeleroCfg.AdditionalBSLBucket, + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + BSLCredentials, veleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) } else { // For Restics By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) }) By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) }) } By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { - Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, + Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName_1, 10*time.Minute)).To(Succeed()) }) By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { - Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, + Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName_2, 10*time.Minute)).To(Succeed()) }) - backupsInBSL1, err := GetBackupsFromBsl(context.Background(), VeleroCfg.VeleroCLI, backupLocation_1) + backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1) Expect(err).To(Succeed()) - backupsInBSL2, err := GetBackupsFromBsl(context.Background(), VeleroCfg.VeleroCLI, backupLocation_2) + backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2) Expect(err).To(Succeed()) backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...) By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() { - backupsBeforeDel, err := GetAllBackups(context.Background(), VeleroCfg.VeleroCLI) + backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { - Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, - VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, + Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName_1, BackupObjectsPrefix)).To(Succeed()) }) By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() { - Expect(DeleteBslResource(context.Background(), VeleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) - Expect(WaitForBackupsToBeDeleted(context.Background(), VeleroCfg.VeleroCLI, backupsInBSL1, 10*time.Minute)).To(Succeed()) + Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) + Expect(WaitForBackupsToBeDeleted(context.Background(), veleroCfg.VeleroCLI, backupsInBSL1, 10*time.Minute)).To(Succeed()) }) By("Get all backups from 2 BSLs after deleting one of them", func() { - backupsAfterDel, err := GetAllBackups(context.Background(), VeleroCfg.VeleroCLI) + backupsAfterDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) // Default BSL is deleted, so backups in additional BSL should be left only Expect(cmp.Diff(backupsInBSL2, backupsAfterDel, cmpopts.SortSlices(less))).Should(BeEmpty()) @@ -300,20 +305,20 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { - Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, - VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, + Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName_1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case // By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() { - // Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.AdditionalBSLCredentials, - // VeleroCfg.AdditionalBSLBucket, VeleroCfg.AdditionalBSLPrefix, VeleroCfg.AdditionalBSLConfig, + // Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials, + // veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig, // backupName_2, BackupObjectsPrefix)).To(Succeed()) // }) if useVolumeSnapshots { - if VeleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == "vsphere" { By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName_1, 1)).To(Succeed()) @@ -325,35 +330,35 @@ func BslDeletionTest(useVolumeSnapshots bool) { snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) }) By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { var BSLCredentials, BSLConfig string - if VeleroCfg.CloudProvider == "vsphere" { - BSLCredentials = VeleroCfg.AdditionalBSLCredentials - BSLConfig = VeleroCfg.AdditionalBSLConfig + if veleroCfg.CloudProvider == "vsphere" { + BSLCredentials = veleroCfg.AdditionalBSLCredentials + BSLConfig = veleroCfg.AdditionalBSLConfig } else { - BSLCredentials = VeleroCfg.CloudCredentialsFile - BSLConfig = VeleroCfg.BSLConfig + BSLCredentials = veleroCfg.CloudCredentialsFile + BSLConfig = veleroCfg.BSLConfig } - snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - BSLCredentials, VeleroCfg.AdditionalBSLBucket, + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + BSLCredentials, veleroCfg.AdditionalBSLBucket, BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) }) } else { // For Restic By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) }) By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) }) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index be0cda010..ebaa4b0a4 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -90,7 +90,7 @@ var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin pro var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest) -var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore) +var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore) // Upgrade test by Kibishi using restic var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) @@ -110,10 +110,10 @@ var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup inclu var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) +var _ = Describe("[Backups][TTL][LongTime] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) -var _ = Describe("[Schedule][BR][Pause] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) +var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) @@ -121,18 +121,19 @@ var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controlle var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) -var _ = Describe("[Migration][Restic]", MigrationWithRestic) - -var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots) +var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic) +var _ = Describe("[Migration][Snapshot] Migrate resources between clusters by snapshot", MigrationWithSnapshots) var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest) -var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) -var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) -var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) +var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) +var _ = Describe("[NamespaceMapping][Single][Snapshot] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) +var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest) var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest) +var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest) + func GetKubeconfigContext() error { var err error var tcDefault, tcStandby TestClient diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index f266e5825..983fc4ec7 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -34,15 +34,18 @@ import ( ) var migrationNamespace string +var veleroCfg VeleroConfig func MigrationWithSnapshots() { - for _, veleroCLI2Version := range GetVersionList(VeleroCfg.MigrateFromVeleroCLI, VeleroCfg.MigrateFromVeleroVersion) { + veleroCfg = VeleroCfg + for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) { MigrationTest(true, veleroCLI2Version) } } func MigrationWithRestic() { - for _, veleroCLI2Version := range GetVersionList(VeleroCfg.MigrateFromVeleroCLI, VeleroCfg.MigrateFromVeleroVersion) { + veleroCfg = VeleroCfg + for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) { MigrationTest(false, veleroCLI2Version) } } @@ -53,41 +56,41 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) backupScName, restoreScName string err error ) - BeforeEach(func() { + veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() migrationNamespace = "migration-workload-" + UUIDgen.String() - if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - if useVolumeSnapshots && VeleroCfg.CloudProvider == "aws" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" { Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool") } - if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" { + if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" { Skip("Migration test needs 2 clusters") } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + DeleteBackups(context.Background(), *veleroCfg.DefaultClient) }) - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) - DeleteNamespace(context.Background(), *VeleroCfg.DefaultClient, migrationNamespace, true) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) + DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true) - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) - DeleteNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace, true) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) + DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) - VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient + By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient }) } @@ -106,19 +109,22 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero", veleroCLI2Version.VeleroVersion), func() { if veleroCLI2Version.VeleroVersion == "self" { - veleroCLI2Version.VeleroCLI = VeleroCfg.VeleroCLI + veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } }) } - OriginVeleroCfg := VeleroCfg - By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() { - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + OriginVeleroCfg := veleroCfg + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed()) OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient + OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots + OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots + // TODO: self means 1.10 and upper version if veleroCLI2Version.VeleroVersion != "self" { fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) OriginVeleroCfg.VeleroImage = "" @@ -129,8 +135,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) OriginVeleroCfg.UseNodeAgent = false OriginVeleroCfg.UseRestic = !useVolumeSnapshots } - fmt.Println(OriginVeleroCfg) - Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed()) + + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed()) @@ -143,14 +149,14 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) restoreScName = restoreName + "-sc" By("Create namespace for sample workload", func() { - Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.DefaultClient, migrationNamespace)).To(Succeed(), + Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace)) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.DefaultClient, VeleroCfg.CloudProvider, - migrationNamespace, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, - VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, + migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, + veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) }) By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() { @@ -166,7 +172,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI, OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "") + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "") return "Fail to backup workload" }) @@ -176,6 +182,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.BackupLocation = "" BackupCfg.Selector = "" + BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots //TODO Remove UseRestic parameter once minor version is 1.10 or upper BackupCfg.UseResticIfFSBackup = true if veleroCLI2Version.VeleroVersion == "self" { @@ -189,26 +196,26 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if VeleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == "vsphere" { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, - migrationNamespace)).To(Succeed()) + migrationNamespace, 2)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*VeleroCfg.DefaultClient, VeleroCfg, 2, + snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, migrationNamespace, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) } - if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") { // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before // v1.0 have issues to fail upgrade case. By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { @@ -218,52 +225,51 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots { + if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() { - ns, err := GetNamespace(context.Background(), *VeleroCfg.DefaultClient, migrationNamespace) + By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() { + ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) Expect(ns.Name).To(Equal(migrationNamespace)) Expect(err).NotTo(HaveOccurred()) - Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) - _, err = GetNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace) + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed()) + _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) Expect(err).To(HaveOccurred()) strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found") fmt.Println(err) - VeleroCfg.ObjectStoreProvider = "" - VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient - VeleroCfg.UseNodeAgent = !useVolumeSnapshots - VeleroCfg.UseRestic = false - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient + veleroCfg.UseNodeAgent = !useVolumeSnapshots + veleroCfg.UseRestic = false + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() { - Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) - Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed()) + By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() { + Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) + Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed()) }) By(fmt.Sprintf("Restore %s", migrationNamespace), func() { - Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, "", restoreName) + Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", restoreName) return "Fail to restore workload" }) - Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace, "", restoreName) + Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace, "", restoreName) return "Fail to restore workload" }) }) By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() { - Expect(KibishiiVerifyAfterRestore(*VeleroCfg.StandbyClient, migrationNamespace, + Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace, oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore") }) }) diff --git a/test/e2e/privilegesmgmt/ssr.go b/test/e2e/privilegesmgmt/ssr.go index 00938ce8a..20b385b6e 100644 --- a/test/e2e/privilegesmgmt/ssr.go +++ b/test/e2e/privilegesmgmt/ssr.go @@ -39,65 +39,66 @@ func SSRTest() { var ( err error ) - + veleroCfg := VeleroCfg BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + veleroCfg.UseVolumeSnapshots = false + if veleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - if VeleroCfg.InstallVelero { - if !VeleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + if veleroCfg.InstallVelero { + if !veleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } } }) - It(fmt.Sprintf("Should create an ssr object in the %s namespace and later removed by controller", VeleroCfg.VeleroNamespace), func() { - defer DeleteNamespace(context.TODO(), *VeleroCfg.ClientToInstallVelero, testNS, false) - ctx, _ := context.WithTimeout(context.Background(), time.Minute*10) + It(fmt.Sprintf("Should create an ssr object in the %s namespace and later removed by controller", veleroCfg.VeleroNamespace), func() { + defer DeleteNamespace(context.TODO(), *veleroCfg.ClientToInstallVelero, testNS, false) + ctx, _ := context.WithTimeout(context.Background(), time.Duration(time.Minute*10)) By(fmt.Sprintf("Create %s namespace", testNS)) - Expect(CreateNamespace(ctx, *VeleroCfg.ClientToInstallVelero, testNS)).To(Succeed(), + Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", testNS)) By(fmt.Sprintf("Get version in %s namespace", testNS), func() { - Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, testNS)).To(Succeed(), + Expect(VeleroVersion(context.Background(), veleroCfg.VeleroCLI, testNS)).To(Succeed(), fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS)) }) - By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace), func() { - Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed(), - fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + By(fmt.Sprintf("Get version in %s namespace", veleroCfg.VeleroNamespace), func() { + Expect(VeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed(), + fmt.Sprintf("Failed to create an ssr object in %s namespace", veleroCfg.VeleroNamespace)) }) ssrListResp := new(v1.ServerStatusRequestList) - By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + By(fmt.Sprintf("Check ssr object in %s namespace", veleroCfg.VeleroNamespace)) err = waitutil.PollImmediate(5*time.Second, time.Minute, func() (bool, error) { - if err = VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { - return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { + return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) } if len(ssrListResp.Items) != 1 { - return false, fmt.Errorf("count of ssr object in %s namespace is not 1", VeleroCfg.VeleroNamespace) + return false, fmt.Errorf("count of ssr object in %s namespace is not 1", veleroCfg.VeleroNamespace) } if ssrListResp.Items[0].Status.ServerVersion == "" { - fmt.Printf("ServerVersion of ssr object in %s namespace should not empty, current response result %v\n", VeleroCfg.VeleroNamespace, ssrListResp) + fmt.Printf("ServerVersion of ssr object in %s namespace should not empty, current response result %v\n", veleroCfg.VeleroNamespace, ssrListResp) return false, nil } if ssrListResp.Items[0].Status.Phase != "Processed" { - return false, fmt.Errorf("phase of ssr object in %s namespace should be Processed but got phase %s", VeleroCfg.VeleroNamespace, ssrListResp.Items[0].Status.Phase) + return false, fmt.Errorf("phase of ssr object in %s namespace should be Processed but got phase %s", veleroCfg.VeleroNamespace, ssrListResp.Items[0].Status.Phase) } return true, nil }) if err == waitutil.ErrWaitTimeout { - fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace) + fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace) } - Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace)) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)) By(fmt.Sprintf("Check ssr object in %s namespace", testNS)) - Expect(VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(), + Expect(veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(), fmt.Sprintf("Failed to list ssr object in %s namespace", testNS)) Expect(len(ssrListResp.Items)).To(BeNumerically("==", 1), fmt.Sprintf("Count of ssr object in %s namespace is not 1", testNS)) @@ -106,10 +107,10 @@ func SSRTest() { Expect(ssrListResp.Items[0].Status.ServerVersion).To(BeEmpty(), fmt.Sprintf("ServerVersion of ssr object in %s namespace should be empty", testNS)) - By(fmt.Sprintf("Waiting ssr object in %s namespace deleted", VeleroCfg.VeleroNamespace)) + By(fmt.Sprintf("Waiting ssr object in %s namespace deleted", veleroCfg.VeleroNamespace)) err = waitutil.PollImmediateInfinite(5*time.Second, func() (bool, error) { - if err = VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { + if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { if apierrors.IsNotFound(err) { return true, nil } @@ -121,6 +122,6 @@ func SSRTest() { return true, nil }) - Expect(err).To(Succeed(), fmt.Sprintf("ssr object in %s namespace is not been deleted by controller", VeleroCfg.VeleroNamespace)) + Expect(err).To(Succeed(), fmt.Sprintf("ssr object in %s namespace is not been deleted by controller", veleroCfg.VeleroNamespace)) }) } diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index 4cef609d3..53f9a2902 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -32,7 +32,10 @@ var OptOutPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_OUT_ func (p *PVBackupFiltering) Init() error { p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute) - p.Client = TestClientInstance + p.VeleroCfg = VeleroCfg + p.Client = *p.VeleroCfg.ClientToInstallVelero + p.VeleroCfg.UseVolumeSnapshots = false + p.VeleroCfg.UseNodeAgent = true p.NSBaseName = "ns" p.NSIncluded = &[]string{fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 1), fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 2)} diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 08def1a7b..f951113b2 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -48,8 +48,9 @@ func (f *FilteringCase) Init() error { f.replica = int32(2) f.labels = map[string]string{"resourcefiltering": "true"} f.labelSelector = "resourcefiltering" - f.Client = TestClientInstance - + //f.Client = TestClientInstance + f.VeleroCfg = VeleroCfg + f.Client = *f.VeleroCfg.ClientToInstallVelero f.NamespacesTotal = 3 f.BackupArgs = []string{ "create", "--namespace", VeleroCfg.VeleroNamespace, "backup", f.BackupName, @@ -84,7 +85,7 @@ func (f *FilteringCase) CreateResources() error { } //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels) + deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil) deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index b5e46621b..8ff0c1755 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error { } //Create deployment: to be included fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2) + deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil) deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index fff56b1b5..2641f60b8 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error { //Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", namespace) - deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels) + deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil) deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace)) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index a95d04a54..5dde07115 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -48,26 +48,29 @@ type OrderedResources struct { } func ScheduleOrderedResources() { + veleroCfg := VeleroCfg BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + if veleroCfg.InstallVelero { + veleroCfg.UseVolumeSnapshots = false + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - if VeleroCfg.InstallVelero && !VeleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + if veleroCfg.InstallVelero && !veleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) } }) It("Create a schedule to backup resources in a specific order should be successful", func() { test := &OrderedResources{} + test.VeleroCfg = VeleroCfg err := test.Init() Expect(err).To(Succeed(), err) defer func() { Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace)) - err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName) + err = VeleroScheduleDelete(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName) Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err)) err = test.DeleteBackups() Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err)) @@ -79,24 +82,24 @@ func ScheduleOrderedResources() { }) By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() { - err = VeleroScheduleCreate(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs) + err = VeleroScheduleCreate(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs) Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err)) }) By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() { - err = CheckScheduleWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap) + err = CheckScheduleWithResourceOrder(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap) Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err)) }) By("Checking resource order in backup cr", func() { backupList := new(velerov1api.BackupList) err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) { - if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { - return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { + return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) } for _, backup := range backupList.Items { - if err = CheckBackupWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil { + if err = CheckBackupWithResourceOrder(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil { return true, nil } } @@ -112,9 +115,8 @@ func ScheduleOrderedResources() { func (o *OrderedResources) Init() error { rand.Seed(time.Now().UnixNano()) UUIDgen, _ = uuid.NewRandom() - client := *VeleroCfg.ClientToInstallVelero - - o.Client = client + o.VeleroCfg = VeleroCfg + o.Client = *o.VeleroCfg.ClientToInstallVelero o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String() o.NSBaseName = "schedule-ordered-resources" o.Namespace = o.NSBaseName + "-" + UUIDgen.String() @@ -136,6 +138,7 @@ func (o *OrderedResources) Init() error { } func (o *OrderedResources) CreateResources() error { + veleroCfg := o.VeleroCfg o.Ctx, _ = context.WithTimeout(context.Background(), 5*time.Minute) label := map[string]string{ "orderedresources": "true", @@ -150,13 +153,13 @@ func (o *OrderedResources) CreateResources() error { return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace) } // add the image pull secret to avoid the image pull limit issue of Docker Hub - if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil { + if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, veleroCfg.RegistryCredentialFile); err != nil { return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace) } //Create deployment deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName) fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace) - deployment := NewDeployment(deploymentName, o.Namespace, 1, label) + deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil) deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err)) @@ -191,12 +194,13 @@ func (o *OrderedResources) CreateResources() error { } func (o *OrderedResources) DeleteBackups() error { + veleroCfg := o.VeleroCfg backupList := new(velerov1api.BackupList) - if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil { - return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err) + if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil { + return fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err) } for _, backup := range backupList.Items { - if err := VeleroBackupDelete(o.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil { + if err := VeleroBackupDelete(o.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name); err != nil { return err } } diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index 0b294221f..8445f6256 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -28,7 +28,9 @@ type ScheduleBackup struct { var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test"}}) func (n *ScheduleBackup) Init() error { - n.Client = TestClientInstance + //n.Client = TestClientInstance + n.VeleroCfg = VeleroCfg + n.Client = *n.VeleroCfg.ClientToInstallVelero n.Period = 3 // Unit is minute n.verifyTimes = 5 // More verify times more confidence n.TestMsg = &TestMSG{ diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 4a754efb8..8a2749daa 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -20,6 +20,7 @@ import ( "context" "flag" "fmt" + "strings" "time" . "github.com/onsi/ginkgo" @@ -27,6 +28,7 @@ import ( "github.com/pkg/errors" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" @@ -69,26 +71,35 @@ type TestCase struct { Ctx context.Context NSIncluded *[]string UseVolumeSnapshots bool + VeleroCfg VeleroConfig + RestorePhaseExpect velerov1api.RestorePhase } var TestClientInstance TestClient func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - By("Create test client instance", func() { - TestClientInstance = *VeleroCfg.ClientToInstallVelero - }) Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") + veleroCfg := test.GetTestCase().VeleroCfg + By("Create test client instance", func() { + TestClientInstance = *veleroCfg.ClientToInstallVelero + }) BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, test.GetTestCase().UseVolumeSnapshots)).To(Succeed()) + veleroCfg := test.GetTestCase().VeleroCfg + // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed + if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") { + Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") + } + if veleroCfg.InstallVelero { + veleroCfg.UseVolumeSnapshots = test.GetTestCase().UseVolumeSnapshots + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } }) AfterEach(func() { - if !VeleroCfg.Debug { - if VeleroCfg.InstallVelero { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + if !veleroCfg.Debug { + if veleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed())) } } }) @@ -101,30 +112,34 @@ func TestFunc(test VeleroBackupRestoreTest) func() { func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { var countIt int - By("Create test client instance", func() { - TestClientInstance = *VeleroCfg.ClientToInstallVelero - }) var useVolumeSnapshots bool + var veleroCfg VeleroConfig for k := range tests { Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) + veleroCfg = tests[k].GetTestCase().VeleroCfg + By("Create test client instance", func() { + TestClientInstance = *veleroCfg.ClientToInstallVelero + }) useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots } BeforeEach(func() { flag.Parse() - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { if countIt == 0 { - Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed()) + veleroCfg.UseVolumeSnapshots = useVolumeSnapshots + veleroCfg.UseNodeAgent = !useVolumeSnapshots + Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed()) } countIt++ } }) AfterEach(func() { - if !VeleroCfg.Debug { - if VeleroCfg.InstallVelero { - if countIt == len(tests) && !VeleroCfg.Debug { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed())) + if !veleroCfg.Debug { + if veleroCfg.InstallVelero { + if countIt == len(tests) && !veleroCfg.Debug { + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed())) } } } @@ -152,22 +167,23 @@ func (t *TestCase) StartRun() error { } func (t *TestCase) Backup() error { - if err := VeleroBackupExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "") + veleroCfg := t.GetTestCase().VeleroCfg + if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") return errors.Wrapf(err, "Failed to backup resources") } return nil } func (t *TestCase) Destroy() error { - err := CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName) - if err != nil { - return errors.Wrap(err, "Could cleanup retrieve namespaces") - } + By(fmt.Sprintf("Start to destroy namespace %s......", t.NSBaseName), func() { + Expect(CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces") + }) return nil } func (t *TestCase) Restore() error { + veleroCfg := t.GetTestCase().VeleroCfg // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed @@ -177,8 +193,11 @@ func (t *TestCase) Restore() error { } By("Start to restore ......", func() { - Expect(VeleroRestoreExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", t.RestoreName) + if t.RestorePhaseExpect == "" { + t.RestorePhaseExpect = velerov1api.RestorePhaseCompleted + } + Expect(VeleroRestoreExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs, t.RestorePhaseExpect)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", t.RestoreName) return "Fail to restore workload" }) }) @@ -190,7 +209,8 @@ func (t *TestCase) Verify() error { } func (t *TestCase) Clean() error { - if !VeleroCfg.Debug { + veleroCfg := t.GetTestCase().VeleroCfg + if !veleroCfg.Debug { By(fmt.Sprintf("Clean namespace with prefix %s after test", t.NSBaseName), func() { CleanupNamespaces(t.Ctx, t.Client, t.NSBaseName) }) diff --git a/test/e2e/types.go b/test/e2e/types.go index c2eb7a950..9e48aa8fd 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -29,43 +29,46 @@ var UUIDgen uuid.UUID var VeleroCfg VeleroConfig type VeleroConfig struct { - VeleroCLI string - VeleroImage string - VeleroVersion string - CloudCredentialsFile string - BSLConfig string - BSLBucket string - BSLPrefix string - VSLConfig string - CloudProvider string - ObjectStoreProvider string - VeleroNamespace string - AdditionalBSLProvider string - AdditionalBSLBucket string - AdditionalBSLPrefix string - AdditionalBSLConfig string - AdditionalBSLCredentials string - RegistryCredentialFile string - RestoreHelperImage string - UpgradeFromVeleroVersion string - UpgradeFromVeleroCLI string - MigrateFromVeleroVersion string - MigrateFromVeleroCLI string - Plugins string - AddBSLPlugins string - InstallVelero bool - KibishiiDirectory string - Features string - Debug bool - GCFrequency string - DefaultCluster string - StandbyCluster string - ClientToInstallVelero *TestClient - DefaultClient *TestClient - StandbyClient *TestClient - UploaderType string - UseNodeAgent bool - UseRestic bool + VeleroCLI string + VeleroImage string + VeleroVersion string + CloudCredentialsFile string + BSLConfig string + BSLBucket string + BSLPrefix string + VSLConfig string + CloudProvider string + ObjectStoreProvider string + VeleroNamespace string + AdditionalBSLProvider string + AdditionalBSLBucket string + AdditionalBSLPrefix string + AdditionalBSLConfig string + AdditionalBSLCredentials string + RegistryCredentialFile string + RestoreHelperImage string + UpgradeFromVeleroVersion string + UpgradeFromVeleroCLI string + MigrateFromVeleroVersion string + MigrateFromVeleroCLI string + Plugins string + AddBSLPlugins string + InstallVelero bool + KibishiiDirectory string + Features string + Debug bool + GCFrequency string + DefaultCluster string + StandbyCluster string + ClientToInstallVelero *TestClient + DefaultClient *TestClient + StandbyClient *TestClient + UploaderType string + UseNodeAgent bool + UseRestic bool + ProvideSnapshotsVolumeParam bool + DefaultVolumesToFsBackup bool + UseVolumeSnapshots bool } type SnapshotCheckPoint struct { @@ -79,17 +82,19 @@ type SnapshotCheckPoint struct { } type BackupConfig struct { - BackupName string - Namespace string - BackupLocation string - UseVolumeSnapshots bool - Selector string - TTL time.Duration - IncludeResources string - ExcludeResources string - IncludeClusterResources bool - OrderedResources string - UseResticIfFSBackup bool + BackupName string + Namespace string + BackupLocation string + UseVolumeSnapshots bool + ProvideSnapshotsVolumeParam bool + Selector string + TTL time.Duration + IncludeResources string + ExcludeResources string + IncludeClusterResources bool + OrderedResources string + UseResticIfFSBackup bool + DefaultVolumesToFsBackup bool } type VeleroCLI2Version struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index ac627fae4..2d91ba039 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -38,14 +38,18 @@ const ( upgradeNamespace = "upgrade-workload" ) +var veleroCfg VeleroConfig + func BackupUpgradeRestoreWithSnapshots() { - for _, upgradeFromVelero := range GetVersionList(VeleroCfg.UpgradeFromVeleroCLI, VeleroCfg.UpgradeFromVeleroVersion) { + veleroCfg = VeleroCfg + for _, upgradeFromVelero := range GetVersionList(veleroCfg.UpgradeFromVeleroCLI, veleroCfg.UpgradeFromVeleroVersion) { BackupUpgradeRestoreTest(true, upgradeFromVelero) } } func BackupUpgradeRestoreWithRestic() { - for _, upgradeFromVelero := range GetVersionList(VeleroCfg.UpgradeFromVeleroCLI, VeleroCfg.UpgradeFromVeleroVersion) { + veleroCfg = VeleroCfg + for _, upgradeFromVelero := range GetVersionList(veleroCfg.UpgradeFromVeleroCLI, veleroCfg.UpgradeFromVeleroVersion) { BackupUpgradeRestoreTest(false, upgradeFromVelero) } } @@ -57,31 +61,32 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC ) BeforeEach(func() { - if !VeleroCfg.InstallVelero { - Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false") + veleroCfg = VeleroCfg + if !veleroCfg.InstallVelero { + Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false") } - if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 { + if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 { Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=") } - if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" { + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } - if VeleroCfg.VeleroCLI == "" { + if veleroCfg.VeleroCLI == "" { Skip("VeleroCLI should be provide") } }) AfterEach(func() { - if !VeleroCfg.Debug { + if !veleroCfg.Debug { By("Clean backups after test", func() { - DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero) + DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero) }) By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() { - DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true) + DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, upgradeNamespace, true) }) - if VeleroCfg.InstallVelero { + if veleroCfg.InstallVelero { By("Uninstall Velero", func() { - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, - VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, + veleroCfg.VeleroNamespace)).To(Succeed()) }) } } @@ -101,12 +106,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC Expect(err).To(Succeed()) }) } - VeleroCfg.GCFrequency = "" + veleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", veleroCLI2Version.VeleroVersion), func() { //Set VeleroImage and RestoreHelperImage to blank //VeleroImage and RestoreHelperImage should be the default value in originalCli - tmpCfgForOldVeleroInstall := VeleroCfg + tmpCfgForOldVeleroInstall := veleroCfg tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI tmpCfgForOldVeleroInstall.VeleroImage = "" @@ -116,25 +121,24 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UseNodeAgent = false tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots - Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, - useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed()) }) backupName = "backup-" + UUIDgen.String() restoreName = "restore-" + UUIDgen.String() - tmpCfg := VeleroCfg + tmpCfg := veleroCfg tmpCfg.UpgradeFromVeleroCLI = veleroCLI2Version.VeleroCLI tmpCfg.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion By("Create namespace for sample workload", func() { - Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, upgradeNamespace)).To(Succeed(), + Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, upgradeNamespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", upgradeNamespace)) }) By("Deploy sample workload of Kibishii", func() { - Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, tmpCfg.CloudProvider, + Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, tmpCfg.CloudProvider, upgradeNamespace, tmpCfg.RegistryCredentialFile, tmpCfg.Features, tmpCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed()) }) @@ -145,6 +149,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BackupCfg.Namespace = upgradeNamespace BackupCfg.BackupLocation = "" BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = "" //TODO: pay attention to this param, remove it when restic is not the default backup tool any more. BackupCfg.UseResticIfFSBackup = true @@ -157,31 +162,31 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) if useVolumeSnapshots { - if VeleroCfg.CloudProvider == "vsphere" { + if veleroCfg.CloudProvider == "vsphere" { // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, - upgradeNamespace)).To(Succeed()) + upgradeNamespace, 2)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 2, + snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, upgradeNamespace, backupName, KibishiiPodNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, - VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, + veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) } By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", upgradeNamespace), func() { - Expect(DeleteNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true)).To(Succeed(), + Expect(DeleteNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, upgradeNamespace, true)).To(Succeed(), fmt.Sprintf("failed to delete namespace %s", upgradeNamespace)) }) - if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { + if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") { // Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before // v1.0 have issues to fail upgrade case. By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { @@ -216,7 +221,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() { - Expect(KibishiiVerifyAfterRestore(*VeleroCfg.ClientToInstallVelero, upgradeNamespace, + Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace, oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore") }) }) diff --git a/test/e2e/util/csi/common.go b/test/e2e/util/csi/common.go index e2e9265b2..51fb5bc2c 100644 --- a/test/e2e/util/csi/common.go +++ b/test/e2e/util/csi/common.go @@ -129,12 +129,13 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) error { +func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) { var err error var snapshotContentNameList []string - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || len(snapshotContentNameList) != expectedCount { - return errors.Wrap(err, "Fail to get Azure CSI snapshot content") + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || + len(snapshotContentNameList) != expectedCount { + return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") } fmt.Println(snapshotContentNameList) - return nil + return snapshotContentNameList, nil } diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 6ea041dfd..09e64b8d1 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -341,3 +341,17 @@ func WaitForCRDEstablished(crdName string) error { } return nil } + +func GetAllService(ctx context.Context) (string, error) { + args := []string{"get", "service", "-A"} + cmd := exec.CommandContext(context.Background(), "kubectl", args...) + fmt.Printf("Kubectl exec cmd =%v\n", cmd) + stdout, stderr, err := veleroexec.RunCommand(cmd) + fmt.Println(stdout) + if err != nil { + fmt.Println(stderr) + fmt.Println(err) + return "", err + } + return stdout, nil +} diff --git a/test/e2e/util/k8s/deployment.go b/test/e2e/util/k8s/deployment.go index bfa1252a2..e9f5572fb 100644 --- a/test/e2e/util/k8s/deployment.go +++ b/test/e2e/util/k8s/deployment.go @@ -36,7 +36,16 @@ const ( ) // newDeployment returns a RollingUpdate Deployment with a fake container image -func NewDeployment(name, ns string, replicas int32, labels map[string]string) *apps.Deployment { +func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment { + if containers == nil { + containers = []v1.Container{ + { + Name: fmt.Sprintf("container-%s", "busybox"), + Image: "gcr.io/velero-gcp/busybox:latest", + Command: []string{"sleep", "1000000"}, + }, + } + } return &apps.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", @@ -59,19 +68,17 @@ func NewDeployment(name, ns string, replicas int32, labels map[string]string) *a Labels: labels, }, Spec: v1.PodSpec{ - Containers: []v1.Container{ - { - Name: name, - Image: "gcr.io/velero-gcp/busybox:latest", - Command: []string{"sleep", "1000000"}, - }, - }, + Containers: containers, }, }, }, } } +func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error { + _, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) + return err +} func CreateDeployment(c clientset.Interface, ns string, deployment *apps.Deployment) (*apps.Deployment, error) { return c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{}) } diff --git a/test/e2e/util/k8s/namespace.go b/test/e2e/util/k8s/namespace.go index 767aba7ba..3135549a3 100644 --- a/test/e2e/util/k8s/namespace.go +++ b/test/e2e/util/k8s/namespace.go @@ -100,7 +100,7 @@ func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseNam if err != nil { return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name) } - fmt.Printf("Delete namespace %s\n", checkNamespace.Name) + fmt.Printf("Namespace %s was deleted\n", checkNamespace.Name) } } return nil diff --git a/test/e2e/util/k8s/service.go b/test/e2e/util/k8s/service.go new file mode 100644 index 000000000..873adfbb3 --- /dev/null +++ b/test/e2e/util/k8s/service.go @@ -0,0 +1,75 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package k8s + +import ( + "context" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + waitutil "k8s.io/apimachinery/pkg/util/wait" +) + +func CreateService(ctx context.Context, client TestClient, namespace string, + service string, labels map[string]string, serviceSpec *corev1.ServiceSpec) error { + se := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: service, + Labels: labels, + Namespace: namespace, + }, + Spec: *serviceSpec, + } + _, err = client.ClientGo.CoreV1().Services(namespace).Create(ctx, se, metav1.CreateOptions{}) + + if err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +func GetService(ctx context.Context, client TestClient, namespace string, service string) (*corev1.Service, error) { + return client.ClientGo.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{}) +} + +func WaitForServiceDelete(client TestClient, ns, name string, deleteFirst bool) error { + if deleteFirst { + if err := client.ClientGo.CoreV1().Services(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete secret in namespace %q", ns)) + } + } + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := client.ClientGo.CoreV1().Services(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("service %q in namespace %q is still being deleted...", name, ns) + return false, nil + }) +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index fc7471f47..de5f2cf69 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -25,6 +25,7 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" + "k8s.io/apimachinery/pkg/util/wait" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test/e2e" @@ -51,15 +52,16 @@ var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2} var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"} // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, - useVolumeSnapshots bool) error { +func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, + useVolumeSnapshots, defaultVolumesToFsBackup bool) error { + client := *veleroCfg.ClientToInstallVelero oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) - veleroCLI := VeleroCfg.VeleroCLI - providerName := VeleroCfg.CloudProvider - veleroNamespace := VeleroCfg.VeleroNamespace - registryCredentialFile := VeleroCfg.RegistryCredentialFile - veleroFeatures := VeleroCfg.Features - kibishiiDirectory := VeleroCfg.KibishiiDirectory + veleroCLI := veleroCfg.VeleroCLI + providerName := veleroCfg.CloudProvider + veleroNamespace := veleroCfg.VeleroNamespace + registryCredentialFile := veleroCfg.RegistryCredentialFile + veleroFeatures := veleroCfg.Features + kibishiiDirectory := veleroCfg.KibishiiDirectory if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil { fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace) if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { @@ -88,32 +90,66 @@ func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, res BackupCfg.Namespace = kibishiiNamespace BackupCfg.BackupLocation = backupLocation BackupCfg.UseVolumeSnapshots = useVolumeSnapshots + BackupCfg.DefaultVolumesToFsBackup = defaultVolumesToFsBackup BackupCfg.Selector = "" + BackupCfg.ProvideSnapshotsVolumeParam = veleroCfg.ProvideSnapshotsVolumeParam if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } var snapshotCheckPoint SnapshotCheckPoint var err error + pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) if useVolumeSnapshots { + if err != nil || len(pvbs) != 0 { + return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) + } if providerName == "vsphere" { // Wait for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") - if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil { + if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) + snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList) if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider, - VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig, + err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } + } else { + if err != nil || len(pvbs) != 2 { + return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) + } + if providerName == "vsphere" { + // Wait for uploads started by the Velero Plug-in for vSphere to complete + // TODO - remove after upload progress monitoring is implemented + + // TODO[High] - uncomment code block below when vSphere plugin PR #500 is included in release version. + // https://github.com/vmware-tanzu/velero-plugin-for-vsphere/pull/500/ + + // fmt.Println("Make sure no vSphere snapshot uploads created") + // if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 0); err != nil { + // return errors.Wrapf(err, "Error get vSphere snapshot uploads") + // } + } else { + if strings.EqualFold(veleroFeatures, "EnableCSI") { + _, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0, + kibishiiNamespace, backupName, KibishiiPodNameList) + } else { + err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, + backupName, snapshotCheckPoint) + if err != nil { + return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + } + } + } } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) @@ -133,6 +169,12 @@ func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, res RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } + if !useVolumeSnapshots { + pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace) + if err != nil || len(pvrs) != 2 { + return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace) + } + } if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") @@ -174,33 +216,55 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler } func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error { - timeout, _ := context.WithTimeout(context.Background(), time.Minute*20) - kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--", - "/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel), - strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength), - strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes)) - fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd) + timeout := 30 * time.Minute + interval := 1 * time.Second + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + timeout, _ := context.WithTimeout(context.Background(), time.Minute*20) + kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--", + "/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel), + strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength), + strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes)) + fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd) + + stdout, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd) + if err != nil || strings.Contains(stderr, "Timeout occurred") || strings.Contains(stderr, "dialing backend") { + fmt.Printf("Kibishi generate stdout Timeout occurred: %s stderr: %s err: %s", stdout, stderr, err) + return false, nil + } + return true, nil + }) - _, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd) if err != nil { - return errors.Wrapf(err, "failed to generate, stderr=%s", stderr) + return errors.Wrapf(err, fmt.Sprintf("Failed to wait generate data in namespace %s", namespace)) } - return nil } func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error { - timeout, _ := context.WithTimeout(context.Background(), time.Minute*10) - kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--", - "/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel), - strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength), - strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), - strconv.Itoa(kibishiiData.ExpectedNodes)) - fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd) + timeout := 10 * time.Minute + interval := 5 * time.Second + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + timeout, _ := context.WithTimeout(context.Background(), time.Minute*20) + kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--", + "/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel), + strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength), + strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), + strconv.Itoa(kibishiiData.ExpectedNodes)) + fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd) + + stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) + if strings.Contains(stderr, "Timeout occurred") { + return false, nil + } + if err != nil { + fmt.Printf("Kibishi verify stdout Timeout occurred: %s stderr: %s err: %s", stdout, stderr, err) + return false, nil + } + return true, nil + }) - stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd) if err != nil { - return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout) + return errors.Wrapf(err, fmt.Sprintf("Failed to wait generate data in namespace %s", namespace)) } return nil } diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 30e99b2ee..93cfb5496 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -49,11 +49,9 @@ type installOptions struct { RestoreHelperImage string } -func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapshots bool) error { +func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error { if veleroCfg.CloudProvider != "kind" { - if veleroCfg.ObjectStoreProvider != "" { - return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud - } + fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider") veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider } else { if veleroCfg.ObjectStoreProvider == "" { @@ -81,14 +79,13 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapsh } } - veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, veleroCfg.Features) + veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, providerPluginsTmp) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider) } - veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots + veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots if !veleroCfg.UseRestic { - veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots + veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent } veleroInstallOptions.UseRestic = veleroCfg.UseRestic veleroInstallOptions.Image = veleroCfg.VeleroImage @@ -179,6 +176,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption if options.UseNodeAgent { args = append(args, "--use-node-agent") } + if options.DefaultVolumesToFsBackup { + args = append(args, "--default-volumes-to-fs-backup") + } if options.UseRestic { args = append(args, "--use-restic") } diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index 5805cdd58..352182e90 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -135,22 +135,14 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin } // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider. -func getProviderVeleroInstallOptions( - pluginProvider, - credentialsFile, - objectStoreBucket, - objectStorePrefix, - bslConfig, - vslConfig string, - plugins []string, - features string, -) (*cliinstall.InstallOptions, error) { +func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, + plugins []string) (*cliinstall.InstallOptions, error) { - if credentialsFile == "" { + if veleroCfg.CloudCredentialsFile == "" { return nil, errors.Errorf("No credentials were supplied to use for E2E tests") } - realPath, err := filepath.Abs(credentialsFile) + realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile) if err != nil { return nil, err } @@ -158,20 +150,22 @@ func getProviderVeleroInstallOptions( io := cliinstall.NewInstallOptions() // always wait for velero and restic pods to be running. io.Wait = true - io.ProviderName = pluginProvider - io.SecretFile = credentialsFile + io.ProviderName = veleroCfg.ObjectStoreProvider + io.SecretFile = veleroCfg.CloudCredentialsFile - io.BucketName = objectStoreBucket - io.Prefix = objectStorePrefix + io.BucketName = veleroCfg.BSLBucket + io.Prefix = veleroCfg.BSLPrefix io.BackupStorageConfig = flag.NewMap() - io.BackupStorageConfig.Set(bslConfig) + io.BackupStorageConfig.Set(veleroCfg.BSLConfig) io.VolumeSnapshotConfig = flag.NewMap() - io.VolumeSnapshotConfig.Set(vslConfig) + io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig) io.SecretFile = realPath io.Plugins = flag.NewStringArray(plugins...) - io.Features = features + io.Features = veleroCfg.Features + io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup + io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots return io, nil } @@ -346,8 +340,11 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin } if backupCfg.UseVolumeSnapshots { - args = append(args, "--snapshot-volumes") - } else { + if backupCfg.ProvideSnapshotsVolumeParam { + args = append(args, "--snapshot-volumes") + } + } + if backupCfg.DefaultVolumesToFsBackup { if backupCfg.UseResticIfFSBackup { args = append(args, "--default-volumes-to-restic") } else { @@ -358,7 +355,9 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin // if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots // for the volumes even though the "--default-volumes-to-fs-backup" is specified // TODO This can be removed if the logic of vSphere plugin bump up to 1.3 - args = append(args, "--snapshot-volumes=false") + if backupCfg.ProvideSnapshotsVolumeParam && !backupCfg.UseVolumeSnapshots { + args = append(args, "--snapshot-volumes=false") + } // if "--snapshot-volumes" is not provide, snapshot should be taken as default behavior. } if backupCfg.BackupLocation != "" { args = append(args, "--storage-location", backupCfg.BackupLocation) @@ -417,15 +416,15 @@ func VeleroRestore(ctx context.Context, veleroCLI, veleroNamespace, restoreName, if includeResources != "" { args = append(args, "--include-resources", includeResources) } - return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args) + return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args, velerov1api.RestorePhaseCompleted) } -func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error { +func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string, phaseExpect velerov1api.RestorePhase) error { if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { return err } - return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, phaseExpect) } func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error { @@ -514,7 +513,7 @@ func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s args = append(args, "--backup", backup) } if len(restore) > 0 { - args = append(args, "--restore", restore) + //args = append(args, "--restore", restore) } fmt.Printf("Generating the debug tarball at %s\n", output) if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { @@ -620,7 +619,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa // WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete // TODO - remove after upload progress monitoring is implemented -func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error { +func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string, expectCount int) error { err := wait.PollImmediate(time.Second*5, timeout, func() (bool, error) { checkSnapshotCmd := exec.CommandContext(ctx, "kubectl", "get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'") @@ -629,10 +628,12 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, if err != nil { fmt.Print(stdout) fmt.Print(stderr) - return false, errors.Wrap(err, "failed to verify") + return false, errors.Wrap(err, "failed to wait for vSphere upload completion") } lines := strings.Split(stdout, "\n") complete := true + actualCount := 0 + for _, curLine := range lines { fmt.Println(curLine) comps := strings.Split(curLine, "=") @@ -651,6 +652,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, // Canceled - the operation was canceled, the snapshot ID is not valid if len(comps) == 2 { phase := comps[1] + actualCount++ switch phase { case "Uploaded": case "New", "InProgress", "Snapshotted", "Uploading": @@ -660,6 +662,14 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, } } } + + if expectCount != actualCount { + fmt.Printf("Snapshot expect count and actual count: %d-%d", expectCount, actualCount) + complete = false + if expectCount == 0 { + return true, nil + } + } return complete, nil }) @@ -1070,24 +1080,15 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) { var snapshotCheckPoint SnapshotCheckPoint - + var err error snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = kibishiiPodNameList if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") { snapshotCheckPoint.EnableCSI = true - if err := util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } - var err error - snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName) - if err != nil { - return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName)) - } - fmt.Println(snapshotCheckPoint) - if len(snapshotCheckPoint.SnapshotIDList) != expectCount { - return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount)) - } } fmt.Println(snapshotCheckPoint) return snapshotCheckPoint, nil @@ -1208,7 +1209,7 @@ func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]stri if veleroCfg.CloudProvider == "vsphere" { args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) } else { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) + args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) } cmd = &common.OsCommandLine{ Cmd: "sed", @@ -1261,7 +1262,7 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string) if veleroCfg.CloudProvider == "vsphere" { args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) } else { - args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) + args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion) } cmd = &common.OsCommandLine{ Cmd: "sed", @@ -1289,3 +1290,34 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string) return common.GetListByCmdPipes(ctx, cmds) } + +func GetVeleroResource(ctx context.Context, veleroNamespace, namespace, resourceName string) ([]string, error) { + cmds := []*common.OsCommandLine{} + cmd := &common.OsCommandLine{ + Cmd: "kubectl", + Args: []string{"get", resourceName, "-n", veleroNamespace}, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "grep", + Args: []string{namespace}, + } + cmds = append(cmds, cmd) + + cmd = &common.OsCommandLine{ + Cmd: "awk", + Args: []string{"{print $1}"}, + } + cmds = append(cmds, cmd) + + return common.GetListByCmdPipes(ctx, cmds) +} + +func GetPVB(ctx context.Context, veleroNamespace, namespace string) ([]string, error) { + return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumebackup") +} + +func GetPVR(ctx context.Context, veleroNamespace, namespace string) ([]string, error) { + return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumerestore") +}