mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-01-04 12:14:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add snapshotvolumes default behavior E2E test

Browse Source 
Signed-off-by: danfengl <danfengl@vmware.com>
This commit is contained in:
danfengl
2023-01-30 03:58:41 +00:00
parent c5efb542d0
commit beed887eeb
32 changed files with 1057 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
test/e2e/backup/backup.go
View File

@@ -39,32 +39,36 @@ func BackupRestoreWithRestic() {
}
func BackupRestoreTest(useVolumeSnapshots bool) {
kibishiiNamespace := "kibishii-workload"
var (
backupName, restoreName string
backupName, restoreName, kibishiiNamespace string
err error
provideSnapshotVolumesParmInBackup bool
veleroCfg VeleroConfig
)
provideSnapshotVolumesParmInBackup = false
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
veleroCfg = VeleroCfg
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
UUIDgen, err = uuid.NewRandom()
kibishiiNamespace = "kibishii-workload" + UUIDgen.String()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
if veleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
}
@@ -72,50 +76,83 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
// TODO[High] - remove code block below when vSphere plugin PR #500 is included in release version.
// because restore will be partiallyFailed when DefaultVolumesToFsBackup is set to true during
// Velero installation with default BSL.
if veleroCfg.CloudProvider == "vsphere" && !useVolumeSnapshots {
Skip("vSphere plugin PR #500 is not included in latest version 1.4.2")
}
if veleroCfg.InstallVelero {
if useVolumeSnapshots {
//Install node agent also
veleroCfg.UseNodeAgent = useVolumeSnapshots
veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots
} else {
veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
}
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(RunKibishiiTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
// if set ProvideSnapshotsVolumeParam to false here, make sure set it true in other tests of this case
veleroCfg.ProvideSnapshotsVolumeParam = provideSnapshotVolumesParmInBackup
// Set DefaultVolumesToFsBackup to false since DefaultVolumesToFsBackup was set to true during installation
Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots, false)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})
It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
if VeleroCfg.AdditionalBSLProvider == "" {
if veleroCfg.AdditionalBSLProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}
if VeleroCfg.AdditionalBSLBucket == "" {
if veleroCfg.AdditionalBSLBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}
if VeleroCfg.AdditionalBSLCredentials == "" {
if veleroCfg.AdditionalBSLCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}
if veleroCfg.InstallVelero {
if useVolumeSnapshots {
veleroCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
} else {
// Install VolumeSnapshots also
veleroCfg.UseVolumeSnapshots = !useVolumeSnapshots
veleroCfg.DefaultVolumesToFsBackup = useVolumeSnapshots
}
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed())
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider,
veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed())
// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider)
secretKey := fmt.Sprintf("creds-%s", veleroCfg.AdditionalBSLProvider)
files := map[string]string{
secretKey: VeleroCfg.AdditionalBSLCredentials,
secretKey: veleroCfg.AdditionalBSLCredentials,
}
Expect(CreateSecretFromFiles(context.TODO(), *VeleroCfg.ClientToInstallVelero, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
Expect(CreateSecretFromFiles(context.TODO(), *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
// Create additional BSL using credential
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
Expect(VeleroCreateBackupLocation(context.TODO(),
VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace,
veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace,
additionalBsl,
VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AdditionalBSLBucket,
VeleroCfg.AdditionalBSLPrefix,
VeleroCfg.AdditionalBSLConfig,
veleroCfg.AdditionalBSLProvider,
veleroCfg.AdditionalBSLBucket,
veleroCfg.AdditionalBSLPrefix,
veleroCfg.AdditionalBSLConfig,
secretName,
secretKey,
)).To(Succeed())
@@ -131,7 +168,8 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
}
Expect(RunKibishiiTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(),
veleroCfg.ProvideSnapshotsVolumeParam = !provideSnapshotVolumesParmInBackup
Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
}
})

64
test/e2e/backups/deletion.go
View File

@@ -48,38 +48,41 @@ func backup_deletion_test(useVolumeSnapshots bool) {
var (
backupName string
err error
veleroCfg VeleroConfig
)
veleroCfg = VeleroCfg
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
if veleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
if veleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
}
})
When("kibishii is the sample workload", func() {
It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
backupName = "backup-" + UUIDgen.String()
Expect(runBackupDeletionTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, backupName, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(),
"Failed to run backup deletion test")
})
})
@@ -89,18 +92,18 @@ func backup_deletion_test(useVolumeSnapshots bool) {
func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
useVolumeSnapshots bool, kibishiiDirectory string) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
veleroCLI := VeleroCfg.VeleroCLI
providerName := VeleroCfg.CloudProvider
veleroNamespace := VeleroCfg.VeleroNamespace
registryCredentialFile := VeleroCfg.RegistryCredentialFile
bslPrefix := VeleroCfg.BSLPrefix
bslConfig := VeleroCfg.BSLConfig
veleroFeatures := VeleroCfg.Features
veleroCLI := veleroCfg.VeleroCLI
providerName := veleroCfg.CloudProvider
veleroNamespace := veleroCfg.VeleroNamespace
registryCredentialFile := veleroCfg.RegistryCredentialFile
bslPrefix := veleroCfg.BSLPrefix
bslConfig := veleroCfg.BSLConfig
veleroFeatures := veleroCfg.Features
if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
}
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
defer func() {
if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
@@ -112,7 +115,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
}
err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}
@@ -121,6 +124,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
BackupCfg.Namespace = deletionTest
BackupCfg.BackupLocation = backupLocation
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
BackupCfg.Selector = ""
By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
@@ -135,20 +139,20 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil {
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest, 2); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
if err != nil {
return err
}
var snapshotCheckPoint SnapshotCheckPoint
if useVolumeSnapshots {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
@@ -159,21 +163,21 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
return err
}
if useVolumeSnapshots {
err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
if err != nil {
return err
}
if useVolumeSnapshots {
if err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint); err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
@@ -189,12 +193,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
})
})
err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
if err != nil {
return err
}
err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}

9
test/e2e/backups/sync_backups.go
View File

@@ -59,7 +59,9 @@ func BackupsSyncTest() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed())
}
})
@@ -105,8 +107,9 @@ func BackupsSyncTest() {
})
By("Install velero", func() {
VeleroCfg.ObjectStoreProvider = ""
Expect(VeleroInstall(test.ctx, &VeleroCfg, false)).To(Succeed())
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(test.ctx, &VeleroCfg)).To(Succeed())
})
By("Check all backups in object storage are synced to Velero", func() {

78
test/e2e/backups/ttl.go
View File

@@ -59,29 +59,33 @@ func (b *TTL) Init() {
func TTLTest() {
var err error
var veleroCfg VeleroConfig
useVolumeSnapshots := true
test := new(TTL)
client := *VeleroCfg.ClientToInstallVelero
veleroCfg = VeleroCfg
client := *veleroCfg.ClientToInstallVelero
//Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
veleroCfg = VeleroCfg
if veleroCfg.InstallVelero {
// Make sure GCFrequency is shorter than backup TTL
VeleroCfg.GCFrequency = "4m0s"
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
veleroCfg.GCFrequency = "4m0s"
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
VeleroCfg.GCFrequency = ""
if !VeleroCfg.Debug {
veleroCfg.GCFrequency = ""
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
if veleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
}
Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
}
@@ -95,9 +99,9 @@ func TTLTest() {
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(test.ctx, client, VeleroCfg.CloudProvider,
test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(test.ctx, client, veleroCfg.CloudProvider,
test.testNS, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
})
var BackupCfg BackupConfig
@@ -109,26 +113,26 @@ func TTLTest() {
BackupCfg.TTL = test.ttl
By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
Expect(VeleroBackupNamespace(test.ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.backupName, "")
return "Fail to backup workload"
})
})
var snapshotCheckPoint SnapshotCheckPoint
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(test.ctx, time.Hour,
test.testNS)).To(Succeed())
test.testNS, 2)).To(Succeed())
})
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}
@@ -137,30 +141,30 @@ func TTLTest() {
fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName))
})
if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...")
time.Sleep(7 * time.Minute)
}
By(fmt.Sprintf("Restore %s", test.testNS), func() {
Expect(VeleroRestore(test.ctx, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string {
RunDebug(test.ctx, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, "", test.restoreName)
Expect(VeleroRestore(test.ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string {
RunDebug(test.ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", test.restoreName)
return "Fail to restore workload"
})
})
By("Associated Restores should be created", func() {
Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
})
By("Check TTL was set correctly", func() {
ttl, err := GetBackupTTL(test.ctx, VeleroCfg.VeleroNamespace, test.backupName)
ttl, err := GetBackupTTL(test.ctx, veleroCfg.VeleroNamespace, test.backupName)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", ""))
fmt.Println(t.Round(time.Minute).String())
@@ -172,28 +176,28 @@ func TTLTest() {
})
By("Check if backups are deleted by GC", func() {
Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
Expect(WaitBackupDeleted(test.ctx, veleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
})
By("Backup file from cloud object storage should be deleted", func() {
Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName,
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
})
By("PersistentVolume snapshots should be deleted", func() {
if useVolumeSnapshots {
Expect(SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}
})
By("Associated Restores should be deleted", func() {
Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
})

145
test/e2e/basic/enable_api_group_versions.go
View File

@@ -40,22 +40,26 @@ import (
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
var veleroCfg VeleroConfig
func APIExtensionsVersionsTest() {
var (
backupName, restoreName string
)
resourceName := "apiextensions.k8s.io"
crdName := "rocknrollbands.music.example.io"
label := "for=backup"
srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
BeforeEach(func() {
if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" {
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
Skip("CRD with apiextension versions migration test needs 2 clusters")
}
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
veleroCfg = VeleroCfg
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName)
Expect(err).ShouldNot(HaveOccurred())
dstVersions, err := GetAPIVersions(VeleroCfg.StandbyClient, resourceName)
dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName)
Expect(err).ShouldNot(HaveOccurred())
Expect(srcVersions).Should(ContainElement("v1"), func() string {
@@ -76,26 +80,26 @@ func APIExtensionsVersionsTest() {
})
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.DefaultClient)
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
})
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
By("Uninstall Velero and delete CRD ", func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
})
}
By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
})
}
@@ -105,19 +109,19 @@ func APIExtensionsVersionsTest() {
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
VeleroCfg.ObjectStoreProvider = ""
VeleroCfg.Features = "EnableAPIGroupVersions"
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
veleroCfg.Features = "EnableAPIGroupVersions"
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
})
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", VeleroCfg.DefaultCluster), func() {
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
Expect(installCRD(context.Background(), srcCrdYaml)).To(Succeed())
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
Expect(WaitForCRDEstablished(crdName)).To(Succeed())
Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed())
// Discovery helper is refreshed every 5 minutes
// Velero server refresh api version data by discovery helper every 5 minutes
time.Sleep(6 * time.Minute)
})
@@ -127,34 +131,33 @@ func APIExtensionsVersionsTest() {
BackupCfg.IncludeResources = "crd"
BackupCfg.IncludeClusterResources = true
BackupCfg.Selector = label
Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, backupName, "")
Expect(VeleroBackupNamespace(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, backupName, "")
return "Fail to backup workload"
})
})
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
VeleroCfg.ObjectStoreProvider = ""
VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() {
Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
})
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, VeleroCfg.StandbyCluster), func() {
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() {
Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
})
By("Restore CRD", func() {
Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, "", restoreName)
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
})
@@ -173,6 +176,7 @@ func APIGropuVersionsTest() {
)
BeforeEach(func() {
veleroCfg = VeleroCfg
resource = "rockbands"
group = "music.example.io"
UUIDgen, err = uuid.NewRandom()
@@ -180,15 +184,16 @@ func APIGropuVersionsTest() {
flag.Parse()
// TODO: install Velero once for the test suite once feature flag is
// removed and velero installation becomes the same as other e2e tests.
if VeleroCfg.InstallVelero {
VeleroCfg.Features = "EnableAPIGroupVersions"
err = VeleroInstall(context.Background(), &VeleroCfg, false)
if veleroCfg.InstallVelero {
veleroCfg.Features = "EnableAPIGroupVersions"
veleroCfg.UseVolumeSnapshots = false
err = VeleroInstall(context.Background(), &veleroCfg)
Expect(err).NotTo(HaveOccurred())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group)
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group)
_, stderr, err := veleroexec.RunCommand(cmd)
@@ -198,12 +203,12 @@ func APIGropuVersionsTest() {
}
Expect(err).NotTo(HaveOccurred())
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
By("Uninstall Velero", func() {
Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
})
}
}
@@ -213,7 +218,7 @@ func APIGropuVersionsTest() {
It("Should back up API group version and restore by version priority", func() {
Expect(runEnableAPIGroupVersionsTests(
ctx,
*VeleroCfg.ClientToInstallVelero,
*veleroCfg.ClientToInstallVelero,
resource,
group,
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
@@ -294,7 +299,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
},
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
tgtVer: "v2beta1",
cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data(
cm: builder.ForConfigMap(veleroCfg.VeleroNamespace, "enableapigroupversions").Data(
"restoreResourcesVersionPriority",
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
).Result(),
@@ -354,12 +359,8 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
tc.namespaces = append(tc.namespaces, ns)
}
// Restart Velero pods in order to recognize music-system CRD right away
// instead of waiting for discovery helper to refresh. See
// https://github.com/vmware-tanzu/velero/issues/3471.
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}
// Velero server refresh api version data by discovery helper every 5 minutes
time.Sleep(6 * time.Minute)
backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
namespacesStr := strings.Join(tc.namespaces, ",")
@@ -371,10 +372,10 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
BackupCfg.UseVolumeSnapshots = false
BackupCfg.Selector = ""
Expect(VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, backup, "")
Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, backup, "")
return "Fail to backup workload"
})
@@ -397,23 +398,21 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
// Apply config map if there is one.
if tc.cm != nil {
_, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
_, err := client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
if err != nil {
return errors.Wrap(err, "create config map with user version priorities")
}
}
// Reset Velero to recognize music-system CRD.
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}
// Velero server refresh api version data by discovery helper every 5 minutes
time.Sleep(6 * time.Minute)
// Restore rockbands namespaces.
restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
if tc.want != nil {
if err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup, ""); err != nil {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore)
if err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, ""); err != nil {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restore)
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
}
@@ -451,7 +450,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
} else {
// No custom resource should have been restored. Expect "no resource found"
// error during restore.
err := VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup, "")
err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, "")
if !strings.Contains(err.Error(), "Unexpected restore phase got PartiallyFailed, expecting Completed") {
return errors.New("expected error but not none")
}
@@ -502,20 +501,6 @@ func deleteCRDByName(ctx context.Context, name string) error {
return nil
}
func restartPods(ctx context.Context, ns string) error {
fmt.Printf("Restart pods in %s namespace.\n", ns)
cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns, "--wait=true")
_, stderr, err := veleroexec.RunCommand(cmd)
if strings.Contains(stderr, "not found") {
return nil
}
if err != nil {
return errors.Wrap(err, stderr)
}
return nil
}
func installCR(ctx context.Context, crFile, ns string) error {
retries := 5
var stderr string

6
test/e2e/basic/namespace-mapping.go
View File

@@ -29,7 +29,11 @@ var OneNamespaceMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase
var MultiNamespacesMappingSnapshotTest func() = TestFunc(&NamespaceMapping{TestCase: TestCase{NSBaseName: NamespaceBaseName, NSIncluded: &[]string{NamespaceBaseName + "1", NamespaceBaseName + "2"}, UseVolumeSnapshots: true}})
func (n *NamespaceMapping) Init() error {
n.Client = TestClientInstance
//n.Client = TestClientInstance
n.VeleroCfg = VeleroCfg
n.Client = *n.VeleroCfg.ClientToInstallVelero
n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots
n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots
n.kibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
backupType := "restic"
if n.UseVolumeSnapshots {

182
test/e2e/basic/nodeport.go Normal file
View File

@@ -0,0 +1,182 @@
package basic
import (
"context"
"fmt"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/intstr"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/test"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)
type NodePort struct {
TestCase
replica int32
labels map[string]string
containers *[]v1.Container
serviceName string
serviceSpec *v1.ServiceSpec
nodePort int32
namespaceToCollision string
namespace string
}
const NodeportBaseName string = "nodeport-"
var NodePortTest func() = TestFunc(&NodePort{namespace: NodeportBaseName + "1", TestCase: TestCase{NSBaseName: NodeportBaseName}})
func (n *NodePort) Init() error {
n.VeleroCfg = VeleroCfg
n.Client = *n.VeleroCfg.ClientToInstallVelero
n.NSBaseName = NodeportBaseName
n.NamespacesTotal = 1
n.TestMsg = &TestMSG{
Desc: fmt.Sprintf("Nodeport preservation"),
FailedMSG: "Failed to restore with nodeport preservation",
Text: fmt.Sprintf("Nodeport can be preserved or omit during restore"),
}
n.BackupName = "backup-label-selector-" + UUIDgen.String()
n.RestoreName = "restore-" + UUIDgen.String()
n.serviceName = "nginx-service-" + UUIDgen.String()
n.labels = map[string]string{"app": "nginx"}
return nil
}
func (n *NodePort) StartRun() error {
n.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", n.BackupName,
"--include-namespaces", n.namespace, "--wait",
}
n.RestoreArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore",
"--from-backup", n.BackupName,
"--wait",
}
return nil
}
func (n *NodePort) CreateResources() error {
n.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, n.namespace), func() {
Expect(CreateNamespace(n.Ctx, n.Client, n.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespace))
Expect(createServiceWithNodeport(n.Ctx, n.Client, n.namespace, n.serviceName, n.labels, 0)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName))
service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName)
Expect(err).To(Succeed())
Expect(len(service.Spec.Ports)).To(Equal(1))
n.nodePort = service.Spec.Ports[0].NodePort
_, err = GetAllService(n.Ctx)
Expect(err).To(Succeed(), "fail to get service")
})
return nil
}
func (n *NodePort) Destroy() error {
By(fmt.Sprintf("Start to destroy namespace %s......", n.NSBaseName), func() {
Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, NodeportBaseName)).To(Succeed(),
fmt.Sprintf("Failed to delete namespace %s", n.NSBaseName))
Expect(WaitForServiceDelete(n.Client, n.namespace, n.serviceName, false)).To(Succeed(), "fail to delete service")
_, err := GetAllService(n.Ctx)
Expect(err).To(Succeed(), "fail to get service")
})
n.namespaceToCollision = NodeportBaseName + "tmp"
By(fmt.Sprintf("Creating a new service which has the same nodeport as backed up service has in a new namespaces for nodeport collision ...%s\n", n.namespaceToCollision), func() {
Expect(CreateNamespace(n.Ctx, n.Client, n.namespaceToCollision)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", n.namespaceToCollision))
Expect(createServiceWithNodeport(n.Ctx, n.Client, n.namespaceToCollision, n.serviceName, n.labels, n.nodePort)).To(Succeed(), fmt.Sprintf("Failed to create service %s", n.serviceName))
_, err := GetAllService(n.Ctx)
Expect(err).To(Succeed(), "fail to get service")
})
return nil
}
func (n *NodePort) Restore() error {
index := 4
restoreName1 := n.RestoreName + "-1"
restoreName2 := restoreName1 + "-1"
args := n.RestoreArgs
args = append(args[:index], append([]string{n.RestoreName}, args[index:]...)...)
args = append(args, "--preserve-nodeports=true")
By(fmt.Sprintf("Start to restore %s with nodeports preservation when port %d is already occupied by other service", n.RestoreName, n.nodePort), func() {
Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI,
n.VeleroCfg.VeleroNamespace, n.RestoreName,
args, velerov1api.RestorePhasePartiallyFailed)).To(
Succeed(),
func() string {
RunDebug(context.Background(), n.VeleroCfg.VeleroCLI,
n.VeleroCfg.VeleroNamespace, "", n.RestoreName)
return "Fail to restore workload"
})
})
args = n.RestoreArgs
args = append(args[:index], append([]string{restoreName1}, args[index:]...)...)
args = append(args, "--preserve-nodeports=false")
By(fmt.Sprintf("Start to restore %s without nodeports preservation ......", restoreName1), func() {
Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace,
restoreName1, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string {
RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName1)
return "Fail to restore workload"
})
})
By(fmt.Sprintf("Delete service %s by deleting namespace %s", n.serviceName, n.namespace), func() {
service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName)
Expect(err).To(Succeed())
Expect(len(service.Spec.Ports)).To(Equal(1))
fmt.Println(service.Spec.Ports)
Expect(DeleteNamespace(n.Ctx, n.Client, n.namespace, true)).To(Succeed())
})
By(fmt.Sprintf("Start to delete service %s in namespace %s ......", n.serviceName, n.namespaceToCollision), func() {
Expect(WaitForServiceDelete(n.Client, n.namespaceToCollision, n.serviceName, true)).To(Succeed(), "fail to delete service")
_, err := GetAllService(n.Ctx)
Expect(err).To(Succeed(), "fail to get service")
})
args = n.RestoreArgs
args = append(args[:index], append([]string{restoreName2}, args[index:]...)...)
args = append(args, "--preserve-nodeports=true")
By(fmt.Sprintf("Start to restore %s with nodeports preservation ......", restoreName2), func() {
Expect(VeleroRestoreExec(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace,
restoreName2, args, velerov1api.RestorePhaseCompleted)).To(Succeed(), func() string {
RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", restoreName2)
return "Fail to restore workload"
})
})
By(fmt.Sprintf("Verify service %s was restore successfully with the origin nodeport.", n.namespace), func() {
service, err := GetService(n.Ctx, n.Client, n.namespace, n.serviceName)
Expect(err).To(Succeed())
Expect(len(service.Spec.Ports)).To(Equal(1))
Expect(service.Spec.Ports[0].NodePort).To(Equal(n.nodePort))
})
return nil
}
func createServiceWithNodeport(ctx context.Context, client TestClient, namespace string,
service string, labels map[string]string, nodePort int32) error {
serviceSpec := &v1.ServiceSpec{
Ports: []v1.ServicePort{
{
Port: 80,
TargetPort: intstr.IntOrString{IntVal: 80},
NodePort: nodePort,
},
},
Selector: nil,
Type: v1.ServiceTypeLoadBalancer,
}
return CreateService(ctx, client, namespace, service, labels, serviceSpec)
}

7
test/e2e/basic/resources-check/namespaces.go
View File

@@ -45,7 +45,8 @@ func (m *MultiNSBackup) Init() error {
m.BackupName = "backup-" + UUIDgen.String()
m.RestoreName = "restore-" + UUIDgen.String()
m.NSBaseName = "nstest-" + UUIDgen.String()
m.Client = TestClientInstance
m.VeleroCfg = VeleroCfg
m.Client = *m.VeleroCfg.ClientToInstallVelero
m.NSExcluded = &[]string{}
if m.IsScalTest {
@@ -79,13 +80,13 @@ func (m *MultiNSBackup) StartRun() error {
}
m.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", m.BackupName,
"create", "--namespace", m.VeleroCfg.VeleroNamespace, "backup", m.BackupName,
"--exclude-namespaces", strings.Join(*m.NSExcluded, ","),
"--default-volumes-to-fs-backup", "--wait",
}
m.RestoreArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", m.RestoreName,
"create", "--namespace", m.VeleroCfg.VeleroNamespace, "restore", m.RestoreName,
"--from-backup", m.BackupName, "--wait",
}
return nil

3
test/e2e/basic/resources-check/namespaces_annotation.go
View File

@@ -43,7 +43,8 @@ func (n *NSAnnotationCase) Init() error {
n.NSBaseName = "namespace-annotations-" + UUIDgen.String()
n.NamespacesTotal = 1
n.NSIncluded = &[]string{}
n.Client = TestClientInstance
n.VeleroCfg = VeleroCfg
n.Client = *n.VeleroCfg.ClientToInstallVelero
for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ {
createNSName := fmt.Sprintf("%s-%00000d", n.NSBaseName, nsNum)
*n.NSIncluded = append(*n.NSIncluded, createNSName)

3
test/e2e/basic/resources-check/rbac.go
View File

@@ -78,7 +78,8 @@ func (r *RBACCase) Init() error {
"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", r.RestoreName,
"--from-backup", r.BackupName, "--wait",
}
r.Client = TestClientInstance
r.VeleroCfg = VeleroCfg
r.Client = *r.VeleroCfg.ClientToInstallVelero
return nil
}

11
test/e2e/basic/resources-check/resources_check.go
View File

@@ -32,13 +32,16 @@ limitations under the License.
package basic
import . "github.com/vmware-tanzu/velero/test/e2e/test"
import (
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/test"
)
func GetResourcesCheckTestCases() []VeleroBackupRestoreTest {
return []VeleroBackupRestoreTest{
&NSAnnotationCase{},
&MultiNSBackup{IsScalTest: false},
&RBACCase{},
&NSAnnotationCase{TestCase{VeleroCfg: VeleroCfg}},
&MultiNSBackup{IsScalTest: false, TestCase: TestCase{VeleroCfg: VeleroCfg}},
&RBACCase{TestCase{VeleroCfg: VeleroCfg}},
}
}

167
test/e2e/bsl-mgmt/deletion.go
View File

@@ -54,36 +54,40 @@ func BslDeletionWithRestic() {
func BslDeletionTest(useVolumeSnapshots bool) {
var (
err error
veleroCfg VeleroConfig
)
veleroCfg = VeleroCfg
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots
less := func(a, b string) bool { return a < b }
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
if veleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.DefaultClient)
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
})
By(fmt.Sprintf("Delete sample workload namespace %s", bslDeletionTestNs), func() {
Expect(DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, bslDeletionTestNs,
Expect(DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, bslDeletionTestNs,
true)).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q",
bslDeletionTestNs))
})
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
By("Uninstall Velero", func() {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
})
}
}
@@ -92,44 +96,44 @@ func BslDeletionTest(useVolumeSnapshots bool) {
When("kibishii is the sample workload", func() {
It("Local backups and restic repos (if Velero was installed with Restic) will be deleted once the corresponding backup storage location is deleted", func() {
if VeleroCfg.AdditionalBSLProvider == "" {
if veleroCfg.AdditionalBSLProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}
if VeleroCfg.AdditionalBSLBucket == "" {
if veleroCfg.AdditionalBSLBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}
if VeleroCfg.AdditionalBSLCredentials == "" {
if veleroCfg.AdditionalBSLCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}
By(fmt.Sprintf("Add an additional plugin for provider %s", VeleroCfg.AdditionalBSLProvider), func() {
Expect(VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AddBSLPlugins, VeleroCfg.Features)).To(Succeed())
By(fmt.Sprintf("Add an additional plugin for provider %s", veleroCfg.AdditionalBSLProvider), func() {
Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, veleroCfg.AdditionalBSLProvider,
veleroCfg.AddBSLPlugins, veleroCfg.Features)).To(Succeed())
})
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider)
secretKey := fmt.Sprintf("creds-%s", veleroCfg.AdditionalBSLProvider)
files := map[string]string{
secretKey: VeleroCfg.AdditionalBSLCredentials,
secretKey: veleroCfg.AdditionalBSLCredentials,
}
By(fmt.Sprintf("Create Secret for additional BSL %s", additionalBsl), func() {
Expect(CreateSecretFromFiles(context.TODO(), *VeleroCfg.ClientToInstallVelero, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
Expect(CreateSecretFromFiles(context.TODO(), *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace, secretName, files)).To(Succeed())
})
By(fmt.Sprintf("Create additional BSL using credential %s", secretName), func() {
Expect(VeleroCreateBackupLocation(context.TODO(),
VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace,
veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace,
additionalBsl,
VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AdditionalBSLBucket,
VeleroCfg.AdditionalBSLPrefix,
VeleroCfg.AdditionalBSLConfig,
veleroCfg.AdditionalBSLProvider,
veleroCfg.AdditionalBSLBucket,
veleroCfg.AdditionalBSLPrefix,
veleroCfg.AdditionalBSLConfig,
secretName,
secretKey,
)).To(Succeed())
@@ -149,13 +153,13 @@ func BslDeletionTest(useVolumeSnapshots bool) {
//label_2 := "for!=1"
label_2 := "for=2"
By("Create namespace for sample workload", func() {
Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed())
Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed())
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, VeleroCfg.CloudProvider,
bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, veleroCfg.CloudProvider,
bslDeletionTestNs, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
})
// Restic can not backup PV only, so pod need to be labeled also
@@ -184,13 +188,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
BackupCfg.Namespace = bslDeletionTestNs
BackupCfg.BackupLocation = backupLocation_1
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
BackupCfg.Selector = label_1
By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() {
// TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command
// TODO move to "runDebug" after we bump up to 1.7 in the upgrade case
Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
return "Fail to backup workload"
})
})
@@ -199,19 +204,19 @@ func BslDeletionTest(useVolumeSnapshots bool) {
BackupCfg.BackupLocation = backupLocation_2
BackupCfg.Selector = label_2
By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() {
Expect(VeleroBackupNamespace(oneHourTimeout, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "")
return "Fail to backup workload"
})
})
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
bslDeletionTestNs)).To(Succeed())
bslDeletionTestNs, 2)).To(Succeed())
})
By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() {
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
@@ -226,73 +231,73 @@ func BslDeletionTest(useVolumeSnapshots bool) {
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() {
snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
var BSLCredentials, BSLConfig string
if VeleroCfg.CloudProvider == "vsphere" {
BSLCredentials = VeleroCfg.AdditionalBSLCredentials
BSLConfig = VeleroCfg.AdditionalBSLConfig
if veleroCfg.CloudProvider == "vsphere" {
BSLCredentials = veleroCfg.AdditionalBSLCredentials
BSLConfig = veleroCfg.AdditionalBSLConfig
} else { // Snapshotting with non-vSphere provider has nothing to do with BSL
BSLCredentials = VeleroCfg.CloudCredentialsFile
BSLConfig = VeleroCfg.BSLConfig
BSLCredentials = veleroCfg.CloudCredentialsFile
BSLConfig = veleroCfg.BSLConfig
}
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
BSLCredentials, veleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})
} else { // For Restics
By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() {
Expect(BackupRepositoriesCountShouldBe(context.Background(),
VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed())
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed())
})
By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() {
Expect(BackupRepositoriesCountShouldBe(context.Background(),
VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
})
}
By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() {
Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI,
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI,
backupName_1, 10*time.Minute)).To(Succeed())
})
By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() {
Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI,
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI,
backupName_2, 10*time.Minute)).To(Succeed())
})
backupsInBSL1, err := GetBackupsFromBsl(context.Background(), VeleroCfg.VeleroCLI, backupLocation_1)
backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)
Expect(err).To(Succeed())
backupsInBSL2, err := GetBackupsFromBsl(context.Background(), VeleroCfg.VeleroCLI, backupLocation_2)
backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2)
Expect(err).To(Succeed())
backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...)
By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() {
backupsBeforeDel, err := GetAllBackups(context.Background(), VeleroCfg.VeleroCLI)
backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI)
Expect(err).To(Succeed())
Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty())
By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() {
Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile,
VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig,
Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
backupName_1, BackupObjectsPrefix)).To(Succeed())
})
By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() {
Expect(DeleteBslResource(context.Background(), VeleroCfg.VeleroCLI, backupLocation_1)).To(Succeed())
Expect(WaitForBackupsToBeDeleted(context.Background(), VeleroCfg.VeleroCLI, backupsInBSL1, 10*time.Minute)).To(Succeed())
Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed())
Expect(WaitForBackupsToBeDeleted(context.Background(), veleroCfg.VeleroCLI, backupsInBSL1, 10*time.Minute)).To(Succeed())
})
By("Get all backups from 2 BSLs after deleting one of them", func() {
backupsAfterDel, err := GetAllBackups(context.Background(), VeleroCfg.VeleroCLI)
backupsAfterDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI)
Expect(err).To(Succeed())
// Default BSL is deleted, so backups in additional BSL should be left only
Expect(cmp.Diff(backupsInBSL2, backupsAfterDel, cmpopts.SortSlices(less))).Should(BeEmpty())
@@ -300,20 +305,20 @@ func BslDeletionTest(useVolumeSnapshots bool) {
})
By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() {
Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile,
VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig,
Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
backupName_1, BackupObjectsPrefix)).To(Succeed())
})
// TODO: Choose additional BSL to be deleted as an new test case
// By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() {
// Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.AdditionalBSLCredentials,
// VeleroCfg.AdditionalBSLBucket, VeleroCfg.AdditionalBSLPrefix, VeleroCfg.AdditionalBSLConfig,
// Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials,
// veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig,
// backupName_2, BackupObjectsPrefix)).To(Succeed())
// })
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() {
Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs,
backupName_1, 1)).To(Succeed())
@@ -325,35 +330,35 @@ func BslDeletionTest(useVolumeSnapshots bool) {
snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() {
snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed())
})
By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() {
var BSLCredentials, BSLConfig string
if VeleroCfg.CloudProvider == "vsphere" {
BSLCredentials = VeleroCfg.AdditionalBSLCredentials
BSLConfig = VeleroCfg.AdditionalBSLConfig
if veleroCfg.CloudProvider == "vsphere" {
BSLCredentials = veleroCfg.AdditionalBSLCredentials
BSLConfig = veleroCfg.AdditionalBSLConfig
} else {
BSLCredentials = VeleroCfg.CloudCredentialsFile
BSLConfig = VeleroCfg.BSLConfig
BSLCredentials = veleroCfg.CloudCredentialsFile
BSLConfig = veleroCfg.BSLConfig
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
BSLCredentials, VeleroCfg.AdditionalBSLBucket,
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
BSLCredentials, veleroCfg.AdditionalBSLBucket,
BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed())
})
} else { // For Restic
By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() {
Expect(BackupRepositoriesCountShouldBe(context.Background(),
VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed())
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed())
})
By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() {
Expect(BackupRepositoriesCountShouldBe(context.Background(),
VeleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed())
})
}
fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n")

13
test/e2e/e2e_suite_test.go
View File

@@ -90,7 +90,7 @@ var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin pro
var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
var _ = Describe("[Scale] Backup/restore of 2500 namespaces", MultiNSBackupRestore)
var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore)
// Upgrade test by Kibishi using restic
var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic)
@@ -110,10 +110,10 @@ var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup inclu
var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
var _ = Describe("[Backups][TTL][LongTime] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
var _ = Describe("[Schedule][BR][Pause] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
@@ -121,9 +121,8 @@ var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controlle
var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
var _ = Describe("[Migration][Restic]", MigrationWithRestic)
var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots)
var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic)
var _ = Describe("[Migration][Snapshot] Migrate resources between clusters by snapshot", MigrationWithSnapshots)
var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
@@ -133,6 +132,8 @@ var _ = Describe("[NamespaceMapping][Multiple][Snapshot] Backup resources shoul
var _ = Describe("[pv-backup][Opt-In] Backup resources should follow the specific order in schedule", OptInPVBackupTest)
var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specific order in schedule", OptOutPVBackupTest)
var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest)
func GetKubeconfigContext() error {
var err error
var tcDefault, tcStandby TestClient

126
test/e2e/migration/migration.go
View File

@@ -34,15 +34,18 @@ import (
)
var migrationNamespace string
var veleroCfg VeleroConfig
func MigrationWithSnapshots() {
for _, veleroCLI2Version := range GetVersionList(VeleroCfg.MigrateFromVeleroCLI, VeleroCfg.MigrateFromVeleroVersion) {
veleroCfg = VeleroCfg
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
MigrationTest(true, veleroCLI2Version)
}
}
func MigrationWithRestic() {
for _, veleroCLI2Version := range GetVersionList(VeleroCfg.MigrateFromVeleroCLI, VeleroCfg.MigrateFromVeleroVersion) {
veleroCfg = VeleroCfg
for _, veleroCLI2Version := range GetVersionList(veleroCfg.MigrateFromVeleroCLI, veleroCfg.MigrateFromVeleroVersion) {
MigrationTest(false, veleroCLI2Version)
}
}
@@ -53,41 +56,41 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
backupScName, restoreScName string
err error
)
BeforeEach(func() {
veleroCfg = VeleroCfg
UUIDgen, err = uuid.NewRandom()
migrationNamespace = "migration-workload-" + UUIDgen.String()
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
if useVolumeSnapshots && VeleroCfg.CloudProvider == "aws" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "aws" {
Skip("Volume snapshots migration not supported on AWS provisioned by Sheperd public pool")
}
if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" {
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
Skip("Migration test needs 2 clusters")
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.DefaultClient)
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
})
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
By(fmt.Sprintf("Uninstall Velero and delete sample workload namespace %s", migrationNamespace), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *VeleroCfg.DefaultClient, migrationNamespace, true)
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true)
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace, true)
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true)
})
}
By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
})
}
@@ -106,19 +109,22 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero",
veleroCLI2Version.VeleroVersion), func() {
if veleroCLI2Version.VeleroVersion == "self" {
veleroCLI2Version.VeleroCLI = VeleroCfg.VeleroCLI
veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI
} else {
veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)
Expect(err).To(Succeed())
}
})
}
OriginVeleroCfg := VeleroCfg
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
OriginVeleroCfg := veleroCfg
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion
OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
// TODO: self means 1.10 and upper version
if veleroCLI2Version.VeleroVersion != "self" {
fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion)
OriginVeleroCfg.VeleroImage = ""
@@ -129,8 +135,8 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
OriginVeleroCfg.UseNodeAgent = false
OriginVeleroCfg.UseRestic = !useVolumeSnapshots
}
fmt.Println(OriginVeleroCfg)
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, useVolumeSnapshots)).To(Succeed())
Expect(VeleroInstall(context.Background(), &OriginVeleroCfg)).To(Succeed())
if veleroCLI2Version.VeleroVersion != "self" {
Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.MigrateFromVeleroVersion)).To(Succeed())
@@ -143,14 +149,14 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
restoreScName = restoreName + "-sc"
By("Create namespace for sample workload", func() {
Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.DefaultClient, migrationNamespace)).To(Succeed(),
Expect(CreateNamespace(oneHourTimeout, *veleroCfg.DefaultClient, migrationNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", migrationNamespace))
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.DefaultClient, VeleroCfg.CloudProvider,
migrationNamespace, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider,
migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
})
By(fmt.Sprintf("Backup namespace %s", migrationNamespace), func() {
@@ -166,7 +172,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
Expect(VeleroBackupNamespace(context.Background(), OriginVeleroCfg.VeleroCLI,
OriginVeleroCfg.VeleroNamespace, BackupStorageClassCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupStorageClassCfg.BackupName, "")
return "Fail to backup workload"
})
@@ -176,6 +182,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.BackupLocation = ""
BackupCfg.Selector = ""
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
//TODO Remove UseRestic parameter once minor version is 1.10 or upper
BackupCfg.UseResticIfFSBackup = true
if veleroCLI2Version.VeleroVersion == "self" {
@@ -189,26 +196,26 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
})
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour,
migrationNamespace)).To(Succeed())
migrationNamespace, 2)).To(Succeed())
})
}
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = migrationNamespace
By("Snapshot should be created in cloud object store", func() {
snapshotCheckPoint, err := GetSnapshotCheckPoint(*VeleroCfg.DefaultClient, VeleroCfg, 2,
snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2,
migrationNamespace, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})
}
if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") {
// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
// v1.0 have issues to fail upgrade case.
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
@@ -218,52 +225,51 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...")
time.Sleep(5 * time.Minute)
}
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() {
ns, err := GetNamespace(context.Background(), *VeleroCfg.DefaultClient, migrationNamespace)
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace)
Expect(ns.Name).To(Equal(migrationNamespace))
Expect(err).NotTo(HaveOccurred())
Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
_, err = GetNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace)
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
_, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace)
Expect(err).To(HaveOccurred())
strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")
fmt.Println(err)
VeleroCfg.ObjectStoreProvider = ""
VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient
VeleroCfg.UseNodeAgent = !useVolumeSnapshots
VeleroCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
veleroCfg.UseNodeAgent = !useVolumeSnapshots
veleroCfg.UseRestic = false
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
})
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() {
Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed())
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupScName, 5*time.Minute)).To(Succeed())
})
By(fmt.Sprintf("Restore %s", migrationNamespace), func() {
Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, "", restoreName)
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreScName, backupScName, "StorageClass")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, "", restoreName)
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", restoreName)
return "Fail to restore workload"
})
})
By(fmt.Sprintf("Verify workload %s after restore ", migrationNamespace), func() {
Expect(KibishiiVerifyAfterRestore(*VeleroCfg.StandbyClient, migrationNamespace,
Expect(KibishiiVerifyAfterRestore(*veleroCfg.StandbyClient, migrationNamespace,
oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
})
})

53
test/e2e/privilegesmgmt/ssr.go
View File

@@ -39,65 +39,66 @@ func SSRTest() {
var (
err error
)
veleroCfg := VeleroCfg
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
veleroCfg.UseVolumeSnapshots = false
if veleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if VeleroCfg.InstallVelero {
if !VeleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
if veleroCfg.InstallVelero {
if !veleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
}
}
})
It(fmt.Sprintf("Should create an ssr object in the %s namespace and later removed by controller", VeleroCfg.VeleroNamespace), func() {
defer DeleteNamespace(context.TODO(), *VeleroCfg.ClientToInstallVelero, testNS, false)
ctx, _ := context.WithTimeout(context.Background(), time.Minute*10)
It(fmt.Sprintf("Should create an ssr object in the %s namespace and later removed by controller", veleroCfg.VeleroNamespace), func() {
defer DeleteNamespace(context.TODO(), *veleroCfg.ClientToInstallVelero, testNS, false)
ctx, _ := context.WithTimeout(context.Background(), time.Duration(time.Minute*10))
By(fmt.Sprintf("Create %s namespace", testNS))
Expect(CreateNamespace(ctx, *VeleroCfg.ClientToInstallVelero, testNS)).To(Succeed(),
Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create %s namespace", testNS))
By(fmt.Sprintf("Get version in %s namespace", testNS), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, testNS)).To(Succeed(),
Expect(VeleroVersion(context.Background(), veleroCfg.VeleroCLI, testNS)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in the %s namespace", testNS))
})
By(fmt.Sprintf("Get version in %s namespace", VeleroCfg.VeleroNamespace), func() {
Expect(VeleroVersion(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in %s namespace", VeleroCfg.VeleroNamespace))
By(fmt.Sprintf("Get version in %s namespace", veleroCfg.VeleroNamespace), func() {
Expect(VeleroVersion(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create an ssr object in %s namespace", veleroCfg.VeleroNamespace))
})
ssrListResp := new(v1.ServerStatusRequestList)
By(fmt.Sprintf("Check ssr object in %s namespace", VeleroCfg.VeleroNamespace))
By(fmt.Sprintf("Check ssr object in %s namespace", veleroCfg.VeleroNamespace))
err = waitutil.PollImmediate(5*time.Second, time.Minute,
func() (bool, error) {
if err = VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
return false, fmt.Errorf("failed to list ssr object in %s namespace with err %v", veleroCfg.VeleroNamespace, err)
}
if len(ssrListResp.Items) != 1 {
return false, fmt.Errorf("count of ssr object in %s namespace is not 1", VeleroCfg.VeleroNamespace)
return false, fmt.Errorf("count of ssr object in %s namespace is not 1", veleroCfg.VeleroNamespace)
}
if ssrListResp.Items[0].Status.ServerVersion == "" {
fmt.Printf("ServerVersion of ssr object in %s namespace should not empty, current response result %v\n", VeleroCfg.VeleroNamespace, ssrListResp)
fmt.Printf("ServerVersion of ssr object in %s namespace should not empty, current response result %v\n", veleroCfg.VeleroNamespace, ssrListResp)
return false, nil
}
if ssrListResp.Items[0].Status.Phase != "Processed" {
return false, fmt.Errorf("phase of ssr object in %s namespace should be Processed but got phase %s", VeleroCfg.VeleroNamespace, ssrListResp.Items[0].Status.Phase)
return false, fmt.Errorf("phase of ssr object in %s namespace should be Processed but got phase %s", veleroCfg.VeleroNamespace, ssrListResp.Items[0].Status.Phase)
}
return true, nil
})
if err == waitutil.ErrWaitTimeout {
fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace)
fmt.Printf("exceed test case deadline and failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace)
}
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", VeleroCfg.VeleroNamespace))
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check ssr object in %s namespace", veleroCfg.VeleroNamespace))
By(fmt.Sprintf("Check ssr object in %s namespace", testNS))
Expect(VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(),
Expect(veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: testNS})).To(Succeed(),
fmt.Sprintf("Failed to list ssr object in %s namespace", testNS))
Expect(len(ssrListResp.Items)).To(BeNumerically("==", 1),
fmt.Sprintf("Count of ssr object in %s namespace is not 1", testNS))
@@ -106,10 +107,10 @@ func SSRTest() {
Expect(ssrListResp.Items[0].Status.ServerVersion).To(BeEmpty(),
fmt.Sprintf("ServerVersion of ssr object in %s namespace should be empty", testNS))
By(fmt.Sprintf("Waiting ssr object in %s namespace deleted", VeleroCfg.VeleroNamespace))
By(fmt.Sprintf("Waiting ssr object in %s namespace deleted", veleroCfg.VeleroNamespace))
err = waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if err = VeleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
if err = veleroCfg.ClientToInstallVelero.Kubebuilder.List(ctx, ssrListResp, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
@@ -121,6 +122,6 @@ func SSRTest() {
return true, nil
})
Expect(err).To(Succeed(), fmt.Sprintf("ssr object in %s namespace is not been deleted by controller", VeleroCfg.VeleroNamespace))
Expect(err).To(Succeed(), fmt.Sprintf("ssr object in %s namespace is not been deleted by controller", veleroCfg.VeleroNamespace))
})
}

5
test/e2e/pv-backup/pv-backup-filter.go
View File

@@ -32,7 +32,10 @@ var OptOutPVBackupTest func() = TestFunc(&PVBackupFiltering{annotation: OPT_OUT_
func (p *PVBackupFiltering) Init() error {
p.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
p.Client = TestClientInstance
p.VeleroCfg = VeleroCfg
p.Client = *p.VeleroCfg.ClientToInstallVelero
p.VeleroCfg.UseVolumeSnapshots = false
p.VeleroCfg.UseNodeAgent = true
p.NSBaseName = "ns"
p.NSIncluded = &[]string{fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 1), fmt.Sprintf("%s-%s-%d", p.NSBaseName, p.id, 2)}

7
test/e2e/resource-filtering/base.go
View File

@@ -48,8 +48,9 @@ func (f *FilteringCase) Init() error {
f.replica = int32(2)
f.labels = map[string]string{"resourcefiltering": "true"}
f.labelSelector = "resourcefiltering"
f.Client = TestClientInstance
//f.Client = TestClientInstance
f.VeleroCfg = VeleroCfg
f.Client = *f.VeleroCfg.ClientToInstallVelero
f.NamespacesTotal = 3
f.BackupArgs = []string{
"create", "--namespace", VeleroCfg.VeleroNamespace, "backup", f.BackupName,
@@ -84,7 +85,7 @@ func (f *FilteringCase) CreateResources() error {
}
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels)
deployment := NewDeployment(f.NSBaseName, namespace, f.replica, f.labels, nil)
deployment, err := CreateDeployment(f.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

2
test/e2e/resource-filtering/exclude_label.go
View File

@@ -100,7 +100,7 @@ func (e *ExcludeFromBackup) CreateResources() error {
}
//Create deployment: to be included
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2)
deployment := NewDeployment(e.NSBaseName, namespace, e.replica, label2, nil)
deployment, err := CreateDeployment(e.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

2
test/e2e/resource-filtering/label_selector.go
View File

@@ -101,7 +101,7 @@ func (l *LabelSelector) CreateResources() error {
//Create deployment
fmt.Printf("Creating deployment in namespaces ...%s\n", namespace)
deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels)
deployment := NewDeployment(l.NSBaseName, namespace, l.replica, labels, nil)
deployment, err := CreateDeployment(l.Client.ClientGo, namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", namespace))

40
test/e2e/schedule/ordered_resources.go
View File

@@ -48,26 +48,29 @@ type OrderedResources struct {
}
func ScheduleOrderedResources() {
veleroCfg := VeleroCfg
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
if veleroCfg.InstallVelero {
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if VeleroCfg.InstallVelero && !VeleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
if veleroCfg.InstallVelero && !veleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
}
})
It("Create a schedule to backup resources in a specific order should be successful", func() {
test := &OrderedResources{}
test.VeleroCfg = VeleroCfg
err := test.Init()
Expect(err).To(Succeed(), err)
defer func() {
Expect(DeleteNamespace(test.Ctx, test.Client, test.Namespace, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.Namespace))
err = VeleroScheduleDelete(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName)
err = VeleroScheduleDelete(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete schedule with err %v", err))
err = test.DeleteBackups()
Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete backups with err %v", err))
@@ -79,24 +82,24 @@ func ScheduleOrderedResources() {
})
By(fmt.Sprintf("Create schedule the workload in %s namespace", test.Namespace), func() {
err = VeleroScheduleCreate(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs)
err = VeleroScheduleCreate(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.ScheduleArgs)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to create schedule %s with err %v", test.ScheduleName, err))
})
By(fmt.Sprintf("Checking resource order in %s schedule cr", test.ScheduleName), func() {
err = CheckScheduleWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap)
err = CheckScheduleWithResourceOrder(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.ScheduleName, test.OrderMap)
Expect(err).To(Succeed(), fmt.Sprintf("Failed to check schedule %s with err %v", test.ScheduleName, err))
})
By("Checking resource order in backup cr", func() {
backupList := new(velerov1api.BackupList)
err = waitutil.PollImmediate(10*time.Second, time.Minute*5, func() (bool, error) {
if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
if err = test.Client.Kubebuilder.List(test.Ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
return false, fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err)
}
for _, backup := range backupList.Items {
if err = CheckBackupWithResourceOrder(test.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil {
if err = CheckBackupWithResourceOrder(test.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name, test.OrderMap); err == nil {
return true, nil
}
}
@@ -112,9 +115,8 @@ func ScheduleOrderedResources() {
func (o *OrderedResources) Init() error {
rand.Seed(time.Now().UnixNano())
UUIDgen, _ = uuid.NewRandom()
client := *VeleroCfg.ClientToInstallVelero
o.Client = client
o.VeleroCfg = VeleroCfg
o.Client = *o.VeleroCfg.ClientToInstallVelero
o.ScheduleName = "schedule-ordered-resources-" + UUIDgen.String()
o.NSBaseName = "schedule-ordered-resources"
o.Namespace = o.NSBaseName + "-" + UUIDgen.String()
@@ -136,6 +138,7 @@ func (o *OrderedResources) Init() error {
}
func (o *OrderedResources) CreateResources() error {
veleroCfg := o.VeleroCfg
o.Ctx, _ = context.WithTimeout(context.Background(), 5*time.Minute)
label := map[string]string{
"orderedresources": "true",
@@ -150,13 +153,13 @@ func (o *OrderedResources) CreateResources() error {
return errors.Wrapf(err, "failed to wait the service account %q created under the namespace %q", serviceAccountName, o.Namespace)
}
// add the image pull secret to avoid the image pull limit issue of Docker Hub
if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, VeleroCfg.RegistryCredentialFile); err != nil {
if err := PatchServiceAccountWithImagePullSecret(o.Ctx, o.Client, o.Namespace, serviceAccountName, veleroCfg.RegistryCredentialFile); err != nil {
return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, o.Namespace)
}
//Create deployment
deploymentName := fmt.Sprintf("deploy-%s", o.NSBaseName)
fmt.Printf("Creating deployment %s in %s namespaces ...\n", deploymentName, o.Namespace)
deployment := NewDeployment(deploymentName, o.Namespace, 1, label)
deployment := NewDeployment(deploymentName, o.Namespace, 1, label, nil)
deployment, err := CreateDeployment(o.Client.ClientGo, o.Namespace, deployment)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to create namespace %q with err %v", o.Namespace, err))
@@ -191,12 +194,13 @@ func (o *OrderedResources) CreateResources() error {
}
func (o *OrderedResources) DeleteBackups() error {
veleroCfg := o.VeleroCfg
backupList := new(velerov1api.BackupList)
if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: VeleroCfg.VeleroNamespace}); err != nil {
return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
if err := o.Client.Kubebuilder.List(o.Ctx, backupList, &kbclient.ListOptions{Namespace: veleroCfg.VeleroNamespace}); err != nil {
return fmt.Errorf("failed to list backup object in %s namespace with err %v", veleroCfg.VeleroNamespace, err)
}
for _, backup := range backupList.Items {
if err := VeleroBackupDelete(o.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil {
if err := VeleroBackupDelete(o.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, backup.Name); err != nil {
return err
}
}

4
test/e2e/schedule/schedule.go
View File

@@ -28,7 +28,9 @@ type ScheduleBackup struct {
var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "schedule-test"}})
func (n *ScheduleBackup) Init() error {
n.Client = TestClientInstance
//n.Client = TestClientInstance
n.VeleroCfg = VeleroCfg
n.Client = *n.VeleroCfg.ClientToInstallVelero
n.Period = 3 // Unit is minute
n.verifyTimes = 5 // More verify times more confidence
n.TestMsg = &TestMSG{

72
test/e2e/test/test.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"flag"
"fmt"
"strings"
"time"
. "github.com/onsi/ginkgo"
@@ -27,6 +28,7 @@ import (
"github.com/pkg/errors"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
. "github.com/vmware-tanzu/velero/test/e2e"
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
@@ -69,26 +71,35 @@ type TestCase struct {
Ctx context.Context
NSIncluded *[]string
UseVolumeSnapshots bool
VeleroCfg VeleroConfig
RestorePhaseExpect velerov1api.RestorePhase
}
var TestClientInstance TestClient
func TestFunc(test VeleroBackupRestoreTest) func() {
return func() {
By("Create test client instance", func() {
TestClientInstance = *VeleroCfg.ClientToInstallVelero
})
Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases")
veleroCfg := test.GetTestCase().VeleroCfg
By("Create test client instance", func() {
TestClientInstance = *veleroCfg.ClientToInstallVelero
})
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, test.GetTestCase().UseVolumeSnapshots)).To(Succeed())
veleroCfg := test.GetTestCase().VeleroCfg
// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().NSBaseName, "nodeport") {
Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
}
if veleroCfg.InstallVelero {
veleroCfg.UseVolumeSnapshots = test.GetTestCase().UseVolumeSnapshots
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if VeleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed()))
if !veleroCfg.Debug {
if veleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed()))
}
}
})
@@ -101,30 +112,34 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() {
return func() {
var countIt int
By("Create test client instance", func() {
TestClientInstance = *VeleroCfg.ClientToInstallVelero
})
var useVolumeSnapshots bool
var veleroCfg VeleroConfig
for k := range tests {
Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc))
veleroCfg = tests[k].GetTestCase().VeleroCfg
By("Create test client instance", func() {
TestClientInstance = *veleroCfg.ClientToInstallVelero
})
useVolumeSnapshots = tests[k].GetTestCase().UseVolumeSnapshots
}
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
if countIt == 0 {
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
countIt++
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if VeleroCfg.InstallVelero {
if countIt == len(tests) && !VeleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To((Succeed()))
if !veleroCfg.Debug {
if veleroCfg.InstallVelero {
if countIt == len(tests) && !veleroCfg.Debug {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To((Succeed()))
}
}
}
@@ -152,22 +167,23 @@ func (t *TestCase) StartRun() error {
}
func (t *TestCase) Backup() error {
if err := VeleroBackupExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName, "")
veleroCfg := t.GetTestCase().VeleroCfg
if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "")
return errors.Wrapf(err, "Failed to backup resources")
}
return nil
}
func (t *TestCase) Destroy() error {
err := CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName)
if err != nil {
return errors.Wrap(err, "Could cleanup retrieve namespaces")
}
By(fmt.Sprintf("Start to destroy namespace %s......", t.NSBaseName), func() {
Expect(CleanupNamespacesWithPoll(t.Ctx, t.Client, t.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces")
})
return nil
}
func (t *TestCase) Restore() error {
veleroCfg := t.GetTestCase().VeleroCfg
// the snapshots of AWS may be still in pending status when do the restore, wait for a while
// to avoid this https://github.com/vmware-tanzu/velero/issues/1799
// TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed
@@ -177,8 +193,11 @@ func (t *TestCase) Restore() error {
}
By("Start to restore ......", func() {
Expect(VeleroRestoreExec(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", t.RestoreName)
if t.RestorePhaseExpect == "" {
t.RestorePhaseExpect = velerov1api.RestorePhaseCompleted
}
Expect(VeleroRestoreExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.RestoreName, t.RestoreArgs, t.RestorePhaseExpect)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", t.RestoreName)
return "Fail to restore workload"
})
})
@@ -190,7 +209,8 @@ func (t *TestCase) Verify() error {
}
func (t *TestCase) Clean() error {
if !VeleroCfg.Debug {
veleroCfg := t.GetTestCase().VeleroCfg
if !veleroCfg.Debug {
By(fmt.Sprintf("Clean namespace with prefix %s after test", t.NSBaseName), func() {
CleanupNamespaces(t.Ctx, t.Client, t.NSBaseName)
})

5
test/e2e/types.go
View File

@@ -66,6 +66,9 @@ type VeleroConfig struct {
UploaderType string
UseNodeAgent bool
UseRestic bool
ProvideSnapshotsVolumeParam bool
DefaultVolumesToFsBackup bool
UseVolumeSnapshots bool
}
type SnapshotCheckPoint struct {
@@ -83,6 +86,7 @@ type BackupConfig struct {
Namespace string
BackupLocation string
UseVolumeSnapshots bool
ProvideSnapshotsVolumeParam bool
Selector string
TTL time.Duration
IncludeResources string
@@ -90,6 +94,7 @@ type BackupConfig struct {
IncludeClusterResources bool
OrderedResources string
UseResticIfFSBackup bool
DefaultVolumesToFsBackup bool
}
type VeleroCLI2Version struct {

63
test/e2e/upgrade/upgrade.go
View File

@@ -38,14 +38,18 @@ const (
upgradeNamespace = "upgrade-workload"
)
var veleroCfg VeleroConfig
func BackupUpgradeRestoreWithSnapshots() {
for _, upgradeFromVelero := range GetVersionList(VeleroCfg.UpgradeFromVeleroCLI, VeleroCfg.UpgradeFromVeleroVersion) {
veleroCfg = VeleroCfg
for _, upgradeFromVelero := range GetVersionList(veleroCfg.UpgradeFromVeleroCLI, veleroCfg.UpgradeFromVeleroVersion) {
BackupUpgradeRestoreTest(true, upgradeFromVelero)
}
}
func BackupUpgradeRestoreWithRestic() {
for _, upgradeFromVelero := range GetVersionList(VeleroCfg.UpgradeFromVeleroCLI, VeleroCfg.UpgradeFromVeleroVersion) {
veleroCfg = VeleroCfg
for _, upgradeFromVelero := range GetVersionList(veleroCfg.UpgradeFromVeleroCLI, veleroCfg.UpgradeFromVeleroVersion) {
BackupUpgradeRestoreTest(false, upgradeFromVelero)
}
}
@@ -57,31 +61,32 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
)
BeforeEach(func() {
if !VeleroCfg.InstallVelero {
Skip("Upgrade test should not be triggered if VeleroCfg.InstallVelero is set to false")
veleroCfg = VeleroCfg
if !veleroCfg.InstallVelero {
Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false")
}
if (len(VeleroCfg.UpgradeFromVeleroVersion)) == 0 {
if (len(veleroCfg.UpgradeFromVeleroVersion)) == 0 {
Skip("An original velero version is required to run upgrade test, please run test with upgrade-from-velero-version=<version>")
}
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
if VeleroCfg.VeleroCLI == "" {
if veleroCfg.VeleroCLI == "" {
Skip("VeleroCLI should be provide")
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
By(fmt.Sprintf("Delete sample workload namespace %s", upgradeNamespace), func() {
DeleteNamespace(context.Background(), *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true)
DeleteNamespace(context.Background(), *veleroCfg.ClientToInstallVelero, upgradeNamespace, true)
})
if VeleroCfg.InstallVelero {
if veleroCfg.InstallVelero {
By("Uninstall Velero", func() {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace)).To(Succeed())
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace)).To(Succeed())
})
}
}
@@ -101,12 +106,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
Expect(err).To(Succeed())
})
}
VeleroCfg.GCFrequency = ""
veleroCfg.GCFrequency = ""
By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade",
veleroCLI2Version.VeleroVersion), func() {
//Set VeleroImage and RestoreHelperImage to blank
//VeleroImage and RestoreHelperImage should be the default value in originalCli
tmpCfgForOldVeleroInstall := VeleroCfg
tmpCfgForOldVeleroInstall := veleroCfg
tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion
tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI
tmpCfgForOldVeleroInstall.VeleroImage = ""
@@ -116,25 +121,24 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
tmpCfgForOldVeleroInstall.UseNodeAgent = false
tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall,
useVolumeSnapshots)).To(Succeed())
Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall)).To(Succeed())
Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI,
tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion)).To(Succeed())
})
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
tmpCfg := VeleroCfg
tmpCfg := veleroCfg
tmpCfg.UpgradeFromVeleroCLI = veleroCLI2Version.VeleroCLI
tmpCfg.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion
By("Create namespace for sample workload", func() {
Expect(CreateNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, upgradeNamespace)).To(Succeed(),
Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, upgradeNamespace)).To(Succeed(),
fmt.Sprintf("Failed to create namespace %s to install Kibishii workload", upgradeNamespace))
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, tmpCfg.CloudProvider,
Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.ClientToInstallVelero, tmpCfg.CloudProvider,
upgradeNamespace, tmpCfg.RegistryCredentialFile, tmpCfg.Features,
tmpCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
})
@@ -145,6 +149,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
BackupCfg.Namespace = upgradeNamespace
BackupCfg.BackupLocation = ""
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
BackupCfg.Selector = ""
//TODO: pay attention to this param, remove it when restic is not the default backup tool any more.
BackupCfg.UseResticIfFSBackup = true
@@ -157,31 +162,31 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
})
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
upgradeNamespace)).To(Succeed())
upgradeNamespace, 2)).To(Succeed())
})
}
var snapshotCheckPoint SnapshotCheckPoint
snapshotCheckPoint.NamespaceBackedUp = upgradeNamespace
By("Snapshot should be created in cloud object store", func() {
snapshotCheckPoint, err := GetSnapshotCheckPoint(*VeleroCfg.ClientToInstallVelero, VeleroCfg, 2,
snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2,
upgradeNamespace, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed())
})
}
By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", upgradeNamespace), func() {
Expect(DeleteNamespace(oneHourTimeout, *VeleroCfg.ClientToInstallVelero, upgradeNamespace, true)).To(Succeed(),
Expect(DeleteNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, upgradeNamespace, true)).To(Succeed(),
fmt.Sprintf("failed to delete namespace %s", upgradeNamespace))
})
if useVolumeSnapshots && VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
if useVolumeSnapshots && veleroCfg.CloudProvider == "azure" && strings.EqualFold(veleroCfg.Features, "EnableCSI") {
// Upgrade test is not running daily since no CSI plugin v1.0 released, because builds before
// v1.0 have issues to fail upgrade case.
By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() {
@@ -216,7 +221,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
})
By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() {
Expect(KibishiiVerifyAfterRestore(*VeleroCfg.ClientToInstallVelero, upgradeNamespace,
Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace,
oneHourTimeout, DefaultKibishiiData)).To(Succeed(), "Fail to verify workload after restore")
})
})

9
test/e2e/util/csi/common.go
View File

@@ -129,12 +129,13 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
}
func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) error {
func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
var err error
var snapshotContentNameList []string
if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil || len(snapshotContentNameList) != expectedCount {
return errors.Wrap(err, "Fail to get Azure CSI snapshot content")
if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil ||
len(snapshotContentNameList) != expectedCount {
return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
}
fmt.Println(snapshotContentNameList)
return nil
return snapshotContentNameList, nil
}

14
test/e2e/util/k8s/common.go
View File

@@ -341,3 +341,17 @@ func WaitForCRDEstablished(crdName string) error {
}
return nil
}
func GetAllService(ctx context.Context) (string, error) {
args := []string{"get", "service", "-A"}
cmd := exec.CommandContext(context.Background(), "kubectl", args...)
fmt.Printf("Kubectl exec cmd =%v\n", cmd)
stdout, stderr, err := veleroexec.RunCommand(cmd)
fmt.Println(stdout)
if err != nil {
fmt.Println(stderr)
fmt.Println(err)
return "", err
}
return stdout, nil
}

23
test/e2e/util/k8s/deployment.go
View File

@@ -36,7 +36,16 @@ const (
)
// newDeployment returns a RollingUpdate Deployment with a fake container image
func NewDeployment(name, ns string, replicas int32, labels map[string]string) *apps.Deployment {
func NewDeployment(name, ns string, replicas int32, labels map[string]string, containers []v1.Container) *apps.Deployment {
if containers == nil {
containers = []v1.Container{
{
Name: fmt.Sprintf("container-%s", "busybox"),
Image: "gcr.io/velero-gcp/busybox:latest",
Command: []string{"sleep", "1000000"},
},
}
}
return &apps.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
@@ -59,19 +68,17 @@ func NewDeployment(name, ns string, replicas int32, labels map[string]string) *a
Labels: labels,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: name,
Image: "gcr.io/velero-gcp/busybox:latest",
Command: []string{"sleep", "1000000"},
},
},
Containers: containers,
},
},
},
}
}
func CreateDeploy(c clientset.Interface, ns string, deployment *apps.Deployment) error {
_, err := c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
return err
}
func CreateDeployment(c clientset.Interface, ns string, deployment *apps.Deployment) (*apps.Deployment, error) {
return c.AppsV1().Deployments(ns).Create(context.TODO(), deployment, metav1.CreateOptions{})
}

2
test/e2e/util/k8s/namespace.go
View File

@@ -100,7 +100,7 @@ func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseNam
if err != nil {
return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
}
fmt.Printf("Delete namespace %s\n", checkNamespace.Name)
fmt.Printf("Namespace %s was deleted\n", checkNamespace.Name)
}
}
return nil

75
test/e2e/util/k8s/service.go Normal file
View File

@@ -0,0 +1,75 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package k8s
import (
"context"
"fmt"
"time"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
waitutil "k8s.io/apimachinery/pkg/util/wait"
)
func CreateService(ctx context.Context, client TestClient, namespace string,
service string, labels map[string]string, serviceSpec *corev1.ServiceSpec) error {
se := &corev1.Service{
TypeMeta: metav1.TypeMeta{
Kind: "Service",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: service,
Labels: labels,
Namespace: namespace,
},
Spec: *serviceSpec,
}
_, err = client.ClientGo.CoreV1().Services(namespace).Create(ctx, se, metav1.CreateOptions{})
if err != nil && !apierrors.IsAlreadyExists(err) {
return err
}
return nil
}
func GetService(ctx context.Context, client TestClient, namespace string, service string) (*corev1.Service, error) {
return client.ClientGo.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
}
func WaitForServiceDelete(client TestClient, ns, name string, deleteFirst bool) error {
if deleteFirst {
if err := client.ClientGo.CoreV1().Services(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
return errors.Wrap(err, fmt.Sprintf("failed to delete secret in namespace %q", ns))
}
}
return waitutil.PollImmediateInfinite(5*time.Second,
func() (bool, error) {
if _, err := client.ClientGo.CoreV1().Services(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {
if apierrors.IsNotFound(err) {
return true, nil
}
return false, err
}
logrus.Debugf("service %q in namespace %q is still being deleted...", name, ns)
return false, nil
})
}

98
test/e2e/util/kibishii/kibishii_utils.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
"golang.org/x/net/context"
"k8s.io/apimachinery/pkg/util/wait"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
@@ -51,15 +52,16 @@ var DefaultKibishiiData = &KibishiiData{2, 10, 10, 1024, 1024, 0, 2}
var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment-1"}
// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
useVolumeSnapshots bool) error {
func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string,
useVolumeSnapshots, defaultVolumesToFsBackup bool) error {
client := *veleroCfg.ClientToInstallVelero
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
veleroCLI := VeleroCfg.VeleroCLI
providerName := VeleroCfg.CloudProvider
veleroNamespace := VeleroCfg.VeleroNamespace
registryCredentialFile := VeleroCfg.RegistryCredentialFile
veleroFeatures := VeleroCfg.Features
kibishiiDirectory := VeleroCfg.KibishiiDirectory
veleroCLI := veleroCfg.VeleroCLI
providerName := veleroCfg.CloudProvider
veleroNamespace := veleroCfg.VeleroNamespace
registryCredentialFile := veleroCfg.RegistryCredentialFile
veleroFeatures := veleroCfg.Features
kibishiiDirectory := veleroCfg.KibishiiDirectory
if _, err := GetNamespace(context.Background(), client, kibishiiNamespace); err == nil {
fmt.Printf("Workload namespace %s exists, delete it first.\n", kibishiiNamespace)
if err = DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil {
@@ -88,32 +90,66 @@ func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, res
BackupCfg.Namespace = kibishiiNamespace
BackupCfg.BackupLocation = backupLocation
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = defaultVolumesToFsBackup
BackupCfg.Selector = ""
BackupCfg.ProvideSnapshotsVolumeParam = veleroCfg.ProvideSnapshotsVolumeParam
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg); err != nil {
RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "")
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
var snapshotCheckPoint SnapshotCheckPoint
var err error
pvbs, err := GetPVB(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace)
if useVolumeSnapshots {
if err != nil || len(pvbs) != 0 {
return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace)
}
if providerName == "vsphere" {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace); err != nil {
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, kibishiiNamespace, backupName, KibishiiPodNameList)
if err != nil {
return errors.Wrap(err, "Fail to get snapshot checkpoint")
}
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
} else {
if err != nil || len(pvbs) != 2 {
return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace)
}
if providerName == "vsphere" {
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
// TODO[High] - uncomment code block below when vSphere plugin PR #500 is included in release version.
// https://github.com/vmware-tanzu/velero-plugin-for-vsphere/pull/500/
// fmt.Println("Make sure no vSphere snapshot uploads created")
// if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 0); err != nil {
// return errors.Wrapf(err, "Error get vSphere snapshot uploads")
// }
} else {
if strings.EqualFold(veleroFeatures, "EnableCSI") {
_, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 0,
kibishiiNamespace, backupName, KibishiiPodNameList)
} else {
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
}
}
fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace)
@@ -133,6 +169,12 @@ func RunKibishiiTests(client TestClient, veleroCfg VeleroConfig, backupName, res
RunDebug(context.Background(), veleroCLI, veleroNamespace, "", restoreName)
return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
}
if !useVolumeSnapshots {
pvrs, err := GetPVR(oneHourTimeout, veleroCfg.VeleroNamespace, kibishiiNamespace)
if err != nil || len(pvrs) != 2 {
return errors.Wrapf(err, "failed to get PVB for namespace %s", kibishiiNamespace)
}
}
if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Error verifying kibishii after restore")
@@ -174,6 +216,9 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler
}
func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
timeout := 30 * time.Minute
interval := 1 * time.Second
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
timeout, _ := context.WithTimeout(context.Background(), time.Minute*20)
kibishiiGenerateCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
"/usr/local/bin/generate.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
@@ -181,16 +226,25 @@ func generateData(ctx context.Context, namespace string, kibishiiData *KibishiiD
strconv.Itoa(kibishiiData.BlockSize), strconv.Itoa(kibishiiData.PassNum), strconv.Itoa(kibishiiData.ExpectedNodes))
fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd)
_, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd)
if err != nil {
return errors.Wrapf(err, "failed to generate, stderr=%s", stderr)
stdout, stderr, err := veleroexec.RunCommand(kibishiiGenerateCmd)
if err != nil || strings.Contains(stderr, "Timeout occurred") || strings.Contains(stderr, "dialing backend") {
fmt.Printf("Kibishi generate stdout Timeout occurred: %s stderr: %s err: %s", stdout, stderr, err)
return false, nil
}
return true, nil
})
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("Failed to wait generate data in namespace %s", namespace))
}
return nil
}
func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiData) error {
timeout, _ := context.WithTimeout(context.Background(), time.Minute*10)
timeout := 10 * time.Minute
interval := 5 * time.Second
err := wait.PollImmediate(interval, timeout, func() (bool, error) {
timeout, _ := context.WithTimeout(context.Background(), time.Minute*20)
kibishiiVerifyCmd := exec.CommandContext(timeout, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
"/usr/local/bin/verify.sh", strconv.Itoa(kibishiiData.Levels), strconv.Itoa(kibishiiData.DirsPerLevel),
strconv.Itoa(kibishiiData.FilesPerLevel), strconv.Itoa(kibishiiData.FileLength),
@@ -199,8 +253,18 @@ func verifyData(ctx context.Context, namespace string, kibishiiData *KibishiiDat
fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
stdout, stderr, err := veleroexec.RunCommand(kibishiiVerifyCmd)
if strings.Contains(stderr, "Timeout occurred") {
return false, nil
}
if err != nil {
return errors.Wrapf(err, "failed to verify, stderr=%s, stdout=%s", stderr, stdout)
fmt.Printf("Kibishi verify stdout Timeout occurred: %s stderr: %s err: %s", stdout, stderr, err)
return false, nil
}
return true, nil
})
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("Failed to wait generate data in namespace %s", namespace))
}
return nil
}

16
test/e2e/util/velero/install.go
View File

@@ -49,11 +49,9 @@ type installOptions struct {
RestoreHelperImage string
}
func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapshots bool) error {
func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
if veleroCfg.CloudProvider != "kind" {
if veleroCfg.ObjectStoreProvider != "" {
return errors.New("For cloud platforms, object store plugin cannot be overridden") // Can't set an object store provider that is different than your cloud
}
fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider")
veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
} else {
if veleroCfg.ObjectStoreProvider == "" {
@@ -81,14 +79,13 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, useVolumeSnapsh
}
}
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, veleroCfg.VSLConfig, providerPluginsTmp, veleroCfg.Features)
veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg, providerPluginsTmp)
if err != nil {
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
veleroInstallOptions.UseVolumeSnapshots = useVolumeSnapshots
veleroInstallOptions.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
if !veleroCfg.UseRestic {
veleroInstallOptions.UseNodeAgent = !useVolumeSnapshots
veleroInstallOptions.UseNodeAgent = veleroCfg.UseNodeAgent
}
veleroInstallOptions.UseRestic = veleroCfg.UseRestic
veleroInstallOptions.Image = veleroCfg.VeleroImage
@@ -179,6 +176,9 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption
if options.UseNodeAgent {
args = append(args, "--use-node-agent")
}
if options.DefaultVolumesToFsBackup {
args = append(args, "--default-volumes-to-fs-backup")
}
if options.UseRestic {
args = append(args, "--use-restic")
}

110
test/e2e/util/velero/velero_utils.go
View File

@@ -135,22 +135,14 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin
}
// getProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
func getProviderVeleroInstallOptions(
pluginProvider,
credentialsFile,
objectStoreBucket,
objectStorePrefix,
bslConfig,
vslConfig string,
plugins []string,
features string,
) (*cliinstall.InstallOptions, error) {
func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
plugins []string) (*cliinstall.InstallOptions, error) {
if credentialsFile == "" {
if veleroCfg.CloudCredentialsFile == "" {
return nil, errors.Errorf("No credentials were supplied to use for E2E tests")
}
realPath, err := filepath.Abs(credentialsFile)
realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile)
if err != nil {
return nil, err
}
@@ -158,20 +150,22 @@ func getProviderVeleroInstallOptions(
io := cliinstall.NewInstallOptions()
// always wait for velero and restic pods to be running.
io.Wait = true
io.ProviderName = pluginProvider
io.SecretFile = credentialsFile
io.ProviderName = veleroCfg.ObjectStoreProvider
io.SecretFile = veleroCfg.CloudCredentialsFile
io.BucketName = objectStoreBucket
io.Prefix = objectStorePrefix
io.BucketName = veleroCfg.BSLBucket
io.Prefix = veleroCfg.BSLPrefix
io.BackupStorageConfig = flag.NewMap()
io.BackupStorageConfig.Set(bslConfig)
io.BackupStorageConfig.Set(veleroCfg.BSLConfig)
io.VolumeSnapshotConfig = flag.NewMap()
io.VolumeSnapshotConfig.Set(vslConfig)
io.VolumeSnapshotConfig.Set(veleroCfg.VSLConfig)
io.SecretFile = realPath
io.Plugins = flag.NewStringArray(plugins...)
io.Features = features
io.Features = veleroCfg.Features
io.DefaultVolumesToFsBackup = veleroCfg.DefaultVolumesToFsBackup
io.UseVolumeSnapshots = veleroCfg.UseVolumeSnapshots
return io, nil
}
@@ -346,8 +340,11 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
}
if backupCfg.UseVolumeSnapshots {
if backupCfg.ProvideSnapshotsVolumeParam {
args = append(args, "--snapshot-volumes")
} else {
}
}
if backupCfg.DefaultVolumesToFsBackup {
if backupCfg.UseResticIfFSBackup {
args = append(args, "--default-volumes-to-restic")
} else {
@@ -358,7 +355,9 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI, veleroNamespace strin
// if the "--snapshot-volumes=false" isn't specified explicitly, the vSphere plugin will always take snapshots
// for the volumes even though the "--default-volumes-to-fs-backup" is specified
// TODO This can be removed if the logic of vSphere plugin bump up to 1.3
if backupCfg.ProvideSnapshotsVolumeParam && !backupCfg.UseVolumeSnapshots {
args = append(args, "--snapshot-volumes=false")
} // if "--snapshot-volumes" is not provide, snapshot should be taken as default behavior.
}
if backupCfg.BackupLocation != "" {
args = append(args, "--storage-location", backupCfg.BackupLocation)
@@ -417,15 +416,15 @@ func VeleroRestore(ctx context.Context, veleroCLI, veleroNamespace, restoreName,
if includeResources != "" {
args = append(args, "--include-resources", includeResources)
}
return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args)
return VeleroRestoreExec(ctx, veleroCLI, veleroNamespace, restoreName, args, velerov1api.RestorePhaseCompleted)
}
func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string) error {
func VeleroRestoreExec(ctx context.Context, veleroCLI, veleroNamespace, restoreName string, args []string, phaseExpect velerov1api.RestorePhase) error {
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
return err
}
return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted)
return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, phaseExpect)
}
func VeleroBackupExec(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, args []string) error {
@@ -514,7 +513,7 @@ func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s
args = append(args, "--backup", backup)
}
if len(restore) > 0 {
args = append(args, "--restore", restore)
//args = append(args, "--restore", restore)
}
fmt.Printf("Generating the debug tarball at %s\n", output)
if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
@@ -620,7 +619,7 @@ func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNa
// WaitForVSphereUploadCompletion waits for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string) error {
func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration, namespace string, expectCount int) error {
err := wait.PollImmediate(time.Second*5, timeout, func() (bool, error) {
checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
"get", "-n", namespace, "snapshots.backupdriver.cnsdp.vmware.com", "-o=jsonpath='{range .items[*]}{.spec.resourceHandle.name}{\"=\"}{.status.phase}{\"\\n\"}{end}'")
@@ -629,10 +628,12 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
if err != nil {
fmt.Print(stdout)
fmt.Print(stderr)
return false, errors.Wrap(err, "failed to verify")
return false, errors.Wrap(err, "failed to wait for vSphere upload completion")
}
lines := strings.Split(stdout, "\n")
complete := true
actualCount := 0
for _, curLine := range lines {
fmt.Println(curLine)
comps := strings.Split(curLine, "=")
@@ -651,6 +652,7 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
// Canceled - the operation was canceled, the snapshot ID is not valid
if len(comps) == 2 {
phase := comps[1]
actualCount++
switch phase {
case "Uploaded":
case "New", "InProgress", "Snapshotted", "Uploading":
@@ -660,6 +662,14 @@ func WaitForVSphereUploadCompletion(ctx context.Context, timeout time.Duration,
}
}
}
if expectCount != actualCount {
fmt.Printf("Snapshot expect count and actual count: %d-%d", expectCount, actualCount)
complete = false
if expectCount == 0 {
return true, nil
}
}
return complete, nil
})
@@ -1070,24 +1080,15 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, kibishiiPodNameList []string) (SnapshotCheckPoint, error) {
var snapshotCheckPoint SnapshotCheckPoint
var err error
snapshotCheckPoint.ExpectCount = expectCount
snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
snapshotCheckPoint.PodName = kibishiiPodNameList
if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
snapshotCheckPoint.EnableCSI = true
if err := util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
}
var err error
snapshotCheckPoint.SnapshotIDList, err = util.GetCsiSnapshotHandle(client, backupName)
if err != nil {
return snapshotCheckPoint, errors.New(fmt.Sprintf("Fail to get CSI SnapshotHandle for backup %s", backupName))
}
fmt.Println(snapshotCheckPoint)
if len(snapshotCheckPoint.SnapshotIDList) != expectCount {
return snapshotCheckPoint, errors.New(fmt.Sprintf("Length of SnapshotIDList is not as expected %d", expectCount))
}
}
fmt.Println(snapshotCheckPoint)
return snapshotCheckPoint, nil
@@ -1208,7 +1209,7 @@ func UpdateVeleroDeployment(ctx context.Context, veleroCfg VeleroConfig) ([]stri
if veleroCfg.CloudProvider == "vsphere" {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
} else {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
}
cmd = &common.OsCommandLine{
Cmd: "sed",
@@ -1261,7 +1262,7 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string)
if veleroCfg.CloudProvider == "vsphere" {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"harbor-repo.vmware.com\\/velero_ci\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
} else {
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"velero\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
args = fmt.Sprintf("s#\\\"image\\\"\\: \\\"velero\\/velero\\:v[0-9]*.[0-9]*.[0-9]\\\"#\\\"image\\\"\\: \\\"gcr.io\\/velero-gcp\\/nightly\\/velero\\:%s\\\"#g", veleroCfg.VeleroVersion)
}
cmd = &common.OsCommandLine{
Cmd: "sed",
@@ -1289,3 +1290,34 @@ func UpdateNodeAgent(ctx context.Context, veleroCfg VeleroConfig, dsjson string)
return common.GetListByCmdPipes(ctx, cmds)
}
func GetVeleroResource(ctx context.Context, veleroNamespace, namespace, resourceName string) ([]string, error) {
cmds := []*common.OsCommandLine{}
cmd := &common.OsCommandLine{
Cmd: "kubectl",
Args: []string{"get", resourceName, "-n", veleroNamespace},
}
cmds = append(cmds, cmd)
cmd = &common.OsCommandLine{
Cmd: "grep",
Args: []string{namespace},
}
cmds = append(cmds, cmd)
cmd = &common.OsCommandLine{
Cmd: "awk",
Args: []string{"{print $1}"},
}
cmds = append(cmds, cmd)
return common.GetListByCmdPipes(ctx, cmds)
}
func GetPVB(ctx context.Context, veleroNamespace, namespace string) ([]string, error) {
return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumebackup")
}
func GetPVR(ctx context.Context, veleroNamespace, namespace string) ([]string, error) {
return GetVeleroResource(ctx, veleroNamespace, namespace, "podvolumerestore")
}