mirrors/velero
1
0
Fork 0
mirror of https://github.com/vmware-tanzu/velero.git synced 2026-10-01 04:05:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add snapshotvolumes default behavior E2E test

Browse Source 
Signed-off-by: danfengl <danfengl@vmware.com>
This commit is contained in:
danfengl
2023-02-19 09:05:20 +00:00
parent c5efb542d0
commit beed887eeb
32 changed files with 1057 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/backups/deletion.go
+34 -30
View File
@@ -48,38 +48,41 @@ func backup_deletion_test(useVolumeSnapshots bool) {
var (
backupName string
err error
veleroCfg VeleroConfig
)
veleroCfg = VeleroCfg
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
veleroCfg.UseNodeAgent = !useVolumeSnapshots
BeforeEach(func() {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
if veleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
if veleroCfg.InstallVelero {
err = VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
}
})
When("kibishii is the sample workload", func() {
It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
backupName = "backup-" + UUIDgen.String()
Expect(runBackupDeletionTests(*VeleroCfg.ClientToInstallVelero, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(),
Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, backupName, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(),
"Failed to run backup deletion test")
})
})
@@ -89,18 +92,18 @@ func backup_deletion_test(useVolumeSnapshots bool) {
func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
useVolumeSnapshots bool, kibishiiDirectory string) error {
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
veleroCLI := VeleroCfg.VeleroCLI
providerName := VeleroCfg.CloudProvider
veleroNamespace := VeleroCfg.VeleroNamespace
registryCredentialFile := VeleroCfg.RegistryCredentialFile
bslPrefix := VeleroCfg.BSLPrefix
bslConfig := VeleroCfg.BSLConfig
veleroFeatures := VeleroCfg.Features
veleroCLI := veleroCfg.VeleroCLI
providerName := veleroCfg.CloudProvider
veleroNamespace := veleroCfg.VeleroNamespace
registryCredentialFile := veleroCfg.RegistryCredentialFile
bslPrefix := veleroCfg.BSLPrefix
bslConfig := veleroCfg.BSLConfig
veleroFeatures := veleroCfg.Features
if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest)
}
if !VeleroCfg.Debug {
if !veleroCfg.Debug {
defer func() {
if err := DeleteNamespace(context.Background(), client, deletionTest, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", deletionTest))
@@ -112,7 +115,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
}
err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}
@@ -121,6 +124,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
BackupCfg.Namespace = deletionTest
BackupCfg.BackupLocation = backupLocation
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
BackupCfg.Selector = ""
By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
@@ -135,20 +139,20 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
// Wait for uploads started by the Velero Plug-in for vSphere to complete
// TODO - remove after upload progress monitoring is implemented
fmt.Println("Waiting for vSphere uploads to complete")
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest); err != nil {
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, deletionTest, 2); err != nil {
return errors.Wrapf(err, "Error waiting for uploads to complete")
}
}
err = ObjectsShouldBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
if err != nil {
return err
}
var snapshotCheckPoint SnapshotCheckPoint
if useVolumeSnapshots {
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, deletionTest, backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
err = SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslConfig,
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
@@ -159,21 +163,21 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
return err
}
if useVolumeSnapshots {
err = SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, veleroCfg.BSLConfig,
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
backupName, snapshotCheckPoint)
if err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
}
err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
if err != nil {
return err
}
if useVolumeSnapshots {
if err := SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
bslConfig, backupName, snapshotCheckPoint); err != nil {
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
}
@@ -189,12 +193,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
})
})
err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
if err != nil {
return err
}
err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
if err != nil {
return err
}
test/e2e/backups/sync_backups.go
+6 -3
View File
@@ -59,7 +59,9 @@ func BackupsSyncTest() {
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(context.Background(), &VeleroCfg)).To(Succeed())
}
})
@@ -105,8 +107,9 @@ func BackupsSyncTest() {
})
By("Install velero", func() {
VeleroCfg.ObjectStoreProvider = ""
Expect(VeleroInstall(test.ctx, &VeleroCfg, false)).To(Succeed())
veleroCfg := VeleroCfg
veleroCfg.UseVolumeSnapshots = false
Expect(VeleroInstall(test.ctx, &VeleroCfg)).To(Succeed())
})
By("Check all backups in object storage are synced to Velero", func() {
test/e2e/backups/ttl.go
+41 -37
View File
@@ -59,29 +59,33 @@ func (b *TTL) Init() {
func TTLTest() {
var err error
var veleroCfg VeleroConfig
useVolumeSnapshots := true
test := new(TTL)
client := *VeleroCfg.ClientToInstallVelero
veleroCfg = VeleroCfg
client := *veleroCfg.ClientToInstallVelero
//Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
BeforeEach(func() {
flag.Parse()
if VeleroCfg.InstallVelero {
veleroCfg = VeleroCfg
if veleroCfg.InstallVelero {
// Make sure GCFrequency is shorter than backup TTL
VeleroCfg.GCFrequency = "4m0s"
Expect(VeleroInstall(context.Background(), &VeleroCfg, useVolumeSnapshots)).To(Succeed())
veleroCfg.GCFrequency = "4m0s"
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
}
})
AfterEach(func() {
VeleroCfg.GCFrequency = ""
if !VeleroCfg.Debug {
veleroCfg.GCFrequency = ""
if !veleroCfg.Debug {
By("Clean backups after test", func() {
DeleteBackups(context.Background(), *VeleroCfg.ClientToInstallVelero)
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
})
if VeleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
if veleroCfg.InstallVelero {
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed())
}
Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
}
@@ -95,9 +99,9 @@ func TTLTest() {
})
By("Deploy sample workload of Kibishii", func() {
Expect(KibishiiPrepareBeforeBackup(test.ctx, client, VeleroCfg.CloudProvider,
test.testNS, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features,
VeleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
Expect(KibishiiPrepareBeforeBackup(test.ctx, client, veleroCfg.CloudProvider,
test.testNS, veleroCfg.RegistryCredentialFile, veleroCfg.Features,
veleroCfg.KibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData)).To(Succeed())
})
var BackupCfg BackupConfig
@@ -109,26 +113,26 @@ func TTLTest() {
BackupCfg.TTL = test.ttl
By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
Expect(VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
Expect(VeleroBackupNamespace(test.ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, test.backupName, "")
return "Fail to backup workload"
})
})
var snapshotCheckPoint SnapshotCheckPoint
if useVolumeSnapshots {
if VeleroCfg.CloudProvider == "vsphere" {
if veleroCfg.CloudProvider == "vsphere" {
// TODO - remove after upload progress monitoring is implemented
By("Waiting for vSphere uploads to complete", func() {
Expect(WaitForVSphereUploadCompletion(test.ctx, time.Hour,
test.testNS)).To(Succeed())
test.testNS, 2)).To(Succeed())
})
}
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, VeleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPodNameList)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(SnapshotsShouldBeCreatedInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}
@@ -137,30 +141,30 @@ func TTLTest() {
fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName))
})
if VeleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
if veleroCfg.CloudProvider == "aws" && useVolumeSnapshots {
fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...")
time.Sleep(7 * time.Minute)
}
By(fmt.Sprintf("Restore %s", test.testNS), func() {
Expect(VeleroRestore(test.ctx, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string {
RunDebug(test.ctx, VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace, "", test.restoreName)
Expect(VeleroRestore(test.ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, test.restoreName, test.backupName, "")).To(Succeed(), func() string {
RunDebug(test.ctx, veleroCfg.VeleroCLI,
veleroCfg.VeleroNamespace, "", test.restoreName)
return "Fail to restore workload"
})
})
By("Associated Restores should be created", func() {
Expect(ObjectsShouldBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
})
By("Check TTL was set correctly", func() {
ttl, err := GetBackupTTL(test.ctx, VeleroCfg.VeleroNamespace, test.backupName)
ttl, err := GetBackupTTL(test.ctx, veleroCfg.VeleroNamespace, test.backupName)
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
t, _ := time.ParseDuration(strings.ReplaceAll(ttl, "'", ""))
fmt.Println(t.Round(time.Minute).String())
@@ -172,28 +176,28 @@ func TTLTest() {
})
By("Check if backups are deleted by GC", func() {
Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
Expect(WaitBackupDeleted(test.ctx, veleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Backup %s was not deleted by GC", test.backupName))
})
By("Backup file from cloud object storage should be deleted", func() {
Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName,
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
})
By("PersistentVolume snapshots should be deleted", func() {
if useVolumeSnapshots {
Expect(SnapshotsShouldNotExistInCloud(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLConfig,
Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
}
})
By("Associated Restores should be deleted", func() {
Expect(ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider,
VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.restoreName,
Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")
})