/* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package backups import ( "context" "flag" "fmt" "strings" "time" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" ) // Test backup and restore of Kibishii using restic func BackupDeletionWithSnapshots() { backup_deletion_test(true) } func BackupDeletionWithRestic() { backup_deletion_test(false) } func backup_deletion_test(useVolumeSnapshots bool) { veleroCfg := VeleroCfg veleroCfg.UseVolumeSnapshots = useVolumeSnapshots veleroCfg.UseNodeAgent = !useVolumeSnapshots BeforeEach(func() { if useVolumeSnapshots && veleroCfg.CloudProvider == Kind { Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind)) } var err error flag.Parse() if InstallVelero { Expect(PrepareVelero(context.Background(), "backup deletion", veleroCfg)).To(Succeed()) } UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) }) AfterEach(func() { if CurrentSpecReport().Failed() && veleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) } }) When("kibishii is the sample workload", func() { It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(), "Failed to run backup deletion test") }) }) } // runUpgradeTests runs upgrade test on the provider by kibishii. func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLocation string, useVolumeSnapshots bool, kibishiiDirectory string) error { var err error var snapshotCheckPoint SnapshotCheckPoint backupName := "backup-" + UUIDgen.String() workloadNamespaceList := []string{"backup-deletion-1-" + UUIDgen.String(), "backup-deletion-2-" + UUIDgen.String()} nsCount := len(workloadNamespaceList) workloadNamespaces := strings.Join(workloadNamespaceList[:], ",") if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { Skip("Volume snapshots not supported on kind") } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() veleroCLI := veleroCfg.VeleroCLI providerName := veleroCfg.CloudProvider veleroNamespace := veleroCfg.VeleroNamespace registryCredentialFile := veleroCfg.RegistryCredentialFile veleroFeatures := veleroCfg.Features for _, ns := range workloadNamespaceList { if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns) } if CurrentSpecReport().Failed() && veleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { defer func() { if err := DeleteNamespace(context.Background(), client, ns, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) } }() } if err := KibishiiPrepareBeforeBackup( oneHourTimeout, client, providerName, ns, registryCredentialFile, veleroFeatures, kibishiiDirectory, DefaultKibishiiData, veleroCfg.ImageRegistryProxy, veleroCfg.WorkerOS, ); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns) } err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } } var BackupCfg BackupConfig BackupCfg.BackupName = backupName BackupCfg.Namespace = workloadNamespaces BackupCfg.BackupLocation = backupLocation BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = "" By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "") return "Fail to backup workload" }) }) for _, ns := range workloadNamespaceList { if useVolumeSnapshots && veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, ns, DefaultKibishiiWorkerCounts); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") } } } err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix) if err != nil { return err } backupVolumeInfo, err := GetVolumeInfo( veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix+"/"+backupName, ) if err != nil { return errors.Wrapf(err, "Failed to get volume info for backup %s", backupName) } if useVolumeSnapshots { // Check for snapshots existence if veleroCfg.CloudProvider == Vsphere { // For vSphere, checking snapshot should base on namespace and backup name for _, ns := range workloadNamespaceList { snapshotCheckPoint, err := BuildSnapshotCheckPointFromVolumeInfo(veleroCfg, backupVolumeInfo, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") err = CheckSnapshotsInProvider( veleroCfg, backupName, snapshotCheckPoint, false, ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } } else { // For public cloud, When using backup name to index VolumeSnapshotContents, make sure count of VolumeSnapshotContents should including PVs in all namespace // so VolumeSnapshotContents count should be equal to "namespace count" * "Kibishii worker count per namespace". snapshotCheckPoint, err := BuildSnapshotCheckPointFromVolumeInfo(veleroCfg, backupVolumeInfo, DefaultKibishiiWorkerCounts*nsCount, "", backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") // Get all snapshots base on backup name, regardless of namespaces err = CheckSnapshotsInProvider( veleroCfg, backupName, snapshotCheckPoint, false, ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } } else { // Check for BackupRepository and DeleteRequest var brList, pvbList []string brList, err = KubectlGetBackupRepository(oneHourTimeout, "kopia", veleroCfg.VeleroNamespace) if err != nil { return err } pvbList, err = KubectlGetPodVolumeBackup(oneHourTimeout, BackupCfg.BackupName, veleroCfg.VeleroNamespace) fmt.Println(brList) fmt.Println(pvbList) if err != nil { return err } } err = DeleteBackup(context.Background(), backupName, &veleroCfg) if err != nil { return err } // Verify snapshots are deleted after backup deletion. if useVolumeSnapshots { snapshotCheckPoint.ExpectCount = 0 err = CheckSnapshotsInProvider( veleroCfg, backupName, snapshotCheckPoint, false, ) if err != nil { return errors.Wrap(err, "fail to verify snapshots are deleted in provider.") } } // Verify backup metadata files are deleted in OSS after backup deletion. err = ObjectsShouldNotBeInBucket( veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 5, ) if err != nil { return err } // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded // Sleep for more than 15 seconds to avoid this issue. time.Sleep(1 * time.Minute) backupName = "backup-1-" + UUIDgen.String() BackupCfg.BackupName = backupName By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "") return "Fail to backup workload" }) }) if err := DeleteObjectsInBucket( veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, ); err != nil { return err } if err := ObjectsShouldNotBeInBucket( veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1, ); err != nil { return err } err = DeleteBackup(context.Background(), backupName, &veleroCfg) if err != nil { return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName) } else { fmt.Printf("|| EXPECTED || - Success to delete backup %s locally\n", backupName) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") return nil }