mirror of
https://github.com/vmware-tanzu/velero.git
synced 2025-12-23 06:15:21 +00:00
264 lines
9.9 KiB
Go
264 lines
9.9 KiB
Go
/*
|
|
Copyright the Velero contributors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
package backups
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
"github.com/pkg/errors"
|
|
|
|
. "github.com/vmware-tanzu/velero/test"
|
|
. "github.com/vmware-tanzu/velero/test/util/k8s"
|
|
. "github.com/vmware-tanzu/velero/test/util/kibishii"
|
|
. "github.com/vmware-tanzu/velero/test/util/providers"
|
|
. "github.com/vmware-tanzu/velero/test/util/velero"
|
|
)
|
|
|
|
// Test backup and restore of Kibishi using restic
|
|
|
|
func BackupDeletionWithSnapshots() {
|
|
backup_deletion_test(true)
|
|
}
|
|
|
|
func BackupDeletionWithRestic() {
|
|
backup_deletion_test(false)
|
|
}
|
|
func backup_deletion_test(useVolumeSnapshots bool) {
|
|
veleroCfg := VeleroCfg
|
|
veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
veleroCfg.UseNodeAgent = !useVolumeSnapshots
|
|
|
|
BeforeEach(func() {
|
|
if useVolumeSnapshots && veleroCfg.CloudProvider == Kind {
|
|
Skip(fmt.Sprintf("Volume snapshots not supported on %s", Kind))
|
|
}
|
|
var err error
|
|
flag.Parse()
|
|
if InstallVelero {
|
|
Expect(PrepareVelero(context.Background(), "backup deletion", veleroCfg)).To(Succeed())
|
|
}
|
|
UUIDgen, err = uuid.NewRandom()
|
|
Expect(err).To(Succeed())
|
|
})
|
|
|
|
AfterEach(func() {
|
|
if CurrentSpecReport().Failed() && veleroCfg.FailFast {
|
|
fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
|
|
} else {
|
|
By("Clean backups after test", func() {
|
|
DeleteAllBackups(context.Background(), &veleroCfg)
|
|
})
|
|
}
|
|
})
|
|
|
|
When("kibishii is the sample workload", func() {
|
|
It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() {
|
|
Expect(runBackupDeletionTests(*veleroCfg.ClientToInstallVelero, veleroCfg, "", useVolumeSnapshots, veleroCfg.KibishiiDirectory)).To(Succeed(),
|
|
"Failed to run backup deletion test")
|
|
})
|
|
})
|
|
}
|
|
|
|
// runUpgradeTests runs upgrade test on the provider by kibishii.
|
|
func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLocation string,
|
|
useVolumeSnapshots bool, kibishiiDirectory string) error {
|
|
var err error
|
|
var snapshotCheckPoint SnapshotCheckPoint
|
|
backupName := "backup-" + UUIDgen.String()
|
|
|
|
workloadNamespaceList := []string{"backup-deletion-1-" + UUIDgen.String(), "backup-deletion-2-" + UUIDgen.String()}
|
|
nsCount := len(workloadNamespaceList)
|
|
workloadNamespaces := strings.Join(workloadNamespaceList[:], ",")
|
|
|
|
if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
|
|
Skip("Volume snapshots not supported on kind")
|
|
}
|
|
oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
|
|
defer ctxCancel()
|
|
veleroCLI := veleroCfg.VeleroCLI
|
|
providerName := veleroCfg.CloudProvider
|
|
veleroNamespace := veleroCfg.VeleroNamespace
|
|
registryCredentialFile := veleroCfg.RegistryCredentialFile
|
|
bslPrefix := veleroCfg.BSLPrefix
|
|
bslConfig := veleroCfg.BSLConfig
|
|
veleroFeatures := veleroCfg.Features
|
|
for _, ns := range workloadNamespaceList {
|
|
if err := CreateNamespace(oneHourTimeout, client, ns); err != nil {
|
|
return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns)
|
|
}
|
|
|
|
if CurrentSpecReport().Failed() && veleroCfg.FailFast {
|
|
fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.")
|
|
} else {
|
|
defer func() {
|
|
if err := DeleteNamespace(context.Background(), client, ns, true); err != nil {
|
|
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
|
|
}
|
|
}()
|
|
}
|
|
|
|
if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, ns,
|
|
registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
|
|
return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns)
|
|
}
|
|
err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
var BackupCfg BackupConfig
|
|
BackupCfg.BackupName = backupName
|
|
BackupCfg.Namespace = workloadNamespaces
|
|
BackupCfg.BackupLocation = backupLocation
|
|
BackupCfg.UseVolumeSnapshots = useVolumeSnapshots
|
|
BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots
|
|
BackupCfg.Selector = ""
|
|
|
|
By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
|
|
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI,
|
|
veleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "")
|
|
return "Fail to backup workload"
|
|
})
|
|
})
|
|
for _, ns := range workloadNamespaceList {
|
|
if providerName == Vsphere && useVolumeSnapshots {
|
|
// Wait for uploads started by the Velero Plugin for vSphere to complete
|
|
// TODO - remove after upload progress monitoring is implemented
|
|
fmt.Println("Waiting for vSphere uploads to complete")
|
|
if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, ns, DefaultKibishiiWorkerCounts); err != nil {
|
|
return errors.Wrapf(err, "Error waiting for uploads to complete")
|
|
}
|
|
}
|
|
}
|
|
err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if useVolumeSnapshots {
|
|
// Check for snapshots existence
|
|
if veleroCfg.CloudProvider == Vsphere {
|
|
// For vSphere, checking snapshot should base on namespace and backup name
|
|
for _, ns := range workloadNamespaceList {
|
|
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList)
|
|
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
|
|
backupName, snapshotCheckPoint)
|
|
if err != nil {
|
|
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
|
}
|
|
}
|
|
} else {
|
|
// For public cloud, When using backup name to index VolumeSnapshotContents, make sure count of VolumeSnapshotContents should including PVs in all namespace
|
|
// so VolumeSnapshotContents count should be equal to "namespace count" * "Kibishii worker count per namespace".
|
|
snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts*nsCount, "", backupName, KibishiiPVCNameList)
|
|
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
|
|
|
|
// Get all snapshots base on backup name, regardless of namespaces
|
|
err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig,
|
|
backupName, snapshotCheckPoint)
|
|
if err != nil {
|
|
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
|
}
|
|
}
|
|
} else {
|
|
// Check for BackupRepository and DeleteRequest
|
|
var brList, pvbList []string
|
|
brList, err = KubectlGetBackupRepository(oneHourTimeout, "kopia", veleroCfg.VeleroNamespace)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pvbList, err = KubectlGetPodVolumeBackup(oneHourTimeout, BackupCfg.BackupName, veleroCfg.VeleroNamespace)
|
|
|
|
fmt.Println(brList)
|
|
fmt.Println(pvbList)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = DeleteBackup(context.Background(), backupName, &veleroCfg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if useVolumeSnapshots {
|
|
err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
|
|
backupName, snapshotCheckPoint)
|
|
if err != nil {
|
|
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
|
}
|
|
}
|
|
|
|
err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if useVolumeSnapshots {
|
|
if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider,
|
|
veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
|
|
bslConfig, backupName, snapshotCheckPoint); err != nil {
|
|
return errors.Wrap(err, "exceed waiting for snapshot created in cloud")
|
|
}
|
|
}
|
|
|
|
// Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded
|
|
// Sleep for more than 15 seconds to avoid this issue.
|
|
time.Sleep(1 * time.Minute)
|
|
|
|
backupName = "backup-1-" + UUIDgen.String()
|
|
BackupCfg.BackupName = backupName
|
|
|
|
By(fmt.Sprintf("Back up workload with name %s", BackupCfg.BackupName), func() {
|
|
Expect(VeleroBackupNamespace(oneHourTimeout, veleroCLI,
|
|
veleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
|
RunDebug(context.Background(), veleroCLI, veleroNamespace, BackupCfg.BackupName, "")
|
|
return "Fail to backup workload"
|
|
})
|
|
})
|
|
|
|
err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = DeleteBackup(context.Background(), backupName, &veleroCfg)
|
|
if err != nil {
|
|
return errors.Wrapf(err, "|| UNEXPECTED || - Failed to delete backup %q", backupName)
|
|
} else {
|
|
fmt.Printf("|| EXPECTED || - Success to delete backup %s locally\n", backupName)
|
|
}
|
|
fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n")
|
|
return nil
|
|
}
|