From caa80db92c72fac66ed68be1b007caeb574947ed Mon Sep 17 00:00:00 2001 From: Ming Date: Tue, 18 Jan 2022 10:18:49 +0800 Subject: [PATCH] Add E2E test of backups sync from BSL Signed-off-by: Ming --- changelogs/unreleased/4545-mqiu | 1 + test/e2e/backups/sync_backups.go | 155 +++++++++++++++++++++++++++ test/e2e/e2e_suite_test.go | 1 + test/e2e/util/velero/velero_utils.go | 32 ++++++ 4 files changed, 189 insertions(+) create mode 100644 changelogs/unreleased/4545-mqiu create mode 100644 test/e2e/backups/sync_backups.go diff --git a/changelogs/unreleased/4545-mqiu b/changelogs/unreleased/4545-mqiu new file mode 100644 index 000000000..bc9eca647 --- /dev/null +++ b/changelogs/unreleased/4545-mqiu @@ -0,0 +1 @@ +Add E2E test of backups sync from BSL diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go new file mode 100644 index 000000000..bd81014d0 --- /dev/null +++ b/test/e2e/backups/sync_backups.go @@ -0,0 +1,155 @@ +/* + * + * Copyright the Velero contributors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +//Refer to https://github.com/vmware-tanzu/velero/issues/4253 +package backups + +import ( + "context" + "flag" + "fmt" + "math/rand" + "time" + + "github.com/google/uuid" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/wait" + + . "github.com/vmware-tanzu/velero/test/e2e" + . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" + . "github.com/vmware-tanzu/velero/test/e2e/util/providers" + . "github.com/vmware-tanzu/velero/test/e2e/util/velero" +) + +type SyncBackups struct { + testNS string + backupName string + ctx context.Context +} + +func (b *SyncBackups) Init() { + rand.Seed(time.Now().UnixNano()) + UUIDgen, _ = uuid.NewRandom() + b.testNS = "sync-bsl-test-" + UUIDgen.String() + b.backupName = "sync-bsl-test-" + UUIDgen.String() + b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*10)) +} + +func BackupsSyncTest() { + test := new(SyncBackups) + client, err := NewTestClient() + if err != nil { + println(err.Error()) + } + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + + BeforeEach(func() { + flag.Parse() + if VeleroCfg.InstallVelero { + Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed()) + } + }) + + AfterEach(func() { + if VeleroCfg.InstallVelero { + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + } + }) + + It("Backups in object storage should be synced to a new Velero successfully", func() { + test.Init() + By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS)) + Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", test.testNS)) + + defer func() { + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + + By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + } + Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + }) + + By("Uninstall velero", func() { + Expect(VeleroUninstall(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + }) + + By("Install velero", func() { + VeleroCfg.ObjectStoreProvider = "" + Expect(VeleroInstall(test.ctx, &VeleroCfg, "", false)).To(Succeed()) + }) + + By("Check all backups in object storage are synced to Velero", func() { + Expect(test.IsBackupsSynced()).To(Succeed(), fmt.Sprintf("Failed to sync backup %s from object storage", test.backupName)) + }) + }) + + It("Deleted backups in object storage are synced to be deleted in Velero", func() { + test.Init() + By(fmt.Sprintf("Prepare workload as target to backup by creating namespace in %s namespace", test.testNS), func() { + Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(), + fmt.Sprintf("Failed to create %s namespace", test.testNS)) + }) + + defer func() { + Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), + fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) + }() + + By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() { + if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "") + } + Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS)) + }) + + By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() { + err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) + }) + + By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() { + err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1) + Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) + }) + + By("Check if backups are deleted as a result of sync from BSL", func() { + Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Failed to check backup %s deleted", test.backupName)) + }) + }) +} + +func (b *SyncBackups) IsBackupsSynced() error { + return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (bool, error) { + if exist, err := IsBackupExist(b.ctx, VeleroCfg.VeleroCLI, b.backupName); err != nil { + return false, err + } else { + if exist { + return true, nil + } else { + return false, nil + } + } + }) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 8540cf619..0c582f600 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -91,6 +91,7 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) var _ = Describe("[Backups][Deletion] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", Backup_deletion_with_restic) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) +var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) func TestE2e(t *testing.T) { // Skip running E2E tests when running only "short" tests because: diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index faa317c72..c1e6046cd 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -596,3 +596,35 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri } return nil } + +func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) { + args := []string{"backup", "get", backupName} + cmd := exec.CommandContext(ctx, veleroCLI, args...) + return veleroexec.RunCommand(cmd) +} + +func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) { + if _, outerr, err := GetBackup(ctx, veleroCLI, backupName); err != nil { + if err != nil { + if strings.Contains(outerr, "not found") { + return false, nil + } + return false, err + } + } + return true, nil +} + +func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error { + return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { + if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil { + return false, err + } else { + if exist { + return false, nil + } else { + return true, nil + } + } + }) +}