Add E2E test of backups sync from BSL

Signed-off-by: Ming <mqiu@vmware.com>

changelogs/unreleased/4545-mqiu

@@ -0,0 +1 @@
+Add E2E test of backups sync from BSL

test/e2e/backups/sync_backups.go

@@ -0,0 +1,155 @@
+/*
+ *
+ * Copyright the Velero contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+//Refer to https://github.com/vmware-tanzu/velero/issues/4253
+package backups
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/providers"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+)
+
+type SyncBackups struct {
+	testNS     string
+	backupName string
+	ctx        context.Context
+}
+
+func (b *SyncBackups) Init() {
+	rand.Seed(time.Now().UnixNano())
+	UUIDgen, _ = uuid.NewRandom()
+	b.testNS = "sync-bsl-test-" + UUIDgen.String()
+	b.backupName = "sync-bsl-test-" + UUIDgen.String()
+	b.ctx, _ = context.WithTimeout(context.Background(), time.Duration(time.Minute*10))
+}
+
+func BackupsSyncTest() {
+	test := new(SyncBackups)
+	client, err := NewTestClient()
+	if err != nil {
+		println(err.Error())
+	}
+	Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")
+
+	BeforeEach(func() {
+		flag.Parse()
+		if VeleroCfg.InstallVelero {
+			Expect(VeleroInstall(context.Background(), &VeleroCfg, "", false)).To(Succeed())
+		}
+	})
+
+	AfterEach(func() {
+		if VeleroCfg.InstallVelero {
+			Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+		}
+	})
+
+	It("Backups in object storage should be synced to a new Velero successfully", func() {
+		test.Init()
+		By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS))
+		Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(),
+			fmt.Sprintf("Failed to create %s namespace", test.testNS))
+
+		defer func() {
+			Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
+		}()
+
+		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
+			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil {
+				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
+			}
+			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))
+		})
+
+		By("Uninstall velero", func() {
+			Expect(VeleroUninstall(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed())
+		})
+
+		By("Install velero", func() {
+			VeleroCfg.ObjectStoreProvider = ""
+			Expect(VeleroInstall(test.ctx, &VeleroCfg, "", false)).To(Succeed())
+		})
+
+		By("Check all backups in object storage are synced to Velero", func() {
+			Expect(test.IsBackupsSynced()).To(Succeed(), fmt.Sprintf("Failed to sync backup %s from object storage", test.backupName))
+		})
+	})
+
+	It("Deleted backups in object storage are synced to be deleted in Velero", func() {
+		test.Init()
+		By(fmt.Sprintf("Prepare workload as target to backup by creating namespace in %s namespace", test.testNS), func() {
+			Expect(CreateNamespace(test.ctx, client, test.testNS)).To(Succeed(),
+				fmt.Sprintf("Failed to create %s namespace", test.testNS))
+		})
+
+		defer func() {
+			Expect(DeleteNamespace(test.ctx, client, test.testNS, false)).To(Succeed(),
+				fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
+		}()
+
+		By(fmt.Sprintf("Backup the workload in %s namespace", test.testNS), func() {
+			if err = VeleroBackupNamespace(test.ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, test.testNS, "", false); err != nil {
+				RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, test.backupName, "")
+			}
+			Expect(err).To(Succeed(), fmt.Sprintf("Failed to backup %s namespace", test.testNS))
+		})
+
+		By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() {
+			err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix)
+			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
+		})
+
+		By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() {
+			err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1)
+			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
+		})
+
+		By("Check if backups are deleted as a result of sync from BSL", func() {
+			Expect(WaitBackupDeleted(test.ctx, VeleroCfg.VeleroCLI, test.backupName, time.Minute*10)).To(Succeed(), fmt.Sprintf("Failed to check backup %s deleted", test.backupName))
+		})
+	})
+}
+
+func (b *SyncBackups) IsBackupsSynced() error {
+	return wait.PollImmediate(10*time.Second, 10*time.Minute, func() (bool, error) {
+		if exist, err := IsBackupExist(b.ctx, VeleroCfg.VeleroCLI, b.backupName); err != nil {
+			return false, err
+		} else {
+			if exist {
+				return true, nil
+			} else {
+				return false, nil
+			}
+		}
+	})
+}

test/e2e/e2e_suite_test.go

@@ -91,6 +91,7 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on
 var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
 var _ = Describe("[Backups][Deletion] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", Backup_deletion_with_restic)
 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
+var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
 
 func TestE2e(t *testing.T) {
 	// Skip running E2E tests when running only "short" tests because:

test/e2e/util/velero/velero_utils.go

@@ -596,3 +596,35 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri
 	}
 	return nil
 }
+
+func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) {
+	args := []string{"backup", "get", backupName}
+	cmd := exec.CommandContext(ctx, veleroCLI, args...)
+	return veleroexec.RunCommand(cmd)
+}
+
+func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) {
+	if _, outerr, err := GetBackup(ctx, veleroCLI, backupName); err != nil {
+		if err != nil {
+			if strings.Contains(outerr, "not found") {
+				return false, nil
+			}
+			return false, err
+		}
+	}
+	return true, nil
+}
+
+func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error {
+	return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) {
+		if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil {
+			return false, err
+		} else {
+			if exist {
+				return false, nil
+			} else {
+				return true, nil
+			}
+		}
+	})
+}