diff --git a/test/Makefile b/test/Makefile index 276cc76af..b3cf6eadd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -41,25 +41,41 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin + # Try to not modify PATH if possible GOBIN := $(REPO_ROOT)/.go/bin + TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) + GINKGO := $(GOBIN)/ginkgo + KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize + OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin + # Please reference to this document for Ginkgo label spec format. # https://onsi.github.io/ginkgo/#spec-labels GINKGO_LABELS ?= + # When --fail-fast is set, the entire suite will stop when the first failure occurs. # Enable --fail-fast by default. # https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure FAIL_FAST ?= false + VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero + VELERO_IMAGE ?= velero/velero:main + PLUGINS ?= + +# Flag used to tell E2E whether the Velero vSphere plugin is installed. +HAS_VSPHERE_PLUGIN ?= false + RESTORE_HELPER_IMAGE ?= + #Released version only UPGRADE_FROM_VELERO_VERSION ?= v1.13.2,v1.14.1 + # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -150,7 +166,8 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - --fail-fast=$(FAIL_FAST) + --fail-fast=$(FAIL_FAST) \ + --has-vsphere-plugin=$(HAS_VSPHERE_PLUGIN) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 65231e01a..331c2920a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -78,6 +78,7 @@ These configuration parameters are expected as values to the following command l 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. +1. `--has-vsphere-plugin`: A switch to indicate whether the Velero vSphere plugin is installed for vSphere environment. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -129,12 +130,13 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. +1. `HAS_VSPHERE_PLUGIN`: `--has-vsphere-plugin`. Optional. ### Examples -Basic examples: +#### Basic examples: 1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider: @@ -208,7 +210,7 @@ ADDITIONAL_CREDS_FILE=/path/to/azure-creds \ make test-e2e ``` -Upgrade examples: +#### Upgrade examples: 1. Run Velero upgrade tests with pre-upgrade version: @@ -234,7 +236,7 @@ UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \ make test-e2e ``` -Migration examples: +#### Migration examples: 1. Migration between 2 cluster of the same provider tests: @@ -275,7 +277,7 @@ GINKGO_LABELS="Migration" \ make test-e2e ``` -## 5. Filtering tests +#### Filtering tests In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. @@ -285,7 +287,6 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE `GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels). -### Examples E2E tests can be run with specific cases to be included and/or excluded using the commands below: 1. Run Velero tests with specific cases to be included: @@ -316,6 +317,44 @@ In this example, cases are labelled as * `Migration` and `Restic` will be skipped. +#### VKS environment test +1. Run the CSI data mover test. + +`HAS_VSPHERE_PLUGIN` should be set to `false` to not install the Velero vSphere plugin. +``` bash +CLOUD_PROVIDER=vsphere \ +DEFAULT_CLUSTER=wl-antreav1301 \ +STANDBY_CLUSTER=wl-antreav1311 \ +DEFAULT_CLUSTER_NAME=192.168.0.4 \ +STANDBY_CLUSTER_NAME=192.168.0.3 \ +FEATURES=EnableCSI \ +PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +HAS_VSPHERE_PLUGIN=false \ +OBJECT_STORE_PROVIDER=aws \ +CREDS_FILE=$HOME/aws-credential \ +BSL_CONFIG=region=us-east-1 \ +BSL_BUCKET=nightly-normal-account4-test \ +BSL_PREFIX=nightly \ +ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ +ADDITIONAL_BSL_CONFIG=region=us-east-1 \ +ADDITIONAL_BSL_BUCKET=nightly-restrict-account-test \ +ADDITIONAL_BSL_PREFIX=nightly \ +ADDITIONAL_CREDS_FILE=$HOME/aws-credential \ +VELERO_IMAGE=gcr.io/velero-gcp/velero:main \ +RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \ +VERSION=main \ +SNAPSHOT_MOVE_DATA=true \ +STANDBY_CLUSTER_CLOUD_PROVIDER=vsphere \ +STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ +STANDBY_CLUSTER_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +DISABLE_INFORMER_CACHE=true \ +REGISTRY_CREDENTIAL_FILE=$HOME/.docker/config.json \ +GINKGO_LABELS=Migration \ +KIBISHII_DIRECTORY=$HOME/kibishii/kubernetes/yaml/ \ +make test-e2e +``` + ## 6. Full Tests execution As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 6f47ed2d9..cfdc89724 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -197,9 +197,9 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { secretKey, )).To(Succeed()) - bsls := []string{"default", additionalBsl} + BSLs := []string{"default", additionalBsl} - for _, bsl := range bsls { + for _, bsl := range BSLs { backupName = fmt.Sprintf("backup-%s", bsl) restoreName = fmt.Sprintf("restore-%s", bsl) // We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370 @@ -209,8 +209,8 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } veleroCfg.ProvideSnapshotsVolumeParam = !provideSnapshotVolumesParmInBackup - workloadNmespace := kibishiiNamespace + bsl - Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, workloadNmespace, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), + workloadNS := kibishiiNamespace + bsl + Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, workloadNS, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0443b7444..e4bf69702 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -34,7 +34,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -// Test backup and restore of Kibishi using restic +// Test backup and restore of Kibishii using restic func BackupDeletionWithSnapshots() { backup_deletion_test(true) @@ -99,8 +99,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc providerName := veleroCfg.CloudProvider veleroNamespace := veleroCfg.VeleroNamespace registryCredentialFile := veleroCfg.RegistryCredentialFile - bslPrefix := veleroCfg.BSLPrefix - bslConfig := veleroCfg.BSLConfig veleroFeatures := veleroCfg.Features for _, ns := range workloadNamespaceList { if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { @@ -143,7 +141,8 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) for _, ns := range workloadNamespaceList { - if providerName == Vsphere && useVolumeSnapshots { + if useVolumeSnapshots && + veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") @@ -152,7 +151,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc } } } - err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix) if err != nil { return err } @@ -164,9 +163,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc for _, ns := range workloadNamespaceList { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -178,9 +180,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") // Get all snapshots base on backup name, regardless of namespaces - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -206,26 +211,34 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc return err } + // Verify snapshots are deleted after backup deletion. if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + snapshotCheckPoint.ExpectCount = 0 + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + return errors.Wrap(err, "fail to verify snapshots are deleted in provider.") } } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + // Verify backup metadata files are deleted in OSS after backup deletion. + err = ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 5, + ) if err != nil { return err } - if useVolumeSnapshots { - if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint); err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - } - } // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded // Sleep for more than 15 seconds to avoid this issue. @@ -242,13 +255,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) - err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) - if err != nil { + if err := DeleteObjectsInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + ); err != nil { return err } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) - if err != nil { + if err := ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 1, + ); err != nil { return err } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 1f101e055..5135c4b0b 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -122,19 +122,31 @@ func TTLTest() { var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(ctx, time.Hour, test.testNS, 2)).To(Succeed()) }) } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList) - Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint, err = GetSnapshotCheckPoint( + client, + veleroCfg, + 2, + test.testNS, + test.backupName, + KibishiiPVCNameList, + ) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + + Expect( + CheckSnapshotsInProvider( + veleroCfg, + test.backupName, + snapshotCheckPoint, + false, + ), + ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() { @@ -188,9 +200,13 @@ func TTLTest() { By("PersistentVolume snapshots should be deleted", func() { if useVolumeSnapshots { - Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint.ExpectCount = 0 + Expect(CheckSnapshotsInProvider( + veleroCfg, + test.backupName, + snapshotCheckPoint, + false, + )).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } }) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 6c3ccee88..51584d85b 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -36,12 +36,12 @@ import ( const ( // Please make sure length of this namespace should be shorter, - // otherwise ResticRepositories name verification will be wrong - // when making combination of ResticRepositories name(max length is 63) + // otherwise BackupRepositories name verification will be wrong + // when making combination of BackupRepositories name(max length is 63) bslDeletionTestNs = "bsl-deletion" ) -// Test backup and restore of Kibishi using restic +// Test backup and restore of Kibishii using restic func BslDeletionWithSnapshots() { BslDeletionTest(true) @@ -134,17 +134,17 @@ func BslDeletionTest(useVolumeSnapshots bool) { )).To(Succeed()) }) - backupName_1 := "backup1-" + UUIDgen.String() - backupName_2 := "backup2-" + UUIDgen.String() - backupLocation_1 := "default" - backupLocation_2 := additionalBsl - podName_1 := "kibishii-deployment-0" - podName_2 := "kibishii-deployment-1" + backupName1 := "backup1-" + UUIDgen.String() + backupName2 := "backup2-" + UUIDgen.String() + backupLocation1 := "default" + backupLocation2 := additionalBsl + podName1 := "kibishii-deployment-0" + podName2 := "kibishii-deployment-1" - label_1 := "for=1" + label1 := "for=1" // TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed - //label_2 := "for!=1" - label_2 := "for=2" + //label2 := "for!=1" + label2 := "for=2" By("Create namespace for sample workload", func() { Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed()) }) @@ -157,33 +157,33 @@ func BslDeletionTest(useVolumeSnapshots bool) { // Restic can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { - Expect(AddLabelToPod(context.Background(), podName_1, bslDeletionTestNs, label_1)).To(Succeed()) - Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label1)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label2)).To(Succeed()) }) By("Get all 2 PVCs of Kibishii and label them separately ", func() { - pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1) + pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName1) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc1 := pvc[0] - pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2) + pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName2) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc2 := pvc[0] - Expect(AddLabelToPvc(context.Background(), pvc1, bslDeletionTestNs, label_1)).To(Succeed()) - Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label_2)).To(Succeed()) + Expect(AddLabelToPvc(context.Background(), pvc1, bslDeletionTestNs, label1)).To(Succeed()) + Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label2)).To(Succeed()) }) var BackupCfg BackupConfig - BackupCfg.BackupName = backupName_1 + BackupCfg.BackupName = backupName1 BackupCfg.Namespace = bslDeletionTestNs - BackupCfg.BackupLocation = backupLocation_1 + BackupCfg.BackupLocation = backupLocation1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots - BackupCfg.Selector = label_1 - By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { + BackupCfg.Selector = label1 + By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, @@ -193,10 +193,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - BackupCfg.BackupName = backupName_2 - BackupCfg.BackupLocation = backupLocation_2 - BackupCfg.Selector = label_2 - By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { + BackupCfg.BackupName = backupName2 + BackupCfg.BackupLocation = backupLocation2 + BackupCfg.Selector = label2 + By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") @@ -205,88 +205,99 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, bslDeletionTestNs, 2)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName1), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_2), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } if veleroCfg.CloudProvider != VanillaZFS { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName1, + []string{podName1}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) } - } else { // For Restics - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } - By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName1), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_1, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName2), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_2, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName2, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1) + backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation1) Expect(err).To(Succeed()) - backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2) + backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation2) Expect(err).To(Succeed()) backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...) - By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() { + By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation1), func() { backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) - By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) - By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() { - Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) + By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation1), func() { + Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation1)).To(Succeed()) Expect(WaitForBackupsToBeDeleted(context.Background(), backupsInBSL1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) @@ -298,10 +309,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case @@ -312,46 +323,53 @@ func BslDeletionTest(useVolumeSnapshots bool) { // }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { + if veleroCfg.HasVspherePlugin { + By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName1, []string{podName1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) - } else { // For Restic - By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be deleted in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 0)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should still exist in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b4cd32de..36cb0b0e5 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -103,6 +103,7 @@ func init() { flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") + flag.BoolVar(&VeleroCfg.HasVspherePlugin, "has-vsphere-plugin", false, "a switch for installing vSphere plugin.") } // Add label [SkipVanillaZfs]: diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 8316126e6..ade423b76 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -25,7 +25,6 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "golang.org/x/mod/semver" . "github.com/vmware-tanzu/velero/test" util "github.com/vmware-tanzu/velero/test/util/csi" @@ -142,33 +141,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if veleroCLI2Version.VeleroVersion == "self" { veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { - fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) - OriginVeleroCfg.VeleroImage = "" - OriginVeleroCfg.RestoreHelperImage = "" - OriginVeleroCfg.Plugins = "" + OriginVeleroCfg, err = SetImagesToDefaultValues( + OriginVeleroCfg, + veleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.") - versionWithoutPatch := semver.MajorMinor(veleroCLI2Version.VeleroVersion) - // Read migration case needs plugins from the PluginsMatrix map. - migrationNeedPlugins, ok := PluginsMatrix[versionWithoutPatch] - Expect(ok).To(BeTrue()) - - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = migrationNeedPlugins[Azure][0] - } - if OriginVeleroCfg.CloudProvider == AWS { - OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] - } - // Because Velero CSI plugin is deprecated in v1.14, - // only need to install it for version lower than v1.14. - if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && - semver.Compare(versionWithoutPatch, "v1.14") < 0 { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] - } - if OriginVeleroCfg.SnapshotMoveData { - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] - } - } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } @@ -253,8 +231,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + // Only wait for the snapshots.backupdriver.cnsdp.vmware.com + // when the vSphere plugin is used. + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, migrationNamespace, kibishiiWorkerCount)).To(Succeed()) @@ -282,9 +261,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index cfda01fef..3c371064c 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -124,14 +124,16 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC veleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", veleroCLI2Version.VeleroVersion), func() { - //Set VeleroImage and RestoreHelperImage to blank - //VeleroImage and RestoreHelperImage should be the default value in originalCli tmpCfgForOldVeleroInstall := veleroCfg tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI - tmpCfgForOldVeleroInstall.VeleroImage = "" - tmpCfgForOldVeleroInstall.RestoreHelperImage = "" - tmpCfgForOldVeleroInstall.Plugins = "" + + tmpCfgForOldVeleroInstall, err = SetImagesToDefaultValues( + tmpCfgForOldVeleroInstall, + veleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), "Fail to set the images for upgrade-from Velero installation.") + tmpCfgForOldVeleroInstall.UploaderType = "" version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true) Expect(err).To(Succeed(), "Fail to get Velero version") @@ -145,9 +147,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots tmpCfgForOldVeleroInstall.UseNodeAgent = false } - //TODO: Remove this setting when upgrade path is from 1.13 to higher - //TODO: version, or self version 1.12 and older versions have no this parameter. - tmpCfgForOldVeleroInstall.WithoutDisableInformerCacheParam = true Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, @@ -190,8 +189,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace, 2)).To(Succeed()) @@ -203,9 +201,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, upgradeNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } diff --git a/test/testdata/volume-snapshot-class/vsphere.yaml b/test/testdata/volume-snapshot-class/vsphere.yaml new file mode 100644 index 000000000..08ea9b222 --- /dev/null +++ b/test/testdata/volume-snapshot-class/vsphere.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: csi.vsphere.vmware.com +kind: VolumeSnapshotClass +metadata: + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" + name: volumesnapshotclass-delete +parameters: + svVolumeSnapshotClass: volumesnapshotclass-delete diff --git a/test/types.go b/test/types.go index 961eebe5d..527068599 100644 --- a/test/types.go +++ b/test/types.go @@ -38,6 +38,8 @@ const AWS = "aws" const GCP = "gcp" const Vsphere = "vsphere" const CSI = "csi" +const Velero = "velero" +const VeleroRestoreHelper = "velero-restore-helper" const UploaderTypeRestic = "restic" @@ -118,6 +120,7 @@ type VeleroConfig struct { ServiceAccountNameToInstall string EKSPolicyARN string FailFast bool + HasVspherePlugin bool } type VeleroCfgInPerf struct { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index a749538d7..be503ff0b 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -65,8 +65,15 @@ func GetKibishiiPVCNameList(workerCount int) []string { } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, - useVolumeSnapshots, defaultVolumesToFsBackup bool) error { +func RunKibishiiTests( + veleroCfg VeleroConfig, + backupName string, + restoreName string, + backupLocation string, + kibishiiNamespace string, + useVolumeSnapshots bool, + defaultVolumesToFsBackup bool, +) error { pvCount := len(KibishiiPVCNameList) client := *veleroCfg.ClientToInstallVelero oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) @@ -120,9 +127,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // Checkpoint for a successful backup if useVolumeSnapshots { - if providerName == Vsphere { + if veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete - // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") @@ -132,9 +138,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -167,9 +176,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrap(err, "failed to get snapshot checkPoint") } } else { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, SnapshotCheckPoint{}) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + SnapshotCheckPoint{}, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 46e42894e..a7c68dc37 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -28,14 +28,14 @@ import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/internal/volume" - . "github.com/vmware-tanzu/velero/test" + velerotest "github.com/vmware-tanzu/velero/test" velero "github.com/vmware-tanzu/velero/test/util/velero" ) type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) } @@ -75,13 +75,13 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case AWS, Vsphere: + case velerotest.AWS, velerotest.Vsphere: aws := AWSStorage("") s = &aws - case GCP: + case velerotest.GCP: gcs := GCSStorage("") s = &gcs - case Azure: + case velerotest.Azure: az := AzureStorage("") s = &az default: @@ -102,7 +102,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl bslPrefix = getFullPrefix(bslPrefix, subPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return false, errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) } @@ -116,66 +116,77 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, } s, err := getProvider(objectStoreProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + return errors.Wrapf(err, "Fail to delete %s", bslPrefix) } return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) +func CheckSnapshotsInProvider( + veleroCfg velerotest.VeleroConfig, + backupName string, + snapshotCheckPoint velerotest.SnapshotCheckPoint, + checkStandbyCluster bool, +) error { + fmt.Printf("|| VERIFICATION || - there should %d snapshots in provider, backup %s\n", + snapshotCheckPoint.ExpectCount, backupName) - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", veleroCfg.CloudProvider) return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) - } - - snapshotCheckPoint.ExpectCount = 0 - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName)) - } - fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName) - return nil -} - -func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) - - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.Vsphere && !veleroCfg.HasVspherePlugin { + fmt.Printf("Skip snapshot check for vSphere environment that doesn't have Velero vSphere plugin.") return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + if veleroCfg.CloudCredentialsFile == "" { + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", veleroCfg.CloudProvider)) } - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName)) + cloudCredentialsFile := veleroCfg.CloudCredentialsFile + bslBucket := veleroCfg.BSLBucket + bslConfig := veleroCfg.BSLConfig + if checkStandbyCluster { + bslBucket = veleroCfg.AdditionalBSLBucket + + // Only vSphere environment's snapshot on standby cluster is related to BSL. + if veleroCfg.CloudProvider == velerotest.Vsphere { + cloudCredentialsFile = veleroCfg.AdditionalBSLCredentials + bslConfig = veleroCfg.AdditionalBSLConfig + } } - fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider) + + err := isSnapshotExisted( + veleroCfg.CloudProvider, + cloudCredentialsFile, + bslBucket, + bslConfig, + backupName, + snapshotCheckPoint, + ) + if err != nil { + return errors.Wrapf(err, "|| UNEXPECTED || - Snapshots count is not as expected after backup %s", backupName) + } + + fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in provider %s\n", backupName, veleroCfg.CloudProvider) return nil } -func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { +func isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error { s, err := getProvider(cloudProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return errors.Wrapf(err, "Cloud provider %s is not valid", cloudProvider) } - if cloudProvider == Vsphere { + if cloudProvider == velerotest.Vsphere { var retSnapshotIDs []string ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2) defer ctxCancel() retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot CRs of backup%s", backupName) } bslPrefix := "plugins" @@ -201,7 +212,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } else { err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot of backup %s", backupName) } } return nil @@ -221,7 +232,7 @@ func GetVolumeInfoMetadataContent( fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider)) + return nil, errors.Wrapf(err, "Cloud provider %s is not valid", objectStoreProvider) } return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName) diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 81204a986..9da1c4b83 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -320,7 +320,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { + if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index cec0d6186..d99824663 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -36,6 +36,7 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/slices" + "golang.org/x/mod/semver" ver "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,67 +55,126 @@ const BackupObjectsPrefix = "backups" const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" -var PluginsMatrix = map[string]map[string][]string{ +var ImagesMatrix = map[string]map[string][]string{ "v1.10": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.10.2"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.10.2"}, }, "v1.11": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.11.1"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.11.1"}, }, "v1.12": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.12.4"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.12.4"}, }, "v1.13": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "velero": {"gcr.io/velero-gcp/velero:v1.13.2"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.13.2"}, }, "v1.14": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "velero": {"gcr.io/velero-gcp/velero:v1.14.1"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.14.1"}, }, "v1.15": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.15.0"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.15.0"}, }, "main": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "velero": {"gcr.io/velero-gcp/velero:main"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:main"}, }, } -func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, needDataMoverPlugin bool) ([]string, error) { +func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig, error) { + fmt.Printf("Get the images for version %s\n", version) + + ret := config + + ret.Plugins = "" + + versionWithoutPatch := semver.MajorMinor(version) + // Read migration case needs images from the PluginsMatrix map. + images, ok := ImagesMatrix[versionWithoutPatch] + if !ok { + return config, fmt.Errorf("fail to read the images for version %s from the ImagesMatrix", + versionWithoutPatch) + } + + ret.VeleroImage = images[Velero][0] + ret.RestoreHelperImage = images[VeleroRestoreHelper][0] + + if ret.CloudProvider == Azure { + ret.Plugins = images[Azure][0] + } + if ret.CloudProvider == AWS { + ret.Plugins = images[AWS][0] + } + // If HasVspherePlugin is false, only install the AWS plugin. + // If do nothing here, the default behavior is + // installing both AWS and vSphere plugins. + if ret.CloudProvider == Vsphere && + !ret.HasVspherePlugin { + ret.Plugins = images[AWS][0] + } + + // Because Velero CSI plugin is deprecated in v1.14, + // only need to install it for version lower than v1.14. + if strings.Contains(ret.Features, FeatureCSI) && + semver.Compare(versionWithoutPatch, "v1.14") < 0 { + ret.Plugins = ret.Plugins + "," + images[CSI][0] + } + if ret.SnapshotMoveData && ret.CloudProvider == Azure { + ret.Plugins = ret.Plugins + "," + images[AWS][0] + } + + return ret, nil +} + +func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { - cloudMap = PluginsMatrix[arr[0]+"."+arr[1]] + cloudMap = ImagesMatrix[arr[0]+"."+arr[1]] } if len(cloudMap) == 0 { - cloudMap = PluginsMatrix["main"] + cloudMap = ImagesMatrix["main"] if len(cloudMap) == 0 { return nil, errors.Errorf("fail to get plugins by version: main") } @@ -133,18 +193,6 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } } - // TODO: Is this section need? - if objectStoreProvider != cloudProvider { - pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) - } - for _, p := range pluginsForObjectStoreProvider { - if !slices.Contains(plugins, p) { - plugins = append(plugins, p) - } - } - } if needDataMoverPlugin { pluginsForDatamover, ok := cloudMap["datamover"] @@ -630,7 +678,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, false) + plugins, err := getPluginsByVersion(version, cloudProvider, false) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) } @@ -673,7 +721,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest { needDataMoverPlugin = true } - plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, needDataMoverPlugin) + plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) }