diff --git a/test/e2e/Makefile b/test/e2e/Makefile index f84ca3b53..f6d8e7afe 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -82,6 +82,8 @@ ADDITIONAL_BSL_BUCKET ?= ADDITIONAL_BSL_PREFIX ?= ADDITIONAL_BSL_CONFIG ?= +FEATURES ?= + .PHONY:ginkgo ginkgo: # Make sure ginkgo is in $GOPATH/bin go get github.com/onsi/ginkgo/ginkgo @@ -115,6 +117,7 @@ run: ginkgo -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ + -features=$(FEATURES) \ -install-velero=$(INSTALL_VELERO) \ -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ -kibishii-directory=$(KIBISHII_DIRECTORY) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 7bf79b9a1..f5f3146d6 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -42,7 +42,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { var ( backupName, restoreName string ) - + kibishiiNamespace := "kibishii-workload" client, err := NewTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") @@ -55,11 +55,14 @@ func BackupRestoreTest(useVolumeSnapshots bool) { UUIDgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if VeleroCfg.InstallVelero { - Expect(VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed()) + Expect(VeleroInstall(context.Background(), &VeleroCfg, "EnableCSI", useVolumeSnapshots)).To(Succeed()) } }) AfterEach(func() { + DeleteNamespace(context.Background(), client, kibishiiNamespace, true) + Expect(err).To(Succeed(), fmt.Sprintf("failed to delete the namespace %q", kibishiiNamespace)) + if VeleroCfg.InstallVelero { err = VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace) Expect(err).To(Succeed()) @@ -72,11 +75,11 @@ func BackupRestoreTest(useVolumeSnapshots bool) { restoreName = "restore-" + UUIDgen.String() // Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on // KinD. So use the kind installation for Kibishii. - Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) - It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() { + XIt("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() { if VeleroCfg.AdditionalBSLProvider == "" { Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests") } @@ -125,7 +128,7 @@ func BackupRestoreTest(useVolumeSnapshots bool) { backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen) restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } - Expect(RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(RunKibishiiTests(client, VeleroCfg, backupName, restoreName, bsl, kibishiiNamespace, useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0f02ef33e..d2c216d7d 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -75,16 +75,23 @@ func backup_deletion_test(useVolumeSnapshots bool) { When("kibishii is the sample workload", func() { It("Deleted backups are deleted from object storage and backups deleted from object storage can be deleted locally", func() { backupName = "backup-" + UUIDgen.String() - Expect(runBackupDeletionTests(client, VeleroCfg.VeleroCLI, VeleroCfg.CloudProvider, VeleroCfg.VeleroNamespace, backupName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, VeleroCfg.KibishiiDirectory)).To(Succeed(), + Expect(runBackupDeletionTests(client, VeleroCfg, backupName, "", useVolumeSnapshots, VeleroCfg.KibishiiDirectory)).To(Succeed(), "Failed to run backup deletion test") }) }) } // runUpgradeTests runs upgrade test on the provider by kibishii. -func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNamespace, backupName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile, bslPrefix, bslConfig, kibishiiDirectory string) error { +func runBackupDeletionTests(client TestClient, veleroCfg VerleroConfig, backupName, backupLocation string, + useVolumeSnapshots bool, kibishiiDirectory string) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + veleroCLI := VeleroCfg.VeleroCLI + providerName := VeleroCfg.CloudProvider + veleroNamespace := VeleroCfg.VeleroNamespace + registryCredentialFile := VeleroCfg.RegistryCredentialFile + bslPrefix := VeleroCfg.BSLPrefix + bslConfig := VeleroCfg.BSLConfig + veleroFeatures := VeleroCfg.Features if err := CreateNamespace(oneHourTimeout, client, deletionTest); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", deletionTest) @@ -95,7 +102,7 @@ func runBackupDeletionTests(client TestClient, veleroCLI, providerName, veleroNa } }() - if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, kibishiiDirectory); err != nil { + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, deletionTest, registryCredentialFile, veleroFeatures, kibishiiDirectory); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) } err := ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 92d85123a..6c7d1b78a 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -140,7 +140,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { By("Deploy sample workload of Kibishii", func() { Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, VeleroCfg.CloudProvider, - bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.KibishiiDirectory)).To(Succeed()) + bslDeletionTestNs, VeleroCfg.RegistryCredentialFile, VeleroCfg.Features, VeleroCfg.KibishiiDirectory)).To(Succeed()) }) // Restic can not backup PV only, so pod need to be labeled also diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 588641b0b..95900a773 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -63,14 +63,15 @@ func init() { flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") + flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") } var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) // Test backup and restore of Kibishi using restic -var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) +var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) -var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) +var _ = Describe("[Basic][Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) var _ = Describe("[Basic] Backup/restore of cluster resources", ResourcesCheckTest) diff --git a/test/e2e/types.go b/test/e2e/types.go index 253ea067b..3e4cd7168 100644 --- a/test/e2e/types.go +++ b/test/e2e/types.go @@ -49,6 +49,7 @@ type VerleroConfig struct { AddBSLPlugins string InstallVelero bool KibishiiDirectory string + Features string } type SnapshotCheckPoint struct { diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 9354d15cc..4aa31d878 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -149,7 +149,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, upgradeFromVelero Upgrade By("Deploy sample workload of Kibishii", func() { Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, client, tmpCfg1.CloudProvider, - upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.KibishiiDirectory)).To(Succeed()) + upgradeNamespace, tmpCfg1.RegistryCredentialFile, tmpCfg1.Features, tmpCfg1.KibishiiDirectory)).To(Succeed()) }) By(fmt.Sprintf("Backup namespace %s", upgradeNamespace), func() { diff --git a/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml b/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml new file mode 100644 index 000000000..6b0d970eb --- /dev/null +++ b/test/e2e/util/csi/AzureVolumeSnapshotClass.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: velero + labels: + velero.io/csi-volumesnapshot-class: "true" +driver: disk.csi.azure.com +deletionPolicy: Retain \ No newline at end of file diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 688084b76..06d7cdaea 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -136,3 +136,8 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error fmt.Println(args) return exec.CommandContext(ctx, "kubectl", args...).Run() } + +func KubectlApplyByFile(ctx context.Context, file string) error { + args := []string{"apply", "-f", file, "--force=true"} + return exec.CommandContext(ctx, "kubectl", args...).Run() +} diff --git a/test/e2e/util/kibishii/kibishii_utils.go b/test/e2e/util/kibishii/kibishii_utils.go index 8d5c49040..5b67e795b 100644 --- a/test/e2e/util/kibishii/kibishii_utils.go +++ b/test/e2e/util/kibishii/kibishii_utils.go @@ -20,38 +20,43 @@ import ( "fmt" "os/exec" "strconv" + "strings" "time" "github.com/pkg/errors" - "golang.org/x/net/context" - veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" + . "github.com/vmware-tanzu/velero/test/e2e" . "github.com/vmware-tanzu/velero/test/e2e/util/k8s" . "github.com/vmware-tanzu/velero/test/e2e/util/velero" + "golang.org/x/net/context" ) const ( - kibishiiNamespace = "kibishii-workload" - jumpPadPod = "jump-pad" + jumpPadPod = "jump-pad" ) // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, - useVolumeSnapshots bool, registryCredentialFile string, kibishiiDirectory string) error { +func RunKibishiiTests(client TestClient, veleroCfg VerleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, + useVolumeSnapshots bool) error { oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + veleroCLI := VeleroCfg.VeleroCLI + providerName := VeleroCfg.CloudProvider + veleroNamespace := VeleroCfg.VeleroNamespace + registryCredentialFile := VeleroCfg.RegistryCredentialFile + veleroFeatures := VeleroCfg.Features + kibishiiDirectory := VeleroCfg.KibishiiDirectory + if err := CreateNamespace(oneHourTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } - defer func() { - if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { - fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) - } - }() - if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory); err != nil { + + if err := KibishiiPrepareBeforeBackup(oneHourTimeout, client, providerName, + kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil { + if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, + kibishiiNamespace, backupLocation, useVolumeSnapshots, ""); err != nil { RunDebug(context.Background(), veleroCLI, veleroNamespace, backupName, "") return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -90,10 +95,14 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac return nil } -func installKibishii(ctx context.Context, namespace string, cloudPlatform string, kibishiiDirectory string) error { +func installKibishii(ctx context.Context, namespace string, cloudPlatform, veleroFeatures, kibishiiDirectory string) error { + if strings.EqualFold(cloudPlatform, "azure") && strings.EqualFold(veleroFeatures, "EnableCSI") { + cloudPlatform = "azure-csi" + } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", - kibishiiDirectory+cloudPlatform) + //"github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/"+cloudPlatform) + "github.com/danfengliu/distributed-data-generator/kubernetes/yaml/"+cloudPlatform) _, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd) if err != nil { return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr) @@ -149,7 +158,7 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } -func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, kibishiiDirectory string) error { +func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClient, providerName, kibishiiNamespace, registryCredentialFile, veleroFeatures, kibishiiDirectory string) error { serviceAccountName := "default" // wait until the service account is created before patch the image pull secret @@ -161,7 +170,7 @@ func KibishiiPrepareBeforeBackup(oneHourTimeout context.Context, client TestClie return errors.Wrapf(err, "failed to patch the service account %q under the namespace %q", serviceAccountName, kibishiiNamespace) } - if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, kibishiiDirectory); err != nil { + if err := installKibishii(oneHourTimeout, kibishiiNamespace, providerName, veleroFeatures, kibishiiDirectory); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index 9c5c30c4a..5d445441f 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "os/exec" + "strings" "time" "github.com/pkg/errors" @@ -199,6 +200,14 @@ func installVeleroServer(ctx context.Context, cli string, options *installOption } if len(options.Features) > 0 { args = append(args, "--features", options.Features) + if strings.EqualFold(options.Features, "EnableCSI") { + if strings.EqualFold(options.ProviderName, "Azure") { + if err := KubectlApplyByFile(ctx, "util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { + return err + } + } + + } } if err := createVelereResources(ctx, cli, namespace, args, options.RegistryCredentialFile, options.ResticHelperImage); err != nil {