Add enable API group on k8s resources E2E test upon issue #5146

Signed-off-by: danfengl <danfengl@vmware.com>

test/e2e/basic/enable_api_group_versions.go

@@ -40,6 +40,128 @@ import (
 	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
 )
 
+func APIExtensionsVersionsTest() {
+	var (
+		backupName, restoreName string
+	)
+	resourceName := "apiextensions.k8s.io"
+	crdName := "rocknrollbands.music.example.io"
+	label := "for=backup"
+	srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
+	BeforeEach(func() {
+		if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" {
+			Skip("CRD with apiextension versions migration test needs 2 clusters")
+		}
+		Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
+		srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
+		Expect(err).ShouldNot(HaveOccurred())
+		dstVersions, err := GetAPIVersions(VeleroCfg.StandbyClient, resourceName)
+		Expect(err).ShouldNot(HaveOccurred())
+
+		Expect(srcVersions).Should(ContainElement("v1"), func() string {
+			Skip("CRD with apiextension versions srcVersions should have v1")
+			return ""
+		})
+		Expect(srcVersions).Should(ContainElement("v1beta1"), func() string {
+			Skip("CRD with apiextension versions srcVersions should have v1")
+			return ""
+		})
+		Expect(dstVersions).Should(ContainElement("v1"), func() string {
+			Skip("CRD with apiextension versions dstVersions should have v1")
+			return ""
+		})
+		Expect(len(srcVersions) > 1 && len(dstVersions) == 1).Should(Equal(true), func() string {
+			Skip("Source cluster should support apiextension v1 and v1beta1, destination cluster should only support apiextension v1")
+			return ""
+		})
+	})
+	AfterEach(func() {
+		if !VeleroCfg.Debug {
+			By("Clean backups after test", func() {
+				DeleteBackups(context.Background(), *VeleroCfg.DefaultClient)
+			})
+			if VeleroCfg.InstallVelero {
+				By("Uninstall Velero and delete CRD ", func() {
+					Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
+					Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
+						VeleroCfg.VeleroNamespace)).To(Succeed())
+					Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
+
+					Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
+					Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI,
+						VeleroCfg.VeleroNamespace)).To(Succeed())
+					Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
+				})
+			}
+			By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() {
+				Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
+				VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
+			})
+		}
+
+	})
+	Context("When EnableAPIGroupVersions flag is set", func() {
+		It("Enable API Group to B/R CRD APIExtensionsVersions", func() {
+			backupName = "backup-" + UUIDgen.String()
+			restoreName = "restore-" + UUIDgen.String()
+
+			By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() {
+				Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
+				VeleroCfg.ObjectStoreProvider = ""
+				VeleroCfg.Features = "EnableAPIGroupVersions"
+				Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
+			})
+
+			By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", VeleroCfg.DefaultCluster), func() {
+				Expect(installCRD(context.Background(), srcCrdYaml)).To(Succeed())
+				Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
+				Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed())
+			})
+
+			By("Backup CRD", func() {
+				var BackupCfg BackupConfig
+				BackupCfg.BackupName = backupName
+				BackupCfg.IncludeResources = "crd"
+				BackupCfg.IncludeClusterResources = true
+				BackupCfg.Selector = label
+				Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI,
+					VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string {
+					VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI,
+						VeleroCfg.VeleroNamespace, backupName)
+					return "Get backup logs"
+				})
+			})
+
+			By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() {
+				Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed())
+				VeleroCfg.ObjectStoreProvider = ""
+				VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient
+				Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed())
+			})
+
+			By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() {
+				Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
+			})
+
+			By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, VeleroCfg.StandbyCluster), func() {
+				Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
+			})
+
+			By("Restore CRD", func() {
+				Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI,
+					VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
+					RunDebug(context.Background(), VeleroCfg.VeleroCLI,
+						VeleroCfg.VeleroNamespace, "", restoreName)
+					return "Fail to restore workload"
+				})
+			})
+
+			By("Verify CRD restore ", func() {
+				Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
+			})
+		})
+	})
+}
 func APIGropuVersionsTest() {
 	var (
 		resource, group string
@@ -302,7 +424,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
 			}
 
 			// Assertion
-			if containsAll(annoSpec["annotations"], tc.want["annotations"]) != true {
+			if !containsAll(annoSpec["annotations"], tc.want["annotations"]) {
 				msg := fmt.Sprintf(
 					"actual annotations: %v, expected annotations: %v",
 					annoSpec["annotations"],
@@ -312,7 +434,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso
 			}
 
 			// Assertion
-			if containsAll(annoSpec["specs"], tc.want["specs"]) != true {
+			if !containsAll(annoSpec["specs"], tc.want["specs"]) {
 				msg := fmt.Sprintf(
 					"actual specs: %v, expected specs: %v",
 					annoSpec["specs"],
@@ -367,6 +489,21 @@ func deleteCRD(ctx context.Context, yaml string) error {
 	return nil
 }
 
+func deleteCRDByName(ctx context.Context, name string) error {
+	fmt.Println("Delete CRD", name)
+	cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", name, "--wait")
+
+	_, stderr, err := veleroexec.RunCommand(cmd)
+	if strings.Contains(stderr, "not found") {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrap(err, stderr)
+	}
+
+	return nil
+}
+
 func restartPods(ctx context.Context, ns string) error {
 	fmt.Printf("Restart pods in %s namespace.\n", ns)
 	cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns, "--wait=true")

test/e2e/e2e_suite_test.go

@@ -80,6 +80,7 @@ func init() {
 }
 
 var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest)
+var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)
 
 // Test backup and restore of Kibishi using restic
 var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
@@ -117,7 +118,6 @@ var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once t
 var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
 
 var _ = Describe("[Migration][Restic]", MigrationWithRestic)
-
 var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots)
 
 var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)

test/e2e/migration/migration.go

@@ -85,7 +85,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 					DeleteNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace, true)
 				})
 			}
-			By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() {
+			By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() {
 				Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed())
 				VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
 			})

test/e2e/testdata/enable_api_group_versions/case-a-source-v1beta1.yaml

@@ -0,0 +1,90 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: music-system/music-serving-cert
+    controller-gen.kubebuilder.io/version: v0.2.5
+  name: rocknrollbands.music.example.io
+spec:
+  group: music.example.io
+  names:
+    kind: RocknrollBand
+    listKind: RocknrollBandList
+    plural: rocknrollbands
+    singular: rocknrollband
+  scope: Namespaced
+  versions:
+    - name: v1
+      schema:
+        openAPIV3Schema:
+          description: RocknrollBand is the Schema for the rocknrollbands API
+          properties:
+            apiVersion:
+              description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources"
+              type: string
+            kind:
+              description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: RocknrollBandSpec defines the desired state of RocknrollBand
+              properties:
+                genre:
+                  type: string
+                leadSinger:
+                  type: string
+                numberComponents:
+                  format: int32
+                  type: integer
+              type: object
+            status:
+              description: RocknrollBandStatus defines the observed state of RocknrollBand
+              properties:
+                lastPlayed:
+                  type: string
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: RocknrollBand is the Schema for the rocknrollbands API
+          properties:
+            apiVersion:
+              description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources"
+              type: string
+            kind:
+              description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: RocknrollBandSpec defines the desired state of RocknrollBand
+              properties:
+                genre:
+                  type: string
+                numberComponents:
+                  format: int32
+                  type: integer
+              type: object
+            status:
+              description: RocknrollBandStatus defines the observed state of RocknrollBand
+              properties:
+                lastPlayed:
+                  type: string
+              required:
+                - lastPlayed
+              type: object
+          type: object
+      served: true
+      storage: false
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

test/e2e/util/k8s/common.go

@@ -49,7 +49,6 @@ func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace str
 
 		data[key] = contents
 	}
-
 	secret := builder.ForSecret(namespace, name).Data(data).Result()
 	_, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
 	return err
@@ -124,6 +123,45 @@ func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) {
 	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
 }
 
+func CRDShouldExist(ctx context.Context, name string) error {
+	return CRDCountShouldBe(ctx, name, 1)
+}
+
+func CRDShouldNotExist(ctx context.Context, name string) error {
+	return CRDCountShouldBe(ctx, name, 0)
+}
+
+func CRDCountShouldBe(ctx context.Context, name string, count int) error {
+	crdList, err := GetCRD(ctx, name)
+	if err != nil {
+		return errors.Wrap(err, "Fail to get CRDs")
+	}
+	len := len(crdList)
+	if len != count {
+		return errors.New(fmt.Sprintf("CRD count is expected as %d instead of %d", count, len))
+	}
+	return nil
+}
+
+func GetCRD(ctx context.Context, name string) ([]string, error) {
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  "kubectl",
+		Args: []string{"get", "crd"},
+	}
+
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "grep",
+		Args: []string{name},
+	}
+
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print $1}"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}
+
 func AddLabelToPv(ctx context.Context, pv, label string) error {
 	return exec.CommandContext(ctx, "kubectl", "label", "pv", pv, label).Run()
 }
@@ -140,6 +178,12 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error
 	return exec.CommandContext(ctx, "kubectl", args...).Run()
 }
 
+func AddLabelToCRD(ctx context.Context, crd, label string) error {
+	args := []string{"label", "crd", crd, label}
+	fmt.Println(args)
+	return exec.CommandContext(ctx, "kubectl", args...).Run()
+}
+
 func KubectlApplyByFile(ctx context.Context, file string) error {
 	args := []string{"apply", "-f", file, "--force=true"}
 	return exec.CommandContext(ctx, "kubectl", args...).Run()
@@ -154,3 +198,22 @@ func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error {
 	fmt.Print(stderr)
 	return err
 }
+
+func GetAPIVersions(client *TestClient, name string) ([]string, error) {
+	var version []string
+	APIGroup, err := client.ClientGo.Discovery().ServerGroups()
+	if err != nil {
+		return nil, errors.Wrap(err, "Fail to get server API groups")
+	}
+	for _, group := range APIGroup.Groups {
+		fmt.Println(group.Name)
+		if group.Name == name {
+			for _, v := range group.Versions {
+				fmt.Println(v.Version)
+				version = append(version, v.Version)
+			}
+			return version, nil
+		}
+	}
+	return nil, errors.New("Server API groups is empty")
+}