mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-01-05 13:05:17 +00:00
Optimize APIGroup E2E test case (#6174)
Signed-off-by: Ming <mqiu@vmware.com>
This commit is contained in:
qiuming
GitHub
157
test/e2e/basic/api-group/enable_api_group_extentions.go
Normal file
157
test/e2e/basic/api-group/enable_api_group_extentions.go
Normal file
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package basic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
func APIExtensionsVersionsTest() {
|
||||
var (
|
||||
backupName, restoreName string
|
||||
)
|
||||
|
||||
resourceName := "apiextensions.k8s.io"
|
||||
crdName := "rocknrollbands.music.example.io"
|
||||
label := "for=backup"
|
||||
srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
|
||||
BeforeEach(func() {
|
||||
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
|
||||
Skip("CRD with apiextension versions migration test needs 2 clusters")
|
||||
}
|
||||
veleroCfg = VeleroCfg
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
Expect(srcVersions).Should(ContainElement("v1"), func() string {
|
||||
Skip("CRD with apiextension versions srcVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(srcVersions).Should(ContainElement("v1beta1"), func() string {
|
||||
Skip("CRD with apiextension versions srcVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(dstVersions).Should(ContainElement("v1"), func() string {
|
||||
Skip("CRD with apiextension versions dstVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(len(srcVersions) > 1 && len(dstVersions) == 1).Should(Equal(true), func() string {
|
||||
Skip("Source cluster should support apiextension v1 and v1beta1, destination cluster should only support apiextension v1")
|
||||
return ""
|
||||
})
|
||||
})
|
||||
AfterEach(func() {
|
||||
if !veleroCfg.Debug {
|
||||
By("Clean backups after test", func() {
|
||||
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
|
||||
})
|
||||
if veleroCfg.InstallVelero {
|
||||
By("Uninstall Velero and delete CRD ", func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace)).To(Succeed())
|
||||
Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
|
||||
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
|
||||
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace)).To(Succeed())
|
||||
Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed())
|
||||
})
|
||||
}
|
||||
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
||||
})
|
||||
}
|
||||
|
||||
})
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
It("Enable API Group to B/R CRD APIExtensionsVersions", func() {
|
||||
backupName = "backup-" + UUIDgen.String()
|
||||
restoreName = "restore-" + UUIDgen.String()
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
veleroCfg.Features = "EnableAPIGroupVersions"
|
||||
veleroCfg.UseVolumeSnapshots = false
|
||||
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
|
||||
Expect(InstallCRD(context.Background(), srcCrdYaml)).To(Succeed())
|
||||
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
|
||||
Expect(WaitForCRDEstablished(crdName)).To(Succeed())
|
||||
Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed())
|
||||
// Velero server refresh api version data by discovery helper every 5 minutes
|
||||
time.Sleep(6 * time.Minute)
|
||||
})
|
||||
|
||||
By("Backup CRD", func() {
|
||||
var BackupCfg BackupConfig
|
||||
BackupCfg.BackupName = backupName
|
||||
BackupCfg.IncludeResources = "crd"
|
||||
BackupCfg.IncludeClusterResources = true
|
||||
BackupCfg.Selector = label
|
||||
Expect(VeleroBackupNamespace(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, backupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
|
||||
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
|
||||
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() {
|
||||
Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
|
||||
})
|
||||
|
||||
By("Restore CRD", func() {
|
||||
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, "", restoreName)
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
})
|
||||
|
||||
By("Verify CRD restore ", func() {
|
||||
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
544
test/e2e/basic/api-group/enable_api_group_versions.go
Normal file
544
test/e2e/basic/api-group/enable_api_group_versions.go
Normal file
@@ -0,0 +1,544 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package basic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/builder"
|
||||
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
var veleroCfg VeleroConfig
|
||||
|
||||
type apiGropuVersionsTest struct {
|
||||
name string
|
||||
namespaces []string
|
||||
srcCrdYaml string
|
||||
srcCRs map[string]string
|
||||
tgtCrdYaml string
|
||||
tgtVer string
|
||||
cm *corev1api.ConfigMap
|
||||
gvs map[string][]string
|
||||
want map[string]map[string]string
|
||||
}
|
||||
|
||||
func APIGropuVersionsTest() {
|
||||
var (
|
||||
group string
|
||||
err error
|
||||
ctx = context.Background()
|
||||
testCaseNum int
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
veleroCfg = VeleroCfg
|
||||
group = "music.example.io"
|
||||
UUIDgen, err = uuid.NewRandom()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
flag.Parse()
|
||||
// TODO: install Velero once for the test suite once feature flag is
|
||||
// removed and velero installation becomes the same as other e2e tests.
|
||||
if veleroCfg.InstallVelero {
|
||||
veleroCfg.Features = "EnableAPIGroupVersions"
|
||||
veleroCfg.UseVolumeSnapshots = false
|
||||
err = VeleroInstall(context.Background(), &veleroCfg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
testCaseNum = 4
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if !veleroCfg.Debug {
|
||||
for i := 0; i < testCaseNum; i++ {
|
||||
curResource := fmt.Sprintf("rockband%ds", i)
|
||||
curGroup := fmt.Sprintf("%s.%d", group, i)
|
||||
By(fmt.Sprintf("Clean up resource: kubectl delete crd %s.%s\n", curResource, curGroup))
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", curResource+"."+curGroup)
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "NotFound") {
|
||||
fmt.Printf("Ignore error: %v\n", stderr)
|
||||
err = nil
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
By("Clean backups after test", func() {
|
||||
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
|
||||
})
|
||||
if veleroCfg.InstallVelero {
|
||||
By("Uninstall Velero", func() {
|
||||
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
It("Should back up API group version and restore by version priority", func() {
|
||||
Expect(runEnableAPIGroupVersionsTests(
|
||||
ctx,
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
group,
|
||||
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, group string) error {
|
||||
tests := []apiGropuVersionsTest{
|
||||
{
|
||||
name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-a-target.yaml",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockband0s.music.example.io.0/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta2",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockband1s.music.example.io.1/originalVersion": "v2beta2",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "No common supported versions means no rockbands custom resource is restored.",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml",
|
||||
tgtVer: "",
|
||||
cm: nil,
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "User config map overrides Priority 3, Case D and restores v2beta1",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta1",
|
||||
cm: builder.ForConfigMap(veleroCfg.VeleroNamespace, "enableapigroupversions").Data(
|
||||
"restoreResourcesVersionPriority",
|
||||
`rockband3s.music.example.io.3=v2beta1,v2beta2,v2`,
|
||||
).Result(),
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockband3s.music.example.io.3/originalVersion": "v2beta1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Restore successful when CRD doesn't (yet) exist in target",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockband4s.music.example.io.4/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
nsListwanted, nsListUnwanted, err := installTestResources(ctx, client, group, tests)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
for i, tc := range tests {
|
||||
for version := range tc.srcCRs {
|
||||
ns := fmt.Sprintf("rockband%ds-src-%s-%d", i, version, i)
|
||||
defer func(namespace string) {
|
||||
if err = DeleteNamespace(ctx, client, namespace, true); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
|
||||
}
|
||||
}(ns)
|
||||
}
|
||||
if tc.cm != nil {
|
||||
defer func(name string) {
|
||||
if err = client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete configmap %q", name))
|
||||
}
|
||||
}(tc.cm.Name)
|
||||
}
|
||||
|
||||
defer func(crdName string) {
|
||||
if err = DeleteCRDByName(ctx, crdName); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete crd %q", crdName))
|
||||
}
|
||||
}(fmt.Sprintf("rockband%ds.music.example.io.%d", i, i))
|
||||
|
||||
}
|
||||
|
||||
time.Sleep(6 * time.Minute)
|
||||
|
||||
BackupCfgWanted := BackupConfig{
|
||||
BackupName: "backup-rockbands-" + UUIDgen.String() + "-wanted",
|
||||
Namespace: nsListwanted,
|
||||
UseVolumeSnapshots: false,
|
||||
}
|
||||
|
||||
Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfgWanted)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfgWanted.BackupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
|
||||
BackupCfgUnwanted := BackupConfig{
|
||||
BackupName: "backup-rockbands-" + UUIDgen.String() + "-unwanted",
|
||||
Namespace: nsListUnwanted,
|
||||
UseVolumeSnapshots: false,
|
||||
}
|
||||
|
||||
Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfgUnwanted)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfgUnwanted.BackupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
|
||||
Expect(reinstallTestResources(ctx, group, client, tests)).NotTo(HaveOccurred())
|
||||
|
||||
time.Sleep(6 * time.Minute)
|
||||
|
||||
restoreName := "restore-rockbands-" + UUIDgen.String() + "-wanted"
|
||||
if err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, BackupCfgWanted.BackupName, ""); err != nil {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restoreName)
|
||||
return errors.Wrapf(err, "restore %s namespaces on target cluster", nsListwanted)
|
||||
}
|
||||
|
||||
restoreName = "restore-rockbands-" + UUIDgen.String() + "-unwanted"
|
||||
err = VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restoreName, BackupCfgUnwanted.BackupName, "")
|
||||
if !strings.Contains(err.Error(), "Unexpected restore phase got PartiallyFailed, expecting Completed") {
|
||||
return errors.New("expected error but not none")
|
||||
}
|
||||
|
||||
for i, tc := range tests {
|
||||
defer func() {
|
||||
_ = deleteTestCRD(ctx, i, group, tc.srcCrdYaml)
|
||||
if tc.tgtCrdYaml != "" {
|
||||
_ = deleteTestCRD(ctx, i, group, tc.tgtCrdYaml)
|
||||
}
|
||||
if tc.cm != nil {
|
||||
client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Delete(ctx, tc.cm.Name, metav1.DeleteOptions{})
|
||||
}
|
||||
}()
|
||||
|
||||
if tc.want != nil {
|
||||
curResource := fmt.Sprintf("rockband%ds", i)
|
||||
annoSpec, err := resourceInfo(ctx, group, tc.tgtVer, curResource, i)
|
||||
if err != nil {
|
||||
return errors.Wrapf(
|
||||
err,
|
||||
"get annotation and spec from %s.%s/%s object",
|
||||
curResource,
|
||||
group,
|
||||
tc.tgtVer,
|
||||
)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if !containsAll(annoSpec["annotations"], tc.want["annotations"]) {
|
||||
msg := fmt.Sprintf(
|
||||
"actual annotations: %v, expected annotations: %v",
|
||||
annoSpec["annotations"],
|
||||
tc.want["annotations"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if !containsAll(annoSpec["specs"], tc.want["specs"]) {
|
||||
msg := fmt.Sprintf(
|
||||
"actual specs: %v, expected specs: %v",
|
||||
annoSpec["specs"],
|
||||
tc.want["specs"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteTestCRD(ctx context.Context, index int, group, path string) error {
|
||||
fileName, err := rerenderTestYaml(index, group, path)
|
||||
defer func() {
|
||||
if fileName != "" {
|
||||
os.Remove(fileName)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return DeleteCRD(ctx, fileName)
|
||||
}
|
||||
|
||||
func installTestCR(ctx context.Context, index int, group, path, ns string) error {
|
||||
fileName, err := rerenderTestYaml(index, group, path)
|
||||
defer func() {
|
||||
if fileName != "" {
|
||||
os.Remove(fileName)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return InstallCR(ctx, fileName, ns)
|
||||
}
|
||||
|
||||
func installTestCRD(ctx context.Context, index int, group, path string) error {
|
||||
fileName, err := rerenderTestYaml(index, group, path)
|
||||
defer func() {
|
||||
if fileName != "" {
|
||||
os.Remove(fileName)
|
||||
}
|
||||
}()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return InstallCRD(ctx, fileName)
|
||||
}
|
||||
|
||||
func rerenderTestYaml(index int, group, path string) (string, error) {
|
||||
content, err := ioutil.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to get %s when install test yaml", path)
|
||||
}
|
||||
|
||||
// replace resource name to new value
|
||||
re := regexp.MustCompile(`\b(RockBand|RockBandList|rockband|rockbands)\b`)
|
||||
newContent := re.ReplaceAllStringFunc(string(content), func(s string) string {
|
||||
if s == "RockBand" {
|
||||
return fmt.Sprintf("RockBand%d", index)
|
||||
} else if s == "RockBandList" {
|
||||
return fmt.Sprintf("RockBand%dList", index)
|
||||
} else if s == "rockbands" {
|
||||
return fmt.Sprintf("rockband%ds", index)
|
||||
} else {
|
||||
return fmt.Sprintf("rockband%d", index)
|
||||
}
|
||||
})
|
||||
|
||||
// replace group name to new value
|
||||
newContent = strings.ReplaceAll(newContent, group, fmt.Sprintf("%s.%d", group, index))
|
||||
|
||||
By(fmt.Sprintf("\n%s\n", newContent))
|
||||
tmpFile, err := ioutil.TempFile("", "test-yaml")
|
||||
if err != nil {
|
||||
return "", errors.Wrapf(err, "failed to create temp file when install storage class")
|
||||
}
|
||||
|
||||
if _, err := tmpFile.WriteString(newContent); err != nil {
|
||||
return "", errors.Wrapf(err, "failed to write content into temp file %s when install storage class", tmpFile.Name())
|
||||
}
|
||||
|
||||
return tmpFile.Name(), nil
|
||||
}
|
||||
|
||||
func resourceInfo(ctx context.Context, g, v, r string, index int) (map[string]map[string]string, error) {
|
||||
rvg := fmt.Sprintf("%s.%s.%s.%d", r, v, g, index)
|
||||
ns := fmt.Sprintf("rockband%ds-src-%s-%d", index, v, index)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "get", rvg, "-n", ns, "-o", "json")
|
||||
stdout, errMsg, err := veleroexec.RunCommand(cmd)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errMsg)
|
||||
}
|
||||
|
||||
var info map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(stdout), &info); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal resource info JSON")
|
||||
}
|
||||
items := info["items"].([]interface{})
|
||||
|
||||
if len(items) < 1 {
|
||||
return nil, errors.New("resource info is empty")
|
||||
}
|
||||
|
||||
item := items[0].(map[string]interface{})
|
||||
metadata := item["metadata"].(map[string]interface{})
|
||||
annotations := metadata["annotations"].(map[string]interface{})
|
||||
specs := item["spec"].(map[string]interface{})
|
||||
|
||||
annoSpec := make(map[string]map[string]string)
|
||||
|
||||
for k, v := range annotations {
|
||||
if annoSpec["annotations"] == nil {
|
||||
annoSpec["annotations"] = map[string]string{
|
||||
k: v.(string),
|
||||
}
|
||||
} else {
|
||||
annoSpec["annotations"][k] = v.(string)
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range specs {
|
||||
if val, ok := v.(string); ok {
|
||||
if annoSpec["specs"] == nil {
|
||||
annoSpec["specs"] = map[string]string{
|
||||
k: val,
|
||||
}
|
||||
} else {
|
||||
annoSpec["specs"][k] = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return annoSpec, nil
|
||||
}
|
||||
|
||||
// containsAll returns true if all the map values in the needles argument
|
||||
// are found in the haystack argument values.
|
||||
func containsAll(haystack, needles map[string]string) bool {
|
||||
for nkey, nval := range needles {
|
||||
|
||||
hval, ok := haystack[nkey]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if hval != nval {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func installTestResources(ctx context.Context, client TestClient, group string, tests []apiGropuVersionsTest) (string, string, error) {
|
||||
var wanted, unwanted []string
|
||||
for i, tc := range tests {
|
||||
fmt.Printf("\n====== Test Case %d: %s ======\n", i, tc.name)
|
||||
|
||||
err := installTestCRD(ctx, i, group, tc.srcCrdYaml)
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "install music-system CRD on source cluster")
|
||||
}
|
||||
|
||||
for version, cr := range tc.srcCRs {
|
||||
ns := fmt.Sprintf("rockband%ds-src-%s-%d", i, version, i)
|
||||
if err := CreateNamespace(ctx, client, ns); err != nil {
|
||||
return "", "", errors.Wrapf(err, "create %s namespace", ns)
|
||||
}
|
||||
|
||||
if err := installTestCR(ctx, i, group, cr, ns); err != nil {
|
||||
return "", "", errors.Wrapf(err, "install %s custom resource on source cluster in namespace %s", cr, ns)
|
||||
}
|
||||
|
||||
if tc.want == nil {
|
||||
unwanted = append(unwanted, ns)
|
||||
} else {
|
||||
wanted = append(wanted, ns)
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(wanted, ","), strings.Join(unwanted, ","), nil
|
||||
}
|
||||
|
||||
func reinstallTestResources(ctx context.Context, group string, client TestClient, tests []apiGropuVersionsTest) error {
|
||||
for i, tc := range tests {
|
||||
By(fmt.Sprintf("Deleting CRD %s", tc.srcCrdYaml))
|
||||
if err := deleteTestCRD(ctx, i, group, tc.srcCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "delete music-system CRD from source cluster")
|
||||
}
|
||||
|
||||
for version := range tc.srcCRs {
|
||||
ns := fmt.Sprintf("rockband%ds-src-%s-%d", i, version, i)
|
||||
By(fmt.Sprintf("Deleting namespace %s", ns))
|
||||
if err := DeleteNamespace(ctx, client, ns, true); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
|
||||
}
|
||||
}
|
||||
// Install music-system CRD for target cluster.
|
||||
if tc.tgtCrdYaml != "" {
|
||||
By(fmt.Sprintf("Installing CRD %s", tc.tgtCrdYaml))
|
||||
if err := installTestCRD(ctx, i, group, tc.tgtCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "install music-system CRD on target cluster")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply config map if there is one.
|
||||
if tc.cm != nil {
|
||||
By(fmt.Sprintf("Creating configmap %s", tc.cm.Name))
|
||||
_, err := client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create config map with user version priorities")
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,591 +0,0 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package basic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/builder"
|
||||
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
var veleroCfg VeleroConfig
|
||||
|
||||
func APIExtensionsVersionsTest() {
|
||||
var (
|
||||
backupName, restoreName string
|
||||
)
|
||||
|
||||
resourceName := "apiextensions.k8s.io"
|
||||
crdName := "rocknrollbands.music.example.io"
|
||||
label := "for=backup"
|
||||
srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml"
|
||||
BeforeEach(func() {
|
||||
if veleroCfg.DefaultCluster == "" && veleroCfg.StandbyCluster == "" {
|
||||
Skip("CRD with apiextension versions migration test needs 2 clusters")
|
||||
}
|
||||
veleroCfg = VeleroCfg
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
dstVersions, err := GetAPIVersions(veleroCfg.StandbyClient, resourceName)
|
||||
Expect(err).ShouldNot(HaveOccurred())
|
||||
|
||||
Expect(srcVersions).Should(ContainElement("v1"), func() string {
|
||||
Skip("CRD with apiextension versions srcVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(srcVersions).Should(ContainElement("v1beta1"), func() string {
|
||||
Skip("CRD with apiextension versions srcVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(dstVersions).Should(ContainElement("v1"), func() string {
|
||||
Skip("CRD with apiextension versions dstVersions should have v1")
|
||||
return ""
|
||||
})
|
||||
Expect(len(srcVersions) > 1 && len(dstVersions) == 1).Should(Equal(true), func() string {
|
||||
Skip("Source cluster should support apiextension v1 and v1beta1, destination cluster should only support apiextension v1")
|
||||
return ""
|
||||
})
|
||||
})
|
||||
AfterEach(func() {
|
||||
if !veleroCfg.Debug {
|
||||
By("Clean backups after test", func() {
|
||||
DeleteBackups(context.Background(), *veleroCfg.DefaultClient)
|
||||
})
|
||||
if veleroCfg.InstallVelero {
|
||||
By("Uninstall Velero and delete CRD ", func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace)).To(Succeed())
|
||||
Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
|
||||
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
|
||||
Expect(VeleroUninstall(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace)).To(Succeed())
|
||||
Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed())
|
||||
})
|
||||
}
|
||||
By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
|
||||
})
|
||||
}
|
||||
|
||||
})
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
It("Enable API Group to B/R CRD APIExtensionsVersions", func() {
|
||||
backupName = "backup-" + UUIDgen.String()
|
||||
restoreName = "restore-" + UUIDgen.String()
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultCluster)).To(Succeed())
|
||||
veleroCfg.Features = "EnableAPIGroupVersions"
|
||||
veleroCfg.UseVolumeSnapshots = false
|
||||
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", veleroCfg.DefaultCluster), func() {
|
||||
Expect(installCRD(context.Background(), srcCrdYaml)).To(Succeed())
|
||||
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
|
||||
Expect(WaitForCRDEstablished(crdName)).To(Succeed())
|
||||
Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed())
|
||||
// Velero server refresh api version data by discovery helper every 5 minutes
|
||||
time.Sleep(6 * time.Minute)
|
||||
})
|
||||
|
||||
By("Backup CRD", func() {
|
||||
var BackupCfg BackupConfig
|
||||
BackupCfg.BackupName = backupName
|
||||
BackupCfg.IncludeResources = "crd"
|
||||
BackupCfg.IncludeClusterResources = true
|
||||
BackupCfg.Selector = label
|
||||
Expect(VeleroBackupNamespace(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, backupName, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyCluster), func() {
|
||||
Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyCluster)).To(Succeed())
|
||||
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
|
||||
Expect(VeleroInstall(context.Background(), &veleroCfg)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", veleroCfg.StandbyCluster), func() {
|
||||
Expect(WaitForBackupToBeCreated(context.Background(), veleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed())
|
||||
})
|
||||
|
||||
By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, veleroCfg.StandbyCluster), func() {
|
||||
Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test")
|
||||
})
|
||||
|
||||
By("Restore CRD", func() {
|
||||
Expect(VeleroRestore(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, "", restoreName)
|
||||
return "Fail to restore workload"
|
||||
})
|
||||
})
|
||||
|
||||
By("Verify CRD restore ", func() {
|
||||
Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed())
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
func APIGropuVersionsTest() {
|
||||
var (
|
||||
resource, group string
|
||||
err error
|
||||
ctx = context.Background()
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
veleroCfg = VeleroCfg
|
||||
resource = "rockbands"
|
||||
group = "music.example.io"
|
||||
UUIDgen, err = uuid.NewRandom()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
flag.Parse()
|
||||
// TODO: install Velero once for the test suite once feature flag is
|
||||
// removed and velero installation becomes the same as other e2e tests.
|
||||
if veleroCfg.InstallVelero {
|
||||
veleroCfg.Features = "EnableAPIGroupVersions"
|
||||
veleroCfg.UseVolumeSnapshots = false
|
||||
err = VeleroInstall(context.Background(), &veleroCfg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
if !veleroCfg.Debug {
|
||||
fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group)
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "NotFound") {
|
||||
fmt.Printf("Ignore error: %v\n", stderr)
|
||||
err = nil
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
By("Clean backups after test", func() {
|
||||
DeleteBackups(context.Background(), *veleroCfg.ClientToInstallVelero)
|
||||
})
|
||||
if veleroCfg.InstallVelero {
|
||||
|
||||
By("Uninstall Velero", func() {
|
||||
Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred())
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
It("Should back up API group version and restore by version priority", func() {
|
||||
Expect(runEnableAPIGroupVersionsTests(
|
||||
ctx,
|
||||
*veleroCfg.ClientToInstallVelero,
|
||||
resource,
|
||||
group,
|
||||
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, resource, group string) error {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespaces []string
|
||||
srcCrdYaml string
|
||||
srcCRs map[string]string
|
||||
tgtCrdYaml string
|
||||
tgtVer string
|
||||
cm *corev1api.ConfigMap
|
||||
gvs map[string][]string
|
||||
want map[string]map[string]string
|
||||
}{
|
||||
{
|
||||
name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-a-target.yaml",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta2",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v2beta2",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "No common supported versions means no rockbands custom resource is restored.",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml",
|
||||
tgtVer: "",
|
||||
cm: nil,
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "User config map overrides Priority 3, Case D and restores v2beta1",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta1",
|
||||
cm: builder.ForConfigMap(veleroCfg.VeleroNamespace, "enableapigroupversions").Data(
|
||||
"restoreResourcesVersionPriority",
|
||||
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
|
||||
).Result(),
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v2beta1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Restore successful when CRD doesn't (yet) exist in target",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range tests {
|
||||
fmt.Printf("\n====== Test Case %d: %s ======\n", i, tc.name)
|
||||
|
||||
err := installCRD(ctx, tc.srcCrdYaml)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "install music-system CRD on source cluster")
|
||||
}
|
||||
|
||||
for version, cr := range tc.srcCRs {
|
||||
ns := resource + "-src-" + version
|
||||
|
||||
if err := CreateNamespace(ctx, client, ns); err != nil {
|
||||
return errors.Wrapf(err, "create %s namespace", ns)
|
||||
}
|
||||
defer func(namespace string) {
|
||||
if err = DeleteNamespace(ctx, client, namespace, true); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
|
||||
}
|
||||
}(ns)
|
||||
|
||||
if err := installCR(ctx, cr, ns); err != nil {
|
||||
return errors.Wrapf(err, "install %s custom resource on source cluster in namespace %s", cr, ns)
|
||||
}
|
||||
|
||||
tc.namespaces = append(tc.namespaces, ns)
|
||||
}
|
||||
|
||||
// Velero server refresh api version data by discovery helper every 5 minutes
|
||||
time.Sleep(6 * time.Minute)
|
||||
|
||||
backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
|
||||
namespacesStr := strings.Join(tc.namespaces, ",")
|
||||
|
||||
var BackupCfg BackupConfig
|
||||
BackupCfg.BackupName = backup
|
||||
BackupCfg.Namespace = namespacesStr
|
||||
BackupCfg.BackupLocation = ""
|
||||
BackupCfg.UseVolumeSnapshots = false
|
||||
BackupCfg.Selector = ""
|
||||
|
||||
Expect(VeleroBackupNamespace(ctx, veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI,
|
||||
veleroCfg.VeleroNamespace, backup, "")
|
||||
return "Fail to backup workload"
|
||||
})
|
||||
|
||||
if err := deleteCRD(ctx, tc.srcCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "delete music-system CRD from source cluster")
|
||||
}
|
||||
|
||||
for _, ns := range tc.namespaces {
|
||||
if err := DeleteNamespace(ctx, client, ns, true); err != nil {
|
||||
return errors.Wrapf(err, "delete %s namespace from source cluster", ns)
|
||||
}
|
||||
}
|
||||
|
||||
// Install music-system CRD for target cluster.
|
||||
if tc.tgtCrdYaml != "" {
|
||||
if err := installCRD(ctx, tc.tgtCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "install music-system CRD on target cluster")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply config map if there is one.
|
||||
if tc.cm != nil {
|
||||
_, err := client.ClientGo.CoreV1().ConfigMaps(veleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create config map with user version priorities")
|
||||
}
|
||||
}
|
||||
|
||||
// Velero server refresh api version data by discovery helper every 5 minutes
|
||||
time.Sleep(6 * time.Minute)
|
||||
|
||||
// Restore rockbands namespaces.
|
||||
restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
|
||||
|
||||
if tc.want != nil {
|
||||
if err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, ""); err != nil {
|
||||
RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", restore)
|
||||
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
|
||||
}
|
||||
|
||||
annoSpec, err := resourceInfo(ctx, group, tc.tgtVer, resource)
|
||||
if err != nil {
|
||||
return errors.Wrapf(
|
||||
err,
|
||||
"get annotation and spec from %s.%s/%s object",
|
||||
resource,
|
||||
group,
|
||||
tc.tgtVer,
|
||||
)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if !containsAll(annoSpec["annotations"], tc.want["annotations"]) {
|
||||
msg := fmt.Sprintf(
|
||||
"actual annotations: %v, expected annotations: %v",
|
||||
annoSpec["annotations"],
|
||||
tc.want["annotations"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if !containsAll(annoSpec["specs"], tc.want["specs"]) {
|
||||
msg := fmt.Sprintf(
|
||||
"actual specs: %v, expected specs: %v",
|
||||
annoSpec["specs"],
|
||||
tc.want["specs"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
} else {
|
||||
// No custom resource should have been restored. Expect "no resource found"
|
||||
// error during restore.
|
||||
err := VeleroRestore(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, restore, backup, "")
|
||||
if !strings.Contains(err.Error(), "Unexpected restore phase got PartiallyFailed, expecting Completed") {
|
||||
return errors.New("expected error but not none")
|
||||
}
|
||||
}
|
||||
|
||||
_ = deleteCRD(ctx, tc.srcCrdYaml)
|
||||
if tc.tgtCrdYaml != "" {
|
||||
_ = deleteCRD(ctx, tc.tgtCrdYaml)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func installCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Printf("Install CRD with %s.\n", yaml)
|
||||
err := KubectlApplyByFile(ctx, yaml)
|
||||
return err
|
||||
}
|
||||
|
||||
func deleteCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Println("Delete CRD", yaml)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", yaml, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteCRDByName(ctx context.Context, name string) error {
|
||||
fmt.Println("Delete CRD", name)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", name, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func installCR(ctx context.Context, crFile, ns string) error {
|
||||
retries := 5
|
||||
var stderr string
|
||||
var err error
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
fmt.Printf("Attempt %d: Install custom resource %s\n", i+1, crFile)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", ns, "-f", crFile)
|
||||
_, stderr, err = veleroexec.RunCommand(cmd)
|
||||
if err == nil {
|
||||
fmt.Printf("Successfully installed CR on %s.\n", ns)
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("Sleep for %ds before next attempt.\n", 20*i)
|
||||
time.Sleep(time.Second * time.Duration(i) * 20)
|
||||
}
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
func resourceInfo(ctx context.Context, g, v, r string) (map[string]map[string]string, error) {
|
||||
rvg := r + "." + v + "." + g
|
||||
ns := r + "-src-" + v
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "get", rvg, "-n", ns, "-o", "json")
|
||||
|
||||
stdout, errMsg, err := veleroexec.RunCommand(cmd)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errMsg)
|
||||
}
|
||||
|
||||
var info map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(stdout), &info); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal resource info JSON")
|
||||
}
|
||||
items := info["items"].([]interface{})
|
||||
|
||||
if len(items) < 1 {
|
||||
return nil, errors.New("resource info is empty")
|
||||
}
|
||||
|
||||
item := items[0].(map[string]interface{})
|
||||
metadata := item["metadata"].(map[string]interface{})
|
||||
annotations := metadata["annotations"].(map[string]interface{})
|
||||
specs := item["spec"].(map[string]interface{})
|
||||
|
||||
annoSpec := make(map[string]map[string]string)
|
||||
|
||||
for k, v := range annotations {
|
||||
if annoSpec["annotations"] == nil {
|
||||
annoSpec["annotations"] = map[string]string{
|
||||
k: v.(string),
|
||||
}
|
||||
} else {
|
||||
annoSpec["annotations"][k] = v.(string)
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range specs {
|
||||
if val, ok := v.(string); ok {
|
||||
if annoSpec["specs"] == nil {
|
||||
annoSpec["specs"] = map[string]string{
|
||||
k: val,
|
||||
}
|
||||
} else {
|
||||
annoSpec["specs"][k] = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return annoSpec, nil
|
||||
}
|
||||
|
||||
// containsAll returns true if all the map values in the needles argument
|
||||
// are found in the haystack argument values.
|
||||
func containsAll(haystack, needles map[string]string) bool {
|
||||
for nkey, nval := range needles {
|
||||
|
||||
hval, ok := haystack[nkey]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if hval != nval {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/backup"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/backups"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/basic"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/basic/api-group"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e/migration"
|
||||
|
||||
85
test/e2e/util/k8s/crd.go
Normal file
85
test/e2e/util/k8s/crd.go
Normal file
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package k8s
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
)
|
||||
|
||||
func InstallCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Printf("Install CRD with %s.\n", yaml)
|
||||
err := KubectlApplyByFile(ctx, yaml)
|
||||
return err
|
||||
}
|
||||
|
||||
func DeleteCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Println("Delete CRD", yaml)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", yaml, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func DeleteCRDByName(ctx context.Context, name string) error {
|
||||
fmt.Println("Delete CRD", name)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", name, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func InstallCR(ctx context.Context, crFile, ns string) error {
|
||||
retries := 5
|
||||
var stderr string
|
||||
var err error
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
fmt.Printf("Attempt %d: Install custom resource %s\n", i+1, crFile)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", ns, "-f", crFile)
|
||||
_, stderr, err = veleroexec.RunCommand(cmd)
|
||||
if err == nil {
|
||||
fmt.Printf("Successfully installed CR on %s.\n", ns)
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("Sleep for %ds before next attempt.\n", 20*i)
|
||||
time.Sleep(time.Second * time.Duration(i) * 20)
|
||||
}
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
Reference in New Issue
Block a user