mirror of
https://github.com/vmware-tanzu/velero.git
synced 2026-01-06 21:36:30 +00:00
Adjust structure of e2e test codes
Put every test moduels into seperate packages and all velero install parameters integrated into one struct Signed-off-by: Ming <mqiu@vmware.com>
This commit is contained in:
ming qiu
Ming
462
test/e2e/basic/enable_api_group_versions.go
Normal file
462
test/e2e/basic/enable_api_group_versions.go
Normal file
@@ -0,0 +1,462 @@
|
||||
/*
|
||||
Copyright the Velero contributors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package basic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
. "github.com/onsi/ginkgo"
|
||||
. "github.com/onsi/gomega"
|
||||
"github.com/pkg/errors"
|
||||
corev1api "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/vmware-tanzu/velero/pkg/builder"
|
||||
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
|
||||
. "github.com/vmware-tanzu/velero/test/e2e"
|
||||
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
|
||||
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
|
||||
)
|
||||
|
||||
func APIGropuVersionsTest() {
|
||||
var (
|
||||
resource, group string
|
||||
err error
|
||||
ctx = context.Background()
|
||||
)
|
||||
|
||||
client, err := k8sutils.NewTestClient()
|
||||
Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")
|
||||
|
||||
BeforeEach(func() {
|
||||
resource = "rockbands"
|
||||
group = "music.example.io"
|
||||
UUIDgen, err = uuid.NewRandom()
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
flag.Parse()
|
||||
// TODO: install Velero once for the test suite once feature flag is
|
||||
// removed and velero installation becomes the same as other e2e tests.
|
||||
if VeleroCfg.InstallVelero {
|
||||
err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
})
|
||||
|
||||
AfterEach(func() {
|
||||
fmt.Printf("Clean up resource: kubectl delete crd %s.%s\n", resource, group)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", resource+"."+group)
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "NotFound") {
|
||||
fmt.Printf("Ignore error: %v\n", stderr)
|
||||
err = nil
|
||||
}
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
if VeleroCfg.InstallVelero {
|
||||
err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
Context("When EnableAPIGroupVersions flag is set", func() {
|
||||
It("Should back up API group version and restore by version priority", func() {
|
||||
Expect(runEnableAPIGroupVersionsTests(
|
||||
ctx,
|
||||
client,
|
||||
resource,
|
||||
group,
|
||||
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error {
|
||||
tests := []struct {
|
||||
name string
|
||||
namespaces []string
|
||||
srcCrdYaml string
|
||||
srcCRs map[string]string
|
||||
tgtCrdYaml string
|
||||
tgtVer string
|
||||
cm *corev1api.ConfigMap
|
||||
gvs map[string][]string
|
||||
want map[string]map[string]string
|
||||
}{
|
||||
{
|
||||
name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-a-target.yaml",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta2",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v2beta2",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "No common supported versions means no rockbands custom resource is restored.",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
"v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml",
|
||||
tgtVer: "",
|
||||
cm: nil,
|
||||
want: nil,
|
||||
},
|
||||
{
|
||||
name: "User config map overrides Priority 3, Case D and restores v2beta1",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml",
|
||||
"v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml",
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
|
||||
tgtVer: "v2beta1",
|
||||
cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data(
|
||||
"restoreResourcesVersionPriority",
|
||||
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
|
||||
).Result(),
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v2beta1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "Restore successful when CRD doesn't (yet) exist in target",
|
||||
srcCrdYaml: "testdata/enable_api_group_versions/case-a-source.yaml",
|
||||
srcCRs: map[string]string{
|
||||
"v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml",
|
||||
},
|
||||
tgtCrdYaml: "",
|
||||
tgtVer: "v1",
|
||||
cm: nil,
|
||||
want: map[string]map[string]string{
|
||||
"annotations": {
|
||||
"rockbands.music.example.io/originalVersion": "v1",
|
||||
},
|
||||
"specs": {
|
||||
"genre": "60s rock",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range tests {
|
||||
fmt.Printf("\n====== Test Case %d: %s ======\n", i, tc.name)
|
||||
|
||||
err := installCRD(ctx, tc.srcCrdYaml)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "install music-system CRD on source cluster")
|
||||
}
|
||||
|
||||
for version, cr := range tc.srcCRs {
|
||||
ns := resource + "-src-" + version
|
||||
|
||||
if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil {
|
||||
return errors.Wrapf(err, "create %s namespace", ns)
|
||||
}
|
||||
defer func(namespace string) {
|
||||
if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil {
|
||||
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
|
||||
}
|
||||
}(ns)
|
||||
|
||||
if err := installCR(ctx, cr, ns); err != nil {
|
||||
return errors.Wrapf(err, "install %s custom resource on source cluster in namespace %s", cr, ns)
|
||||
}
|
||||
|
||||
tc.namespaces = append(tc.namespaces, ns)
|
||||
}
|
||||
|
||||
// Restart Velero pods in order to recognize music-system CRD right away
|
||||
// instead of waiting for discovery helper to refresh. See
|
||||
// https://github.com/vmware-tanzu/velero/issues/3471.
|
||||
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
|
||||
return errors.Wrapf(err, "restart Velero pods")
|
||||
}
|
||||
|
||||
backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
|
||||
namespacesStr := strings.Join(tc.namespaces, ",")
|
||||
|
||||
err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false)
|
||||
if err != nil {
|
||||
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
|
||||
return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
|
||||
}
|
||||
|
||||
if err := deleteCRD(ctx, tc.srcCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "delete music-system CRD from source cluster")
|
||||
}
|
||||
|
||||
for _, ns := range tc.namespaces {
|
||||
if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil {
|
||||
return errors.Wrapf(err, "delete %s namespace from source cluster", ns)
|
||||
}
|
||||
}
|
||||
|
||||
// Install music-system CRD for target cluster.
|
||||
if tc.tgtCrdYaml != "" {
|
||||
if err := installCRD(ctx, tc.tgtCrdYaml); err != nil {
|
||||
return errors.Wrapf(err, "install music-system CRD on target cluster")
|
||||
}
|
||||
}
|
||||
|
||||
// Apply config map if there is one.
|
||||
if tc.cm != nil {
|
||||
_, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create config map with user version priorities")
|
||||
}
|
||||
}
|
||||
|
||||
// Reset Velero to recognize music-system CRD.
|
||||
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
|
||||
return errors.Wrapf(err, "restart Velero pods")
|
||||
}
|
||||
|
||||
// Restore rockbands namespaces.
|
||||
restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
|
||||
|
||||
if tc.want != nil {
|
||||
if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil {
|
||||
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore)
|
||||
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
|
||||
}
|
||||
|
||||
annoSpec, err := resourceInfo(ctx, group, tc.tgtVer, resource)
|
||||
if err != nil {
|
||||
return errors.Wrapf(
|
||||
err,
|
||||
"get annotation and spec from %s.%s/%s object",
|
||||
resource,
|
||||
group,
|
||||
tc.tgtVer,
|
||||
)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if containsAll(annoSpec["annotations"], tc.want["annotations"]) != true {
|
||||
msg := fmt.Sprintf(
|
||||
"actual annotations: %v, expected annotations: %v",
|
||||
annoSpec["annotations"],
|
||||
tc.want["annotations"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
// Assertion
|
||||
if containsAll(annoSpec["specs"], tc.want["specs"]) != true {
|
||||
msg := fmt.Sprintf(
|
||||
"actual specs: %v, expected specs: %v",
|
||||
annoSpec["specs"],
|
||||
tc.want["specs"],
|
||||
)
|
||||
return errors.New(msg)
|
||||
}
|
||||
|
||||
} else {
|
||||
// No custom resource should have been restored. Expect "no resource found"
|
||||
// error during restore.
|
||||
err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup)
|
||||
|
||||
if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" {
|
||||
return errors.New("expected error but not none")
|
||||
}
|
||||
}
|
||||
|
||||
_ = deleteCRD(ctx, tc.srcCrdYaml)
|
||||
if tc.tgtCrdYaml != "" {
|
||||
_ = deleteCRD(ctx, tc.tgtCrdYaml)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func installCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Printf("Install CRD with %s.\n", yaml)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", yaml)
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteCRD(ctx context.Context, yaml string) error {
|
||||
fmt.Println("Delete CRD", yaml)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", yaml, "--wait")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func restartPods(ctx context.Context, ns string) error {
|
||||
fmt.Printf("Restart pods in %s namespace.\n", ns)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns, "--wait=true")
|
||||
|
||||
_, stderr, err := veleroexec.RunCommand(cmd)
|
||||
if strings.Contains(stderr, "not found") {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func installCR(ctx context.Context, crFile, ns string) error {
|
||||
retries := 5
|
||||
var stderr string
|
||||
var err error
|
||||
|
||||
for i := 0; i < retries; i++ {
|
||||
fmt.Printf("Attempt %d: Install custom resource %s\n", i+1, crFile)
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", ns, "-f", crFile)
|
||||
_, stderr, err = veleroexec.RunCommand(cmd)
|
||||
if err == nil {
|
||||
fmt.Printf("Successfully installed CR on %s.\n", ns)
|
||||
return nil
|
||||
}
|
||||
|
||||
fmt.Printf("Sleep for %ds before next attempt.\n", 20*i)
|
||||
time.Sleep(time.Second * time.Duration(i) * 20)
|
||||
}
|
||||
return errors.Wrap(err, stderr)
|
||||
}
|
||||
|
||||
func resourceInfo(ctx context.Context, g, v, r string) (map[string]map[string]string, error) {
|
||||
rvg := r + "." + v + "." + g
|
||||
ns := r + "-src-" + v
|
||||
cmd := exec.CommandContext(ctx, "kubectl", "get", rvg, "-n", ns, "-o", "json")
|
||||
|
||||
stdout, errMsg, err := veleroexec.RunCommand(cmd)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errMsg)
|
||||
}
|
||||
|
||||
var info map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(stdout), &info); err != nil {
|
||||
return nil, errors.Wrap(err, "unmarshal resource info JSON")
|
||||
}
|
||||
items := info["items"].([]interface{})
|
||||
|
||||
if len(items) < 1 {
|
||||
return nil, errors.New("resource info is empty")
|
||||
}
|
||||
|
||||
item := items[0].(map[string]interface{})
|
||||
metadata := item["metadata"].(map[string]interface{})
|
||||
annotations := metadata["annotations"].(map[string]interface{})
|
||||
specs := item["spec"].(map[string]interface{})
|
||||
|
||||
annoSpec := make(map[string]map[string]string)
|
||||
|
||||
for k, v := range annotations {
|
||||
if annoSpec["annotations"] == nil {
|
||||
annoSpec["annotations"] = map[string]string{
|
||||
k: v.(string),
|
||||
}
|
||||
} else {
|
||||
annoSpec["annotations"][k] = v.(string)
|
||||
}
|
||||
}
|
||||
|
||||
for k, v := range specs {
|
||||
if val, ok := v.(string); ok {
|
||||
if annoSpec["specs"] == nil {
|
||||
annoSpec["specs"] = map[string]string{
|
||||
k: val,
|
||||
}
|
||||
} else {
|
||||
annoSpec["specs"][k] = val
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return annoSpec, nil
|
||||
}
|
||||
|
||||
// containsAll returns true if all the map values in the needles argument
|
||||
// are found in the haystack argument values.
|
||||
func containsAll(haystack, needles map[string]string) bool {
|
||||
for nkey, nval := range needles {
|
||||
|
||||
hval, ok := haystack[nkey]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
if hval != nval {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
Reference in New Issue
Block a user