Add an E2E test framework to test Velero across cloud platforms (#3060)

* Basic end-to-end tests, generate data/backup/remove/restore/verify Uses distributed data generator Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * Moved backup/restore into velero_utils, started using a name for the restore Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * remove checked in binary and update test/e2e Makefile Signed-off-by: Ashish Amarnath <ashisham@vmware.com> * Ran make update Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * Save Signed-off-by: Ashish Amarnath <ashisham@vmware.com> * Ran make update Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * Basic end-to-end test, generate data/backup/remove/restore/verify Uses distributed data generator Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * Changed tests/e2e Makefile to just use go get to install ginkgo in the GOPATH/bin Updated to ginkgo 1.14.2 Put cobra back to v0.0.7 Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * Added CLOUD_PLATFORM env variable to Makefile, updated README, removed ginkgo from .gitignore Signed-off-by: Dave Smith-Uchida <dsmithuchida@vmware.com> * choose velero CLI binary based on local env Signed-off-by: Ashish Amarnath <ashisham@vmware.com> Co-authored-by: Ashish Amarnath <ashisham@vmware.com>
2026-09-20 06:54:32 +00:00 · 2020-11-24 14:12:52 -05:00
parent b321838c72
commit aa47309700
14 changed files with 587 additions and 5 deletions
@@ -0,0 +1,61 @@
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# If you update this file, please follow:
+# https://suva.sh/posts/well-documented-makefiles/
+
+# Use GOPROXY environment variable if set
+
+.DEFAULT_GOAL:=help
+
+ARCH ?= $(shell go env GOOS)-$(shell go env GOARCH)
+platform_temp = $(subst -, ,$(ARCH))
+GOOS = $(word 1, $(platform_temp))
+GOARCH = $(word 2, $(platform_temp))
+
+GOPROXY := $(shell go env GOPROXY)
+ifeq ($(GOPROXY),)
+GOPROXY := https://proxy.golang.org
+endif
+export GOPROXY
+
+REPO_ROOT := $(shell git rev-parse --show-toplevel)
+
+help:  ## Display this help
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf "  \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+## --------------------------------------
+## Binaries
+## --------------------------------------
+
+TOOLS_DIR := $(REPO_ROOT)/hack/tools
+BIN_DIR := bin
+TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
+GINKGO := $(GOPATH)/bin/ginkgo
+KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
+OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
+GINKGO_FOCUS ?=
+CLOUD_PLATFORM ?= kind
+
+.PHONY:ginkgo
+ginkgo: # Make sure ginkgo is in $GOPATH/bin
+	go get github.com/onsi/ginkgo/ginkgo
+
+.PHONY: run
+run: ginkgo
+	$(GINKGO) -v -focus="$(GINKGO_FOCUS)" . -- -velerocli=../../_output/bin/$(GOOS)/$(GOARCH)/velero -cloudplatform=$(CLOUD_PLATFORM)
+
+build: ginkgo
+	mkdir -p $(OUTPUT_DIR)
+	$(GINKGO) build . 
@@ -0,0 +1,46 @@
+# End-to-end tests
+
+Document for running Velero end-to-end test suite.
+
+## Command line flags for E2E tests
+
+Command line flags can be set after
+```
+velerocli - the velero CLI to use
+kibishiins - the namespace to install kibishii in
+cloudplatform - the cloud platform the tests will be run against (aws, vsphere, azure)
+```
+
+## Running tests locally
+
+1. From Velero repository root
+
+    ```
+    make test-e2e
+    ```
+
+1. From `test/e2e/` directory
+
+    ```
+    make run
+    ```
+
+## Running tests based on cloud platforms
+
+1. Running Velero E2E tests on KinD
+
+    ```
+    CLOUD_PLATFORM=kind make test-e2e
+    ```
+
+1. Running Velero E2E tests on AWS
+
+    ```
+    CLOUD_PLATFORM=aws make test-e2e
+    ```
+
+1. Running Velero E2E tests on Azure
+
+    ```
+    CLOUD_PLATFORM=azure make test-e2e
+    ```
@@ -0,0 +1,24 @@
+package e2e
+
+import (
+	"context"
+	"flag"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Testing Velero on an aws cluster", func() {
+	BeforeEach(func() {
+		flag.Parse()
+		ctx := context.TODO()
+		err := EnsureClusterExists(ctx)
+		Expect(err).NotTo(HaveOccurred())
+	})
+	Describe("", func() {
+		Context("Dummy test", func() {
+			It("is a dummy test", func() {
+			})
+		})
+	})
+})
@@ -0,0 +1,24 @@
+package e2e
+
+import (
+	"context"
+	"flag"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Testing Velero on an azure cluster", func() {
+	BeforeEach(func() {
+		flag.Parse()
+		ctx := context.TODO()
+		err := EnsureClusterExists(ctx)
+		Expect(err).NotTo(HaveOccurred())
+	})
+	Describe("Dummy test", func() {
+		Context("Dummy test", func() {
+			It("is a dummy test", func() {
+			})
+		})
+	})
+})
@@ -0,0 +1,83 @@
+package e2e
+
+import (
+	"flag"
+	"time"
+
+	"github.com/google/uuid"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"golang.org/x/net/context"
+
+	velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+var (
+	backupName  string
+	restoreName string
+)
+
+var _ = Describe("Backup Restore test using Kibishii to generate/verify data", func() {
+
+	BeforeEach(func() {
+		flag.Parse()
+	})
+	Describe("backing up and restoring namespace with data", func() {
+		Context("when the backup is successful", func() {
+			It("generates data, backups up the namespace, deletes the namespace, restores the namespace and verifies data", func() {
+				backupUUID, err := uuid.NewRandom()
+				Expect(err).NotTo(HaveOccurred())
+				backupName = "backup-" + backupUUID.String()
+				restoreName = "restore-" + backupUUID.String()
+				println("backupName = " + backupName)
+				println("creating namespace " + kibishiNamespace)
+				timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute)
+				err = CreateNamespace(timeoutCTX, kibishiNamespace)
+				Expect(err).NotTo(HaveOccurred())
+
+				println("installing kibishii in namespace " + kibishiNamespace)
+				timeoutCTX, _ = context.WithTimeout(context.Background(), 30*time.Minute)
+				err = InstallKibishii(timeoutCTX, kibishiNamespace, cloudPlatform)
+				Expect(err).NotTo(HaveOccurred())
+
+				println("running kibishii generate")
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute*60)
+
+				err = GenerateData(timeoutCTX, kibishiNamespace, 2, 10, 10, 1024, 1024, 0, 2)
+				Expect(err).NotTo(HaveOccurred())
+
+				println("executing backup")
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute*30)
+
+				err = BackupNamespace(timeoutCTX, veleroCLI, backupName, kibishiNamespace)
+				Expect(err).NotTo(HaveOccurred())
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute)
+				err = CheckBackupPhase(timeoutCTX, veleroCLI, backupName, velerov1.BackupPhaseCompleted)
+
+				Expect(err).NotTo(HaveOccurred())
+
+				println("removing namespace " + kibishiNamespace)
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute)
+				err = RemoveNamespace(timeoutCTX, kibishiNamespace)
+				Expect(err).NotTo(HaveOccurred())
+
+				println("restoring namespace")
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute*30)
+				err = RestoreNamespace(timeoutCTX, veleroCLI, restoreName, backupName)
+				Expect(err).NotTo(HaveOccurred())
+				println("Checking that namespace is present")
+				// TODO - check that namespace exists
+				println("running kibishii verify")
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute*60)
+
+				err = VerifyData(timeoutCTX, kibishiNamespace, 2, 10, 10, 1024, 1024, 0, 2)
+				Expect(err).NotTo(HaveOccurred())
+
+				println("removing namespace " + kibishiNamespace)
+				timeoutCTX, _ = context.WithTimeout(context.Background(), time.Minute)
+				err = RemoveNamespace(timeoutCTX, kibishiNamespace)
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+	})
+})
@@ -0,0 +1,27 @@
+package e2e
+
+import (
+	"os/exec"
+
+	"golang.org/x/net/context"
+)
+
+func EnsureClusterExists(ctx context.Context) error {
+	return exec.CommandContext(ctx, "kubectl", "cluster-info").Run()
+}
+
+func CreateNamespace(ctx context.Context, namespace string) error {
+	// TODO - should we talk directly to the API server?
+	err := exec.CommandContext(ctx, "kubectl", "create", "namespace", namespace).Run()
+	return err
+}
+
+func RemoveNamespace(ctx context.Context, namespace string) error {
+	// TODO - should we talk directly to the API server?
+	err := exec.CommandContext(ctx, "kubectl", "delete", "namespace", namespace).Run()
+	return err
+}
+
+func NamespaceExists(ctx context.Context, namespace string) (bool, error) {
+	return false, nil
+}
@@ -0,0 +1,26 @@
+package e2e
+
+import (
+	"flag"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	veleroCLI        string
+	kibishiNamespace string
+	cloudPlatform    string // aws, vsphere, azure
+)
+
+func init() {
+	flag.StringVar(&veleroCLI, "velerocli", "velero", "path to the velero application to use")
+	flag.StringVar(&kibishiNamespace, "kibishiins", "kibishii", "namespace to use for Kibishii distributed data generator")
+	flag.StringVar(&cloudPlatform, "cloudplatform", "aws", "cloud platform we are deploying on (aws, vsphere, azure)")
+}
+
+func TestE2e(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "E2e Suite")
+}
@@ -0,0 +1,145 @@
+package e2e
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+func InstallKibishii(ctx context.Context, namespace string, cloudPlatform string) error {
+	// We use kustomize to generate YAML for Kibishii from the checked-in yaml directories
+	kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k",
+		"github.com/vmware-tanzu-labs/distributed-data-generator/kubernetes/yaml/"+cloudPlatform)
+	stdoutPipe, err := kibishiiInstallCmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	err = kibishiiInstallCmd.Start()
+	if err != nil {
+		return err
+	}
+
+	defer stdoutPipe.Close()
+	// copy the data written to the PipeReader via the cmd to stdout
+	_, err = io.Copy(os.Stdout, stdoutPipe)
+
+	if err != nil {
+		return err
+	}
+	err = kibishiiInstallCmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	kibishiiSetWaitCmd := exec.CommandContext(ctx, "kubectl", "rollout", "status", "statefulset.apps/kibishii-deployment",
+		"-n", namespace, "-w", "--timeout=30m")
+	err = kibishiiSetWaitCmd.Run()
+
+	if err != nil {
+		return err
+	}
+
+	jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad")
+	err = jumpPadWaitCmd.Run()
+
+	return err
+}
+
+func GenerateData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
+	blockSize int, passNum int, expectedNodes int) error {
+	kibishiiGenerateCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+		"/usr/local/bin/generate.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
+		strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
+	fmt.Printf("kibishiiGenerateCmd cmd =%v\n", kibishiiGenerateCmd)
+	stdoutPipe, err := kibishiiGenerateCmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	err = kibishiiGenerateCmd.Start()
+	if err != nil {
+		return err
+	}
+	defer stdoutPipe.Close()
+
+	stdoutReader := bufio.NewReader(stdoutPipe)
+	var readErr error
+	for true {
+		buf, isPrefix, err := stdoutReader.ReadLine()
+		if err != nil {
+			readErr = err
+			break
+		} else {
+			if isPrefix {
+				readErr = errors.New("line returned exceeded max length")
+				break
+			}
+			line := strings.TrimSpace(string(buf))
+			if line == "success" {
+				break
+			}
+			if line == "failed" {
+				readErr = errors.New("generate failed")
+				break
+			}
+			fmt.Println(line)
+		}
+	}
+	err = kibishiiGenerateCmd.Wait()
+	if readErr != nil {
+		err = readErr // Squash the Wait err, the read error is probably more interesting
+	}
+	return err
+}
+
+func VerifyData(ctx context.Context, namespace string, levels int, filesPerLevel int, dirsPerLevel int, fileSize int,
+	blockSize int, passNum int, expectedNodes int) error {
+	kibishiiVerifyCmd := exec.CommandContext(ctx, "kubectl", "exec", "-n", namespace, "jump-pad", "--",
+		"/usr/local/bin/verify.sh", strconv.Itoa(levels), strconv.Itoa(filesPerLevel), strconv.Itoa(dirsPerLevel), strconv.Itoa(fileSize),
+		strconv.Itoa(blockSize), strconv.Itoa(passNum), strconv.Itoa(expectedNodes))
+	fmt.Printf("kibishiiVerifyCmd cmd =%v\n", kibishiiVerifyCmd)
+	stdoutPipe, err := kibishiiVerifyCmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	err = kibishiiVerifyCmd.Start()
+	if err != nil {
+		return err
+	}
+	defer stdoutPipe.Close()
+
+	stdoutReader := bufio.NewReader(stdoutPipe)
+	var readErr error
+	for true {
+		buf, isPrefix, err := stdoutReader.ReadLine()
+		if err != nil {
+			readErr = err
+			break
+		} else {
+			if isPrefix {
+				readErr = errors.New("line returned exceeded max length")
+				break
+			}
+			line := strings.TrimSpace(string(buf))
+			if line == "success" {
+				break
+			}
+			if line == "failed" {
+				readErr = errors.New("generate failed")
+				break
+			}
+			fmt.Println(line)
+		}
+	}
+	err = kibishiiVerifyCmd.Wait()
+	if readErr != nil {
+		err = readErr // Squash the Wait err, the read error is probably more interesting
+	}
+	return err
+}
@@ -0,0 +1,24 @@
+package e2e
+
+import (
+	"context"
+	"flag"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Testing Velero on a kind cluster", func() {
+	BeforeEach(func() {
+		flag.Parse()
+		ctx := context.TODO()
+		err := EnsureClusterExists(ctx)
+		Expect(err).NotTo(HaveOccurred())
+	})
+	Describe("Dummy test", func() {
+		Context("Dummy test", func() {
+			It("is a dummy test", func() {
+			})
+		})
+	})
+})
@@ -0,0 +1,114 @@
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os/exec"
+
+	"github.com/pkg/errors"
+
+	v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+)
+
+func CheckBackupPhase(ctx context.Context, veleroCLI string, backupName string, expectedPhase v1.BackupPhase) error {
+	checkCMD := exec.CommandContext(ctx, veleroCLI, "backup", "get", "-o", "json", backupName)
+	fmt.Printf("get backup cmd =%v\n", checkCMD)
+	stdoutPipe, err := checkCMD.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
+
+	err = checkCMD.Start()
+	if err != nil {
+		return err
+	}
+
+	bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return err
+	}
+	if bytesRead == len(jsonBuf) {
+		return errors.New("yaml returned bigger than max allowed")
+	}
+
+	jsonBuf = jsonBuf[0:bytesRead]
+	err = checkCMD.Wait()
+	if err != nil {
+		return err
+	}
+	backup := v1.Backup{}
+	err = json.Unmarshal(jsonBuf, &backup)
+	if err != nil {
+		return err
+	}
+	if backup.Status.Phase != expectedPhase {
+		return errors.Errorf("Unexpected backup phase got %s, expecting %s", backup.Status.Phase, expectedPhase)
+	}
+	return nil
+}
+
+func CheckRestorePhase(ctx context.Context, veleroCLI string, restoreName string, expectedPhase v1.RestorePhase) error {
+	checkCMD := exec.CommandContext(ctx, veleroCLI, "restore", "get", "-o", "json", restoreName)
+	fmt.Printf("get restore cmd =%v\n", checkCMD)
+	stdoutPipe, err := checkCMD.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	jsonBuf := make([]byte, 16*1024) // If the YAML is bigger than 16K, there's probably something bad happening
+
+	err = checkCMD.Start()
+	if err != nil {
+		return err
+	}
+
+	bytesRead, err := io.ReadFull(stdoutPipe, jsonBuf)
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return err
+	}
+	if bytesRead == len(jsonBuf) {
+		return errors.New("yaml returned bigger than max allowed")
+	}
+
+	jsonBuf = jsonBuf[0:bytesRead]
+	err = checkCMD.Wait()
+	if err != nil {
+		return err
+	}
+	restore := v1.Restore{}
+	err = json.Unmarshal(jsonBuf, &restore)
+	if err != nil {
+		return err
+	}
+	if restore.Status.Phase != expectedPhase {
+		return errors.Errorf("Unexpected restore phase got %s, expecting %s", restore.Status.Phase, expectedPhase)
+	}
+	return nil
+}
+
+func BackupNamespace(ctx context.Context, veleroCLI string, backupName string, namespace string) error {
+	backupCmd := exec.CommandContext(ctx, veleroCLI, "create", "backup", backupName, "--include-namespaces", namespace,
+		"--default-volumes-to-restic", "--wait")
+	fmt.Printf("backup cmd =%v\n", backupCmd)
+	err := backupCmd.Run()
+	if err != nil {
+		return err
+	}
+	return CheckBackupPhase(ctx, veleroCLI, backupName, v1.BackupPhaseCompleted)
+}
+
+func RestoreNamespace(ctx context.Context, veleroCLI string, restoreName string, backupName string) error {
+	restoreCmd := exec.CommandContext(ctx, veleroCLI, "create", "restore", restoreName, "--from-backup", backupName, "--wait")
+	fmt.Printf("restore cmd =%v\n", restoreCmd)
+	err := restoreCmd.Run()
+	if err != nil {
+		return err
+	}
+	return CheckRestorePhase(ctx, veleroCLI, restoreName, v1.RestorePhaseCompleted)
+}