From 55a9b65c17dd1cb97c3fb4c2e645956e141ccff8 Mon Sep 17 00:00:00 2001 From: Ashish Amarnath Date: Tue, 9 Mar 2021 13:32:30 -0800 Subject: [PATCH] Prefer conditional waiting over magic sleep (#3527) * prefer conditional waiting over magic sleep Signed-off-by: Ashish Amarnath * update go modules Signed-off-by: Ashish Amarnath --- test/e2e/common.go | 19 ++++++++++++++ test/e2e/kibishii_tests.go | 51 +++++++++++++++++++++++++++++++------- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index c479bc159..41e4f5002 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,11 +1,14 @@ package e2e import ( + "fmt" "os/exec" + "time" "golang.org/x/net/context" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" @@ -25,3 +28,19 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac } return err } + +// WaitForNamespaceDeletion Waits for namespace to be deleted. +func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { + err := wait.PollImmediate(interval, timeout, func() (bool, error) { + _, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + fmt.Printf("Namespace %s is still being deleted...\n", ns) + return false, nil + }) + return err +} diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 0f025f1e1..868322f0f 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -6,17 +6,20 @@ import ( "strconv" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/pkg/errors" "golang.org/x/net/context" "k8s.io/client-go/kubernetes" + corev1api "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" ) const ( kibishiiNamespace = "kibishii-workload" + jumpPadPod = "jump-pad" ) func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error { @@ -37,15 +40,18 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform string return err } - fmt.Printf("Waiting for kibishii jump-pad ready\n") + fmt.Printf("Waiting for kibishii jump-pad pod to be ready\n") jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad") _, _, err = veleroexec.RunCommand(jumpPadWaitCmd) + if err != nil { + return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s", namespace, jumpPadPod) + } // TODO - Fix kibishii so we can check that it is ready to go - if err == nil { - fmt.Printf("Sleeping for Kibishii startup\n") - time.Sleep(time.Minute * 1) - } + // Wait for etcd run as kibishii workload to be ready + fmt.Printf("Waiting for etcd workload pods to be ready\n") + exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "pod/etcd0", "pod/etcd1", "pod/etcd2") + return err } @@ -82,6 +88,8 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) + timeout := 10 * time.Minute + interval := 5 * time.Second if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) @@ -104,12 +112,33 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, "Failed to simulate a disaster") } + // wait for ns delete + err := WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) + } if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } + // wait for kibishii pod startup + err = wait.PollImmediate(interval, timeout, func() (bool, error) { + kp, err := client.CoreV1().Pods(kibishiiNamespace).Get(context.TODO(), jumpPadPod, metav1.GetOptions{}) + if err != nil { + return false, errors.Wrapf(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", kibishiiNamespace, jumpPadPod, corev1api.PodRunning)) + } + if kp.Status.Phase != corev1api.PodRunning { + fmt.Printf("Pod %s is in state %s waiting for it to be %s\n", jumpPadPod, kp.Status.Phase, corev1api.PodRunning) + return false, nil + } + return true, nil + }) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Failed to wait for pod %s/%s to start running", kibishiiNamespace, jumpPadPod)) + } + // TODO - check that namespace exists fmt.Printf("running kibishii verify\n") if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil { @@ -119,7 +148,11 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } - - // kibishii test completed successfully + // wait for ns delete + err = WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) + if err != nil { + return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) + } + fmt.Printf("kibishii test completed successfully\n") return nil }