Prefer conditional waiting over magic sleep (#3527)

* prefer conditional waiting over magic sleep

Signed-off-by: Ashish Amarnath <ashisham@vmware.com>

* update go modules

Signed-off-by: Ashish Amarnath <ashisham@vmware.com>

test/e2e/common.go

@@ -1,11 +1,14 @@
 package e2e
 
 import (
+	"fmt"
 	"os/exec"
+	"time"
 
 	"golang.org/x/net/context"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/vmware-tanzu/velero/pkg/builder"
@@ -25,3 +28,19 @@ func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespac
 	}
 	return err
 }
+
+// WaitForNamespaceDeletion Waits for namespace to be deleted.
+func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error {
+	err := wait.PollImmediate(interval, timeout, func() (bool, error) {
+		_, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		fmt.Printf("Namespace %s is still being deleted...\n", ns)
+		return false, nil
+	})
+	return err
+}

test/e2e/kibishii_tests.go

@@ -6,17 +6,20 @@ import (
 	"strconv"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	"k8s.io/client-go/kubernetes"
 
+	corev1api "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
 	veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
 )
 
 const (
 	kibishiiNamespace = "kibishii-workload"
+	jumpPadPod        = "jump-pad"
 )
 
 func installKibishii(ctx context.Context, namespace string, cloudPlatform string) error {
@@ -37,15 +40,18 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform string
 		return err
 	}
 
-	fmt.Printf("Waiting for kibishii jump-pad ready\n")
+	fmt.Printf("Waiting for kibishii jump-pad pod to be ready\n")
 	jumpPadWaitCmd := exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "-n", namespace, "pod/jump-pad")
 	_, _, err = veleroexec.RunCommand(jumpPadWaitCmd)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to wait for ready status of pod %s/%s", namespace, jumpPadPod)
+	}
 
 	// TODO - Fix kibishii so we can check that it is ready to go
-	if err == nil {
-		fmt.Printf("Sleeping for Kibishii startup\n")
-		time.Sleep(time.Minute * 1)
-	}
+	// Wait for etcd run as kibishii workload to be ready
+	fmt.Printf("Waiting for etcd workload pods to be ready\n")
+	exec.CommandContext(ctx, "kubectl", "wait", "--for=condition=ready", "pod/etcd0", "pod/etcd1", "pod/etcd2")
+
 	return err
 }
 
@@ -82,6 +88,8 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
 func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error {
 	fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
+	timeout := 10 * time.Minute
+	interval := 5 * time.Second
 
 	if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil {
 		return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace)
@@ -104,12 +112,33 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel
 	if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrap(err, "Failed to simulate a disaster")
 	}
+	// wait for ns delete
+	err := WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace)
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace))
+	}
 
 	if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil {
 		VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName)
 		return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName)
 	}
 
+	// wait for kibishii pod startup
+	err = wait.PollImmediate(interval, timeout, func() (bool, error) {
+		kp, err := client.CoreV1().Pods(kibishiiNamespace).Get(context.TODO(), jumpPadPod, metav1.GetOptions{})
+		if err != nil {
+			return false, errors.Wrapf(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", kibishiiNamespace, jumpPadPod, corev1api.PodRunning))
+		}
+		if kp.Status.Phase != corev1api.PodRunning {
+			fmt.Printf("Pod %s is in state %s waiting for it to be %s\n", jumpPadPod, kp.Status.Phase, corev1api.PodRunning)
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Failed to wait for pod %s/%s to start running", kibishiiNamespace, jumpPadPod))
+	}
+
 	// TODO - check that namespace exists
 	fmt.Printf("running kibishii verify\n")
 	if err := verifyData(oneHourTimeout, kibishiiNamespace, 2, 10, 10, 1024, 1024, 0, 2); err != nil {
@@ -119,7 +148,11 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel
 	if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil {
 		return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace)
 	}
-
-	// kibishii test completed successfully
+	// wait for ns delete
+	err = WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace)
+	if err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace))
+	}
+	fmt.Printf("kibishii test completed successfully\n")
 	return nil
 }