test: fix various E2E test issues (#5576)

* Don't use state sync for nodes starting at initial height.
* Also remove stopped containers when cleaning up.
* Start nodes in order of startAt, mode, name to avoid full nodes starting before their seeds.
* Tweak network waiting to avoid halts caused by validator changes and perturbations.
* Disable most tests for seed nodes, which aren't always able to join consensus.
* Disable `blockchain/v2` due to known bugs.

test/e2e/runner/cleanup.go

@@ -32,7 +32,7 @@ func cleanupDocker() error {
 	xargsR := `$(if [[ $OSTYPE == "linux-gnu"* ]]; then echo -n "-r"; fi)`
 
 	err := exec("bash", "-c", fmt.Sprintf(
-		"docker container ls -q --filter label=e2e | xargs %v docker container rm -f", xargsR))
+		"docker container ls -qa --filter label=e2e | xargs %v docker container rm -f", xargsR))
 	if err != nil {
 		return err
 	}

test/e2e/runner/main.go

@@ -69,25 +69,33 @@ func NewCLI() *CLI {
 			if err := Start(cli.testnet); err != nil {
 				return err
 			}
+
 			if lastMisbehavior := cli.testnet.LastMisbehaviorHeight(); lastMisbehavior > 0 {
-				// wait for misbehaviors before starting perturbations
-				if err := WaitUntil(cli.testnet, lastMisbehavior+5); err != nil {
+				// wait for misbehaviors before starting perturbations. We do a separate
+				// wait for another 5 blocks, since the last misbehavior height may be
+				// in the past depending on network startup ordering.
+				if err := WaitUntil(cli.testnet, lastMisbehavior); err != nil {
 					return err
 				}
 			}
-			if err := Perturb(cli.testnet); err != nil {
-				return err
-			}
 			if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
 				return err
 			}
 
+			if cli.testnet.HasPerturbations() {
+				if err := Perturb(cli.testnet); err != nil {
+					return err
+				}
+				if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
+					return err
+				}
+			}
+
 			loadCancel()
 			if err := <-chLoadResult; err != nil {
 				return err
 			}
-			// wait for network to settle before tests
-			if err := Wait(cli.testnet, 5); err != nil {
+			if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
 				return err
 			}
 			if err := Test(cli.testnet); err != nil {

test/e2e/runner/start.go

@@ -10,8 +10,21 @@ import (
 
 func Start(testnet *e2e.Testnet) error {
 
-	// Sort nodes by starting order
+	// Nodes are already sorted by name. Sort them by name then startAt,
+	// which gives the overall order startAt, mode, name.
 	nodeQueue := testnet.Nodes
+	sort.SliceStable(nodeQueue, func(i, j int) bool {
+		a, b := nodeQueue[i], nodeQueue[j]
+		switch {
+		case a.Mode == b.Mode:
+			return false
+		case a.Mode == e2e.ModeSeed:
+			return true
+		case a.Mode == e2e.ModeValidator && b.Mode == e2e.ModeFull:
+			return true
+		}
+		return false
+	})
 	sort.SliceStable(nodeQueue, func(i, j int) bool {
 		return nodeQueue[i].StartAt < nodeQueue[j].StartAt
 	})