Merge branch 'master' into callum/p2p-provider

test/e2e/runner/evidence.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -29,7 +30,21 @@ const lightClientEvidenceRatio = 4
 // DuplicateVoteEvidence.
 func InjectEvidence(testnet *e2e.Testnet, amount int) error {
 	// select a random node
-	targetNode := testnet.RandomNode()
+	var targetNode *e2e.Node
+
+	for i := 0; i < len(testnet.Nodes)-1; i++ {
+		targetNode = testnet.Nodes[rand.Intn(len(testnet.Nodes))] // nolint: gosec
+		if targetNode.Mode == e2e.ModeSeed {
+			targetNode = nil
+			continue
+		}
+
+		break
+	}
+
+	if targetNode == nil {
+		return errors.New("could not find node to inject evidence into")
+	}
 
 	logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount))
 

test/e2e/runner/load.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"container/ring"
 	"context"
 	"crypto/rand"
 	"errors"
@@ -93,34 +94,64 @@ func loadGenerate(ctx context.Context, chTx chan<- types.Tx, multiplier int, siz
 
 // loadProcess processes transactions
 func loadProcess(ctx context.Context, testnet *e2e.Testnet, chTx <-chan types.Tx, chSuccess chan<- types.Tx) {
-	// Each worker gets its own client to each node, which allows for some
-	// concurrency while still bounding it.
-	clients := map[string]*rpchttp.HTTP{}
+	// Each worker gets its own client to each usable node, which
+	// allows for some concurrency while still bounding it.
+	clients := make([]*rpchttp.HTTP, 0, len(testnet.Nodes))
 
-	var err error
-	for tx := range chTx {
-		node := testnet.RandomNode()
-
-		client, ok := clients[node.Name]
-		if !ok {
-			client, err = node.Client()
-			if err != nil {
-				continue
-			}
-
-			// check that the node is up
-			_, err = client.Health(ctx)
-			if err != nil {
-				continue
-			}
-
-			clients[node.Name] = client
-		}
-
-		if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
+	for idx := range testnet.Nodes {
+		// Construct a list of usable nodes for the creating
+		// load. Don't send load through seed nodes because
+		// they do not provide the RPC endpoints required to
+		// broadcast transaction.
+		if testnet.Nodes[idx].Mode == e2e.ModeSeed {
 			continue
 		}
 
-		chSuccess <- tx
+		client, err := testnet.Nodes[idx].Client()
+		if err != nil {
+			continue
+		}
+
+		clients = append(clients, client)
+	}
+
+	if len(clients) == 0 {
+		panic("no clients to process load")
+	}
+
+	// Put the clients in a ring so they can be used in a
+	// round-robin fashion.
+	clientRing := ring.New(len(clients))
+	for idx := range clients {
+		clientRing.Value = clients[idx]
+		clientRing = clientRing.Next()
+	}
+
+	var err error
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case tx := <-chTx:
+			clientRing = clientRing.Next()
+			client := clientRing.Value.(*rpchttp.HTTP)
+
+			if _, err := client.Health(ctx); err != nil {
+				continue
+			}
+
+			if _, err = client.BroadcastTxSync(ctx, tx); err != nil {
+				continue
+			}
+
+			select {
+			case chSuccess <- tx:
+				continue
+			case <-ctx.Done():
+				return
+			}
+
+		}
 	}
 }

test/e2e/runner/setup.go

@@ -238,7 +238,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	cfg.RPC.PprofListenAddress = ":6060"
 	cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
 	cfg.P2P.AddrBookStrict = false
-	cfg.P2P.DisableLegacy = node.DisableLegacyP2P
+	cfg.P2P.UseLegacy = node.UseLegacyP2P
 	cfg.P2P.QueueType = node.QueueType
 	cfg.DBBackend = node.Database
 	cfg.StateSync.DiscoveryTime = 5 * time.Second
@@ -345,17 +345,17 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 // MakeAppConfig generates an ABCI application config for a node.
 func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 	cfg := map[string]interface{}{
-		"chain_id":           node.Testnet.Name,
-		"dir":                "data/app",
-		"listen":             AppAddressUNIX,
-		"mode":               node.Mode,
-		"proxy_port":         node.ProxyPort,
-		"protocol":           "socket",
-		"persist_interval":   node.PersistInterval,
-		"snapshot_interval":  node.SnapshotInterval,
-		"retain_blocks":      node.RetainBlocks,
-		"key_type":           node.PrivvalKey.Type(),
-		"disable_legacy_p2p": node.DisableLegacyP2P,
+		"chain_id":          node.Testnet.Name,
+		"dir":               "data/app",
+		"listen":            AppAddressUNIX,
+		"mode":              node.Mode,
+		"proxy_port":        node.ProxyPort,
+		"protocol":          "socket",
+		"persist_interval":  node.PersistInterval,
+		"snapshot_interval": node.SnapshotInterval,
+		"retain_blocks":     node.RetainBlocks,
+		"key_type":          node.PrivvalKey.Type(),
+		"use_legacy_p2p":    node.UseLegacyP2P,
 	}
 	switch node.ABCIProtocol {
 	case e2e.ProtocolUNIX:

test/e2e/runner/start.go

@@ -9,6 +9,9 @@ import (
 )
 
 func Start(testnet *e2e.Testnet) error {
+	if len(testnet.Nodes) == 0 {
+		return fmt.Errorf("no nodes in testnet")
+	}
 
 	// Nodes are already sorted by name. Sort them by name then startAt,
 	// which gives the overall order startAt, mode, name.
@@ -25,12 +28,11 @@ func Start(testnet *e2e.Testnet) error {
 		}
 		return false
 	})
+
 	sort.SliceStable(nodeQueue, func(i, j int) bool {
 		return nodeQueue[i].StartAt < nodeQueue[j].StartAt
 	})
-	if len(nodeQueue) == 0 {
-		return fmt.Errorf("no nodes in testnet")
-	}
+
 	if nodeQueue[0].StartAt > 0 {
 		return fmt.Errorf("no initial nodes in testnet")
 	}
@@ -49,9 +51,15 @@ func Start(testnet *e2e.Testnet) error {
 		logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
 	}
 
+	networkHeight := testnet.InitialHeight
+
 	// Wait for initial height
-	logger.Info(fmt.Sprintf("Waiting for initial height %v...", testnet.InitialHeight))
-	block, blockID, err := waitForHeight(testnet, testnet.InitialHeight)
+	logger.Info("Waiting for initial height",
+		"height", networkHeight,
+		"nodes", len(testnet.Nodes)-len(nodeQueue),
+		"pending", len(nodeQueue))
+
+	block, blockID, err := waitForHeight(testnet, networkHeight)
 	if err != nil {
 		return err
 	}
@@ -66,12 +74,28 @@ func Start(testnet *e2e.Testnet) error {
 		}
 	}
 
-	// Start up remaining nodes
 	for _, node := range nodeQueue {
-		logger.Info(fmt.Sprintf("Starting node %v at height %v...", node.Name, node.StartAt))
-		if _, _, err := waitForHeight(testnet, node.StartAt); err != nil {
-			return err
+		if node.StartAt > networkHeight {
+			// if we're starting a node that's ahead of
+			// the last known height of the network, then
+			// we should make sure that the rest of the
+			// network has reached at least the height
+			// that this node will start at before we
+			// start the node.
+
+			networkHeight = node.StartAt
+
+			logger.Info("Waiting for network to advance before starting catch up node",
+				"node", node.Name,
+				"height", networkHeight)
+
+			if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
+				return err
+			}
 		}
+
+		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
+
 		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}