e2e: add contexts and e2e generator tests (#9426)

2026-07-20 06:52:30 +00:00 · 2022-11-10 12:29:59 +01:00
parent 4f3e87b2e4
commit f8e453a245
14 changed files with 196 additions and 89 deletions
@@ -7,7 +7,27 @@ make
 ./build/runner -f networks/ci.toml
 ```

-This creates and runs a testnet named `ci` under `networks/ci/` (determined by the manifest filename).
+This creates and runs a testnet named `ci` under `networks/ci/`.
+
+## Conceptual Overview
+
+End-to-end testnets are used to test Tendermint functionality as a user would use it, by spinning up a set of nodes with various configurations and making sure the nodes and network behave correctly. The background for the E2E test suite is outlined in [RFC-001](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-066-e2e-testing.md).
+
+The end-to-end tests can be thought of in this manner:
+
+1. Does a certain (valid!) testnet configuration result in a block-producing network where all nodes eventually reach the latest height?
+
+2. If so, does each node in that network satisfy all invariants specified by the Go E2E tests?
+
+The above should hold for any arbitrary, valid network configuration, and that configuration space  should be searched and tested by randomly generating testnets.
+
+A testnet configuration is specified as a TOML testnet manifest (see below). The testnet runner uses the manifest to configure a set of Docker containers and start them in some order. The manifests can be written manually (to test specific configurations) or generated randomly by the testnet generator (to test a wide range of configuration permutations).
+
+When running a testnet, the runner will first start the Docker nodes in some sequence, submit random transactions, and wait for the nodes to come online and the first blocks to be produced. This may involve e.g. waiting for nodes to block sync and/or state sync. If specified, it will then run any misbehaviors (e.g. double-signing) and perturbations (e.g. killing or disconnecting nodes). It then waits for the testnet to stabilize, with all nodes online and having reached the latest height.
+
+Once the testnet stabilizes, a set of Go end-to-end tests are run against the live testnet to verify network invariants (for example that blocks are identical across nodes). These use the RPC client to interact with the network, and should consider the entire network as a black box (i.e. it should not test any network or node internals, only externally visible behavior via RPC). The tests may use the `testNode()` helper to run parallel tests against each individual testnet node, and/or inspect the full blockchain history via `fetchBlockChain()`.
+
+The tests must take into account the network and/or node configuration, and tolerate that the network is still live and producing blocks. For example, validator tests should only run against nodes that are actually validators, and take into account the node's block retention and/or state sync configuration to not query blocks that don't exist.

 ## Testnet Manifests

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+// TestGenerator tests that only valid manifests are generated
+func TestGenerator(t *testing.T) {
+	manifests, err := Generate(rand.New(rand.NewSource(randomSeed)))
+	require.NoError(t, err)
+	require.True(t, len(manifests) >= 24, "insufficient combinations %d", len(manifests))
+
+	for idx, m := range manifests {
+		t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) {
+			_, err := e2e.NewTestnetFromManifest(m, filepath.Join(t.TempDir(), fmt.Sprintf("Case%04d", idx)))
+			require.NoError(t, err)
+		})
+	}
+}
@@ -1,5 +1,4 @@
 [node.validator01]
 [node.validator02]
 [node.validator03]
-[node.validator04]
-
+[node.validator04]
@@ -23,7 +23,6 @@ type Config struct {
 	PrivValServer    string                      `toml:"privval_server"`
 	PrivValKey       string                      `toml:"privval_key"`
 	PrivValState     string                      `toml:"privval_state"`
-	Misbehaviors     map[string]string           `toml:"misbehaviors"`
 	KeyType          string                      `toml:"key_type"`
 }

@@ -124,8 +124,8 @@ type ManifestNode struct {
 	SnapshotInterval uint64 `toml:"snapshot_interval"`

 	// RetainBlocks specifies the number of recent blocks to retain. Defaults to
-	// 0, which retains all blocks. Must be greater that PersistInterval and
-	// SnapshotInterval.
+	// 0, which retains all blocks. Must be greater that PersistInterval,
+	// SnapshotInterval and EvidenceAgeHeight.
 	RetainBlocks uint64 `toml:"retain_blocks"`

 	// Perturb lists perturbations to apply to the node after it has been
@@ -105,6 +105,11 @@ func LoadTestnet(file string) (*Testnet, error) {
 	if err != nil {
 		return nil, err
 	}
+	return NewTestnetFromManifest(manifest, file)
+}
+
+// NewTestnetFromManifest creates and validates a testnet from a manifest
+func NewTestnetFromManifest(manifest Manifest, file string) (*Testnet, error) {
 	dir := strings.TrimSuffix(file, filepath.Ext(file))

 	// Set up resource generators. These must be deterministic.
@@ -19,7 +19,7 @@ FAILED=()

 for MANIFEST in "$@"; do
 	START=$SECONDS
-	echo "==> Running testnet $MANIFEST..."
+	echo "==> Running testnet: $MANIFEST"

 	if ! ./build/runner -f "$MANIFEST"; then
 		echo "==> Testnet $MANIFEST failed, dumping manifest..."
@@ -2,8 +2,10 @@ package main

 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
+	"path/filepath"
 	"time"

 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -19,26 +21,28 @@ import (
 //
 // Metrics are based of the `benchmarkLength`, the amount of consecutive blocks
 // sampled from in the testnet
-func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
-	block, _, err := waitForHeight(testnet, 0)
+func Benchmark(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) error {
+	block, _, err := waitForHeight(ctx, testnet, 0)
 	if err != nil {
 		return err
 	}

 	logger.Info("Beginning benchmark period...", "height", block.Height)
+	startAt := time.Now()

 	// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
 	// which should be sufficient.
 	waitingTime := time.Duration(benchmarkLength*5) * time.Second
-	endHeight, err := waitForAllNodes(testnet, block.Height+benchmarkLength, waitingTime)
+	endHeight, err := waitForAllNodes(ctx, testnet, block.Height+benchmarkLength, waitingTime)
 	if err != nil {
 		return err
 	}
+	dur := time.Since(startAt)

 	logger.Info("Ending benchmark period", "height", endHeight)

 	// fetch a sample of blocks
-	blocks, err := fetchBlockChainSample(testnet, benchmarkLength)
+	blocks, err := fetchBlockChainSample(ctx, testnet, benchmarkLength)
 	if err != nil {
 		return err
 	}
@@ -46,18 +50,29 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	// slice into time intervals and collate data
 	timeIntervals := splitIntoBlockIntervals(blocks)
 	testnetStats := extractTestnetStats(timeIntervals)
+	testnetStats.populateTxns(blocks)
+	testnetStats.totalTime = dur
 	testnetStats.startHeight = blocks[0].Header.Height
 	testnetStats.endHeight = blocks[len(blocks)-1].Header.Height

 	// print and return
-	logger.Info(testnetStats.String())
+	logger.Info(testnetStats.OutputJSON(testnet))
 	return nil
 }

+func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) {
+	t.numtxns = 0
+	for _, b := range blocks {
+		t.numtxns += int64(b.NumTxs)
+	}
+}
+
 type testnetStats struct {
 	startHeight int64
 	endHeight   int64

+	numtxns   int64
+	totalTime time.Duration
 	// average time to produce a block
 	mean time.Duration
 	// standard deviation of block production
@@ -68,6 +83,28 @@ type testnetStats struct {
 	min time.Duration
 }

+func (t *testnetStats) OutputJSON(net *e2e.Testnet) string {
+	jsn, err := json.Marshal(map[string]interface{}{
+		"case":         filepath.Base(net.File),
+		"start_height": t.startHeight,
+		"end_height":   t.endHeight,
+		"blocks":       t.endHeight - t.startHeight,
+		"stddev":       t.std,
+		"mean":         t.mean.Seconds(),
+		"max":          t.max.Seconds(),
+		"min":          t.min.Seconds(),
+		"size":         len(net.Nodes),
+		"txns":         t.numtxns,
+		"dur":          t.totalTime.Seconds(),
+	})
+
+	if err != nil {
+		return ""
+	}
+
+	return string(jsn)
+}
+
 func (t *testnetStats) String() string {
 	return fmt.Sprintf(`Benchmarked from height %v to %v
 	Mean Block Interval: %v
@@ -86,7 +123,7 @@ func (t *testnetStats) String() string {

 // fetchBlockChainSample waits for `benchmarkLength` amount of blocks to pass, fetching
 // all of the headers for these blocks from an archive node and returning it.
-func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
+func fetchBlockChainSample(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
 	var blocks []*types.BlockMeta

 	// Find the first archive node
@@ -97,7 +134,6 @@ func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*type
 	}

 	// find the latest height
-	ctx := context.Background()
 	s, err := c.Status(ctx)
 	if err != nil {
 		return nil, err
@@ -82,7 +82,7 @@ func InjectEvidence(ctx context.Context, r *rand.Rand, testnet *e2e.Testnet, amo

 	// wait for the node to reach the height above the forged height so that
 	// it is able to validate the evidence
-	_, err = waitForNode(targetNode, waitHeight, time.Minute)
+	_, err = waitForNode(ctx, targetNode, waitHeight, time.Minute)
 	if err != nil {
 		return err
 	}
@@ -110,7 +110,7 @@ func InjectEvidence(ctx context.Context, r *rand.Rand, testnet *e2e.Testnet, amo

 	// wait for the node to reach the height above the forged height so that
 	// it is able to validate the evidence
-	_, err = waitForNode(targetNode, blockRes.Block.Height+2, 30*time.Second)
+	_, err = waitForNode(ctx, targetNode, blockRes.Block.Height+2, 30*time.Second)
 	if err != nil {
 		return err
 	}
@@ -70,19 +70,19 @@ func NewCLI() *CLI {
 				chLoadResult <- err
 			}()

-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cmd.Context(), cli.testnet); err != nil {
 				return err
 			}

-			if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
+			if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
 				return err
 			}

 			if cli.testnet.HasPerturbations() {
-				if err := Perturb(cli.testnet); err != nil {
+				if err := Perturb(cmd.Context(), cli.testnet); err != nil {
 					return err
 				}
-				if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
+				if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
 					return err
 				}
 			}
@@ -91,7 +91,7 @@ func NewCLI() *CLI {
 				if err := InjectEvidence(ctx, r, cli.testnet, cli.testnet.Evidence); err != nil {
 					return err
 				}
-				if err := Wait(cli.testnet, 5); err != nil { // ensure chain progress
+				if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // ensure chain progress
 					return err
 				}
 			}
@@ -100,7 +100,7 @@ func NewCLI() *CLI {
 			if err := <-chLoadResult; err != nil {
 				return err
 			}
-			if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
+			if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // wait for network to settle before tests
 				return err
 			}
 			if err := Test(cli.testnet); err != nil {
@@ -140,7 +140,7 @@ func NewCLI() *CLI {
 			if err != nil {
 				return err
 			}
-			return Start(cli.testnet)
+			return Start(cmd.Context(), cli.testnet)
 		},
 	})

@@ -148,7 +148,7 @@ func NewCLI() *CLI {
 		Use:   "perturb",
 		Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Perturb(cli.testnet)
+			return Perturb(cmd.Context(), cli.testnet)
 		},
 	})

@@ -156,7 +156,7 @@ func NewCLI() *CLI {
 		Use:   "wait",
 		Short: "Waits for a few blocks to be produced and all nodes to catch up",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Wait(cli.testnet, 5)
+			return Wait(cmd.Context(), cli.testnet, 5)
 		},
 	})

@@ -263,7 +263,7 @@ Does not run any perbutations.
 			}

 			chLoadResult := make(chan error)
-			ctx, loadCancel := context.WithCancel(context.Background())
+			ctx, loadCancel := context.WithCancel(cmd.Context())
 			defer loadCancel()
 			go func() {
 				err := Load(ctx, cli.testnet, 1)
@@ -273,16 +273,16 @@ Does not run any perbutations.
 				chLoadResult <- err
 			}()

-			if err := Start(cli.testnet); err != nil {
+			if err := Start(cmd.Context(), cli.testnet); err != nil {
 				return err
 			}

-			if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
+			if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
 				return err
 			}

 			// we benchmark performance over the next 100 blocks
-			if err := Benchmark(cli.testnet, 100); err != nil {
+			if err := Benchmark(cmd.Context(), cli.testnet, 100); err != nil {
 				return err
 			}

@@ -1,6 +1,7 @@
 package main

 import (
+	"context"
 	"fmt"
 	"time"

@@ -10,10 +11,10 @@ import (
 )

 // Perturbs a running testnet.
-func Perturb(testnet *e2e.Testnet) error {
+func Perturb(ctx context.Context, testnet *e2e.Testnet) error {
 	for _, node := range testnet.Nodes {
 		for _, perturbation := range node.Perturbations {
-			_, err := PerturbNode(node, perturbation)
+			_, err := PerturbNode(ctx, node, perturbation)
 			if err != nil {
 				return err
 			}
@@ -25,7 +26,7 @@ func Perturb(testnet *e2e.Testnet) error {

 // PerturbNode perturbs a node with a given perturbation, returning its status
 // after recovering.
-func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
+func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
 	testnet := node.Testnet
 	switch perturbation {
 	case e2e.PerturbationDisconnect:
@@ -67,7 +68,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
 		return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
 	}

-	status, err := waitForNode(node, 0, 20*time.Second)
+	status, err := waitForNode(ctx, node, 0, 20*time.Second)
 	if err != nil {
 		return nil, err
 	}
@@ -15,7 +15,7 @@ import (
 // waitForHeight waits for the network to reach a certain height (or above),
 // returning the highest height seen. Errors if the network is not making
 // progress at all.
-func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
+func waitForHeight(ctx context.Context, testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
 	var (
 		err          error
 		maxResult    *rpctypes.ResultBlock
@@ -23,73 +23,92 @@ func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.Blo
 		lastIncrease = time.Now()
 	)

+	timer := time.NewTimer(0)
+	defer timer.Stop()
 	for {
-		for _, node := range testnet.Nodes {
-			if node.Mode == e2e.ModeSeed {
-				continue
-			}
-			client, ok := clients[node.Name]
-			if !ok {
-				client, err = node.Client()
+		select {
+		case <-ctx.Done():
+			return nil, nil, ctx.Err()
+		case <-timer.C:
+			for _, node := range testnet.Nodes {
+				if node.Stateless() {
+					continue
+				}
+				client, ok := clients[node.Name]
+				if !ok {
+					client, err = node.Client()
+					if err != nil {
+						continue
+					}
+					clients[node.Name] = client
+				}
+
+				subctx, cancel := context.WithTimeout(ctx, 1*time.Second)
+				defer cancel()
+				result, err := client.Block(subctx, nil)
+				if err == context.DeadlineExceeded || err == context.Canceled {
+					return nil, nil, ctx.Err()
+				}
 				if err != nil {
 					continue
 				}
-				clients[node.Name] = client
+				if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
+					maxResult = result
+					lastIncrease = time.Now()
+				}
+				if maxResult != nil && maxResult.Block.Height >= height {
+					return maxResult.Block, &maxResult.BlockID, nil
+				}
 			}

-			ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
-			defer cancel()
-			result, err := client.Block(ctx, nil)
-			if err != nil {
-				continue
+			if len(clients) == 0 {
+				return nil, nil, errors.New("unable to connect to any network nodes")
 			}
-			if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
-				maxResult = result
-				lastIncrease = time.Now()
-			}
-			if maxResult != nil && maxResult.Block.Height >= height {
-				return maxResult.Block, &maxResult.BlockID, nil
+			if time.Since(lastIncrease) >= 20*time.Second {
+				if maxResult == nil {
+					return nil, nil, errors.New("chain stalled at unknown height")
+				}
+				return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
 			}
+			timer.Reset(1 * time.Second)
 		}

-		if len(clients) == 0 {
-			return nil, nil, errors.New("unable to connect to any network nodes")
-		}
-		if time.Since(lastIncrease) >= 20*time.Second {
-			if maxResult == nil {
-				return nil, nil, errors.New("chain stalled at unknown height")
-			}
-			return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
-		}
-		time.Sleep(1 * time.Second)
 	}
 }

 // waitForNode waits for a node to become available and catch up to the given block height.
-func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
+func waitForNode(ctx context.Context, node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
 	client, err := node.Client()
 	if err != nil {
 		return nil, err
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	for {
-		status, err := client.Status(ctx)
-		switch {
-		case errors.Is(err, context.DeadlineExceeded):
-			return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
-		case errors.Is(err, context.Canceled):
-			return nil, err
-		case err == nil && status.SyncInfo.LatestBlockHeight >= height:
-			return status, nil
-		}

-		time.Sleep(300 * time.Millisecond)
+	timer := time.NewTimer(0)
+	defer timer.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-timer.C:
+			subctx, cancel := context.WithTimeout(ctx, timeout)
+			defer cancel()
+			status, err := client.Status(subctx)
+			switch {
+			case errors.Is(err, context.DeadlineExceeded):
+				return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
+			case errors.Is(err, context.Canceled):
+				return nil, err
+			case err == nil && status.SyncInfo.LatestBlockHeight >= height:
+				return status, nil
+			}
+
+			timer.Reset(300 * time.Millisecond)
+		}
 	}
 }

 // waitForAllNodes waits for all nodes to become available and catch up to the given block height.
-func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
+func waitForAllNodes(ctx context.Context, testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
 	var lastHeight int64

 	for _, node := range testnet.Nodes {
@@ -97,7 +116,7 @@ func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration)
 			continue
 		}

-		status, err := waitForNode(node, height, timeout)
+		status, err := waitForNode(ctx, node, height, timeout)
 		if err != nil {
 			return 0, err
 		}
@@ -1,6 +1,7 @@
 package main

 import (
+	"context"
 	"fmt"
 	"sort"
 	"time"
@@ -9,7 +10,7 @@ import (
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )

-func Start(testnet *e2e.Testnet) error {
+func Start(ctx context.Context, testnet *e2e.Testnet) error {
 	if len(testnet.Nodes) == 0 {
 		return fmt.Errorf("no nodes in testnet")
 	}
@@ -46,7 +47,7 @@ func Start(testnet *e2e.Testnet) error {
 		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
-		if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
+		if _, err := waitForNode(ctx, node, 0, 15*time.Second); err != nil {
 			return err
 		}
 		logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
@@ -60,7 +61,7 @@ func Start(testnet *e2e.Testnet) error {
 		"nodes", len(testnet.Nodes)-len(nodeQueue),
 		"pending", len(nodeQueue))

-	block, blockID, err := waitForHeight(testnet, networkHeight)
+	block, blockID, err := waitForHeight(ctx, testnet, networkHeight)
 	if err != nil {
 		return err
 	}
@@ -90,7 +91,7 @@ func Start(testnet *e2e.Testnet) error {
 				"node", node.Name,
 				"height", networkHeight)

-			if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
+			if _, _, err := waitForHeight(ctx, testnet, networkHeight); err != nil {
 				return err
 			}
 		}
@@ -100,7 +101,7 @@ func Start(testnet *e2e.Testnet) error {
 		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err
 		}
-		status, err := waitForNode(node, node.StartAt, 3*time.Minute)
+		status, err := waitForNode(ctx, node, node.StartAt, 3*time.Minute)
 		if err != nil {
 			return err
 		}
@@ -1,6 +1,7 @@
 package main

 import (
+	"context"
 	"time"

 	"github.com/tendermint/tendermint/libs/log"
@@ -9,18 +10,18 @@ import (

 // Wait waits for a number of blocks to be produced, and for all nodes to catch
 // up with it.
-func Wait(testnet *e2e.Testnet, blocks int64) error {
-	block, _, err := waitForHeight(testnet, 0)
+func Wait(ctx context.Context, testnet *e2e.Testnet, blocks int64) error {
+	block, _, err := waitForHeight(ctx, testnet, 0)
 	if err != nil {
 		return err
 	}
-	return WaitUntil(testnet, block.Height+blocks)
+	return WaitUntil(ctx, testnet, block.Height+blocks)
 }

 // WaitUntil waits until a given height has been reached.
-func WaitUntil(testnet *e2e.Testnet, height int64) error {
+func WaitUntil(ctx context.Context, testnet *e2e.Testnet, height int64) error {
 	logger.Info("wait until", "msg", log.NewLazySprintf("Waiting for all nodes to reach height %v...", height))
-	_, err := waitForAllNodes(testnet, height, waitingTime(len(testnet.Nodes)))
+	_, err := waitForAllNodes(ctx, testnet, height, waitingTime(len(testnet.Nodes)))
 	if err != nil {
 		return err
 	}