e2e: add benchmarking functionality (bp #6210) (#6216)

test/e2e/README.md

@@ -50,6 +50,8 @@ The test runner has the following stages, which can also be executed explicitly
 
 * `cleanup`: removes configuration files and Docker containers/networks.
 
+Auxiliary commands:
+
 * `logs`: outputs all node logs.
 
 * `tail`: tails (follows) node logs until cancelled.
@@ -119,3 +121,11 @@ Docker does not enable IPv6 by default. To do so, enter the following in
   "fixed-cidr-v6": "2001:db8:1::/64"
 }
 ```
+
+## Benchmarking testnets
+
+It is also possible to run a simple benchmark on a testnet. This is done through the `benchmark` command. This manages the entire process: setting up the environment, starting the test net, waiting for a considerable amount of blocks to be used (currently 100), and then returning the following metrics from the sample of the blockchain:
+
+- Average time to produce a block
+- Standard deviation of producing a block
+- Minimum and maximum time to produce a block

test/e2e/runner/benchmark.go

@@ -0,0 +1,192 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"time"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Benchmark is a simple function for fetching, calculating and printing
+// the following metrics:
+// 1. Average block production time
+// 2. Block interval standard deviation
+// 3. Max block interval (slowest block)
+// 4. Min block interval (fastest block)
+//
+// Metrics are based of the `benchmarkLength`, the amount of consecutive blocks
+// sampled from in the testnet
+func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
+	block, _, err := waitForHeight(testnet, 0)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Beginning benchmark period...", "height", block.Height)
+
+	// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
+	// which should be sufficient.
+	waitingTime := time.Duration(benchmarkLength*5) * time.Second
+	endHeight, err := waitForAllNodes(testnet, block.Height+benchmarkLength, waitingTime)
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Ending benchmark period", "height", endHeight)
+
+	// fetch a sample of blocks
+	blocks, err := fetchBlockChainSample(testnet, benchmarkLength)
+	if err != nil {
+		return err
+	}
+
+	// slice into time intervals and collate data
+	timeIntervals := splitIntoBlockIntervals(blocks)
+	testnetStats := extractTestnetStats(timeIntervals)
+	testnetStats.startHeight = blocks[0].Header.Height
+	testnetStats.endHeight = blocks[len(blocks)-1].Header.Height
+
+	// print and return
+	logger.Info(testnetStats.String())
+	return nil
+}
+
+type testnetStats struct {
+	startHeight int64
+	endHeight   int64
+
+	// average time to produce a block
+	mean time.Duration
+	// standard deviation of block production
+	std float64
+	// longest time to produce a block
+	max time.Duration
+	// shortest time to produce a block
+	min time.Duration
+}
+
+func (t *testnetStats) String() string {
+	return fmt.Sprintf(`Benchmarked from height %v to %v
+	Mean Block Interval: %v
+	Standard Deviation: %f
+	Max Block Interval: %v
+	Min Block Interval: %v
+	`,
+		t.startHeight,
+		t.endHeight,
+		t.mean,
+		t.std,
+		t.max,
+		t.min,
+	)
+}
+
+// fetchBlockChainSample waits for `benchmarkLength` amount of blocks to pass, fetching
+// all of the headers for these blocks from an archive node and returning it.
+func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
+	var blocks []*types.BlockMeta
+
+	// Find the first archive node
+	archiveNode := testnet.ArchiveNodes()[0]
+	c, err := archiveNode.Client()
+	if err != nil {
+		return nil, err
+	}
+
+	// find the latest height
+	ctx := context.Background()
+	s, err := c.Status(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	to := s.SyncInfo.LatestBlockHeight
+	from := to - benchmarkLength + 1
+	if from <= testnet.InitialHeight {
+		return nil, fmt.Errorf("tesnet was unable to reach required height for benchmarking (latest height %d)", to)
+	}
+
+	// Fetch blocks
+	for from < to {
+		// fetch the blockchain metas. Currently we can only fetch 20 at a time
+		resp, err := c.BlockchainInfo(ctx, from, min(from+19, to))
+		if err != nil {
+			return nil, err
+		}
+
+		blockMetas := resp.BlockMetas
+		// we receive blocks in descending order so we have to add them in reverse
+		for i := len(blockMetas) - 1; i >= 0; i-- {
+			if blockMetas[i].Header.Height != from {
+				return nil, fmt.Errorf("node gave us another header. Wanted %d, got %d",
+					from,
+					blockMetas[i].Header.Height,
+				)
+			}
+			from++
+			blocks = append(blocks, blockMetas[i])
+		}
+	}
+
+	return blocks, nil
+}
+
+func splitIntoBlockIntervals(blocks []*types.BlockMeta) []time.Duration {
+	intervals := make([]time.Duration, len(blocks)-1)
+	lastTime := blocks[0].Header.Time
+	for i, block := range blocks {
+		// skip the first block
+		if i == 0 {
+			continue
+		}
+
+		intervals[i-1] = block.Header.Time.Sub(lastTime)
+		lastTime = block.Header.Time
+	}
+	return intervals
+}
+
+func extractTestnetStats(intervals []time.Duration) testnetStats {
+	var (
+		sum, mean time.Duration
+		std       float64
+		max       = intervals[0]
+		min       = intervals[0]
+	)
+
+	for _, interval := range intervals {
+		sum += interval
+
+		if interval > max {
+			max = interval
+		}
+
+		if interval < min {
+			min = interval
+		}
+	}
+	mean = sum / time.Duration(len(intervals))
+
+	for _, interval := range intervals {
+		diff := (interval - mean).Seconds()
+		std += math.Pow(diff, 2)
+	}
+	std = math.Sqrt(std / float64(len(intervals)))
+
+	return testnetStats{
+		mean: mean,
+		std:  std,
+		max:  max,
+		min:  min,
+	}
+}
+
+func min(a, b int64) int64 {
+	if a > b {
+		return b
+	}
+	return a
+}

test/e2e/runner/main.go

@@ -204,6 +204,63 @@ func NewCLI() *CLI {
 		},
 	})
 
+	cli.root.AddCommand(&cobra.Command{
+		Use:   "benchmark",
+		Short: "Benchmarks testnet",
+		Long: `Benchmarks the following metrics:
+	Mean Block Interval
+	Standard Deviation
+	Min Block Interval
+	Max Block Interval
+over a 100 block sampling period.
+		
+Does not run any perbutations.
+		`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := Cleanup(cli.testnet); err != nil {
+				return err
+			}
+			if err := Setup(cli.testnet); err != nil {
+				return err
+			}
+
+			chLoadResult := make(chan error)
+			ctx, loadCancel := context.WithCancel(context.Background())
+			defer loadCancel()
+			go func() {
+				err := Load(ctx, cli.testnet)
+				if err != nil {
+					logger.Error(fmt.Sprintf("Transaction load failed: %v", err.Error()))
+				}
+				chLoadResult <- err
+			}()
+
+			if err := Start(cli.testnet); err != nil {
+				return err
+			}
+
+			if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
+				return err
+			}
+
+			// we benchmark performance over the next 100 blocks
+			if err := Benchmark(cli.testnet, 100); err != nil {
+				return err
+			}
+
+			loadCancel()
+			if err := <-chLoadResult; err != nil {
+				return err
+			}
+
+			if err := Cleanup(cli.testnet); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	})
+
 	return cli
 }
 

test/e2e/runner/wait.go

@@ -20,9 +20,15 @@ func Wait(testnet *e2e.Testnet, blocks int64) error {
 // WaitUntil waits until a given height has been reached.
 func WaitUntil(testnet *e2e.Testnet, height int64) error {
 	logger.Info(fmt.Sprintf("Waiting for all nodes to reach height %v...", height))
-	_, err := waitForAllNodes(testnet, height, 20*time.Second)
+	_, err := waitForAllNodes(testnet, height, waitingTime(len(testnet.Nodes)))
 	if err != nil {
 		return err
 	}
 	return nil
 }
+
+// waitingTime estimates how long it should take for a node to reach the height.
+// More nodes in a network implies we may expect a slower network and may have to wait longer.
+func waitingTime(nodes int) time.Duration {
+	return time.Duration(20+(nodes*2)) * time.Second
+}