mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-09-27 10:24:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: add contexts and e2e generator tests (#9426)

Browse Source
This commit is contained in:
Callum Waters
2022-11-10 12:29:59 +01:00
committed by GitHub
parent 4f3e87b2e4
commit f8e453a245
14 changed files with 196 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/runner/benchmark.go
+43 -7
View File
@@ -2,8 +2,10 @@ package main
import (
"context"
"encoding/json"
"fmt"
"math"
"path/filepath"
"time"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -19,26 +21,28 @@ import (
//
// Metrics are based of the `benchmarkLength`, the amount of consecutive blocks
// sampled from in the testnet
func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
block, _, err := waitForHeight(testnet, 0)
func Benchmark(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) error {
block, _, err := waitForHeight(ctx, testnet, 0)
if err != nil {
return err
}
logger.Info("Beginning benchmark period...", "height", block.Height)
startAt := time.Now()
// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
// which should be sufficient.
waitingTime := time.Duration(benchmarkLength*5) * time.Second
endHeight, err := waitForAllNodes(testnet, block.Height+benchmarkLength, waitingTime)
endHeight, err := waitForAllNodes(ctx, testnet, block.Height+benchmarkLength, waitingTime)
if err != nil {
return err
}
dur := time.Since(startAt)
logger.Info("Ending benchmark period", "height", endHeight)
// fetch a sample of blocks
blocks, err := fetchBlockChainSample(testnet, benchmarkLength)
blocks, err := fetchBlockChainSample(ctx, testnet, benchmarkLength)
if err != nil {
return err
}
@@ -46,18 +50,29 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
// slice into time intervals and collate data
timeIntervals := splitIntoBlockIntervals(blocks)
testnetStats := extractTestnetStats(timeIntervals)
testnetStats.populateTxns(blocks)
testnetStats.totalTime = dur
testnetStats.startHeight = blocks[0].Header.Height
testnetStats.endHeight = blocks[len(blocks)-1].Header.Height
// print and return
logger.Info(testnetStats.String())
logger.Info(testnetStats.OutputJSON(testnet))
return nil
}
func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) {
t.numtxns = 0
for _, b := range blocks {
t.numtxns += int64(b.NumTxs)
}
}
type testnetStats struct {
startHeight int64
endHeight int64
numtxns int64
totalTime time.Duration
// average time to produce a block
mean time.Duration
// standard deviation of block production
@@ -68,6 +83,28 @@ type testnetStats struct {
min time.Duration
}
func (t *testnetStats) OutputJSON(net *e2e.Testnet) string {
jsn, err := json.Marshal(map[string]interface{}{
"case": filepath.Base(net.File),
"start_height": t.startHeight,
"end_height": t.endHeight,
"blocks": t.endHeight - t.startHeight,
"stddev": t.std,
"mean": t.mean.Seconds(),
"max": t.max.Seconds(),
"min": t.min.Seconds(),
"size": len(net.Nodes),
"txns": t.numtxns,
"dur": t.totalTime.Seconds(),
})
if err != nil {
return ""
}
return string(jsn)
}
func (t *testnetStats) String() string {
return fmt.Sprintf(`Benchmarked from height %v to %v
Mean Block Interval: %v
@@ -86,7 +123,7 @@ func (t *testnetStats) String() string {
// fetchBlockChainSample waits for `benchmarkLength` amount of blocks to pass, fetching
// all of the headers for these blocks from an archive node and returning it.
func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
func fetchBlockChainSample(ctx context.Context, testnet *e2e.Testnet, benchmarkLength int64) ([]*types.BlockMeta, error) {
var blocks []*types.BlockMeta
// Find the first archive node
@@ -97,7 +134,6 @@ func fetchBlockChainSample(testnet *e2e.Testnet, benchmarkLength int64) ([]*type
}
// find the latest height
ctx := context.Background()
s, err := c.Status(ctx)
if err != nil {
return nil, err
test/e2e/runner/evidence.go
+2 -2
View File
@@ -82,7 +82,7 @@ func InjectEvidence(ctx context.Context, r *rand.Rand, testnet *e2e.Testnet, amo
// wait for the node to reach the height above the forged height so that
// it is able to validate the evidence
_, err = waitForNode(targetNode, waitHeight, time.Minute)
_, err = waitForNode(ctx, targetNode, waitHeight, time.Minute)
if err != nil {
return err
}
@@ -110,7 +110,7 @@ func InjectEvidence(ctx context.Context, r *rand.Rand, testnet *e2e.Testnet, amo
// wait for the node to reach the height above the forged height so that
// it is able to validate the evidence
_, err = waitForNode(targetNode, blockRes.Block.Height+2, 30*time.Second)
_, err = waitForNode(ctx, targetNode, blockRes.Block.Height+2, 30*time.Second)
if err != nil {
return err
}
test/e2e/runner/main.go
+13 -13
View File
@@ -70,19 +70,19 @@ func NewCLI() *CLI {
chLoadResult <- err
}()
if err := Start(cli.testnet); err != nil {
if err := Start(cmd.Context(), cli.testnet); err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
return err
}
if cli.testnet.HasPerturbations() {
if err := Perturb(cli.testnet); err != nil {
if err := Perturb(cmd.Context(), cli.testnet); err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
return err
}
}
@@ -91,7 +91,7 @@ func NewCLI() *CLI {
if err := InjectEvidence(ctx, r, cli.testnet, cli.testnet.Evidence); err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // ensure chain progress
if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // ensure chain progress
return err
}
}
@@ -100,7 +100,7 @@ func NewCLI() *CLI {
if err := <-chLoadResult; err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // wait for network to settle before tests
return err
}
if err := Test(cli.testnet); err != nil {
@@ -140,7 +140,7 @@ func NewCLI() *CLI {
if err != nil {
return err
}
return Start(cli.testnet)
return Start(cmd.Context(), cli.testnet)
},
})
@@ -148,7 +148,7 @@ func NewCLI() *CLI {
Use: "perturb",
Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes",
RunE: func(cmd *cobra.Command, args []string) error {
return Perturb(cli.testnet)
return Perturb(cmd.Context(), cli.testnet)
},
})
@@ -156,7 +156,7 @@ func NewCLI() *CLI {
Use: "wait",
Short: "Waits for a few blocks to be produced and all nodes to catch up",
RunE: func(cmd *cobra.Command, args []string) error {
return Wait(cli.testnet, 5)
return Wait(cmd.Context(), cli.testnet, 5)
},
})
@@ -263,7 +263,7 @@ Does not run any perbutations.
}
chLoadResult := make(chan error)
ctx, loadCancel := context.WithCancel(context.Background())
ctx, loadCancel := context.WithCancel(cmd.Context())
defer loadCancel()
go func() {
err := Load(ctx, cli.testnet, 1)
@@ -273,16 +273,16 @@ Does not run any perbutations.
chLoadResult <- err
}()
if err := Start(cli.testnet); err != nil {
if err := Start(cmd.Context(), cli.testnet); err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
if err := Wait(cmd.Context(), cli.testnet, 5); err != nil { // allow some txs to go through
return err
}
// we benchmark performance over the next 100 blocks
if err := Benchmark(cli.testnet, 100); err != nil {
if err := Benchmark(cmd.Context(), cli.testnet, 100); err != nil {
return err
}
test/e2e/runner/perturb.go
+5 -4
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"time"
@@ -10,10 +11,10 @@ import (
)
// Perturbs a running testnet.
func Perturb(testnet *e2e.Testnet) error {
func Perturb(ctx context.Context, testnet *e2e.Testnet) error {
for _, node := range testnet.Nodes {
for _, perturbation := range node.Perturbations {
_, err := PerturbNode(node, perturbation)
_, err := PerturbNode(ctx, node, perturbation)
if err != nil {
return err
}
@@ -25,7 +26,7 @@ func Perturb(testnet *e2e.Testnet) error {
// PerturbNode perturbs a node with a given perturbation, returning its status
// after recovering.
func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
func PerturbNode(ctx context.Context, node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
testnet := node.Testnet
switch perturbation {
case e2e.PerturbationDisconnect:
@@ -67,7 +68,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul
return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
}
status, err := waitForNode(node, 0, 20*time.Second)
status, err := waitForNode(ctx, node, 0, 20*time.Second)
if err != nil {
return nil, err
}
test/e2e/runner/rpc.go
+65 -46
View File
@@ -15,7 +15,7 @@ import (
// waitForHeight waits for the network to reach a certain height (or above),
// returning the highest height seen. Errors if the network is not making
// progress at all.
func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
func waitForHeight(ctx context.Context, testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
var (
err error
maxResult *rpctypes.ResultBlock
@@ -23,73 +23,92 @@ func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.Blo
lastIncrease = time.Now()
)
timer := time.NewTimer(0)
defer timer.Stop()
for {
for _, node := range testnet.Nodes {
if node.Mode == e2e.ModeSeed {
continue
}
client, ok := clients[node.Name]
if !ok {
client, err = node.Client()
select {
case <-ctx.Done():
return nil, nil, ctx.Err()
case <-timer.C:
for _, node := range testnet.Nodes {
if node.Stateless() {
continue
}
client, ok := clients[node.Name]
if !ok {
client, err = node.Client()
if err != nil {
continue
}
clients[node.Name] = client
}
subctx, cancel := context.WithTimeout(ctx, 1*time.Second)
defer cancel()
result, err := client.Block(subctx, nil)
if err == context.DeadlineExceeded || err == context.Canceled {
return nil, nil, ctx.Err()
}
if err != nil {
continue
}
clients[node.Name] = client
if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
maxResult = result
lastIncrease = time.Now()
}
if maxResult != nil && maxResult.Block.Height >= height {
return maxResult.Block, &maxResult.BlockID, nil
}
}
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
defer cancel()
result, err := client.Block(ctx, nil)
if err != nil {
continue
if len(clients) == 0 {
return nil, nil, errors.New("unable to connect to any network nodes")
}
if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
maxResult = result
lastIncrease = time.Now()
}
if maxResult != nil && maxResult.Block.Height >= height {
return maxResult.Block, &maxResult.BlockID, nil
if time.Since(lastIncrease) >= 20*time.Second {
if maxResult == nil {
return nil, nil, errors.New("chain stalled at unknown height")
}
return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
}
timer.Reset(1 * time.Second)
}
if len(clients) == 0 {
return nil, nil, errors.New("unable to connect to any network nodes")
}
if time.Since(lastIncrease) >= 20*time.Second {
if maxResult == nil {
return nil, nil, errors.New("chain stalled at unknown height")
}
return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
}
time.Sleep(1 * time.Second)
}
}
// waitForNode waits for a node to become available and catch up to the given block height.
func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
func waitForNode(ctx context.Context, node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
client, err := node.Client()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
for {
status, err := client.Status(ctx)
switch {
case errors.Is(err, context.DeadlineExceeded):
return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
case errors.Is(err, context.Canceled):
return nil, err
case err == nil && status.SyncInfo.LatestBlockHeight >= height:
return status, nil
}
time.Sleep(300 * time.Millisecond)
timer := time.NewTimer(0)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case <-timer.C:
subctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
status, err := client.Status(subctx)
switch {
case errors.Is(err, context.DeadlineExceeded):
return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
case errors.Is(err, context.Canceled):
return nil, err
case err == nil && status.SyncInfo.LatestBlockHeight >= height:
return status, nil
}
timer.Reset(300 * time.Millisecond)
}
}
}
// waitForAllNodes waits for all nodes to become available and catch up to the given block height.
func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
func waitForAllNodes(ctx context.Context, testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
var lastHeight int64
for _, node := range testnet.Nodes {
@@ -97,7 +116,7 @@ func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration)
continue
}
status, err := waitForNode(node, height, timeout)
status, err := waitForNode(ctx, node, height, timeout)
if err != nil {
return 0, err
}
test/e2e/runner/start.go
+6 -5
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"sort"
"time"
@@ -9,7 +10,7 @@ import (
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)
func Start(testnet *e2e.Testnet) error {
func Start(ctx context.Context, testnet *e2e.Testnet) error {
if len(testnet.Nodes) == 0 {
return fmt.Errorf("no nodes in testnet")
}
@@ -46,7 +47,7 @@ func Start(testnet *e2e.Testnet) error {
if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
return err
}
if _, err := waitForNode(node, 0, 15*time.Second); err != nil {
if _, err := waitForNode(ctx, node, 0, 15*time.Second); err != nil {
return err
}
logger.Info("start", "msg", log.NewLazySprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
@@ -60,7 +61,7 @@ func Start(testnet *e2e.Testnet) error {
"nodes", len(testnet.Nodes)-len(nodeQueue),
"pending", len(nodeQueue))
block, blockID, err := waitForHeight(testnet, networkHeight)
block, blockID, err := waitForHeight(ctx, testnet, networkHeight)
if err != nil {
return err
}
@@ -90,7 +91,7 @@ func Start(testnet *e2e.Testnet) error {
"node", node.Name,
"height", networkHeight)
if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
if _, _, err := waitForHeight(ctx, testnet, networkHeight); err != nil {
return err
}
}
@@ -100,7 +101,7 @@ func Start(testnet *e2e.Testnet) error {
if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
return err
}
status, err := waitForNode(node, node.StartAt, 3*time.Minute)
status, err := waitForNode(ctx, node, node.StartAt, 3*time.Minute)
if err != nil {
return err
}
test/e2e/runner/wait.go
+6 -5
View File
@@ -1,6 +1,7 @@
package main
import (
"context"
"time"
"github.com/tendermint/tendermint/libs/log"
@@ -9,18 +10,18 @@ import (
// Wait waits for a number of blocks to be produced, and for all nodes to catch
// up with it.
func Wait(testnet *e2e.Testnet, blocks int64) error {
block, _, err := waitForHeight(testnet, 0)
func Wait(ctx context.Context, testnet *e2e.Testnet, blocks int64) error {
block, _, err := waitForHeight(ctx, testnet, 0)
if err != nil {
return err
}
return WaitUntil(testnet, block.Height+blocks)
return WaitUntil(ctx, testnet, block.Height+blocks)
}
// WaitUntil waits until a given height has been reached.
func WaitUntil(testnet *e2e.Testnet, height int64) error {
func WaitUntil(ctx context.Context, testnet *e2e.Testnet, height int64) error {
logger.Info("wait until", "msg", log.NewLazySprintf("Waiting for all nodes to reach height %v...", height))
_, err := waitForAllNodes(testnet, height, waitingTime(len(testnet.Nodes)))
_, err := waitForAllNodes(ctx, testnet, height, waitingTime(len(testnet.Nodes)))
if err != nil {
return err
}