mirror of
https://github.com/tendermint/tendermint.git
synced 2026-08-21 06:36:19 +00:00
test: add end-to-end testing framework (#5435)
Partial fix for #5291. For details, see [README.md](https://github.com/tendermint/tendermint/blob/erik/e2e-tests/test/e2e/README.md) and [RFC-001](https://github.com/tendermint/tendermint/blob/master/docs/rfc/rfc-001-end-to-end-testing.md). This only includes a single test case under `test/e2e/tests/`, as a proof of concept - additional test cases will be submitted separately. A randomized testnet generator will also be submitted separately, there a currently just a handful of static testnets under `test/e2e/networks/`. This will eventually replace the current P2P tests and run in CI.
This commit is contained in:
Erik Grinaker
committed by
Erik Grinaker
Erik Grinaker
parent
1b733ea28d
commit
a58454e788
@@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// Cleanup removes the Docker Compose containers and testnet directory.
|
||||
func Cleanup(testnet *e2e.Testnet) error {
|
||||
if testnet.Dir == "" {
|
||||
return errors.New("no directory set")
|
||||
}
|
||||
_, err := os.Stat(testnet.Dir)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info("Removing Docker containers and networks")
|
||||
err = execCompose(testnet.Dir, "down")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("Removing testnet directory %q", testnet.Dir))
|
||||
err = os.RemoveAll(testnet.Dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
//nolint: gosec
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
osexec "os/exec"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// execute executes a shell command.
|
||||
func exec(args ...string) error {
|
||||
cmd := osexec.Command(args[0], args[1:]...)
|
||||
out, err := cmd.CombinedOutput()
|
||||
switch err := err.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case *osexec.ExitError:
|
||||
return fmt.Errorf("failed to run %q:\n%v", args, string(out))
|
||||
default:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// execVerbose executes a shell command while displaying its output.
|
||||
func execVerbose(args ...string) error {
|
||||
cmd := osexec.Command(args[0], args[1:]...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
// execCompose runs a Docker Compose command for a testnet.
|
||||
func execCompose(dir string, args ...string) error {
|
||||
return exec(append(
|
||||
[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
|
||||
args...)...)
|
||||
}
|
||||
|
||||
// execComposeVerbose runs a Docker Compose command for a testnet and displays its output.
|
||||
func execComposeVerbose(dir string, args ...string) error {
|
||||
return execVerbose(append(
|
||||
[]string{"docker-compose", "-f", filepath.Join(dir, "docker-compose.yml")},
|
||||
args...)...)
|
||||
}
|
||||
|
||||
// execDocker runs a Docker command.
|
||||
func execDocker(args ...string) error {
|
||||
return exec(append([]string{"docker"}, args...)...)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
"github.com/tendermint/tendermint/types"
|
||||
)
|
||||
|
||||
// Load generates transactions against the network until the given
|
||||
// context is cancelled.
|
||||
func Load(ctx context.Context, testnet *e2e.Testnet) error {
|
||||
concurrency := 50
|
||||
initialTimeout := 1 * time.Minute
|
||||
stallTimeout := 15 * time.Second
|
||||
|
||||
chTx := make(chan types.Tx)
|
||||
chSuccess := make(chan types.Tx)
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Spawn job generator and processors.
|
||||
logger.Info("Starting transaction load...")
|
||||
started := time.Now()
|
||||
|
||||
go loadGenerate(ctx, chTx)
|
||||
|
||||
for w := 0; w < concurrency; w++ {
|
||||
go loadProcess(ctx, testnet, chTx, chSuccess)
|
||||
}
|
||||
|
||||
// Monitor successful transactions, and abort on stalls.
|
||||
success := 0
|
||||
timeout := initialTimeout
|
||||
for {
|
||||
select {
|
||||
case <-chSuccess:
|
||||
success++
|
||||
timeout = stallTimeout
|
||||
case <-time.After(timeout):
|
||||
return fmt.Errorf("unable to submit transactions for %v", timeout)
|
||||
case <-ctx.Done():
|
||||
if success == 0 {
|
||||
return errors.New("failed to submit any transactions")
|
||||
}
|
||||
logger.Info(fmt.Sprintf("Ending transaction load after %v txs (%.1f tx/s)...",
|
||||
success, float64(success)/time.Since(started).Seconds()))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadGenerate generates jobs until the context is cancelled
|
||||
func loadGenerate(ctx context.Context, chTx chan<- types.Tx) {
|
||||
for i := 0; i < math.MaxInt64; i++ {
|
||||
// We keep generating the same 1000 keys over and over, with different values.
|
||||
// This gives a reasonable load without putting too much data in the app.
|
||||
id := i % 1000
|
||||
|
||||
bz := make([]byte, 2048) // 4kb hex-encoded
|
||||
_, err := rand.Read(bz)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to read random bytes: %v", err))
|
||||
}
|
||||
tx := types.Tx(fmt.Sprintf("load-%X=%x", id, bz))
|
||||
|
||||
select {
|
||||
case chTx <- tx:
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
case <-ctx.Done():
|
||||
close(chTx)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadProcess processes transactions
|
||||
func loadProcess(ctx context.Context, testnet *e2e.Testnet, chTx <-chan types.Tx, chSuccess chan<- types.Tx) {
|
||||
// Each worker gets its own client to each node, which allows for some
|
||||
// concurrency while still bounding it.
|
||||
clients := map[string]*rpchttp.HTTP{}
|
||||
|
||||
var err error
|
||||
for tx := range chTx {
|
||||
node := testnet.RandomNode()
|
||||
client, ok := clients[node.Name]
|
||||
if !ok {
|
||||
client, err = node.Client()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
clients[node.Name] = client
|
||||
}
|
||||
_, err = client.BroadcastTxCommit(ctx, tx)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
chSuccess <- tx
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/tendermint/tendermint/libs/log"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
|
||||
|
||||
func main() {
|
||||
NewCLI().Run()
|
||||
}
|
||||
|
||||
// CLI is the Cobra-based command-line interface.
|
||||
type CLI struct {
|
||||
root *cobra.Command
|
||||
testnet *e2e.Testnet
|
||||
}
|
||||
|
||||
// NewCLI sets up the CLI.
|
||||
func NewCLI() *CLI {
|
||||
cli := &CLI{}
|
||||
cli.root = &cobra.Command{
|
||||
Use: "runner",
|
||||
Short: "End-to-end test runner",
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true, // we'll output them ourselves in Run()
|
||||
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
||||
file, err := cmd.Flags().GetString("file")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
testnet, err := e2e.LoadTestnet(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cli.testnet = testnet
|
||||
return nil
|
||||
},
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := Cleanup(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Setup(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
chLoadResult := make(chan error)
|
||||
ctx, loadCancel := context.WithCancel(context.Background())
|
||||
defer loadCancel()
|
||||
go func() {
|
||||
err := Load(ctx, cli.testnet)
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("Transaction load failed: %v", err.Error()))
|
||||
}
|
||||
chLoadResult <- err
|
||||
}()
|
||||
|
||||
if err := Start(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Perturb(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle
|
||||
return err
|
||||
}
|
||||
|
||||
loadCancel()
|
||||
if err := <-chLoadResult; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Wait(cli.testnet, 3); err != nil { // wait for last txs to commit
|
||||
return err
|
||||
}
|
||||
if err := Test(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := Cleanup(cli.testnet); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest")
|
||||
_ = cli.root.MarkPersistentFlagRequired("file")
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "setup",
|
||||
Short: "Generates the testnet directory and configuration",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Setup(cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "start",
|
||||
Short: "Starts the Docker testnet, waiting for nodes to become available",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
_, err := os.Stat(cli.testnet.Dir)
|
||||
if os.IsNotExist(err) {
|
||||
err = Setup(cli.testnet)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return Start(cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "perturb",
|
||||
Short: "Perturbs the Docker testnet, e.g. by restarting or disconnecting nodes",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Perturb(cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "wait",
|
||||
Short: "Waits for a few blocks to be produced and all nodes to catch up",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Wait(cli.testnet, 5)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "stop",
|
||||
Short: "Stops the Docker testnet",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
logger.Info("Stopping testnet")
|
||||
return execCompose(cli.testnet.Dir, "down")
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "load",
|
||||
Short: "Generates transaction load until the command is cancelled",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Load(context.Background(), cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "test",
|
||||
Short: "Runs test cases against a running testnet",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Test(cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "cleanup",
|
||||
Short: "Removes the testnet directory",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return Cleanup(cli.testnet)
|
||||
},
|
||||
})
|
||||
|
||||
cli.root.AddCommand(&cobra.Command{
|
||||
Use: "logs",
|
||||
Short: "Shows the testnet logs",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return execComposeVerbose(cli.testnet.Dir, "logs", "--follow")
|
||||
},
|
||||
})
|
||||
|
||||
return cli
|
||||
}
|
||||
|
||||
// Run runs the CLI.
|
||||
func (cli *CLI) Run() {
|
||||
if err := cli.root.Execute(); err != nil {
|
||||
logger.Error(err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
rpctypes "github.com/tendermint/tendermint/rpc/core/types"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// Perturbs a running testnet.
|
||||
func Perturb(testnet *e2e.Testnet) error {
|
||||
for _, node := range testnet.Nodes {
|
||||
for _, perturbation := range node.Perturbations {
|
||||
_, err := PerturbNode(node, perturbation)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
time.Sleep(3 * time.Second) // give network some time to recover between each
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// PerturbNode perturbs a node with a given perturbation, returning its status
|
||||
// after recovering.
|
||||
func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.ResultStatus, error) {
|
||||
testnet := node.Testnet
|
||||
switch perturbation {
|
||||
case e2e.PerturbationDisconnect:
|
||||
logger.Info(fmt.Sprintf("Disconnecting node %v...", node.Name))
|
||||
if err := execDocker("network", "disconnect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
if err := execDocker("network", "connect", testnet.Name+"_"+testnet.Name, node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
case e2e.PerturbationKill:
|
||||
logger.Info(fmt.Sprintf("Killing node %v...", node.Name))
|
||||
if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := execCompose(testnet.Dir, "start", node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
case e2e.PerturbationPause:
|
||||
logger.Info(fmt.Sprintf("Pausing node %v...", node.Name))
|
||||
if err := execCompose(testnet.Dir, "pause", node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
time.Sleep(10 * time.Second)
|
||||
if err := execCompose(testnet.Dir, "unpause", node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
case e2e.PerturbationRestart:
|
||||
logger.Info(fmt.Sprintf("Restarting node %v...", node.Name))
|
||||
if err := execCompose(testnet.Dir, "restart", node.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected perturbation %q", perturbation)
|
||||
}
|
||||
|
||||
status, err := waitForNode(node, 0, 10*time.Second)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logger.Info(fmt.Sprintf("Node %v recovered at height %v", node.Name, status.SyncInfo.LatestBlockHeight))
|
||||
return status, nil
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
|
||||
rpctypes "github.com/tendermint/tendermint/rpc/core/types"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
"github.com/tendermint/tendermint/types"
|
||||
)
|
||||
|
||||
// waitForHeight waits for the network to reach a certain height (or above),
|
||||
// returning the highest height seen. Errors if the network is not making
|
||||
// progress at all.
|
||||
func waitForHeight(testnet *e2e.Testnet, height int64) (*types.Block, *types.BlockID, error) {
|
||||
var (
|
||||
err error
|
||||
maxResult *rpctypes.ResultBlock
|
||||
clients = map[string]*rpchttp.HTTP{}
|
||||
lastIncrease = time.Now()
|
||||
)
|
||||
|
||||
for {
|
||||
for _, node := range testnet.Nodes {
|
||||
if node.Mode == e2e.ModeSeed {
|
||||
continue
|
||||
}
|
||||
client, ok := clients[node.Name]
|
||||
if !ok {
|
||||
client, err = node.Client()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
clients[node.Name] = client
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
result, err := client.Block(ctx, nil)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if result.Block != nil && (maxResult == nil || result.Block.Height >= maxResult.Block.Height) {
|
||||
maxResult = result
|
||||
lastIncrease = time.Now()
|
||||
}
|
||||
if maxResult != nil && maxResult.Block.Height >= height {
|
||||
return maxResult.Block, &maxResult.BlockID, nil
|
||||
}
|
||||
}
|
||||
|
||||
if len(clients) == 0 {
|
||||
return nil, nil, errors.New("unable to connect to any network nodes")
|
||||
}
|
||||
if time.Since(lastIncrease) >= 10*time.Second {
|
||||
if maxResult == nil {
|
||||
return nil, nil, errors.New("chain stalled at unknown height")
|
||||
}
|
||||
return nil, nil, fmt.Errorf("chain stalled at height %v", maxResult.Block.Height)
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForNode waits for a node to become available and catch up to the given block height.
|
||||
func waitForNode(node *e2e.Node, height int64, timeout time.Duration) (*rpctypes.ResultStatus, error) {
|
||||
client, err := node.Client()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
for {
|
||||
status, err := client.Status(ctx)
|
||||
switch {
|
||||
case errors.Is(err, context.DeadlineExceeded):
|
||||
return nil, fmt.Errorf("timed out waiting for %v to reach height %v", node.Name, height)
|
||||
case errors.Is(err, context.Canceled):
|
||||
return nil, err
|
||||
case err == nil && status.SyncInfo.LatestBlockHeight >= height:
|
||||
return status, nil
|
||||
}
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// waitForAllNodes waits for all nodes to become available and catch up to the given block height.
|
||||
func waitForAllNodes(testnet *e2e.Testnet, height int64, timeout time.Duration) (int64, error) {
|
||||
lastHeight := int64(0)
|
||||
for _, node := range testnet.Nodes {
|
||||
if node.Mode == e2e.ModeSeed {
|
||||
continue
|
||||
}
|
||||
status, err := waitForNode(node, height, 20*time.Second)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if status.SyncInfo.LatestBlockHeight > lastHeight {
|
||||
lastHeight = status.SyncInfo.LatestBlockHeight
|
||||
}
|
||||
}
|
||||
return lastHeight, nil
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
// nolint: gosec
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/tendermint/tendermint/config"
|
||||
"github.com/tendermint/tendermint/crypto/ed25519"
|
||||
"github.com/tendermint/tendermint/p2p"
|
||||
"github.com/tendermint/tendermint/privval"
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
"github.com/tendermint/tendermint/types"
|
||||
)
|
||||
|
||||
const (
|
||||
AppAddressTCP = "tcp://127.0.0.1:30000"
|
||||
AppAddressUNIX = "unix:///var/run/app.sock"
|
||||
|
||||
PrivvalAddressTCP = "tcp://0.0.0.0:27559"
|
||||
PrivvalAddressUNIX = "unix:///var/run/privval.sock"
|
||||
PrivvalKeyFile = "config/priv_validator_key.json"
|
||||
PrivvalStateFile = "data/priv_validator_state.json"
|
||||
PrivvalDummyKeyFile = "config/dummy_validator_key.json"
|
||||
PrivvalDummyStateFile = "data/dummy_validator_state.json"
|
||||
)
|
||||
|
||||
// Setup sets up the testnet configuration.
|
||||
func Setup(testnet *e2e.Testnet) error {
|
||||
logger.Info(fmt.Sprintf("Generating testnet files in %q", testnet.Dir))
|
||||
|
||||
err := os.MkdirAll(testnet.Dir, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compose, err := MakeDockerCompose(testnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioutil.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
genesis, err := MakeGenesis(testnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, node := range testnet.Nodes {
|
||||
nodeDir := filepath.Join(testnet.Dir, node.Name)
|
||||
dirs := []string{
|
||||
filepath.Join(nodeDir, "config"),
|
||||
filepath.Join(nodeDir, "data"),
|
||||
filepath.Join(nodeDir, "data", "app"),
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
err := os.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg, err := MakeConfig(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
config.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), cfg) // panics
|
||||
|
||||
appCfg, err := MakeAppConfig(node)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioutil.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = (&p2p.NodeKey{PrivKey: node.Key}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
(privval.NewFilePV(node.Key,
|
||||
filepath.Join(nodeDir, PrivvalKeyFile),
|
||||
filepath.Join(nodeDir, PrivvalStateFile),
|
||||
)).Save()
|
||||
|
||||
// Set up a dummy validator. Tendermint requires a file PV even when not used, so we
|
||||
// give it a dummy such that it will fail if it actually tries to use it.
|
||||
(privval.NewFilePV(ed25519.GenPrivKey(),
|
||||
filepath.Join(nodeDir, PrivvalDummyKeyFile),
|
||||
filepath.Join(nodeDir, PrivvalDummyStateFile),
|
||||
)).Save()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MakeDockerCompose generates a Docker Compose config for a testnet.
|
||||
func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
|
||||
// Must use version 2 Docker Compose format, to support IPv6.
|
||||
tmpl, err := template.New("docker-compose").Parse(`version: '2.4'
|
||||
|
||||
networks:
|
||||
{{ .Name }}:
|
||||
driver: bridge
|
||||
{{- if .IPv6 }}
|
||||
enable_ipv6: true
|
||||
{{- end }}
|
||||
ipam:
|
||||
driver: default
|
||||
config:
|
||||
- subnet: {{ .IP }}
|
||||
|
||||
services:
|
||||
{{- range .Nodes }}
|
||||
{{ .Name }}:
|
||||
container_name: {{ .Name }}
|
||||
image: tendermint/e2e-node
|
||||
{{- if eq .ABCIProtocol "builtin" }}
|
||||
entrypoint: /usr/bin/entrypoint-builtin
|
||||
{{- end }}
|
||||
init: true
|
||||
ports:
|
||||
- 26656
|
||||
- {{ if .ProxyPort }}{{ .ProxyPort }}:{{ end }}26657
|
||||
volumes:
|
||||
- ./{{ .Name }}:/tendermint
|
||||
networks:
|
||||
{{ $.Name }}:
|
||||
ipv{{ if $.IPv6 }}6{{ else }}4{{ end}}_address: {{ .IP }}
|
||||
|
||||
{{end}}`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
err = tmpl.Execute(&buf, testnet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// MakeGenesis generates a genesis document.
|
||||
func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
|
||||
genesis := types.GenesisDoc{
|
||||
GenesisTime: time.Now(),
|
||||
ChainID: testnet.Name,
|
||||
ConsensusParams: types.DefaultConsensusParams(),
|
||||
InitialHeight: testnet.InitialHeight,
|
||||
}
|
||||
for validator, power := range testnet.Validators {
|
||||
genesis.Validators = append(genesis.Validators, types.GenesisValidator{
|
||||
Name: validator.Name,
|
||||
Address: validator.Key.PubKey().Address(),
|
||||
PubKey: validator.Key.PubKey(),
|
||||
Power: power,
|
||||
})
|
||||
}
|
||||
// The validator set will be sorted internally by Tendermint ranked by power,
|
||||
// but we sort it here as well so that all genesis files are identical.
|
||||
sort.Slice(genesis.Validators, func(i, j int) bool {
|
||||
return strings.Compare(genesis.Validators[i].Name, genesis.Validators[j].Name) == -1
|
||||
})
|
||||
if len(testnet.InitialState) > 0 {
|
||||
appState, err := json.Marshal(testnet.InitialState)
|
||||
if err != nil {
|
||||
return genesis, err
|
||||
}
|
||||
genesis.AppState = appState
|
||||
}
|
||||
return genesis, genesis.ValidateAndComplete()
|
||||
}
|
||||
|
||||
// MakeConfig generates a Tendermint config for a node.
|
||||
func MakeConfig(node *e2e.Node) (*config.Config, error) {
|
||||
cfg := config.DefaultConfig()
|
||||
cfg.Moniker = node.Name
|
||||
cfg.ProxyApp = AppAddressTCP
|
||||
cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657"
|
||||
cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
|
||||
cfg.P2P.AddrBookStrict = false
|
||||
cfg.DBBackend = node.Database
|
||||
cfg.StateSync.DiscoveryTime = 5 * time.Second
|
||||
|
||||
switch node.ABCIProtocol {
|
||||
case e2e.ProtocolUNIX:
|
||||
cfg.ProxyApp = AppAddressUNIX
|
||||
case e2e.ProtocolTCP:
|
||||
cfg.ProxyApp = AppAddressTCP
|
||||
case e2e.ProtocolGRPC:
|
||||
cfg.ProxyApp = AppAddressTCP
|
||||
cfg.ABCI = "grpc"
|
||||
case e2e.ProtocolBuiltin:
|
||||
cfg.ProxyApp = ""
|
||||
cfg.ABCI = ""
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
|
||||
}
|
||||
|
||||
// Tendermint errors if it does not have a privval key set up, regardless of whether
|
||||
// it's actually needed (e.g. for remote KMS or non-validators). We set up a dummy
|
||||
// key here by default, and use the real key for actual validators that should use
|
||||
// the file privval.
|
||||
cfg.PrivValidatorListenAddr = ""
|
||||
cfg.PrivValidatorKey = PrivvalDummyKeyFile
|
||||
cfg.PrivValidatorState = PrivvalDummyStateFile
|
||||
|
||||
switch node.Mode {
|
||||
case e2e.ModeValidator:
|
||||
switch node.PrivvalProtocol {
|
||||
case e2e.ProtocolFile:
|
||||
cfg.PrivValidatorKey = PrivvalKeyFile
|
||||
cfg.PrivValidatorState = PrivvalStateFile
|
||||
case e2e.ProtocolUNIX:
|
||||
cfg.PrivValidatorListenAddr = PrivvalAddressUNIX
|
||||
case e2e.ProtocolTCP:
|
||||
cfg.PrivValidatorListenAddr = PrivvalAddressTCP
|
||||
default:
|
||||
return nil, fmt.Errorf("invalid privval protocol setting %q", node.PrivvalProtocol)
|
||||
}
|
||||
case e2e.ModeSeed:
|
||||
cfg.P2P.SeedMode = true
|
||||
cfg.P2P.PexReactor = true
|
||||
case e2e.ModeFull:
|
||||
// Don't need to do anything, since we're using a dummy privval key by default.
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected mode %q", node.Mode)
|
||||
}
|
||||
|
||||
if node.FastSync == "" {
|
||||
cfg.FastSyncMode = false
|
||||
} else {
|
||||
cfg.FastSync.Version = node.FastSync
|
||||
}
|
||||
|
||||
if node.StateSync {
|
||||
cfg.StateSync.Enable = true
|
||||
cfg.StateSync.RPCServers = []string{}
|
||||
for _, peer := range node.Testnet.ArchiveNodes() {
|
||||
if peer.Name == node.Name {
|
||||
continue
|
||||
}
|
||||
cfg.StateSync.RPCServers = append(cfg.StateSync.RPCServers, peer.AddressRPC())
|
||||
}
|
||||
if len(cfg.StateSync.RPCServers) < 2 {
|
||||
return nil, errors.New("unable to find 2 suitable state sync RPC servers")
|
||||
}
|
||||
}
|
||||
|
||||
cfg.P2P.Seeds = ""
|
||||
for _, seed := range node.Seeds {
|
||||
if len(cfg.P2P.Seeds) > 0 {
|
||||
cfg.P2P.Seeds += ","
|
||||
}
|
||||
cfg.P2P.Seeds += seed.AddressP2P(true)
|
||||
}
|
||||
cfg.P2P.PersistentPeers = ""
|
||||
for _, peer := range node.PersistentPeers {
|
||||
if len(cfg.P2P.PersistentPeers) > 0 {
|
||||
cfg.P2P.PersistentPeers += ","
|
||||
}
|
||||
cfg.P2P.PersistentPeers += peer.AddressP2P(true)
|
||||
}
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// MakeAppConfig generates an ABCI application config for a node.
|
||||
func MakeAppConfig(node *e2e.Node) ([]byte, error) {
|
||||
cfg := map[string]interface{}{
|
||||
"chain_id": node.Testnet.Name,
|
||||
"dir": "data/app",
|
||||
"listen": AppAddressUNIX,
|
||||
"protocol": "socket",
|
||||
"persist_interval": node.PersistInterval,
|
||||
"snapshot_interval": node.SnapshotInterval,
|
||||
"retain_blocks": node.RetainBlocks,
|
||||
}
|
||||
switch node.ABCIProtocol {
|
||||
case e2e.ProtocolUNIX:
|
||||
cfg["listen"] = AppAddressUNIX
|
||||
case e2e.ProtocolTCP:
|
||||
cfg["listen"] = AppAddressTCP
|
||||
case e2e.ProtocolGRPC:
|
||||
cfg["listen"] = AppAddressTCP
|
||||
cfg["protocol"] = "grpc"
|
||||
case e2e.ProtocolBuiltin:
|
||||
delete(cfg, "listen")
|
||||
cfg["protocol"] = "builtin"
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol)
|
||||
}
|
||||
switch node.PrivvalProtocol {
|
||||
case e2e.ProtocolFile:
|
||||
case e2e.ProtocolTCP:
|
||||
cfg["privval_server"] = PrivvalAddressTCP
|
||||
cfg["privval_key"] = PrivvalKeyFile
|
||||
cfg["privval_state"] = PrivvalStateFile
|
||||
case e2e.ProtocolUNIX:
|
||||
cfg["privval_server"] = PrivvalAddressUNIX
|
||||
cfg["privval_key"] = PrivvalKeyFile
|
||||
cfg["privval_state"] = PrivvalStateFile
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected privval protocol setting %q", node.PrivvalProtocol)
|
||||
}
|
||||
|
||||
if len(node.Testnet.ValidatorUpdates) > 0 {
|
||||
validatorUpdates := map[string]map[string]int64{}
|
||||
for height, validators := range node.Testnet.ValidatorUpdates {
|
||||
updateVals := map[string]int64{}
|
||||
for node, power := range validators {
|
||||
updateVals[base64.StdEncoding.EncodeToString(node.Key.PubKey().Bytes())] = power
|
||||
}
|
||||
validatorUpdates[fmt.Sprintf("%v", height)] = updateVals
|
||||
}
|
||||
cfg["validator_update"] = validatorUpdates
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
err := toml.NewEncoder(&buf).Encode(cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to generate app config: %w", err)
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UpdateConfigStateSync updates the state sync config for a node.
|
||||
func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error {
|
||||
cfgPath := filepath.Join(node.Testnet.Dir, node.Name, "config", "config.toml")
|
||||
|
||||
// FIXME Apparently there's no function to simply load a config file without
|
||||
// involving the entire Viper apparatus, so we'll just resort to regexps.
|
||||
bz, err := ioutil.ReadFile(cfgPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bz = regexp.MustCompile(`(?m)^trust_height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust_height = %v`, height)))
|
||||
bz = regexp.MustCompile(`(?m)^trust_hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust_hash = "%X"`, hash)))
|
||||
return ioutil.WriteFile(cfgPath, bz, 0644)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
func Start(testnet *e2e.Testnet) error {
|
||||
|
||||
// Sort nodes by starting order
|
||||
nodeQueue := testnet.Nodes
|
||||
sort.SliceStable(nodeQueue, func(i, j int) bool {
|
||||
return nodeQueue[i].StartAt < nodeQueue[j].StartAt
|
||||
})
|
||||
|
||||
// Start initial nodes (StartAt: 0)
|
||||
logger.Info("Starting initial network nodes...")
|
||||
for len(nodeQueue) > 0 && nodeQueue[0].StartAt == 0 {
|
||||
node := nodeQueue[0]
|
||||
nodeQueue = nodeQueue[1:]
|
||||
if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := waitForNode(node, 0, 10*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v", node.Name, node.ProxyPort))
|
||||
}
|
||||
|
||||
// Wait for initial height
|
||||
logger.Info(fmt.Sprintf("Waiting for initial height %v...", testnet.InitialHeight))
|
||||
block, blockID, err := waitForHeight(testnet, testnet.InitialHeight)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Update any state sync nodes with a trusted height and hash
|
||||
for _, node := range nodeQueue {
|
||||
if node.StateSync {
|
||||
err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start up remaining nodes
|
||||
for _, node := range nodeQueue {
|
||||
logger.Info(fmt.Sprintf("Starting node %v at height %v...", node.Name, node.StartAt))
|
||||
if _, _, err := waitForHeight(testnet, node.StartAt); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
status, err := waitForNode(node, node.StartAt, 30*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Info(fmt.Sprintf("Node %v up on http://127.0.0.1:%v at height %v",
|
||||
node.Name, node.ProxyPort, status.SyncInfo.LatestBlockHeight))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// Test runs test cases under tests/
|
||||
func Test(testnet *e2e.Testnet) error {
|
||||
logger.Info("Running tests in ./tests/...")
|
||||
|
||||
err := os.Setenv("E2E_MANIFEST", testnet.File)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return execVerbose("go", "test", "-count", "1", "./tests/...")
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
|
||||
)
|
||||
|
||||
// Wait waits for a number of blocks to be produced, and for all nodes to catch
|
||||
// up with it.
|
||||
func Wait(testnet *e2e.Testnet, blocks int64) error {
|
||||
block, _, err := waitForHeight(testnet, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waitFor := block.Height + blocks
|
||||
logger.Info(fmt.Sprintf("Waiting for all nodes to reach height %v...", waitFor))
|
||||
_, err = waitForAllNodes(testnet, waitFor, 20*time.Second)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user