test/e2e: add random testnet generator (#5479)

Closes #5291. Adds a randomized testnet generator. Nightly CI job will be submitted separately. A few of the testnets can be a bit flaky, even after disabling known-faulty behavior and making minor tweaks, and the larger networks may be too resource-intensive to run in CI - this will be optimized separately.

test/e2e/generator/generate.go

@@ -0,0 +1,227 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+var (
+	// testnetCombinations defines global testnet options, where we generate a
+	// separate testnet for each combination (Cartesian product) of options.
+	testnetCombinations = map[string][]interface{}{
+		"topology":      {"single", "quad", "large"},
+		"ipv6":          {false, true},
+		"initialHeight": {0, 1000},
+		"initialState": {
+			map[string]string{},
+			map[string]string{"initial01": "a", "initial02": "b", "initial03": "c"},
+		},
+		"validators": {"genesis", "initchain"},
+	}
+
+	// The following specify randomly chosen values for testnet nodes.
+	nodeDatabases = uniformChoice{"goleveldb", "cleveldb", "rocksdb", "boltdb", "badgerdb"}
+	// FIXME disabled grpc due to https://github.com/tendermint/tendermint/issues/5439
+	nodeABCIProtocols    = uniformChoice{"unix", "tcp", "builtin"} // "grpc"
+	nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"}
+	// FIXME disabled v1 due to https://github.com/tendermint/tendermint/issues/5444
+	nodeFastSyncs         = uniformChoice{"", "v0", "v2"} // "v1"
+	nodeStateSyncs        = uniformChoice{false, true}
+	nodePersistIntervals  = uniformChoice{0, 1, 5}
+	nodeSnapshotIntervals = uniformChoice{0, 3}
+	nodeRetainBlocks      = uniformChoice{0, 1, 5}
+	nodePerturbations     = probSetChoice{
+		"disconnect": 0.1,
+		"pause":      0.1,
+		// FIXME disabled due to https://github.com/tendermint/tendermint/issues/5422
+		// "kill":       0.1,
+		// "restart":    0.1,
+	}
+)
+
+// Generate generates random testnets using the given RNG.
+func Generate(r *rand.Rand) ([]e2e.Manifest, error) {
+	manifests := []e2e.Manifest{}
+	for _, opt := range combinations(testnetCombinations) {
+		manifest, err := generateTestnet(r, opt)
+		if err != nil {
+			return nil, err
+		}
+		manifests = append(manifests, manifest)
+	}
+	return manifests, nil
+}
+
+// generateTestnet generates a single testnet with the given options.
+func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, error) {
+	manifest := e2e.Manifest{
+		IPv6:             opt["ipv6"].(bool),
+		InitialHeight:    int64(opt["initialHeight"].(int)),
+		InitialState:     opt["initialState"].(map[string]string),
+		Validators:       &map[string]int64{},
+		ValidatorUpdates: map[string]map[string]int64{},
+		Nodes:            map[string]*e2e.ManifestNode{},
+	}
+
+	var numSeeds, numValidators, numFulls int
+	switch opt["topology"].(string) {
+	case "single":
+		numValidators = 1
+	case "quad":
+		numValidators = 4
+	case "large":
+		// FIXME Networks are kept small since large ones use too much CPU.
+		numSeeds = r.Intn(4)
+		numValidators = 4 + r.Intn(7)
+		numFulls = r.Intn(5)
+	default:
+		return manifest, fmt.Errorf("unknown topology %q", opt["topology"])
+	}
+
+	// First we generate seed nodes, starting at the initial height.
+	for i := 1; i <= numSeeds; i++ {
+		manifest.Nodes[fmt.Sprintf("seed%02d", i)] = generateNode(r, e2e.ModeSeed, 0, false)
+	}
+
+	// Next, we generate validators. We make sure a BFT quorum of validators start
+	// at the initial height, and that we have two archive nodes. We also set up
+	// the initial validator set, and validator set updates for delayed nodes.
+	nextStartAt := manifest.InitialHeight + 5
+	quorum := numValidators*2/3 + 1
+	for i := 1; i <= numValidators; i++ {
+		startAt := int64(0)
+		if i > quorum {
+			startAt = nextStartAt
+			nextStartAt += 5
+		}
+		name := fmt.Sprintf("validator%02d", i)
+		manifest.Nodes[name] = generateNode(r, e2e.ModeValidator, startAt, i <= 2)
+
+		if startAt == 0 {
+			(*manifest.Validators)[name] = int64(30 + r.Intn(71))
+		} else {
+			manifest.ValidatorUpdates[fmt.Sprint(startAt+5)] = map[string]int64{
+				name: int64(30 + r.Intn(71)),
+			}
+		}
+	}
+
+	// Move validators to InitChain if specified.
+	switch opt["validators"].(string) {
+	case "genesis":
+	case "initchain":
+		manifest.ValidatorUpdates["0"] = *manifest.Validators
+		manifest.Validators = &map[string]int64{}
+	default:
+		return manifest, fmt.Errorf("invalid validators option %q", opt["validators"])
+	}
+
+	// Finally, we generate random full nodes.
+	for i := 1; i <= numFulls; i++ {
+		startAt := int64(0)
+		if r.Float64() >= 0.5 {
+			startAt = nextStartAt
+			nextStartAt += 5
+		}
+		manifest.Nodes[fmt.Sprintf("full%02d", i)] = generateNode(r, e2e.ModeFull, startAt, false)
+	}
+
+	// We now set up peer discovery for nodes. Seed nodes are fully meshed with
+	// each other, while non-seed nodes either use a set of random seeds or a
+	// set of random peers that start before themselves.
+	var seedNames, peerNames []string
+	for name, node := range manifest.Nodes {
+		if node.Mode == string(e2e.ModeSeed) {
+			seedNames = append(seedNames, name)
+		} else {
+			peerNames = append(peerNames, name)
+		}
+	}
+
+	for _, name := range seedNames {
+		for _, otherName := range seedNames {
+			if name != otherName {
+				manifest.Nodes[name].Seeds = append(manifest.Nodes[name].Seeds, otherName)
+			}
+		}
+	}
+
+	sort.Slice(peerNames, func(i, j int) bool {
+		iName, jName := peerNames[i], peerNames[j]
+		switch {
+		case manifest.Nodes[iName].StartAt < manifest.Nodes[jName].StartAt:
+			return true
+		case manifest.Nodes[iName].StartAt > manifest.Nodes[jName].StartAt:
+			return false
+		default:
+			return strings.Compare(iName, jName) == -1
+		}
+	})
+	for i, name := range peerNames {
+		if len(seedNames) > 0 && (i == 0 || r.Float64() >= 0.5) {
+			manifest.Nodes[name].Seeds = uniformSetChoice(seedNames).Choose(r)
+		} else if i > 0 {
+			manifest.Nodes[name].PersistentPeers = uniformSetChoice(peerNames[:i]).Choose(r)
+		}
+	}
+
+	return manifest, nil
+}
+
+// generateNode randomly generates a node, with some constraints to avoid
+// generating invalid configurations. We do not set Seeds or PersistentPeers
+// here, since we need to know the overall network topology and startup
+// sequencing.
+func generateNode(r *rand.Rand, mode e2e.Mode, startAt int64, forceArchive bool) *e2e.ManifestNode {
+	node := e2e.ManifestNode{
+		Mode:             string(mode),
+		StartAt:          startAt,
+		Database:         nodeDatabases.Choose(r).(string),
+		ABCIProtocol:     nodeABCIProtocols.Choose(r).(string),
+		PrivvalProtocol:  nodePrivvalProtocols.Choose(r).(string),
+		FastSync:         nodeFastSyncs.Choose(r).(string),
+		StateSync:        nodeStateSyncs.Choose(r).(bool) && startAt > 0,
+		PersistInterval:  ptrUint64(uint64(nodePersistIntervals.Choose(r).(int))),
+		SnapshotInterval: uint64(nodeSnapshotIntervals.Choose(r).(int)),
+		RetainBlocks:     uint64(nodeRetainBlocks.Choose(r).(int)),
+		Perturb:          nodePerturbations.Choose(r),
+	}
+
+	// If this node is forced to be an archive node, retain all blocks and
+	// enable state sync snapshotting.
+	if forceArchive {
+		node.RetainBlocks = 0
+		node.SnapshotInterval = 3
+	}
+
+	// If a node which does not persist state also does not retain blocks, randomly
+	// choose to either persist state or retain all blocks.
+	if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 {
+		if r.Float64() > 0.5 {
+			node.RetainBlocks = 0
+		} else {
+			node.PersistInterval = ptrUint64(node.RetainBlocks)
+		}
+	}
+
+	// If either PersistInterval or SnapshotInterval are greater than RetainBlocks,
+	// expand the block retention time.
+	if node.RetainBlocks > 0 {
+		if node.PersistInterval != nil && node.RetainBlocks < *node.PersistInterval {
+			node.RetainBlocks = *node.PersistInterval
+		}
+		if node.RetainBlocks < node.SnapshotInterval {
+			node.RetainBlocks = node.SnapshotInterval
+		}
+	}
+
+	return &node
+}
+
+func ptrUint64(i uint64) *uint64 {
+	return &i
+}

test/e2e/generator/main.go

@@ -0,0 +1,97 @@
+//nolint: gosec
+package main
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/tendermint/tendermint/libs/log"
+)
+
+const (
+	randomSeed int64 = 4827085738
+)
+
+var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+
+func main() {
+	NewCLI().Run()
+}
+
+// CLI is the Cobra-based command-line interface.
+type CLI struct {
+	root *cobra.Command
+}
+
+// NewCLI sets up the CLI.
+func NewCLI() *CLI {
+	cli := &CLI{}
+	cli.root = &cobra.Command{
+		Use:           "generator",
+		Short:         "End-to-end testnet generator",
+		SilenceUsage:  true,
+		SilenceErrors: true, // we'll output them ourselves in Run()
+		RunE: func(cmd *cobra.Command, args []string) error {
+			dir, err := cmd.Flags().GetString("dir")
+			if err != nil {
+				return err
+			}
+			groups, err := cmd.Flags().GetInt("groups")
+			if err != nil {
+				return err
+			}
+			return cli.generate(dir, groups)
+		},
+	}
+
+	cli.root.PersistentFlags().StringP("dir", "d", "", "Output directory for manifests")
+	_ = cli.root.MarkPersistentFlagRequired("dir")
+	cli.root.PersistentFlags().IntP("groups", "g", 0, "Number of groups")
+
+	return cli
+}
+
+// generate generates manifests in a directory.
+func (cli *CLI) generate(dir string, groups int) error {
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return err
+	}
+
+	manifests, err := Generate(rand.New(rand.NewSource(randomSeed)))
+	if err != nil {
+		return err
+	}
+	if groups <= 0 {
+		for i, manifest := range manifests {
+			err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-%04d.toml", i)))
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		groupSize := int(math.Ceil(float64(len(manifests)) / float64(groups)))
+		for g := 0; g < groups; g++ {
+			for i := 0; i < groupSize && g*groupSize+i < len(manifests); i++ {
+				manifest := manifests[g*groupSize+i]
+				err = manifest.Save(filepath.Join(dir, fmt.Sprintf("gen-group%02d-%04d.toml", g, i)))
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// Run runs the CLI.
+func (cli *CLI) Run() {
+	if err := cli.root.Execute(); err != nil {
+		logger.Error(err.Error())
+		os.Exit(1)
+	}
+}

test/e2e/generator/random.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"math/rand"
+	"sort"
+)
+
+// combinations takes input in the form of a map of item lists, and returns a
+// list of all combinations of each item for each key. E.g.:
+//
+// {"foo": [1, 2, 3], "bar": [4, 5, 6]}
+//
+// Will return the following maps:
+//
+// {"foo": 1, "bar": 4}
+// {"foo": 1, "bar": 5}
+// {"foo": 1, "bar": 6}
+// {"foo": 2, "bar": 4}
+// {"foo": 2, "bar": 5}
+// {"foo": 2, "bar": 6}
+// {"foo": 3, "bar": 4}
+// {"foo": 3, "bar": 5}
+// {"foo": 3, "bar": 6}
+func combinations(items map[string][]interface{}) []map[string]interface{} {
+	keys := []string{}
+	for key := range items {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	return combiner(map[string]interface{}{}, keys, items)
+}
+
+// combiner is a utility function for combinations.
+func combiner(head map[string]interface{}, pending []string, items map[string][]interface{}) []map[string]interface{} {
+	if len(pending) == 0 {
+		return []map[string]interface{}{head}
+	}
+	key, pending := pending[0], pending[1:]
+
+	result := []map[string]interface{}{}
+	for _, value := range items[key] {
+		path := map[string]interface{}{}
+		for k, v := range head {
+			path[k] = v
+		}
+		path[key] = value
+		result = append(result, combiner(path, pending, items)...)
+	}
+	return result
+}
+
+// uniformChoice chooses a single random item from the argument list, uniformly weighted.
+type uniformChoice []interface{}
+
+func (uc uniformChoice) Choose(r *rand.Rand) interface{} {
+	return uc[r.Intn(len(uc))]
+}
+
+// weightedChoice chooses a single random key from a map of keys and weights.
+type weightedChoice map[interface{}]uint // nolint:unused
+
+func (wc weightedChoice) Choose(r *rand.Rand) interface{} {
+	total := 0
+	choices := make([]interface{}, 0, len(wc))
+	for choice, weight := range wc {
+		total += int(weight)
+		choices = append(choices, choice)
+	}
+
+	rem := r.Intn(total)
+	for _, choice := range choices {
+		rem -= int(wc[choice])
+		if rem <= 0 {
+			return choice
+		}
+	}
+
+	return nil
+}
+
+// probSetChoice picks a set of strings based on each string's probability (0-1).
+type probSetChoice map[string]float64
+
+func (pc probSetChoice) Choose(r *rand.Rand) []string {
+	choices := []string{}
+	for item, prob := range pc {
+		if r.Float64() <= prob {
+			choices = append(choices, item)
+		}
+	}
+	return choices
+}
+
+// uniformSetChoice picks a set of strings with uniform probability, picking at least one.
+type uniformSetChoice []string
+
+func (usc uniformSetChoice) Choose(r *rand.Rand) []string {
+	choices := []string{}
+	indexes := r.Perm(len(usc))
+	if len(indexes) > 1 {
+		indexes = indexes[:1+r.Intn(len(indexes)-1)]
+	}
+	for _, i := range indexes {
+		choices = append(choices, usc[i])
+	}
+	return choices
+}

test/e2e/generator/random_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCombinations(t *testing.T) {
+	input := map[string][]interface{}{
+		"bool":   {false, true},
+		"int":    {1, 2, 3},
+		"string": {"foo", "bar"},
+	}
+
+	c := combinations(input)
+	assert.Equal(t, []map[string]interface{}{
+		{"bool": false, "int": 1, "string": "foo"},
+		{"bool": false, "int": 1, "string": "bar"},
+		{"bool": false, "int": 2, "string": "foo"},
+		{"bool": false, "int": 2, "string": "bar"},
+		{"bool": false, "int": 3, "string": "foo"},
+		{"bool": false, "int": 3, "string": "bar"},
+		{"bool": true, "int": 1, "string": "foo"},
+		{"bool": true, "int": 1, "string": "bar"},
+		{"bool": true, "int": 2, "string": "foo"},
+		{"bool": true, "int": 2, "string": "bar"},
+		{"bool": true, "int": 3, "string": "foo"},
+		{"bool": true, "int": 3, "string": "bar"},
+	}, c)
+}