e2e: add generator tests (#7008)

.github/workflows/coverage.yml

@@ -4,7 +4,6 @@ on:
   push:
     paths:
       - "**.go"
-      - "!test/"
     branches:
       - master
       - release/**

test/e2e/generator/generate.go

@@ -73,7 +73,15 @@ func Generate(r *rand.Rand, opts Options) ([]e2e.Manifest, error) {
 	manifests := []e2e.Manifest{}
 	switch opts.P2P {
 	case NewP2PMode, LegacyP2PMode, HybridP2PMode:
+		defer func() {
+			// avoid modifying the global state.
+			original := make([]interface{}, len(testnetCombinations["p2p"]))
+			copy(original, testnetCombinations["p2p"])
+			testnetCombinations["p2p"] = original
+		}()
+
 		testnetCombinations["p2p"] = []interface{}{opts.P2P}
+
 	default:
 		testnetCombinations["p2p"] = []interface{}{NewP2PMode, LegacyP2PMode, HybridP2PMode}
 	}
@@ -154,13 +162,15 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 	case "large":
 		// FIXME Networks are kept small since large ones use too much CPU.
 		numSeeds = r.Intn(2)
-		numLightClients = r.Intn(3)
+		numLightClients = r.Intn(2)
 		numValidators = 4 + r.Intn(4)
 		numFulls = r.Intn(4)
 	default:
 		return manifest, fmt.Errorf("unknown topology %q", opt["topology"])
 	}
 
+	const legacyP2PFactor float64 = 0.5
+
 	// First we generate seed nodes, starting at the initial height.
 	for i := 1; i <= numSeeds; i++ {
 		node := generateNode(r, manifest, e2e.ModeSeed, 0, false)
@@ -169,18 +179,23 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() < legacyP2PFactor
 		}
 
 		manifest.Nodes[fmt.Sprintf("seed%02d", i)] = node
 	}
 
+	var (
+		numSyncingNodes = 0
+		hybridNumNew    = 0
+		hybridNumLegacy = 0
+	)
+
 	// Next, we generate validators. We make sure a BFT quorum of validators start
 	// at the initial height, and that we have two archive nodes. We also set up
 	// the initial validator set, and validator set updates for delayed nodes.
 	nextStartAt := manifest.InitialHeight + 5
 	quorum := numValidators*2/3 + 1
-	numSyncingNodes := 0
 	for i := 1; i <= numValidators; i++ {
 		startAt := int64(0)
 		if i > quorum && numSyncingNodes < 2 && r.Float64() >= 0.25 {
@@ -195,7 +210,23 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() < legacyP2PFactor
+			if node.UseLegacyP2P {
+				hybridNumLegacy++
+				if hybridNumNew == 0 {
+					hybridNumNew++
+					hybridNumLegacy--
+					node.UseLegacyP2P = false
+				}
+			} else {
+				hybridNumNew++
+				if hybridNumLegacy == 0 {
+					hybridNumNew--
+					hybridNumLegacy++
+					node.UseLegacyP2P = true
+
+				}
+			}
 		}
 
 		manifest.Nodes[name] = node
@@ -222,7 +253,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 	// Finally, we generate random full nodes.
 	for i := 1; i <= numFulls; i++ {
 		startAt := int64(0)
-		if r.Float64() >= 0.5 {
+		if numSyncingNodes < 2 && r.Float64() >= 0.5 {
+			numSyncingNodes++
 			startAt = nextStartAt
 			nextStartAt += 5
 		}
@@ -232,7 +264,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		case LegacyP2PMode:
 			node.UseLegacyP2P = true
 		case HybridP2PMode:
-			node.UseLegacyP2P = r.Intn(5) < 2
+			node.UseLegacyP2P = r.Float64() > legacyP2PFactor
 		}
 
 		manifest.Nodes[fmt.Sprintf("full%02d", i)] = node

test/e2e/generator/generate_test.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+)
+
+func TestGenerator(t *testing.T) {
+	manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), Options{P2P: MixedP2PMode})
+	require.NoError(t, err)
+	require.True(t, len(manifests) >= 64, "insufficient combinations")
+
+	// this just means that the numbers reported by the test
+	// failures map to the test cases that you'd see locally.
+	e2e.SortManifests(manifests, false /* ascending */)
+
+	for idx, m := range manifests {
+		t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) {
+			numStateSyncs := 0
+			for name, node := range m.Nodes {
+				if node.StateSync != e2e.StateSyncDisabled {
+					numStateSyncs++
+				}
+				t.Run(name, func(t *testing.T) {
+					if node.StartAt > m.InitialHeight+5 && !node.Stateless() {
+						require.NotEqual(t, node.StateSync, e2e.StateSyncDisabled)
+					}
+					if node.StateSync != e2e.StateSyncDisabled {
+						require.Zero(t, node.Seeds, node.StateSync)
+						require.True(t, len(node.PersistentPeers) >= 2)
+						require.Equal(t, "v0", node.BlockSync)
+					}
+
+				})
+			}
+			require.True(t, numStateSyncs <= 2)
+		})
+	}
+
+	t.Run("Hybrid", func(t *testing.T) {
+		manifests, err := Generate(rand.New(rand.NewSource(randomSeed)), Options{P2P: HybridP2PMode})
+		require.NoError(t, err)
+		require.True(t, len(manifests) >= 16, "insufficient combinations: %d", len(manifests))
+
+		for idx, m := range manifests {
+			t.Run(fmt.Sprintf("Case%04d", idx), func(t *testing.T) {
+				require.True(t, len(m.Nodes) > 1)
+
+				var numLegacy, numNew int
+				for _, node := range m.Nodes {
+					if node.UseLegacyP2P {
+						numLegacy++
+					} else {
+						numNew++
+					}
+				}
+
+				assert.True(t, numLegacy >= 1, "not enough legacy nodes [%d/%d]",
+					numLegacy, len(m.Nodes))
+				assert.True(t, numNew >= 1, "not enough new nodes [%d/%d]",
+					numNew, len(m.Nodes))
+			})
+		}
+	})
+}

test/e2e/pkg/manifest.go

@@ -150,6 +150,11 @@ type ManifestNode struct {
 	UseLegacyP2P bool `toml:"use_legacy_p2p"`
 }
 
+// Stateless reports whether m is a node that does not own state, including light and seed nodes.
+func (m ManifestNode) Stateless() bool {
+	return m.Mode == string(ModeLight) || m.Mode == string(ModeSeed)
+}
+
 // Save saves the testnet manifest to a file.
 func (m Manifest) Save(file string) error {
 	f, err := os.Create(file)