statesync: implement p2p state provider (#6807)

test/e2e/generator/generate.go

@@ -48,7 +48,7 @@ var (
 	// FIXME: v2 disabled due to flake
 	nodeBlockSyncs        = uniformChoice{"v0"} // "v2"
 	nodeMempools          = uniformChoice{"v0", "v1"}
-	nodeStateSyncs        = uniformChoice{false, true}
+	nodeStateSyncs        = uniformChoice{e2e.StateSyncDisabled, e2e.StateSyncP2P, e2e.StateSyncRPC}
 	nodePersistIntervals  = uniformChoice{0, 1, 5}
 	nodeSnapshotIntervals = uniformChoice{0, 3}
 	nodeRetainBlocks      = uniformChoice{0, int(e2e.EvidenceAgeHeight), int(e2e.EvidenceAgeHeight) + 5}
@@ -291,13 +291,17 @@ func generateNode(
 		PrivvalProtocol:  nodePrivvalProtocols.Choose(r),
 		BlockSync:        nodeBlockSyncs.Choose(r).(string),
 		Mempool:          nodeMempools.Choose(r).(string),
-		StateSync:        nodeStateSyncs.Choose(r).(bool) && startAt > 0,
+		StateSync:        e2e.StateSyncDisabled,
 		PersistInterval:  ptrUint64(uint64(nodePersistIntervals.Choose(r).(int))),
 		SnapshotInterval: uint64(nodeSnapshotIntervals.Choose(r).(int)),
 		RetainBlocks:     uint64(nodeRetainBlocks.Choose(r).(int)),
 		Perturb:          nodePerturbations.Choose(r),
 	}
 
+	if startAt > 0 {
+		node.StateSync = nodeStateSyncs.Choose(r).(string)
+	}
+
 	// If this node is forced to be an archive node, retain all blocks and
 	// enable state sync snapshotting.
 	if forceArchive {
@@ -326,7 +330,7 @@ func generateNode(
 		}
 	}
 
-	if node.StateSync {
+	if node.StateSync != e2e.StateSyncDisabled {
 		node.BlockSync = "v0"
 	}
 

test/e2e/networks/ci.toml

@@ -55,6 +55,7 @@ retain_blocks = 7
 
 [node.validator04]
 abci_protocol = "builtin"
+snapshot_interval = 5
 database = "rocksdb"
 persistent_peers = ["validator01"]
 perturb = ["pause"]
@@ -62,6 +63,7 @@ perturb = ["pause"]
 [node.validator05]
 database = "cleveldb" 
 block_sync = "v0" 
+state_sync = "p2p"
 seeds = ["seed01"] 
 start_at = 1005 # Becomes part of the validator set at 1010
 abci_protocol = "grpc"
@@ -73,10 +75,10 @@ mode = "full"
 start_at = 1010
 # FIXME: should be v2, disabled due to flake
 block_sync = "v0"
-persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
+persistent_peers = ["validator01", "validator02", "validator03", "validator04"]
 perturb = ["restart"]
 retain_blocks = 7
-state_sync = true
+state_sync = "rpc"
 
 [node.light01]
 mode = "light"

test/e2e/pkg/manifest.go

@@ -117,7 +117,8 @@ type ManifestNode struct {
 	// block hashes and RPC servers. At least one node in the network must have
 	// SnapshotInterval set to non-zero, and the state syncing node must have
 	// StartAt set to an appropriate height where a snapshot is available.
-	StateSync bool `toml:"state_sync"`
+	// StateSync can either be "p2p" or "rpc" or an empty string to disable
+	StateSync string `toml:"state_sync"`
 
 	// PersistInterval specifies the height interval at which the application
 	// will persist state to disk. Defaults to 1 (every height), setting this to

test/e2e/pkg/testnet.go

@@ -50,6 +50,10 @@ const (
 
 	EvidenceAgeHeight int64         = 7
 	EvidenceAgeTime   time.Duration = 500 * time.Millisecond
+
+	StateSyncP2P      = "p2p"
+	StateSyncRPC      = "rpc"
+	StateSyncDisabled = ""
 )
 
 // Testnet represents a single testnet.
@@ -81,7 +85,7 @@ type Node struct {
 	StartAt          int64
 	BlockSync        string
 	Mempool          string
-	StateSync        bool
+	StateSync        string
 	Database         string
 	ABCIProtocol     Protocol
 	PrivvalProtocol  Protocol
@@ -333,6 +337,11 @@ func (n Node) Validate(testnet Testnet) error {
 	default:
 		return fmt.Errorf("invalid block sync setting %q", n.BlockSync)
 	}
+	switch n.StateSync {
+	case StateSyncDisabled, StateSyncP2P, StateSyncRPC:
+	default:
+		return fmt.Errorf("invalid state sync setting %q", n.StateSync)
+	}
 	switch n.Mempool {
 	case "", "v0", "v1":
 	default:
@@ -366,7 +375,7 @@ func (n Node) Validate(testnet Testnet) error {
 		return fmt.Errorf("cannot start at height %v lower than initial height %v",
 			n.StartAt, n.Testnet.InitialHeight)
 	}
-	if n.StateSync && n.StartAt == 0 {
+	if n.StateSync != StateSyncDisabled && n.StartAt == 0 {
 		return errors.New("state synced nodes cannot start at the initial height")
 	}
 	if n.RetainBlocks != 0 && n.RetainBlocks < uint64(EvidenceAgeHeight) {

test/e2e/runner/main.go

@@ -168,12 +168,21 @@ func NewCLI() *CLI {
 		},
 	})
 
+	cli.root.AddCommand(&cobra.Command{
+		Use:   "pause",
+		Short: "Pauses the Docker testnet",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			logger.Info("Pausing testnet")
+			return execCompose(cli.testnet.Dir, "pause")
+		},
+	})
+
 	cli.root.AddCommand(&cobra.Command{
 		Use:   "resume",
 		Short: "Resumes the Docker testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			logger.Info("Resuming testnet")
-			return execCompose(cli.testnet.Dir, "up")
+			return execCompose(cli.testnet.Dir, "unpause")
 		},
 	})
 

test/e2e/runner/setup.go

@@ -302,10 +302,13 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 		cfg.BlockSync.Version = node.BlockSync
 	}
 
-	if node.StateSync {
+	switch node.StateSync {
+	case e2e.StateSyncP2P:
+		cfg.StateSync.Enable = true
+		cfg.StateSync.UseP2P = true
+	case e2e.StateSyncRPC:
 		cfg.StateSync.Enable = true
 		cfg.StateSync.RPCServers = []string{}
-
 		for _, peer := range node.Testnet.ArchiveNodes() {
 			if peer.Name == node.Name {
 				continue

test/e2e/runner/start.go

@@ -64,16 +64,6 @@ func Start(testnet *e2e.Testnet) error {
 		return err
 	}
 
-	// Update any state sync nodes with a trusted height and hash
-	for _, node := range nodeQueue {
-		if node.StateSync || node.Mode == e2e.ModeLight {
-			err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes())
-			if err != nil {
-				return err
-			}
-		}
-	}
-
 	for _, node := range nodeQueue {
 		if node.StartAt > networkHeight {
 			// if we're starting a node that's ahead of
@@ -85,16 +75,19 @@ func Start(testnet *e2e.Testnet) error {
 
 			networkHeight = node.StartAt
 
-			logger.Info("Waiting for network to advance before starting catch up node",
-				"node", node.Name,
-				"height", networkHeight)
-
-			if _, _, err := waitForHeight(testnet, networkHeight); err != nil {
+			block, blockID, err = waitForHeight(testnet, networkHeight)
+			if err != nil {
 				return err
 			}
 		}
 
-		logger.Info("Starting catch up node", "node", node.Name, "height", node.StartAt)
+		// Update any state sync nodes with a trusted height and hash
+		if node.StateSync != e2e.StateSyncDisabled || node.Mode == e2e.ModeLight {
+			err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes())
+			if err != nil {
+				return err
+			}
+		}
 
 		if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil {
 			return err

test/e2e/tests/block_test.go

@@ -34,7 +34,7 @@ func TestBlock_Header(t *testing.T) {
 			}
 			// the first blocks after state sync come from the backfill process
 			// and are therefore not complete
-			if node.StateSync && block.Header.Height <= first+e2e.EvidenceAgeHeight+1 {
+			if node.StateSync != e2e.StateSyncDisabled && block.Header.Height <= first+e2e.EvidenceAgeHeight+1 {
 				continue
 			}
 			if block.Header.Height > last {
@@ -70,7 +70,7 @@ func TestBlock_Range(t *testing.T) {
 		switch {
 		// if the node state synced we ignore any assertions because it's hard to know how far back
 		// the node ran reverse sync for
-		case node.StateSync:
+		case node.StateSync != e2e.StateSyncDisabled:
 			break
 		case node.RetainBlocks > 0 && int64(node.RetainBlocks) < (last-node.Testnet.InitialHeight+1):
 			// Delta handles race conditions in reading first/last heights.
@@ -83,7 +83,7 @@ func TestBlock_Range(t *testing.T) {
 		}
 
 		for h := first; h <= last; h++ {
-			if node.StateSync && h <= first+e2e.EvidenceAgeHeight+1 {
+			if node.StateSync != e2e.StateSyncDisabled && h <= first+e2e.EvidenceAgeHeight+1 {
 				continue
 			}
 			resp, err := client.Block(ctx, &(h))