e2e: integrate light clients (bp #6196)

integrate light clients (#6196)
fix e2e app test (#6223)
fix light client generator (#6236)

test/e2e/app/app.go

@@ -117,7 +117,7 @@ func (app *Application) Commit() abci.ResponseCommit {
 		if err != nil {
 			panic(err)
 		}
-		logger.Info("Created state sync snapshot", "height", snapshot.Height)
+		app.logger.Info("Created state sync snapshot", "height", snapshot.Height)
 	}
 	retainHeight := int64(0)
 	if app.cfg.RetainBlocks > 0 {

test/e2e/app/config.go

@@ -14,6 +14,7 @@ type Config struct {
 	Listen           string
 	Protocol         string
 	Dir              string
+	Mode             string                      `toml:"mode"`
 	PersistInterval  uint64                      `toml:"persist_interval"`
 	SnapshotInterval uint64                      `toml:"snapshot_interval"`
 	RetainBlocks     uint64                      `toml:"retain_blocks"`

test/e2e/app/main.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/spf13/viper"
@@ -16,10 +19,16 @@ import (
 	tmflags "github.com/tendermint/tendermint/libs/cli/flags"
 	"github.com/tendermint/tendermint/libs/log"
 	tmnet "github.com/tendermint/tendermint/libs/net"
+	"github.com/tendermint/tendermint/light"
+	lproxy "github.com/tendermint/tendermint/light/proxy"
+	lrpc "github.com/tendermint/tendermint/light/rpc"
+	dbs "github.com/tendermint/tendermint/light/store/db"
 	"github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
+	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 	mcs "github.com/tendermint/tendermint/test/maverick/consensus"
 	maverick "github.com/tendermint/tendermint/test/maverick/node"
 )
@@ -66,7 +75,11 @@ func run(configFile string) error {
 		err = startApp(cfg)
 	case "builtin":
 		if len(cfg.Misbehaviors) == 0 {
-			err = startNode(cfg)
+			if cfg.Mode == string(e2e.ModeLight) {
+				err = startLightClient(cfg)
+			} else {
+				err = startNode(cfg)
+			}
 		} else {
 			err = startMaverick(cfg)
 		}
@@ -131,6 +144,64 @@ func startNode(cfg *Config) error {
 	return n.Start()
 }
 
+func startLightClient(cfg *Config) error {
+	tmcfg, nodeLogger, _, err := setupNode()
+	if err != nil {
+		return err
+	}
+
+	dbContext := &node.DBContext{ID: "light", Config: tmcfg}
+	lightDB, err := node.DefaultDBProvider(dbContext)
+	if err != nil {
+		return err
+	}
+
+	providers := rpcEndpoints(tmcfg.P2P.PersistentPeers)
+
+	c, err := light.NewHTTPClient(
+		context.Background(),
+		cfg.ChainID,
+		light.TrustOptions{
+			Period: tmcfg.StateSync.TrustPeriod,
+			Height: tmcfg.StateSync.TrustHeight,
+			Hash:   tmcfg.StateSync.TrustHashBytes(),
+		},
+		providers[0],
+		providers[1:],
+		dbs.New(lightDB, "light"),
+		light.Logger(nodeLogger),
+	)
+	if err != nil {
+		return err
+	}
+
+	rpccfg := rpcserver.DefaultConfig()
+	rpccfg.MaxBodyBytes = tmcfg.RPC.MaxBodyBytes
+	rpccfg.MaxHeaderBytes = tmcfg.RPC.MaxHeaderBytes
+	rpccfg.MaxOpenConnections = tmcfg.RPC.MaxOpenConnections
+	// If necessary adjust global WriteTimeout to ensure it's greater than
+	// TimeoutBroadcastTxCommit.
+	// See https://github.com/tendermint/tendermint/issues/3435
+	if rpccfg.WriteTimeout <= tmcfg.RPC.TimeoutBroadcastTxCommit {
+		rpccfg.WriteTimeout = tmcfg.RPC.TimeoutBroadcastTxCommit + 1*time.Second
+	}
+
+	p, err := lproxy.NewProxy(c, tmcfg.RPC.ListenAddress, providers[0], rpccfg, nodeLogger,
+		lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn()))
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Starting proxy...", "laddr", tmcfg.RPC.ListenAddress)
+	if err := p.ListenAndServe(); err != http.ErrServerClosed {
+		// Error starting or closing listener:
+		logger.Error("proxy ListenAndServe", "err", err)
+	}
+
+	return nil
+}
+
+// FIXME: Temporarily disconnected maverick until it is redesigned
 // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
 // configuration is in $TMHOME/config/tendermint.toml.
 func startMaverick(cfg *Config) error {
@@ -238,3 +309,19 @@ func setupNode() (*config.Config, log.Logger, *p2p.NodeKey, error) {
 
 	return tmcfg, nodeLogger, nodeKey, nil
 }
+
+// rpcEndpoints takes a list of persistent peers and splits them into a list of rpc endpoints
+// using 26657 as the port number
+func rpcEndpoints(peers string) []string {
+	arr := strings.Split(peers, ",")
+	endpoints := make([]string, len(arr))
+	for i, v := range arr {
+		urlString := strings.SplitAfter(v, "@")[1]
+		hostName := strings.Split(urlString, ":26656")[0]
+		// use RPC port instead
+		port := 26657
+		rpcEndpoint := "http://" + hostName + ":" + fmt.Sprint(port)
+		endpoints[i] = rpcEndpoint
+	}
+	return endpoints
+}

test/e2e/generator/generate.go

@@ -74,7 +74,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		Nodes:            map[string]*e2e.ManifestNode{},
 	}
 
-	var numSeeds, numValidators, numFulls int
+	var numSeeds, numValidators, numFulls, numLightClients int
 	switch opt["topology"].(string) {
 	case "single":
 		numValidators = 1
@@ -82,7 +82,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		numValidators = 4
 	case "large":
 		// FIXME Networks are kept small since large ones use too much CPU.
-		numSeeds = r.Intn(4)
+		numSeeds = r.Intn(3)
+		numLightClients = r.Intn(3)
 		numValidators = 4 + r.Intn(7)
 		numFulls = r.Intn(5)
 	default:
@@ -143,11 +144,16 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 	// We now set up peer discovery for nodes. Seed nodes are fully meshed with
 	// each other, while non-seed nodes either use a set of random seeds or a
 	// set of random peers that start before themselves.
-	var seedNames, peerNames []string
+	var seedNames, peerNames, lightProviders []string
 	for name, node := range manifest.Nodes {
 		if node.Mode == string(e2e.ModeSeed) {
 			seedNames = append(seedNames, name)
 		} else {
+			// if the full node or validator is an ideal candidate, it is added as a light provider.
+			// There are at least two archive nodes so there should be at least two ideal candidates
+			if (node.StartAt == 0 || node.StartAt == manifest.InitialHeight) && node.RetainBlocks == 0 {
+				lightProviders = append(lightProviders, name)
+			}
 			peerNames = append(peerNames, name)
 		}
 	}
@@ -179,6 +185,14 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		}
 	}
 
+	// lastly, set up the light clients
+	for i := 1; i <= numLightClients; i++ {
+		startAt := manifest.InitialHeight + 5
+		manifest.Nodes[fmt.Sprintf("light%02d", i)] = generateLightNode(
+			r, startAt+(5*int64(i)), lightProviders,
+		)
+	}
+
 	return manifest, nil
 }
 
@@ -210,7 +224,7 @@ func generateNode(
 		node.SnapshotInterval = 3
 	}
 
-	if node.Mode == "validator" {
+	if node.Mode == string(e2e.ModeValidator) {
 		misbehaveAt := startAt + 5 + int64(r.Intn(10))
 		if startAt == 0 {
 			misbehaveAt += initialHeight - 1
@@ -245,6 +259,17 @@ func generateNode(
 	return &node
 }
 
+func generateLightNode(r *rand.Rand, startAt int64, providers []string) *e2e.ManifestNode {
+	return &e2e.ManifestNode{
+		Mode:            string(e2e.ModeLight),
+		StartAt:         startAt,
+		Database:        nodeDatabases.Choose(r).(string),
+		ABCIProtocol:    "builtin",
+		PersistInterval: ptrUint64(0),
+		PersistentPeers: providers,
+	}
+}
+
 func ptrUint64(i uint64) *uint64 {
 	return &i
 }

test/e2e/networks/ci.toml

@@ -1,6 +1,7 @@
 # This testnet is run by CI, and attempts to cover a broad range of
 # functionality with a single network.
 
+ipv6 = true
 initial_height = 1000
 initial_state = { initial01 = "a", initial02 = "b", initial03 = "c" }
 
@@ -89,3 +90,13 @@ fast_sync = "v0"
 state_sync = true
 seeds = ["seed01"]
 perturb = ["restart"]
+
+[node.light01]
+mode= "light"
+start_at= 1005
+persistent_peers = ["validator01", "validator02", "validator03"]
+
+[node.light02]
+mode= "light"
+start_at= 1015
+persistent_peers = ["validator04", "full01", "validator05"]

test/e2e/networks/simple.toml

@@ -2,4 +2,4 @@
 [node.validator02]
 [node.validator03]
 [node.validator04]
-  
+

test/e2e/pkg/manifest.go

@@ -54,9 +54,9 @@ type Manifest struct {
 
 // ManifestNode represents a node in a testnet manifest.
 type ManifestNode struct {
-	// Mode specifies the type of node: "validator", "full", or "seed". Defaults to
-	// "validator". Full nodes do not get a signing key (a dummy key is generated),
-	// and seed nodes run in seed mode with the PEX reactor enabled.
+	// Mode specifies the type of node: "validator", "full", "light" or "seed".
+	// Defaults to "validator". Full nodes do not get a signing key (a dummy key
+	// is generated), and seed nodes run in seed mode with the PEX reactor enabled.
 	Mode string `toml:"mode"`
 
 	// Seeds is the list of node names to use as P2P seed nodes. Defaults to none.
@@ -64,7 +64,8 @@ type ManifestNode struct {
 
 	// PersistentPeers is a list of node names to maintain persistent P2P
 	// connections to. If neither seeds nor persistent peers are specified,
-	// this defaults to all other nodes in the network.
+	// this defaults to all other nodes in the network. For light clients,
+	// this relates to the providers the light client is connected to.
 	PersistentPeers []string `toml:"persistent_peers"`
 
 	// Database specifies the database backend: "goleveldb", "cleveldb",

test/e2e/pkg/testnet.go

@@ -33,6 +33,7 @@ type Perturbation string
 const (
 	ModeValidator Mode = "validator"
 	ModeFull      Mode = "full"
+	ModeLight     Mode = "light"
 	ModeSeed      Mode = "seed"
 
 	ProtocolBuiltin Protocol = "builtin"
@@ -144,7 +145,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 			ProxyPort:        proxyPortGen.Next(),
 			Mode:             ModeValidator,
 			Database:         "goleveldb",
-			ABCIProtocol:     ProtocolUNIX,
+			ABCIProtocol:     ProtocolBuiltin,
 			PrivvalProtocol:  ProtocolFile,
 			StartAt:          nodeManifest.StartAt,
 			FastSync:         nodeManifest.FastSync,
@@ -311,6 +312,9 @@ func (n Node) Validate(testnet Testnet) error {
 	default:
 		return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol)
 	}
+	if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin {
+		return errors.New("light client must use builtin protocol")
+	}
 	switch n.PrivvalProtocol {
 	case ProtocolFile, ProtocolUNIX, ProtocolTCP:
 	default:
@@ -385,7 +389,7 @@ func (t Testnet) LookupNode(name string) *Node {
 func (t Testnet) ArchiveNodes() []*Node {
 	nodes := []*Node{}
 	for _, node := range t.Nodes {
-		if node.Mode != ModeSeed && node.StartAt == 0 && node.RetainBlocks == 0 {
+		if !node.Stateless() && node.StartAt == 0 && node.RetainBlocks == 0 {
 			nodes = append(nodes, node)
 		}
 	}
@@ -459,6 +463,11 @@ func (n Node) Client() (*rpchttp.HTTP, error) {
 	return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort), "/websocket")
 }
 
+// Stateless returns true if the node is either a seed node or a light node
+func (n Node) Stateless() bool {
+	return n.Mode == ModeLight || n.Mode == ModeSeed
+}
+
 // keyGenerator generates pseudorandom Ed25519 keys based on a seed.
 type keyGenerator struct {
 	random *rand.Rand

test/e2e/runner/setup.go

@@ -64,23 +64,23 @@ func Setup(testnet *e2e.Testnet) error {
 
 	for _, node := range testnet.Nodes {
 		nodeDir := filepath.Join(testnet.Dir, node.Name)
+
 		dirs := []string{
 			filepath.Join(nodeDir, "config"),
 			filepath.Join(nodeDir, "data"),
 			filepath.Join(nodeDir, "data", "app"),
 		}
 		for _, dir := range dirs {
+			// light clients don't need an app directory
+			if node.Mode == e2e.ModeLight && strings.Contains(dir, "app") {
+				continue
+			}
 			err := os.MkdirAll(dir, 0755)
 			if err != nil {
 				return err
 			}
 		}
 
-		err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
-		if err != nil {
-			return err
-		}
-
 		cfg, err := MakeConfig(node)
 		if err != nil {
 			return err
@@ -96,6 +96,16 @@ func Setup(testnet *e2e.Testnet) error {
 			return err
 		}
 
+		if node.Mode == e2e.ModeLight {
+			// stop early if a light client
+			continue
+		}
+
+		err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
+		if err != nil {
+			return err
+		}
+
 		err = (&p2p.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json"))
 		if err != nil {
 			return err
@@ -266,7 +276,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	case e2e.ModeSeed:
 		cfg.P2P.SeedMode = true
 		cfg.P2P.PexReactor = true
-	case e2e.ModeFull:
+	case e2e.ModeFull, e2e.ModeLight:
 		// Don't need to do anything, since we're using a dummy privval key by default.
 	default:
 		return nil, fmt.Errorf("unexpected mode %q", node.Mode)
@@ -315,6 +325,8 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 		"chain_id":          node.Testnet.Name,
 		"dir":               "data/app",
 		"listen":            AppAddressUNIX,
+		"mode":              node.Mode,
+		"proxy_port":        node.ProxyPort,
 		"protocol":          "socket",
 		"persist_interval":  node.PersistInterval,
 		"snapshot_interval": node.SnapshotInterval,

test/e2e/runner/start.go

@@ -58,7 +58,7 @@ func Start(testnet *e2e.Testnet) error {
 
 	// Update any state sync nodes with a trusted height and hash
 	for _, node := range nodeQueue {
-		if node.StateSync {
+		if node.StateSync || node.Mode == e2e.ModeLight {
 			err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes())
 			if err != nil {
 				return err

test/e2e/tests/app_test.go

@@ -16,7 +16,7 @@ import (
 // Tests that any initial state given in genesis has made it into the app.
 func TestApp_InitialState(t *testing.T) {
 	testNode(t, func(t *testing.T, node e2e.Node) {
-		if node.Mode == e2e.ModeSeed {
+		if node.Stateless() {
 			return
 		}
 		if len(node.Testnet.InitialState) == 0 {
@@ -81,12 +81,27 @@ func TestApp_Tx(t *testing.T) {
 		value := fmt.Sprintf("%x", bz)
 		tx := types.Tx(fmt.Sprintf("%v=%v", key, value))
 
-		_, err = client.BroadcastTxCommit(ctx, tx)
+		resp, err := client.BroadcastTxCommit(ctx, tx)
 		require.NoError(t, err)
 
-		resp, err := client.ABCIQuery(ctx, "", []byte(key))
+		// wait for the tx to be persisted in the tx indexer
+		time.Sleep(500 * time.Millisecond)
+
+		hash := tx.Hash()
+		txResp, err := client.Tx(ctx, hash, false)
 		require.NoError(t, err)
-		assert.Equal(t, key, string(resp.Response.Key))
-		assert.Equal(t, value, string(resp.Response.Value))
+		assert.Equal(t, txResp.Tx, tx)
+		assert.Equal(t, txResp.Height, resp.Height)
+
+		// NOTE: we don't test abci query of the light client
+		if node.Mode == e2e.ModeLight {
+			return
+		}
+
+		abciResp, err := client.ABCIQuery(ctx, "", []byte(key))
+		require.NoError(t, err)
+		assert.Equal(t, key, string(abciResp.Response.Key))
+		assert.Equal(t, value, string(abciResp.Response.Value))
+
 	})
 }