e2e: integrate light clients (#6196)

cmd/tendermint/commands/light.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"time"
 
@@ -16,7 +15,6 @@ import (
 
 	dbm "github.com/tendermint/tm-db"
 
-	"github.com/tendermint/tendermint/crypto/merkle"
 	"github.com/tendermint/tendermint/libs/log"
 	tmmath "github.com/tendermint/tendermint/libs/math"
 	tmos "github.com/tendermint/tendermint/libs/os"
@@ -24,7 +22,6 @@ import (
 	lproxy "github.com/tendermint/tendermint/light/proxy"
 	lrpc "github.com/tendermint/tendermint/light/rpc"
 	dbs "github.com/tendermint/tendermint/light/store/db"
-	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 )
 
@@ -217,17 +214,11 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		cfg.WriteTimeout = config.RPC.TimeoutBroadcastTxCommit + 1*time.Second
 	}
 
-	rpcClient, err := rpchttp.NewWithTimeout(primaryAddr, cfg.WriteTimeout)
+	p, err := lproxy.NewProxy(c, listenAddr, primaryAddr, cfg, logger, lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn()))
 	if err != nil {
-		return fmt.Errorf("failed to create http client for %s: %w", primaryAddr, err)
+		return err
 	}
 
-	p := lproxy.Proxy{
-		Addr:   listenAddr,
-		Config: cfg,
-		Client: lrpc.NewClient(rpcClient, c, lrpc.KeyPathFn(defaultMerkleKeyPathFn())),
-		Logger: logger,
-	}
 	// Stop upon receiving SIGTERM or CTRL-C.
 	tmos.TrapSignal(logger, func() {
 		p.Listener.Close()
@@ -266,21 +257,3 @@ func saveProviders(db dbm.DB, primaryAddr, witnessesAddrs string) error {
 	}
 	return nil
 }
-
-func defaultMerkleKeyPathFn() lrpc.KeyPathFunc {
-	// regexp for extracting store name from /abci_query path
-	storeNameRegexp := regexp.MustCompile(`\/store\/(.+)\/key`)
-
-	return func(path string, key []byte) (merkle.KeyPath, error) {
-		matches := storeNameRegexp.FindStringSubmatch(path)
-		if len(matches) != 2 {
-			return nil, fmt.Errorf("can't find store name in %s using %s", path, storeNameRegexp)
-		}
-		storeName := matches[1]
-
-		kp := merkle.KeyPath{}
-		kp = kp.AppendKey([]byte(storeName), merkle.KeyEncodingURL)
-		kp = kp.AppendKey(key, merkle.KeyEncodingURL)
-		return kp, nil
-	}
-}

light/client.go

@@ -861,6 +861,8 @@ func (c *Client) cleanupAfter(height int64) error {
 }
 
 func (c *Client) updateTrustedLightBlock(l *types.LightBlock) error {
+	c.logger.Debug("updating trusted light block", "light_block", l)
+
 	if err := c.trustedStore.SaveLightBlock(l); err != nil {
 		return fmt.Errorf("failed to save trusted header: %w", err)
 	}

light/proxy/proxy.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/tendermint/tendermint/libs/log"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/light"
 	lrpc "github.com/tendermint/tendermint/light/rpc"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
 	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 )
 
@@ -21,6 +23,28 @@ type Proxy struct {
 	Listener net.Listener
 }
 
+// NewProxy creates the struct used to run an HTTP server for serving light
+// client rpc requests.
+func NewProxy(
+	lightClient *light.Client,
+	listenAddr, providerAddr string,
+	config *rpcserver.Config,
+	logger log.Logger,
+	opts ...lrpc.Option,
+) (*Proxy, error) {
+	rpcClient, err := rpchttp.NewWithTimeout(providerAddr, config.WriteTimeout)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create http client for %s: %w", providerAddr, err)
+	}
+
+	return &Proxy{
+		Addr:   listenAddr,
+		Config: config,
+		Client: lrpc.NewClient(rpcClient, lightClient, opts...),
+		Logger: logger,
+	}, nil
+}
+
 // ListenAndServe configures the rpcserver.WebsocketManager, sets up the RPC
 // routes to proxy via Client, and starts up an HTTP server on the TCP network
 // address p.Addr.

light/rpc/client.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"regexp"
 	"time"
 
 	"github.com/gogo/protobuf/proto"
@@ -59,6 +60,27 @@ func KeyPathFn(fn KeyPathFunc) Option {
 	}
 }
 
+// DefaultMerkleKeyPathFn creates a function used to generate merkle key paths
+// from a path string and a key. This is the default used by the cosmos SDK.
+// This merkle key paths are required when verifying /abci_query calls
+func DefaultMerkleKeyPathFn() KeyPathFunc {
+	// regexp for extracting store name from /abci_query path
+	storeNameRegexp := regexp.MustCompile(`\/store\/(.+)\/key`)
+
+	return func(path string, key []byte) (merkle.KeyPath, error) {
+		matches := storeNameRegexp.FindStringSubmatch(path)
+		if len(matches) != 2 {
+			return nil, fmt.Errorf("can't find store name in %s using %s", path, storeNameRegexp)
+		}
+		storeName := matches[1]
+
+		kp := merkle.KeyPath{}
+		kp = kp.AppendKey([]byte(storeName), merkle.KeyEncodingURL)
+		kp = kp.AppendKey(key, merkle.KeyEncodingURL)
+		return kp, nil
+	}
+}
+
 // NewClient returns a new client.
 func NewClient(next rpcclient.Client, lc LightClient, opts ...Option) *Client {
 	c := &Client{

test/e2e/app/config.go

@@ -14,6 +14,7 @@ type Config struct {
 	Listen           string
 	Protocol         string
 	Dir              string
+	Mode             string                      `toml:"mode"`
 	PersistInterval  uint64                      `toml:"persist_interval"`
 	SnapshotInterval uint64                      `toml:"snapshot_interval"`
 	RetainBlocks     uint64                      `toml:"retain_blocks"`

test/e2e/app/main.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/spf13/viper"
@@ -17,12 +20,18 @@ import (
 	tmflags "github.com/tendermint/tendermint/libs/cli/flags"
 	"github.com/tendermint/tendermint/libs/log"
 	tmnet "github.com/tendermint/tendermint/libs/net"
+	"github.com/tendermint/tendermint/light"
+	lproxy "github.com/tendermint/tendermint/light/proxy"
+	lrpc "github.com/tendermint/tendermint/light/rpc"
+	dbs "github.com/tendermint/tendermint/light/store/db"
 	"github.com/tendermint/tendermint/node"
 	"github.com/tendermint/tendermint/p2p"
 	"github.com/tendermint/tendermint/privval"
 	grpcprivval "github.com/tendermint/tendermint/privval/grpc"
 	privvalproto "github.com/tendermint/tendermint/proto/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
+	rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )
 
 var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
@@ -66,10 +75,13 @@ func run(configFile string) error {
 	case "socket", "grpc":
 		err = startApp(cfg)
 	case "builtin":
+		if cfg.Mode == string(e2e.ModeLight) {
+			err = startLightClient(cfg)
+		} else {
+			err = startNode(cfg)
+		}
 		// FIXME: Temporarily remove maverick until it is redesigned
 		// if len(cfg.Misbehaviors) == 0 {
-		err = startNode(cfg)
-		// } else {
 		// 	err = startMaverick(cfg)
 		// }
 	default:
@@ -137,6 +149,63 @@ func startNode(cfg *Config) error {
 	return n.Start()
 }
 
+func startLightClient(cfg *Config) error {
+	tmcfg, nodeLogger, _, err := setupNode()
+	if err != nil {
+		return err
+	}
+
+	dbContext := &node.DBContext{ID: "light", Config: tmcfg}
+	lightDB, err := node.DefaultDBProvider(dbContext)
+	if err != nil {
+		return err
+	}
+
+	providers := rpcEndpoints(tmcfg.P2P.PersistentPeers)
+
+	c, err := light.NewHTTPClient(
+		context.Background(),
+		cfg.ChainID,
+		light.TrustOptions{
+			Period: tmcfg.StateSync.TrustPeriod,
+			Height: tmcfg.StateSync.TrustHeight,
+			Hash:   tmcfg.StateSync.TrustHashBytes(),
+		},
+		providers[0],
+		providers[1:],
+		dbs.New(lightDB),
+		light.Logger(nodeLogger),
+	)
+	if err != nil {
+		return err
+	}
+
+	rpccfg := rpcserver.DefaultConfig()
+	rpccfg.MaxBodyBytes = tmcfg.RPC.MaxBodyBytes
+	rpccfg.MaxHeaderBytes = tmcfg.RPC.MaxHeaderBytes
+	rpccfg.MaxOpenConnections = tmcfg.RPC.MaxOpenConnections
+	// If necessary adjust global WriteTimeout to ensure it's greater than
+	// TimeoutBroadcastTxCommit.
+	// See https://github.com/tendermint/tendermint/issues/3435
+	if rpccfg.WriteTimeout <= tmcfg.RPC.TimeoutBroadcastTxCommit {
+		rpccfg.WriteTimeout = tmcfg.RPC.TimeoutBroadcastTxCommit + 1*time.Second
+	}
+
+	p, err := lproxy.NewProxy(c, tmcfg.RPC.ListenAddress, providers[0], rpccfg, nodeLogger,
+		lrpc.KeyPathFn(lrpc.DefaultMerkleKeyPathFn()))
+	if err != nil {
+		return err
+	}
+
+	logger.Info("Starting proxy...", "laddr", tmcfg.RPC.ListenAddress)
+	if err := p.ListenAndServe(); err != http.ErrServerClosed {
+		// Error starting or closing listener:
+		logger.Error("proxy ListenAndServe", "err", err)
+	}
+
+	return nil
+}
+
 // FIXME: Temporarily disconnected maverick until it is redesigned
 // startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
 // configuration is in $TMHOME/config/tendermint.toml.
@@ -267,3 +336,21 @@ func setupNode() (*config.Config, log.Logger, *p2p.NodeKey, error) {
 
 	return tmcfg, nodeLogger, &nodeKey, nil
 }
+
+// rpcEndpoints takes a list of persistent peers and splits them into a list of rpc endpoints
+// using 26657 as the port number
+func rpcEndpoints(peers string) []string {
+	arr := strings.Split(peers, ",")
+	endpoints := make([]string, len(arr))
+	for i, v := range arr {
+		addr, err := p2p.ParseNodeAddress(v)
+		if err != nil {
+			panic(err)
+		}
+		// use RPC port instead
+		addr.Port = 26657
+		rpcEndpoint := "http://" + addr.Hostname + ":" + fmt.Sprint(addr.Port)
+		endpoints[i] = rpcEndpoint
+	}
+	return endpoints
+}

test/e2e/generator/generate.go

@@ -77,7 +77,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		KeyType:          opt["keyType"].(string),
 	}
 
-	var numSeeds, numValidators, numFulls int
+	var numSeeds, numValidators, numFulls, numLightClients int
 	switch opt["topology"].(string) {
 	case "single":
 		numValidators = 1
@@ -85,7 +85,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		numValidators = 4
 	case "large":
 		// FIXME Networks are kept small since large ones use too much CPU.
-		numSeeds = r.Intn(4)
+		numSeeds = r.Intn(3)
+		numLightClients = r.Intn(3)
 		numValidators = 4 + r.Intn(7)
 		numFulls = r.Intn(5)
 	default:
@@ -143,6 +144,13 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 			r, e2e.ModeFull, startAt, manifest.InitialHeight, false)
 	}
 
+	for i := 1; i <= numLightClients; i++ {
+		startAt := manifest.InitialHeight + 5
+		manifest.Nodes[fmt.Sprintf("light%02d", i)] = generateNode(
+			r, e2e.ModeLight, startAt+(5*int64(i)), manifest.InitialHeight, false,
+		)
+	}
+
 	// We now set up peer discovery for nodes. Seed nodes are fully meshed with
 	// each other, while non-seed nodes either use a set of random seeds or a
 	// set of random peers that start before themselves.
@@ -175,6 +183,10 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		}
 	})
 	for i, name := range peerNames {
+		// we skip over light clients - they connect to all peers initially
+		if manifest.Nodes[name].Mode == string(e2e.ModeLight) {
+			continue
+		}
 		if len(seedNames) > 0 && (i == 0 || r.Float64() >= 0.5) {
 			manifest.Nodes[name].Seeds = uniformSetChoice(seedNames).Choose(r)
 		} else if i > 0 {
@@ -213,7 +225,7 @@ func generateNode(
 		node.SnapshotInterval = 3
 	}
 
-	if node.Mode == "validator" {
+	if node.Mode == string(e2e.ModeValidator) {
 		misbehaveAt := startAt + 5 + int64(r.Intn(10))
 		if startAt == 0 {
 			misbehaveAt += initialHeight - 1
@@ -224,6 +236,11 @@ func generateNode(
 		}
 	}
 
+	if node.Mode == string(e2e.ModeLight) {
+		node.ABCIProtocol = "builtin"
+		node.StateSync = false
+	}
+
 	// If a node which does not persist state also does not retain blocks, randomly
 	// choose to either persist state or retain all blocks.
 	if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 {

test/e2e/networks/ci.toml

@@ -90,3 +90,13 @@ fast_sync = "v0"
 state_sync = true
 seeds = ["seed01"]
 perturb = ["restart"]
+
+[node.light01]
+mode= "light"
+start_at= 1005
+persistent_peers = ["validator01", "validator02", "validator03"]
+
+[node.light02]
+mode= "light"
+start_at= 1015
+persistent_peers = ["validator04", "full01", "validator05"]

test/e2e/networks/simple.toml

@@ -2,4 +2,4 @@
 [node.validator02]
 [node.validator03]
 [node.validator04]
-  
+

test/e2e/pkg/manifest.go

@@ -58,9 +58,9 @@ type Manifest struct {
 
 // ManifestNode represents a node in a testnet manifest.
 type ManifestNode struct {
-	// Mode specifies the type of node: "validator", "full", or "seed". Defaults to
-	// "validator". Full nodes do not get a signing key (a dummy key is generated),
-	// and seed nodes run in seed mode with the PEX reactor enabled.
+	// Mode specifies the type of node: "validator", "full", "light" or "seed".
+	// Defaults to "validator". Full nodes do not get a signing key (a dummy key
+	// is generated), and seed nodes run in seed mode with the PEX reactor enabled.
 	Mode string `toml:"mode"`
 
 	// Seeds is the list of node names to use as P2P seed nodes. Defaults to none.
@@ -68,7 +68,8 @@ type ManifestNode struct {
 
 	// PersistentPeers is a list of node names to maintain persistent P2P
 	// connections to. If neither seeds nor persistent peers are specified,
-	// this defaults to all other nodes in the network.
+	// this defaults to all other nodes in the network. For light clients,
+	// this relates to the providers the light client is connected to.
 	PersistentPeers []string `toml:"persistent_peers"`
 
 	// Database specifies the database backend: "goleveldb", "cleveldb",

test/e2e/pkg/testnet.go

@@ -33,6 +33,7 @@ type Perturbation string
 const (
 	ModeValidator Mode = "validator"
 	ModeFull      Mode = "full"
+	ModeLight     Mode = "light"
 	ModeSeed      Mode = "seed"
 
 	ProtocolBuiltin Protocol = "builtin"
@@ -151,7 +152,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 			ProxyPort:        proxyPortGen.Next(),
 			Mode:             ModeValidator,
 			Database:         "goleveldb",
-			ABCIProtocol:     ProtocolUNIX,
+			ABCIProtocol:     ProtocolBuiltin,
 			PrivvalProtocol:  ProtocolFile,
 			StartAt:          nodeManifest.StartAt,
 			FastSync:         nodeManifest.FastSync,
@@ -327,6 +328,9 @@ func (n Node) Validate(testnet Testnet) error {
 	default:
 		return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol)
 	}
+	if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin {
+		return errors.New("light client must use builtin protocol")
+	}
 	switch n.PrivvalProtocol {
 	case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX:
 	default:
@@ -402,7 +406,7 @@ func (t Testnet) LookupNode(name string) *Node {
 func (t Testnet) ArchiveNodes() []*Node {
 	nodes := []*Node{}
 	for _, node := range t.Nodes {
-		if node.Mode != ModeSeed && node.StartAt == 0 && node.RetainBlocks == 0 {
+		if !node.Stateless() && node.StartAt == 0 && node.RetainBlocks == 0 {
 			nodes = append(nodes, node)
 		}
 	}
@@ -476,6 +480,11 @@ func (n Node) Client() (*rpchttp.HTTP, error) {
 	return rpchttp.New(fmt.Sprintf("http://127.0.0.1:%v", n.ProxyPort))
 }
 
+// Stateless returns true if the node is either a seed node or a light node
+func (n Node) Stateless() bool {
+	return n.Mode == ModeLight || n.Mode == ModeSeed
+}
+
 // keyGenerator generates pseudorandom Ed25519 keys based on a seed.
 type keyGenerator struct {
 	random *rand.Rand

test/e2e/runner/setup.go

@@ -65,23 +65,23 @@ func Setup(testnet *e2e.Testnet) error {
 
 	for _, node := range testnet.Nodes {
 		nodeDir := filepath.Join(testnet.Dir, node.Name)
+
 		dirs := []string{
 			filepath.Join(nodeDir, "config"),
 			filepath.Join(nodeDir, "data"),
 			filepath.Join(nodeDir, "data", "app"),
 		}
 		for _, dir := range dirs {
+			// light clients don't need an app directory
+			if node.Mode == e2e.ModeLight && strings.Contains(dir, "app") {
+				continue
+			}
 			err := os.MkdirAll(dir, 0755)
 			if err != nil {
 				return err
 			}
 		}
 
-		err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
-		if err != nil {
-			return err
-		}
-
 		cfg, err := MakeConfig(node)
 		if err != nil {
 			return err
@@ -97,6 +97,16 @@ func Setup(testnet *e2e.Testnet) error {
 			return err
 		}
 
+		if node.Mode == e2e.ModeLight {
+			// stop early if a light client
+			continue
+		}
+
+		err = genesis.SaveAs(filepath.Join(nodeDir, "config", "genesis.json"))
+		if err != nil {
+			return err
+		}
+
 		err = (&p2p.NodeKey{PrivKey: node.NodeKey}).SaveAs(filepath.Join(nodeDir, "config", "node_key.json"))
 		if err != nil {
 			return err
@@ -168,8 +178,7 @@ services:
     image: tendermint/e2e-node
 {{- if eq .ABCIProtocol "builtin" }}
     entrypoint: /usr/bin/entrypoint-builtin
-{{- end }}
-{{- if ne .ABCIProtocol "builtin"}}
+{{- else }}
     command: {{ startCommands .Misbehaviors .LogLevel }}
 {{- end }}
     init: true
@@ -289,7 +298,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	case e2e.ModeSeed:
 		cfg.P2P.SeedMode = true
 		cfg.P2P.PexReactor = true
-	case e2e.ModeFull:
+	case e2e.ModeFull, e2e.ModeLight:
 		// Don't need to do anything, since we're using a dummy privval key by default.
 	default:
 		return nil, fmt.Errorf("unexpected mode %q", node.Mode)
@@ -338,6 +347,8 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
 		"chain_id":          node.Testnet.Name,
 		"dir":               "data/app",
 		"listen":            AppAddressUNIX,
+		"mode":              node.Mode,
+		"proxy_port":        node.ProxyPort,
 		"protocol":          "socket",
 		"persist_interval":  node.PersistInterval,
 		"snapshot_interval": node.SnapshotInterval,

test/e2e/runner/start.go

@@ -58,7 +58,7 @@ func Start(testnet *e2e.Testnet) error {
 
 	// Update any state sync nodes with a trusted height and hash
 	for _, node := range nodeQueue {
-		if node.StateSync {
+		if node.StateSync || node.Mode == e2e.ModeLight {
 			err = UpdateConfigStateSync(node, block.Height, blockID.Hash.Bytes())
 			if err != nil {
 				return err

test/e2e/tests/app_test.go

@@ -16,7 +16,7 @@ import (
 // Tests that any initial state given in genesis has made it into the app.
 func TestApp_InitialState(t *testing.T) {
 	testNode(t, func(t *testing.T, node e2e.Node) {
-		if node.Mode == e2e.ModeSeed {
+		if node.Stateless() {
 			return
 		}
 		if len(node.Testnet.InitialState) == 0 {
@@ -84,9 +84,24 @@ func TestApp_Tx(t *testing.T) {
 		_, err = client.BroadcastTxCommit(ctx, tx)
 		require.NoError(t, err)
 
-		resp, err := client.ABCIQuery(ctx, "", []byte(key))
+		// wait for the tx to go through
+		time.Sleep(1 * time.Second)
+
+		hash := tx.Hash()
+		txResp, err := client.Tx(ctx, hash, false)
 		require.NoError(t, err)
-		assert.Equal(t, key, string(resp.Response.Key))
-		assert.Equal(t, value, string(resp.Response.Value))
+		assert.Equal(t, txResp.Tx, tx)
+
+		if node.Mode == e2e.ModeLight {
+			return
+		}
+
+		abciResp, err := client.ABCIQuery(ctx, "", []byte(key))
+		require.NoError(t, err)
+		assert.Equal(t, key, string(abciResp.Response.Key))
+		assert.Equal(t, value, string(abciResp.Response.Value))
+
+		assert.Equal(t, txResp.Height, abciResp.Response.Height)
+
 	})
 }