Merge branch 'master' into abci++

test/e2e/README.md

@@ -23,7 +23,7 @@ The above should hold for any arbitrary, valid network configuration, and that c
 
 A testnet configuration is specified as a TOML testnet manifest (see below). The testnet runner uses the manifest to configure a set of Docker containers and start them in some order. The manifests can be written manually (to test specific configurations) or generated randomly by the testnet generator (to test a wide range of configuration permutations).
 
-When running a testnet, the runner will first start the Docker nodes in some sequence, submit random transactions, and wait for the nodes to come online and the first blocks to be produced. This may involve e.g. waiting for nodes to fast sync and/or state sync. If specified, it will then run any misbehaviors (e.g. double-signing) and perturbations (e.g. killing or disconnecting nodes). It then waits for the testnet to stabilize, with all nodes online and having reached the latest height.
+When running a testnet, the runner will first start the Docker nodes in some sequence, submit random transactions, and wait for the nodes to come online and the first blocks to be produced. This may involve e.g. waiting for nodes to block sync and/or state sync. If specified, it will then run any misbehaviors (e.g. double-signing) and perturbations (e.g. killing or disconnecting nodes). It then waits for the testnet to stabilize, with all nodes online and having reached the latest height.
 
 Once the testnet stabilizes, a set of Go end-to-end tests are run against the live testnet to verify network invariants (for example that blocks are identical across nodes). These use the RPC client to interact with the network, and should consider the entire network as a black box (i.e. it should not test any network or node internals, only externally visible behavior via RPC). The tests may use the `testNode()` helper to run parallel tests against each individual testnet node, and/or inspect the full blockchain history via `fetchBlockChain()`.
 

test/e2e/generator/generate.go

@@ -30,7 +30,7 @@ var (
 	nodeABCIProtocols    = uniformChoice{"unix", "tcp", "builtin", "grpc"}
 	nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp", "grpc"}
 	// FIXME: v2 disabled due to flake
-	nodeFastSyncs         = uniformChoice{"v0"} // "v2"
+	nodeBlockSyncs        = uniformChoice{"v0"} // "v2"
 	nodeMempools          = uniformChoice{"v0", "v1"}
 	nodeStateSyncs        = uniformChoice{false, true}
 	nodePersistIntervals  = uniformChoice{0, 1, 5}
@@ -273,7 +273,7 @@ func generateNode(
 		Database:         nodeDatabases.Choose(r).(string),
 		ABCIProtocol:     nodeABCIProtocols.Choose(r).(string),
 		PrivvalProtocol:  nodePrivvalProtocols.Choose(r).(string),
-		FastSync:         nodeFastSyncs.Choose(r).(string),
+		BlockSync:        nodeBlockSyncs.Choose(r).(string),
 		Mempool:          nodeMempools.Choose(r).(string),
 		StateSync:        nodeStateSyncs.Choose(r).(bool) && startAt > 0,
 		PersistInterval:  ptrUint64(uint64(nodePersistIntervals.Choose(r).(int))),
@@ -311,7 +311,7 @@ func generateNode(
 	}
 
 	if node.StateSync {
-		node.FastSync = "v0"
+		node.BlockSync = "v0"
 	}
 
 	return &node

test/e2e/networks/ci.toml

@@ -30,11 +30,6 @@ validator05 = 50
 [node.seed01]
 mode = "seed"
 perturb = ["restart"]
-seeds = ["seed02"]
-
-[node.seed02]
-mode = "seed"
-seeds = ["seed01"]
 
 [node.validator01]
 perturb = ["disconnect"]
@@ -47,7 +42,7 @@ database = "boltdb"
 persist_interval = 0
 perturb = ["restart"]
 privval_protocol = "tcp"
-seeds = ["seed02"]
+seeds = ["seed01"]
 
 [node.validator03]
 database = "badgerdb"
@@ -66,29 +61,21 @@ perturb = ["pause"]
 
 [node.validator05]
 database = "cleveldb" 
-fast_sync = "v0" 
-seeds = ["seed02"] 
+block_sync = "v0" 
+seeds = ["seed01"] 
 start_at = 1005 # Becomes part of the validator set at 1010
 abci_protocol = "grpc"
-perturb = ["kill", "pause", "disconnect", "restart"]
+perturb = ["pause", "disconnect", "restart"]
 privval_protocol = "tcp"
 
 [node.full01]
 mode = "full"
 start_at = 1010
 # FIXME: should be v2, disabled due to flake
-fast_sync = "v0"
+block_sync = "v0"
 persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
 perturb = ["restart"]
 retain_blocks = 7
-
-[node.full02]
-mode = "full"
-start_at = 1015
-# FIXME: should be v2, disabled due to flake
-fast_sync = "v0"
-perturb = ["restart"]
-seeds = ["seed01"]
 state_sync = true
 
 [node.light01]

test/e2e/pkg/manifest.go

@@ -106,9 +106,9 @@ type ManifestNode struct {
 	// runner will wait for the network to reach at least this block height.
 	StartAt int64 `toml:"start_at"`
 
-	// FastSync specifies the fast sync mode: "" (disable), "v0" or "v2".
+	// BlockSync specifies the block sync mode: "" (disable), "v0" or "v2".
 	// Defaults to disabled.
-	FastSync string `toml:"fast_sync"`
+	BlockSync string `toml:"block_sync"`
 
 	// Mempool specifies which version of mempool to use. Either "v0" or "v1"
 	Mempool string `toml:"mempool_version"`

test/e2e/pkg/testnet.go

@@ -79,7 +79,7 @@ type Node struct {
 	IP               net.IP
 	ProxyPort        uint32
 	StartAt          int64
-	FastSync         string
+	BlockSync        string
 	Mempool          string
 	StateSync        bool
 	Database         string
@@ -168,7 +168,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 			ABCIProtocol:     ProtocolBuiltin,
 			PrivvalProtocol:  ProtocolFile,
 			StartAt:          nodeManifest.StartAt,
-			FastSync:         nodeManifest.FastSync,
+			BlockSync:        nodeManifest.BlockSync,
 			Mempool:          nodeManifest.Mempool,
 			StateSync:        nodeManifest.StateSync,
 			PersistInterval:  1,
@@ -328,10 +328,10 @@ func (n Node) Validate(testnet Testnet) error {
 			}
 		}
 	}
-	switch n.FastSync {
+	switch n.BlockSync {
 	case "", "v0", "v2":
 	default:
-		return fmt.Errorf("invalid fast sync setting %q", n.FastSync)
+		return fmt.Errorf("invalid block sync setting %q", n.BlockSync)
 	}
 	switch n.Mempool {
 	case "", "v0", "v1":

test/e2e/runner/cleanup.go

@@ -15,11 +15,7 @@ func Cleanup(testnet *e2e.Testnet) error {
 	if err != nil {
 		return err
 	}
-	err = cleanupDir(testnet.Dir)
-	if err != nil {
-		return err
-	}
-	return nil
+	return cleanupDir(testnet.Dir)
 }
 
 // cleanupDocker removes all E2E resources (with label e2e=True), regardless
@@ -37,13 +33,8 @@ func cleanupDocker() error {
 		return err
 	}
 
-	err = exec("bash", "-c", fmt.Sprintf(
+	return exec("bash", "-c", fmt.Sprintf(
 		"docker network ls -q --filter label=e2e | xargs %v docker network rm", xargsR))
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
 // cleanupDir cleans up a testnet directory
@@ -74,10 +65,5 @@ func cleanupDir(dir string) error {
 		return err
 	}
 
-	err = os.RemoveAll(dir)
-	if err != nil {
-		return err
-	}
-
-	return nil
+	return os.RemoveAll(dir)
 }

test/e2e/runner/setup.go

@@ -296,10 +296,10 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 		cfg.Mempool.Version = node.Mempool
 	}
 
-	if node.FastSync == "" {
+	if node.BlockSync == "" {
 		cfg.FastSyncMode = false
 	} else {
-		cfg.FastSync.Version = node.FastSync
+		cfg.BlockSync.Version = node.BlockSync
 	}
 
 	if node.StateSync {

test/e2e/runner/wait.go

@@ -21,10 +21,7 @@ func Wait(testnet *e2e.Testnet, blocks int64) error {
 func WaitUntil(testnet *e2e.Testnet, height int64) error {
 	logger.Info(fmt.Sprintf("Waiting for all nodes to reach height %v...", height))
 	_, err := waitForAllNodes(testnet, height, waitingTime(len(testnet.Nodes)))
-	if err != nil {
-		return err
-	}
-	return nil
+	return err
 }
 
 // waitingTime estimates how long it should take for a node to reach the height.

test/fuzz/Makefile

@@ -1,8 +1,15 @@
 #!/usr/bin/make -f
 
-.PHONY: fuzz-mempool
-fuzz-mempool:
-	cd mempool && \
+.PHONY: fuzz-mempool-v1
+fuzz-mempool-v1:
+	cd mempool/v1 && \
+		rm -f *-fuzz.zip && \
+		go-fuzz-build && \
+		go-fuzz
+
+.PHONY: fuzz-mempool-v0
+fuzz-mempool-v0:
+	cd mempool/v0 && \
 		rm -f *-fuzz.zip && \
 		go-fuzz-build && \
 		go-fuzz
@@ -37,3 +44,9 @@ fuzz-rpc-server:
 		rm -f *-fuzz.zip && \
 		go-fuzz-build && \
 		go-fuzz
+
+clean:
+	find . -name corpus -type d -exec rm -rf {} +;
+	find . -name crashers -type d -exec rm -rf {} +;
+	find . -name suppressions -type d -exec rm -rf {} +;
+	find . -name *\.zip -type f -delete

test/fuzz/mempool/v0/checktx.go

@@ -1,4 +1,4 @@
-package checktx
+package v0
 
 import (
 	"context"

test/fuzz/mempool/v0/fuzz_test.go

@@ -0,0 +1,33 @@
+package v0_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	mempoolv0 "github.com/tendermint/tendermint/test/fuzz/mempool/v0"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestMempoolTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			mempoolv0.Fuzz(input)
+		})
+	}
+}

test/fuzz/mempool/v1/checktx.go

@@ -0,0 +1,37 @@
+package v1
+
+import (
+	"context"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/mempool"
+	mempoolv1 "github.com/tendermint/tendermint/internal/mempool/v0"
+	"github.com/tendermint/tendermint/proxy"
+)
+
+var mp mempool.Mempool
+
+func init() {
+	app := kvstore.NewApplication()
+	cc := proxy.NewLocalClientCreator(app)
+	appConnMem, _ := cc.NewABCIClient()
+	err := appConnMem.Start()
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := config.DefaultMempoolConfig()
+	cfg.Broadcast = false
+
+	mp = mempoolv1.NewCListMempool(cfg, appConnMem, 0)
+}
+
+func Fuzz(data []byte) int {
+	err := mp.CheckTx(context.Background(), data, nil, mempool.TxInfo{})
+	if err != nil {
+		return 0
+	}
+
+	return 1
+}

test/fuzz/mempool/v1/fuzz_test.go

@@ -0,0 +1,33 @@
+package v1_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	mempoolv1 "github.com/tendermint/tendermint/test/fuzz/mempool/v1"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestMempoolTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			mempoolv1.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/addrbook/fuzz.go

@@ -1,5 +1,5 @@
 // nolint: gosec
-package addr
+package addrbook
 
 import (
 	"encoding/json"

test/fuzz/p2p/addrbook/fuzz_test.go

@@ -0,0 +1,33 @@
+package addrbook_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/addrbook"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestAddrbookTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			addrbook.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/pex/fuzz_test.go

@@ -0,0 +1,33 @@
+package pex_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/pex"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestPexTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			pex.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/secretconnection/fuzz_test.go

@@ -0,0 +1,33 @@
+package secretconnection_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/secretconnection"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestSecretConnectionTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			secretconnection.Fuzz(input)
+		})
+	}
+}

test/fuzz/rpc/jsonrpc/server/fuzz_test.go

@@ -0,0 +1,33 @@
+package server_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/rpc/jsonrpc/server"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestServerTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			server.Fuzz(input)
+		})
+	}
+}

test/fuzz/rpc/jsonrpc/server/handler.go

@@ -1,4 +1,4 @@
-package handler
+package server
 
 import (
 	"bytes"
@@ -39,11 +39,25 @@ func Fuzz(data []byte) int {
 	if err := res.Body.Close(); err != nil {
 		panic(err)
 	}
-	if len(blob) > 0 {
-		recv := new(types.RPCResponse)
-		if err := json.Unmarshal(blob, recv); err != nil {
+	if len(blob) == 0 {
+		return 1
+	}
+
+	if outputJSONIsSlice(blob) {
+		recv := []types.RPCResponse{}
+		if err := json.Unmarshal(blob, &recv); err != nil {
 			panic(err)
 		}
+		return 1
+	}
+	recv := &types.RPCResponse{}
+	if err := json.Unmarshal(blob, recv); err != nil {
+		panic(err)
 	}
 	return 1
 }
+
+func outputJSONIsSlice(input []byte) bool {
+	slice := []interface{}{}
+	return json.Unmarshal(input, &slice) == nil
+}

test/fuzz/rpc/jsonrpc/server/testdata/1184f5b8d4b6dd08709cf1513f26744167065e0d

@@ -0,0 +1 @@
+[0]

test/fuzz/rpc/jsonrpc/server/testdata/cases/1184f5b8d4b6dd08709cf1513f26744167065e0d

@@ -0,0 +1 @@
+[0]

test/fuzz/rpc/jsonrpc/server/testdata/cases/bbcffb1cdb2cea50fd3dd8c1524905551d0b2e79

@@ -0,0 +1 @@
+[0,0]

test/fuzz/rpc/jsonrpc/server/testdata/cases/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176

@@ -0,0 +1 @@
+[{"iD":7},{"iD":7}]

test/fuzz/rpc/jsonrpc/server/testdata/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176

@@ -0,0 +1 @@
+[{"iD":7},{"iD":7}]