test/fuzz: add mechanism to reproduce found fuzz errors (#6768)

* test/fuzz: add test to reproduce found fuzz errors

.github/workflows/fuzz-nightly.yml

@@ -23,9 +23,14 @@ jobs:
         working-directory: test/fuzz
         run: go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
 
-      - name: Fuzz mempool
+      - name: Fuzz mempool-v1
         working-directory: test/fuzz
-        run: timeout -s SIGINT --preserve-status 10m make fuzz-mempool
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-mempool-v1
+        continue-on-error: true
+
+      - name: Fuzz mempool-v0
+        working-directory: test/fuzz
+        run: timeout -s SIGINT --preserve-status 10m make fuzz-mempool-v0
         continue-on-error: true
 
       - name: Fuzz p2p-addrbook

test/fuzz/Makefile

@@ -1,8 +1,15 @@
 #!/usr/bin/make -f
 
-.PHONY: fuzz-mempool
-fuzz-mempool:
-	cd mempool && \
+.PHONY: fuzz-mempool-v1
+fuzz-mempool-v1:
+	cd mempool/v1 && \
+		rm -f *-fuzz.zip && \
+		go-fuzz-build && \
+		go-fuzz
+
+.PHONY: fuzz-mempool-v0
+fuzz-mempool-v0:
+	cd mempool/v0 && \
 		rm -f *-fuzz.zip && \
 		go-fuzz-build && \
 		go-fuzz
@@ -37,3 +44,9 @@ fuzz-rpc-server:
 		rm -f *-fuzz.zip && \
 		go-fuzz-build && \
 		go-fuzz
+
+clean:
+	find . -name corpus -type d -exec rm -rf {} +;
+	find . -name crashers -type d -exec rm -rf {} +;
+	find . -name suppressions -type d -exec rm -rf {} +;
+	find . -name *\.zip -type f -delete

test/fuzz/mempool/v0/checktx.go

@@ -1,4 +1,4 @@
-package checktx
+package v0
 
 import (
 	"context"

test/fuzz/mempool/v0/fuzz_test.go

@@ -0,0 +1,33 @@
+package v0_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	mempoolv0 "github.com/tendermint/tendermint/test/fuzz/mempool/v0"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestMempoolTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			mempoolv0.Fuzz(input)
+		})
+	}
+}

test/fuzz/mempool/v1/checktx.go

@@ -0,0 +1,37 @@
+package v1
+
+import (
+	"context"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/mempool"
+	mempoolv1 "github.com/tendermint/tendermint/internal/mempool/v0"
+	"github.com/tendermint/tendermint/proxy"
+)
+
+var mp mempool.Mempool
+
+func init() {
+	app := kvstore.NewApplication()
+	cc := proxy.NewLocalClientCreator(app)
+	appConnMem, _ := cc.NewABCIClient()
+	err := appConnMem.Start()
+	if err != nil {
+		panic(err)
+	}
+
+	cfg := config.DefaultMempoolConfig()
+	cfg.Broadcast = false
+
+	mp = mempoolv1.NewCListMempool(cfg, appConnMem, 0)
+}
+
+func Fuzz(data []byte) int {
+	err := mp.CheckTx(context.Background(), data, nil, mempool.TxInfo{})
+	if err != nil {
+		return 0
+	}
+
+	return 1
+}

test/fuzz/mempool/v1/fuzz_test.go

@@ -0,0 +1,33 @@
+package v1_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	mempoolv1 "github.com/tendermint/tendermint/test/fuzz/mempool/v1"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestMempoolTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			mempoolv1.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/addrbook/fuzz.go

@@ -1,5 +1,5 @@
 // nolint: gosec
-package addr
+package addrbook
 
 import (
 	"encoding/json"

test/fuzz/p2p/addrbook/fuzz_test.go

@@ -0,0 +1,33 @@
+package addrbook_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/addrbook"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestAddrbookTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			addrbook.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/pex/fuzz_test.go

@@ -0,0 +1,33 @@
+package pex_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/pex"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestPexTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			pex.Fuzz(input)
+		})
+	}
+}

test/fuzz/p2p/secretconnection/fuzz_test.go

@@ -0,0 +1,33 @@
+package secretconnection_test
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/test/fuzz/p2p/secretconnection"
+)
+
+const testdataCasesDir = "testdata/cases"
+
+func TestSecretConnectionTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
+	require.NoError(t, err)
+
+	for _, e := range entries {
+		entry := e
+		t.Run(entry.Name(), func(t *testing.T) {
+			defer func() {
+				r := recover()
+				require.Nilf(t, r, "testdata/cases test panic")
+			}()
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
+			require.NoError(t, err)
+			input, err := ioutil.ReadAll(f)
+			require.NoError(t, err)
+			secretconnection.Fuzz(input)
+		})
+	}
+}

test/fuzz/rpc/jsonrpc/server/fuzz_test.go

@@ -10,10 +10,10 @@ import (
 	"github.com/tendermint/tendermint/test/fuzz/rpc/jsonrpc/server"
 )
 
-const testdataDir = "testdata"
+const testdataCasesDir = "testdata/cases"
 
-func TestServerOnTestData(t *testing.T) {
-	entries, err := os.ReadDir(testdataDir)
+func TestServerTestdataCases(t *testing.T) {
+	entries, err := os.ReadDir(testdataCasesDir)
 	require.NoError(t, err)
 
 	for _, e := range entries {
@@ -21,9 +21,9 @@ func TestServerOnTestData(t *testing.T) {
 		t.Run(entry.Name(), func(t *testing.T) {
 			defer func() {
 				r := recover()
-				require.Nilf(t, r, "testdata test panic")
+				require.Nilf(t, r, "testdata/cases test panic")
 			}()
-			f, err := os.Open(filepath.Join(testdataDir, entry.Name()))
+			f, err := os.Open(filepath.Join(testdataCasesDir, entry.Name()))
 			require.NoError(t, err)
 			input, err := ioutil.ReadAll(f)
 			require.NoError(t, err)

test/fuzz/rpc/jsonrpc/server/handler.go

@@ -43,7 +43,7 @@ func Fuzz(data []byte) int {
 		return 1
 	}
 
-	if inputJSONIsMultiElementSlice(data) {
+	if outputJSONIsSlice(blob) {
 		recv := []types.RPCResponse{}
 		if err := json.Unmarshal(blob, &recv); err != nil {
 			panic(err)
@@ -57,8 +57,8 @@ func Fuzz(data []byte) int {
 	return 1
 }
 
-func inputJSONIsMultiElementSlice(input []byte) bool {
+func outputJSONIsSlice(input []byte) bool {
 	slice := []interface{}{}
 	err := json.Unmarshal(input, &slice)
-	return err == nil && len(slice) > 1
+	return err == nil
 }

test/fuzz/rpc/jsonrpc/server/testdata/cases/1184f5b8d4b6dd08709cf1513f26744167065e0d

@@ -0,0 +1 @@
+[0]

test/fuzz/rpc/jsonrpc/server/testdata/cases/bbcffb1cdb2cea50fd3dd8c1524905551d0b2e79

@@ -0,0 +1 @@
+[0,0]

test/fuzz/rpc/jsonrpc/server/testdata/cases/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176

@@ -0,0 +1 @@
+[{"iD":7},{"iD":7}]