improve ResetTestRootWithChainID() concurrency safety (#3291)

* improve ResetTestRootWithChainID() concurrency safety

Rely on ioutil.TempDir() to create test root directories and ensure
multiple same-chain id test cases can run in parallel.

* Update config/toml.go

Co-Authored-By: alessio <quadrispro@ubuntu.com>

* clean up test directories after completion

Closes: #1034

* Remove redundant EnsureDir call

* s/PanicSafety()/panic()/s

* Put create dir functionality back in ResetTestRootWithChainID

* Place test directories in OS's tempdir

In modern UNIX and UNIX-like systems /tmp is very often
mounted as tmpfs. This might speed test execution a bit.

* Set 0700 to a const

* rootsDirs -> configRootDirs

* Don't double remove directories

* Avoid global variables

* Fix consensus tests

* Reduce defer stack

* Address review comments

* Try to fix tests

* Update CHANGELOG_PENDING.md

Co-Authored-By: alessio <quadrispro@ubuntu.com>

* Update consensus/common_test.go

Co-Authored-By: alessio <quadrispro@ubuntu.com>

* Update consensus/common_test.go

Co-Authored-By: alessio <quadrispro@ubuntu.com>

mempool/bench_test.go

@@ -11,7 +11,8 @@ import (
 func BenchmarkReap(b *testing.B) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 
 	size := 10000
 	for i := 0; i < size; i++ {

mempool/mempool_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-func newMempoolWithApp(cc proxy.ClientCreator) *Mempool {
+func newMempoolWithApp(cc proxy.ClientCreator) (*Mempool, func()) {
 	config := cfg.ResetTestRoot("mempool_test")
 
 	appConnMem, _ := cc.NewABCIClient()
@@ -36,7 +36,7 @@ func newMempoolWithApp(cc proxy.ClientCreator) *Mempool {
 	}
 	mempool := NewMempool(config.Mempool, appConnMem, 0)
 	mempool.SetLogger(log.TestingLogger())
-	return mempool
+	return mempool, func() { os.RemoveAll(config.RootDir) }
 }
 
 func ensureNoFire(t *testing.T, ch <-chan struct{}, timeoutMS int) {
@@ -82,7 +82,8 @@ func checkTxs(t *testing.T, mempool *Mempool, count int) types.Txs {
 func TestReapMaxBytesMaxGas(t *testing.T) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 
 	// Ensure gas calculation behaves as expected
 	checkTxs(t, mempool, 1)
@@ -130,7 +131,8 @@ func TestReapMaxBytesMaxGas(t *testing.T) {
 func TestMempoolFilters(t *testing.T) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 	emptyTxArr := []types.Tx{[]byte{}}
 
 	nopPreFilter := func(tx types.Tx) error { return nil }
@@ -168,7 +170,8 @@ func TestMempoolFilters(t *testing.T) {
 func TestMempoolUpdateAddsTxsToCache(t *testing.T) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 	mempool.Update(1, []types.Tx{[]byte{0x01}}, nil, nil)
 	err := mempool.CheckTx([]byte{0x01}, nil)
 	if assert.Error(t, err) {
@@ -179,7 +182,8 @@ func TestMempoolUpdateAddsTxsToCache(t *testing.T) {
 func TestTxsAvailable(t *testing.T) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 	mempool.EnableTxsAvailable()
 
 	timeoutMS := 500
@@ -224,7 +228,9 @@ func TestSerialReap(t *testing.T) {
 	app.SetOption(abci.RequestSetOption{Key: "serial", Value: "on"})
 	cc := proxy.NewLocalClientCreator(app)
 
-	mempool := newMempoolWithApp(cc)
+	mempool, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
+
 	appConnCon, _ := cc.NewABCIClient()
 	appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus"))
 	err := appConnCon.Start()
@@ -364,6 +370,7 @@ func TestMempoolCloseWAL(t *testing.T) {
 	// 3. Create the mempool
 	wcfg := cfg.DefaultMempoolConfig()
 	wcfg.RootDir = rootDir
+	defer os.RemoveAll(wcfg.RootDir)
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
 	appConnMem, _ := cc.NewABCIClient()
@@ -406,7 +413,8 @@ func txMessageSize(tx types.Tx) int {
 func TestMempoolMaxMsgSize(t *testing.T) {
 	app := kvstore.NewKVStoreApplication()
 	cc := proxy.NewLocalClientCreator(app)
-	mempl := newMempoolWithApp(cc)
+	mempl, cleanup := newMempoolWithApp(cc)
+	defer cleanup()
 
 	testCases := []struct {
 		len int

mempool/reactor_test.go

@@ -49,7 +49,8 @@ func makeAndConnectMempoolReactors(config *cfg.Config, N int) []*MempoolReactor
 	for i := 0; i < N; i++ {
 		app := kvstore.NewKVStoreApplication()
 		cc := proxy.NewLocalClientCreator(app)
-		mempool := newMempoolWithApp(cc)
+		mempool, cleanup := newMempoolWithApp(cc)
+		defer cleanup()
 
 		reactors[i] = NewMempoolReactor(config.Mempool, mempool) // so we dont start the consensus states
 		reactors[i].SetLogger(logger.With("validator", i))