Merge branch 'master' into marko/remove-apphash

.github/CODEOWNERS

@@ -7,17 +7,5 @@
 # global owners are only requested if there isn't a more specific
 # codeowner specified below. For this reason, the global codeowners
 # are often repeated in package-level definitions.
-*       @alexanderbez @ebuchman @tessr
+*       @alexanderbez @ebuchman @cmwaters @tessr @tychoish
 
-# Overrides for tooling packages
-.github/ @marbar3778 @alexanderbez @ebuchman @tessr
-DOCKER/ @marbar3778 @alexanderbez @ebuchman @tessr
-
-# Overrides for core Tendermint packages
-abci/  @marbar3778 @alexanderbez @ebuchman @tessr @tychoish
-evidence/ @cmwaters @ebuchman @tessr @tychoish
-light/ @cmwaters @ebuchman @tessr @tychoish
-
-# Overrides for docs
-*.md @marbar3778 @alexanderbez @ebuchman @tessr
-docs/ @marbar3778 @alexanderbez @ebuchman @tessr

.github/workflows/coverage.yml

@@ -121,7 +121,7 @@ jobs:
       - run: |
           cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt
         if: env.GIT_DIFF
-      - uses: codecov/codecov-action@v1.3.2
+      - uses: codecov/codecov-action@v1.4.1
         with:
           file: ./coverage.txt
         if: env.GIT_DIFF

.github/workflows/docker.yml

@@ -40,7 +40,7 @@ jobs:
           platforms: all
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.1.2
+        uses: docker/setup-buildx-action@v1.2.0
 
       - name: Login to DockerHub
         if: ${{ github.event_name != 'pull_request' }}

.github/workflows/proto-docker.yml

@@ -34,7 +34,7 @@ jobs:
           echo ::set-output name=tags::${TAGS}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1.1.2
+        uses: docker/setup-buildx-action@v1.2.0
 
       - name: Login to DockerHub
         uses: docker/login-action@v1

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v0.34.10
+
+*April 14, 2021*
+
+This release fixes a bug where peers would sometimes try to send messages 
+on incorrect channels. Special thanks to our friends at Oasis Labs for surfacing
+this issue! 
+
+Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermint).
+
+- [p2p/node] [\#6339](https://github.com/tendermint/tendermint/issues/6339) Fix bug with using custom channels (@cmwaters)
+- [light] [\#6346](https://github.com/tendermint/tendermint/issues/6346) Correctly handle too high errors to improve client robustness (@cmwaters)
+
 ## v0.34.9
 
 *April 8, 2021*

CHANGELOG_PENDING.md

@@ -16,6 +16,9 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
   - [rpc] \#6019 standardise RPC errors and return the correct status code (@bipulprasad & @cmwaters)
   - [rpc] \#6168 Change default sorting to desc for `/tx_search` results (@melekes)
   - [cli] \#6282 User must specify the node mode when using `tendermint init` (@cmwaters)
+  - [state/indexer] \#6382 reconstruct indexer, move txindex into the indexer package (@JayT106)
+  - [cli] \#6372 Introduce `BootstrapPeers` as part of the new p2p stack. Peers to be connected on
+    startup (@cmwaters)
 
 - Apps
   - [ABCI] \#5447 Remove `SetOption` method from `ABCI.Client` interface
@@ -34,15 +37,16 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
   - [store] \#5848 Remove block store state in favor of using the db iterators directly (@cmwaters)
   - [state] \#5864 Use an iterator when pruning state (@cmwaters)
   - [types] \#6023 Remove `tm2pb.Header`, `tm2pb.BlockID`, `tm2pb.PartSetHeader` and `tm2pb.NewValidatorUpdate`.
-    - Each of the above types has a `ToProto` and `FromProto` method or function which replaced this logic.
+	- Each of the above types has a `ToProto` and `FromProto` method or function which replaced this logic.
   - [light] \#6054 Move `MaxRetryAttempt` option from client to provider.
-    - `NewWithOptions` now sets the max retry attempts and timeouts (@cmwaters)
+	- `NewWithOptions` now sets the max retry attempts and timeouts (@cmwaters)
   - [all] \#6077 Change spelling from British English to American (@cmwaters)
-    - Rename "Subscription.Cancelled()" to "Subscription.Canceled()" in libs/pubsub
-    - Rename "behaviour" pkg to "behavior" and internalized it in blockchain v2
+	- Rename "Subscription.Cancelled()" to "Subscription.Canceled()" in libs/pubsub
+	- Rename "behaviour" pkg to "behavior" and internalized it in blockchain v2
   - [rpc/client/http] \#6176 Remove `endpoint` arg from `New`, `NewWithTimeout` and `NewWithClient` (@melekes)
   - [rpc/client/http] \#6176 Unexpose `WSEvents` (@melekes)
   - [rpc/jsonrpc/client/ws_client] \#6176 `NewWS` no longer accepts options (use `NewWSWithOptions` and `OnReconnect` funcs to configure the client) (@melekes)
+  - [libs/rand] \#6364 Removed most of libs/rand in favour of standard lib's `math/rand` (@liamsi)
 
 - Blockchain Protocol
 
@@ -52,7 +56,11 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
 ### FEATURES
 
 - [config] Add `--mode` flag and config variable. See [ADR-52](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-052-tendermint-mode.md) @dongsam
-- [rpc] /#6329 Don't cap page size in unsafe mode (@gotjoshua, @cmwaters)
+- [rpc] \#6329 Don't cap page size in unsafe mode (@gotjoshua, @cmwaters)
+- [pex] \#6305 v2 pex reactor with backwards compatability. Introduces two new pex messages to
+  accomodate for the new p2p stack. Removes the notion of seeds and crawling. All peer
+  exchange reactors behave the same. (@cmwaters)
+- [crypto] \#6376 Enable sr25519 as a validator key
 
 ### IMPROVEMENTS
 
@@ -76,10 +84,12 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
 - [node] \#6059 Validate and complete genesis doc before saving to state store (@silasdavis)
 - [state] \#6067 Batch save state data (@githubsands & @cmwaters)
 - [crypto] \#6120 Implement batch verification interface for ed25519 and sr25519. (@marbar3778)
-- [types] \#6120 use batch verification for verifying commits signatures. 
-  - If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature. 
+- [types] \#6120 use batch verification for verifying commits signatures.
+  - If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature.
 - [privval/file] \#6185 Return error on `LoadFilePV`, `LoadFilePVEmptyState`. Allows for better programmatic control of Tendermint.
-- [privval] /#6240 Add `context.Context` to privval interface. 
+- [privval] \#6240 Add `context.Context` to privval interface.
+- [rpc] \#6265 set cache control in http-rpc response header (@JayT106)
+- [statesync] \#6378 Retry requests for snapshots and add a minimum discovery time (5s) for new snapshots.
 
 ### BUG FIXES
 
@@ -87,4 +97,4 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi
 - [privval] \#5638 Increase read/write timeout to 5s and calculate ping interval based on it (@JoeKash)
 - [blockchain/v1] [\#5701](https://github.com/tendermint/tendermint/pull/5701) Handle peers without blocks (@melekes)
 - [blockchain/v1] \#5711 Fix deadlock (@melekes)
-- [light] \#6346 Correctly handle too high errors to improve client robustness (@cmwaters)
+- [evidence] \#6375 Fix bug with inconsistent LightClientAttackEvidence hashing (cmwaters)

UPGRADING.md

@@ -21,6 +21,10 @@ This guide provides instructions for upgrading to specific versions of Tendermin
 
 * Added `--mode` flag and `mode` config variable on `config.toml` for setting Mode of the Node: `full` | `validator` | `seed` (default: `full`)
   [ADR-52](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-052-tendermint-mode.md)
+  
+* `BootstrapPeers` has been added as part of the new p2p stack. This will eventually replace
+  `Seeds`. Bootstrap peers are connected with on startup if needed for peer discovery. Unlike
+  persistent peers, there's no gaurantee that the node will remain connected with these peers. 
 
 ### CLI Changes
 

abci/client/socket_client_test.go

@@ -6,13 +6,14 @@ import (
 	"testing"
 	"time"
 
+	"math/rand"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	abcicli "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/server"
 	"github.com/tendermint/tendermint/abci/types"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
 	"github.com/tendermint/tendermint/libs/service"
 )
 
@@ -101,7 +102,7 @@ func TestHangingSyncCalls(t *testing.T) {
 func setupClientServer(t *testing.T, app types.Application) (
 	service.Service, abcicli.Client) {
 	// some port between 20k and 30k
-	port := 20000 + tmrand.Int32()%10000
+	port := 20000 + rand.Int31()%10000
 	addr := fmt.Sprintf("localhost:%d", port)
 
 	s, err := server.NewServer(addr, "socket", app)

abci/example/kvstore/helpers.go

@@ -1,6 +1,8 @@
 package kvstore
 
 import (
+	mrand "math/rand"
+
 	"github.com/tendermint/tendermint/abci/types"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 )
@@ -9,7 +11,8 @@ import (
 // from the input value
 func RandVal(i int) types.ValidatorUpdate {
 	pubkey := tmrand.Bytes(32)
-	power := tmrand.Uint16() + 1
+	// Random value between [0, 2^16 - 1]
+	power := mrand.Uint32() & (1<<16 - 1) // nolint:gosec // G404: Use of weak random number generator
 	v := types.UpdateValidator(pubkey, int64(power), "")
 	return v
 }

abci/tests/server/client.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	mrand "math/rand"
 
 	abcicli "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/types"
@@ -18,7 +19,8 @@ func InitChain(client abcicli.Client) error {
 	vals := make([]types.ValidatorUpdate, total)
 	for i := 0; i < total; i++ {
 		pubkey := tmrand.Bytes(33)
-		power := tmrand.Int()
+		// nolint:gosec // G404: Use of weak random number generator
+		power := mrand.Int()
 		vals[i] = types.UpdateValidator(pubkey, int64(power), "")
 	}
 	_, err := client.InitChainSync(ctx, types.RequestInitChain{

abci/types/pubkey.go

@@ -6,6 +6,7 @@ import (
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
 	"github.com/tendermint/tendermint/crypto/secp256k1"
+	"github.com/tendermint/tendermint/crypto/sr25519"
 )
 
 func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate {
@@ -17,7 +18,6 @@ func Ed25519ValidatorUpdate(pk []byte, power int64) ValidatorUpdate {
 	}
 
 	return ValidatorUpdate{
-		// Address:
 		PubKey: pkp,
 		Power:  power,
 	}
@@ -34,7 +34,16 @@ func UpdateValidator(pk []byte, power int64, keyType string) ValidatorUpdate {
 			panic(err)
 		}
 		return ValidatorUpdate{
-			// Address:
+			PubKey: pkp,
+			Power:  power,
+		}
+	case sr25519.KeyType:
+		pke := sr25519.PubKey(pk)
+		pkp, err := cryptoenc.PubKeyToProto(pke)
+		if err != nil {
+			panic(err)
+		}
+		return ValidatorUpdate{
 			PubKey: pkp,
 			Power:  power,
 		}

blockchain/v0/pool_test.go

@@ -2,6 +2,7 @@ package v0
 
 import (
 	"fmt"
+	mrand "math/rand"
 	"testing"
 	"time"
 
@@ -67,7 +68,7 @@ func makePeers(numPeers int, minHeight, maxHeight int64) testPeers {
 	peers := make(testPeers, numPeers)
 	for i := 0; i < numPeers; i++ {
 		peerID := p2p.NodeID(tmrand.Str(12))
-		height := minHeight + tmrand.Int63n(maxHeight-minHeight)
+		height := minHeight + mrand.Int63n(maxHeight-minHeight)
 		base := minHeight + int64(i)
 		if base > height {
 			base = height

blockchain/v0/reactor_test.go

@@ -300,7 +300,10 @@ func TestReactor_BadBlockStopsPeer(t *testing.T) {
 	// XXX: This causes a potential race condition.
 	// See: https://github.com/tendermint/tendermint/issues/6005
 	otherGenDoc, otherPrivVals := randGenesisDoc(config, 1, false, 30)
-	newNode := rts.network.MakeNode(t)
+	newNode := rts.network.MakeNode(t, p2ptest.NodeOptions{
+		MaxPeers:     uint16(len(rts.nodes) + 1),
+		MaxConnected: uint16(len(rts.nodes) + 1),
+	})
 	rts.addNode(t, newNode.NodeID, otherGenDoc, otherPrivVals[0], maxBlockHeight)
 
 	// add a fake peer just so we do not wait for the consensus ticker to timeout

config/config.go

@@ -551,8 +551,16 @@ type P2PConfig struct { //nolint: maligned
 
 	// Comma separated list of seed nodes to connect to
 	// We only use these if we can’t connect to peers in the addrbook
+	// NOTE: not used by the new PEX reactor. Please use BootstrapPeers instead.
+	// TODO: Remove once p2p refactor is complete
+	// ref: https://github.com/tendermint/tendermint/issues/5670
 	Seeds string `mapstructure:"seeds"`
 
+	// Comma separated list of peers to be added to the peer store
+	// on startup. Either BootstrapPeers or PersistentPeers are
+	// needed for peer discovery
+	BootstrapPeers string `mapstructure:"bootstrap-peers"`
+
 	// Comma separated list of nodes to keep persistent connections to
 	PersistentPeers string `mapstructure:"persistent-peers"`
 
@@ -567,11 +575,25 @@ type P2PConfig struct { //nolint: maligned
 	AddrBookStrict bool `mapstructure:"addr-book-strict"`
 
 	// Maximum number of inbound peers
+	//
+	// TODO: Remove once p2p refactor is complete in favor of MaxConnections.
+	// ref: https://github.com/tendermint/tendermint/issues/5670
 	MaxNumInboundPeers int `mapstructure:"max-num-inbound-peers"`
 
-	// Maximum number of outbound peers to connect to, excluding persistent peers
+	// Maximum number of outbound peers to connect to, excluding persistent peers.
+	//
+	// TODO: Remove once p2p refactor is complete in favor of MaxConnections.
+	// ref: https://github.com/tendermint/tendermint/issues/5670
 	MaxNumOutboundPeers int `mapstructure:"max-num-outbound-peers"`
 
+	// MaxConnections defines the maximum number of connected peers (inbound and
+	// outbound).
+	MaxConnections uint16 `mapstructure:"max-connections"`
+
+	// MaxIncomingConnectionAttempts rate limits the number of incoming connection
+	// attempts per IP address.
+	MaxIncomingConnectionAttempts uint `mapstructure:"max-incoming-connection-attempts"`
+
 	// List of node IDs, to which a connection will be (re)established ignoring any existing limits
 	UnconditionalPeerIDs string `mapstructure:"unconditional-peer-ids"`
 
@@ -608,9 +630,9 @@ type P2PConfig struct { //nolint: maligned
 	// Force dial to fail
 	TestDialFail bool `mapstructure:"test-dial-fail"`
 
-	// Mostly for testing, use rather than environment variables
-	// to turn on the new P2P stack.
-	UseNewP2P bool `mapstructure:"use-new-p2p"`
+	// DisableLegacy is used mostly for testing to enable or disable the legacy
+	// P2P stack.
+	DisableLegacy bool `mapstructure:"disable-legacy"`
 
 	// Makes it possible to configure which queue backend the p2p
 	// layer uses. Options are: "fifo", "priority" and "wdrr",
@@ -621,15 +643,17 @@ type P2PConfig struct { //nolint: maligned
 // DefaultP2PConfig returns a default configuration for the peer-to-peer layer
 func DefaultP2PConfig() *P2PConfig {
 	return &P2PConfig{
-		ListenAddress:                "tcp://0.0.0.0:26656",
-		ExternalAddress:              "",
-		UPNP:                         false,
-		AddrBook:                     defaultAddrBookPath,
-		AddrBookStrict:               true,
-		MaxNumInboundPeers:           40,
-		MaxNumOutboundPeers:          10,
-		PersistentPeersMaxDialPeriod: 0 * time.Second,
-		FlushThrottleTimeout:         100 * time.Millisecond,
+		ListenAddress:                 "tcp://0.0.0.0:26656",
+		ExternalAddress:               "",
+		UPNP:                          false,
+		AddrBook:                      defaultAddrBookPath,
+		AddrBookStrict:                true,
+		MaxNumInboundPeers:            40,
+		MaxNumOutboundPeers:           10,
+		MaxConnections:                64,
+		MaxIncomingConnectionAttempts: 100,
+		PersistentPeersMaxDialPeriod:  0 * time.Second,
+		FlushThrottleTimeout:          100 * time.Millisecond,
 		// The MTU (Maximum Transmission Unit) for Ethernet is 1500 bytes.
 		// The IP header and the TCP header take up 20 bytes each at least (unless
 		// optional header fields are used) and thus the max for (non-Jumbo frame)
@@ -818,6 +842,10 @@ func (cfg *StateSyncConfig) ValidateBasic() error {
 				return errors.New("found empty rpc-servers entry")
 			}
 		}
+		if cfg.DiscoveryTime != 0 && cfg.DiscoveryTime < 5*time.Second {
+			return errors.New("discovery time must be 0s or greater than five seconds")
+		}
+
 		if cfg.TrustPeriod <= 0 {
 			return errors.New("trusted-period is required")
 		}

config/toml.go

@@ -262,7 +262,7 @@ pprof-laddr = "{{ .RPC.PprofListenAddress }}"
 [p2p]
 
 # Enable the new p2p layer.
-use-new-p2p = {{ .P2P.UseNewP2P }}
+disable-legacy = {{ .P2P.DisableLegacy }}
 
 # Select the p2p internal queue
 queue-type = "{{ .P2P.QueueType }}"
@@ -277,8 +277,17 @@ laddr = "{{ .P2P.ListenAddress }}"
 external-address = "{{ .P2P.ExternalAddress }}"
 
 # Comma separated list of seed nodes to connect to
+# We only use these if we can’t connect to peers in the addrbook
+# NOTE: not used by the new PEX reactor. Please use BootstrapPeers instead.
+# TODO: Remove once p2p refactor is complete
+# ref: https:#github.com/tendermint/tendermint/issues/5670
 seeds = "{{ .P2P.Seeds }}"
 
+# Comma separated list of peers to be added to the peer store
+# on startup. Either BootstrapPeers or PersistentPeers are
+# needed for peer discovery
+bootstrap-peers = "{{ .P2P.BootstrapPeers }}"
+
 # Comma separated list of nodes to keep persistent connections to
 persistent-peers = "{{ .P2P.PersistentPeers }}"
 
@@ -293,11 +302,23 @@ addr-book-file = "{{ js .P2P.AddrBook }}"
 addr-book-strict = {{ .P2P.AddrBookStrict }}
 
 # Maximum number of inbound peers
+#
+# TODO: Remove once p2p refactor is complete in favor of MaxConnections.
+# ref: https://github.com/tendermint/tendermint/issues/5670
 max-num-inbound-peers = {{ .P2P.MaxNumInboundPeers }}
 
 # Maximum number of outbound peers to connect to, excluding persistent peers
+#
+# TODO: Remove once p2p refactor is complete in favor of MaxConnections.
+# ref: https://github.com/tendermint/tendermint/issues/5670
 max-num-outbound-peers = {{ .P2P.MaxNumOutboundPeers }}
 
+# Maximum number of connections (inbound and outbound).
+max-connections = {{ .P2P.MaxConnections }}
+
+# Rate limits the number of incoming connection attempts per IP address.
+max-incoming-connection-attempts = {{ .P2P.MaxIncomingConnectionAttempts }}
+
 # List of node IDs, to which a connection will be (re)established ignoring any existing limits
 unconditional-peer-ids = "{{ .P2P.UnconditionalPeerIDs }}"
 

consensus/wal_generator.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	mrand "math/rand"
 	"path/filepath"
 	"testing"
 	"time"
@@ -15,7 +16,6 @@ import (
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/libs/log"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
 	"github.com/tendermint/tendermint/privval"
 	"github.com/tendermint/tendermint/proxy"
 	sm "github.com/tendermint/tendermint/state"
@@ -140,7 +140,8 @@ func WALWithNBlocks(t *testing.T, numBlocks int) (data []byte, err error) {
 func randPort() int {
 	// returns between base and base + spread
 	base, spread := 20000, 20000
-	return base + tmrand.Intn(spread)
+	// nolint:gosec // G404: Use of weak random number generator
+	return base + mrand.Intn(spread)
 }
 
 func makeAddrs() (string, string, string) {

crypto/encoding/codec.go

@@ -6,6 +6,7 @@ import (
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/crypto/secp256k1"
+	"github.com/tendermint/tendermint/crypto/sr25519"
 	"github.com/tendermint/tendermint/libs/json"
 	pc "github.com/tendermint/tendermint/proto/tendermint/crypto"
 )
@@ -32,6 +33,12 @@ func PubKeyToProto(k crypto.PubKey) (pc.PublicKey, error) {
 				Secp256K1: k,
 			},
 		}
+	case sr25519.PubKey:
+		kp = pc.PublicKey{
+			Sum: &pc.PublicKey_Sr25519{
+				Sr25519: k,
+			},
+		}
 	default:
 		return kp, fmt.Errorf("toproto: key type %v is not supported", k)
 	}
@@ -57,6 +64,14 @@ func PubKeyFromProto(k pc.PublicKey) (crypto.PubKey, error) {
 		pk := make(secp256k1.PubKey, secp256k1.PubKeySize)
 		copy(pk, k.Secp256K1)
 		return pk, nil
+	case *pc.PublicKey_Sr25519:
+		if len(k.Sr25519) != sr25519.PubKeySize {
+			return nil, fmt.Errorf("invalid size for PubKeySr25519. Got %d, expected %d",
+				len(k.Sr25519), sr25519.PubKeySize)
+		}
+		pk := make(sr25519.PubKey, sr25519.PubKeySize)
+		copy(pk, k.Sr25519)
+		return pk, nil
 	default:
 		return nil, fmt.Errorf("fromproto: key type %v is not supported", k)
 	}

docs/nodes/configuration.md

@@ -233,6 +233,12 @@ max-num-inbound-peers = 40
 # Maximum number of outbound peers to connect to, excluding persistent peers
 max-num-outbound-peers = 10
 
+# Maximum number of connections (inbound and outbound).
+max-connections = 64
+
+# Rate limits the number of incoming connection attempts per IP address.
+max-incoming-connection-attempts = 100
+
 # List of node IDs, to which a connection will be (re)established ignoring any existing limits
 unconditional-peer-ids = ""
 

evidence/pool.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"sort"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -203,9 +202,11 @@ func (evpool *Pool) CheckEvidence(evList types.EvidenceList) error {
 	hashes := make([][]byte, len(evList))
 	for idx, ev := range evList {
 
-		ok := evpool.fastCheck(ev)
+		_, isLightEv := ev.(*types.LightClientAttackEvidence)
 
-		if !ok {
+		// We must verify light client attack evidence regardless because there could be a
+		// different conflicting block with the same hash.
+		if isLightEv || !evpool.isPending(ev) {
 			// check that the evidence isn't already committed
 			if evpool.isCommitted(ev) {
 				return &types.ErrInvalidEvidence{Evidence: ev, Reason: errors.New("evidence was already committed")}
@@ -222,7 +223,7 @@ func (evpool *Pool) CheckEvidence(evList types.EvidenceList) error {
 				evpool.logger.Error("failed to add evidence to pending list", "err", err, "evidence", ev)
 			}
 
-			evpool.logger.Info("verified new evidence of byzantine behavior", "evidence", ev)
+			evpool.logger.Info("check evidence: verified evidence of byzantine behavior", "evidence", ev)
 		}
 
 		// check for duplicate evidence. We cache hashes so we don't have to work them out again.
@@ -260,78 +261,6 @@ func (evpool *Pool) State() sm.State {
 	return evpool.state
 }
 
-// fastCheck leverages the fact that the evidence pool may have already verified
-// the evidence to see if it can quickly conclude that the evidence is already
-// valid.
-func (evpool *Pool) fastCheck(ev types.Evidence) bool {
-	if lcae, ok := ev.(*types.LightClientAttackEvidence); ok {
-		key := keyPending(ev)
-		evBytes, err := evpool.evidenceStore.Get(key)
-		if evBytes == nil { // the evidence is not in the nodes pending list
-			return false
-		}
-
-		if err != nil {
-			evpool.logger.Error("failed to load light client attack evidence", "err", err, "key(height/hash)", key)
-			return false
-		}
-
-		var trustedPb tmproto.LightClientAttackEvidence
-
-		if err = trustedPb.Unmarshal(evBytes); err != nil {
-			evpool.logger.Error(
-				"failed to convert light client attack evidence from bytes",
-				"key(height/hash)", key,
-				"err", err,
-			)
-			return false
-		}
-
-		trustedEv, err := types.LightClientAttackEvidenceFromProto(&trustedPb)
-		if err != nil {
-			evpool.logger.Error(
-				"failed to convert light client attack evidence from protobuf",
-				"key(height/hash)", key,
-				"err", err,
-			)
-			return false
-		}
-
-		// Ensure that all the byzantine validators that the evidence pool has match
-		// the byzantine validators in this evidence.
-		if trustedEv.ByzantineValidators == nil && lcae.ByzantineValidators != nil {
-			return false
-		}
-
-		if len(trustedEv.ByzantineValidators) != len(lcae.ByzantineValidators) {
-			return false
-		}
-
-		byzValsCopy := make([]*types.Validator, len(lcae.ByzantineValidators))
-		for i, v := range lcae.ByzantineValidators {
-			byzValsCopy[i] = v.Copy()
-		}
-
-		// ensure that both validator arrays are in the same order
-		sort.Sort(types.ValidatorsByVotingPower(byzValsCopy))
-
-		for idx, val := range trustedEv.ByzantineValidators {
-			if !bytes.Equal(byzValsCopy[idx].Address, val.Address) {
-				return false
-			}
-			if byzValsCopy[idx].VotingPower != val.VotingPower {
-				return false
-			}
-		}
-
-		return true
-	}
-
-	// For all other evidence the evidence pool just checks if it is already in
-	// the pending db.
-	return evpool.isPending(ev)
-}
-
 // IsExpired checks whether evidence or a polc is expired by checking whether a height and time is older
 // than set by the evidence consensus parameters
 func (evpool *Pool) isExpired(height int64, time time.Time) bool {
@@ -396,7 +325,7 @@ func (evpool *Pool) markEvidenceAsCommitted(evidence types.EvidenceList, height
 	for _, ev := range evidence {
 		if evpool.isPending(ev) {
 			if err := batch.Delete(keyPending(ev)); err != nil {
-				evpool.logger.Error("failed to batch pending evidence", "err", err)
+				evpool.logger.Error("failed to batch delete pending evidence", "err", err)
 			}
 			blockEvidenceMap[evMapKey(ev)] = struct{}{}
 		}
@@ -546,7 +475,7 @@ func (evpool *Pool) batchExpiredPendingEvidence(batch dbm.Batch) (int64, time.Ti
 
 		// else add to the batch
 		if err := batch.Delete(iter.Key()); err != nil {
-			evpool.logger.Error("failed to batch evidence", "err", err, "ev", ev)
+			evpool.logger.Error("failed to batch delete evidence", "err", err, "ev", ev)
 			continue
 		}
 

evidence/pool_test.go

@@ -14,7 +14,6 @@ import (
 	"github.com/tendermint/tendermint/evidence"
 	"github.com/tendermint/tendermint/evidence/mocks"
 	"github.com/tendermint/tendermint/libs/log"
-	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	sm "github.com/tendermint/tendermint/state"
 	smmocks "github.com/tendermint/tendermint/state/mocks"
 	"github.com/tendermint/tendermint/store"
@@ -166,6 +165,9 @@ func TestReportConflictingVotes(t *testing.T) {
 	// should be able to retrieve evidence from pool
 	evList, _ = pool.PendingEvidence(defaultEvidenceMaxBytes)
 	require.Equal(t, []types.Evidence{ev}, evList)
+
+	next = pool.EvidenceFront()
+	require.NotNil(t, next)
 }
 
 func TestEvidencePoolUpdate(t *testing.T) {
@@ -262,84 +264,61 @@ func TestVerifyDuplicatedEvidenceFails(t *testing.T) {
 
 // check that valid light client evidence is correctly validated and stored in
 // evidence pool
-func TestCheckEvidenceWithLightClientAttack(t *testing.T) {
+func TestLightClientAttackEvidenceLifecycle(t *testing.T) {
 	var (
-		nValidators          = 5
-		validatorPower int64 = 10
-		height         int64 = 10
+		height       int64 = 100
+		commonHeight int64 = 90
 	)
 
-	conflictingVals, conflictingPrivVals := types.RandValidatorSet(nValidators, validatorPower)
-	trustedHeader := makeHeaderRandom(height)
-	trustedHeader.Time = defaultEvidenceTime
-
-	conflictingHeader := makeHeaderRandom(height)
-	conflictingHeader.ValidatorsHash = conflictingVals.Hash()
-
-	trustedHeader.ValidatorsHash = conflictingHeader.ValidatorsHash
-	trustedHeader.NextValidatorsHash = conflictingHeader.NextValidatorsHash
-	trustedHeader.ConsensusHash = conflictingHeader.ConsensusHash
-	trustedHeader.AppHash = conflictingHeader.AppHash
-	trustedHeader.LastResultsHash = conflictingHeader.LastResultsHash
-
-	// For simplicity we are simulating a duplicate vote attack where all the
-	// validators in the conflictingVals set voted twice.
-	blockID := makeBlockID(conflictingHeader.Hash(), 1000, []byte("partshash"))
-	voteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
-
-	commit, err := types.MakeCommit(blockID, height, 1, voteSet, conflictingPrivVals, defaultEvidenceTime)
-	require.NoError(t, err)
-
-	ev := &types.LightClientAttackEvidence{
-		ConflictingBlock: &types.LightBlock{
-			SignedHeader: &types.SignedHeader{
-				Header: conflictingHeader,
-				Commit: commit,
-			},
-			ValidatorSet: conflictingVals,
-		},
-		CommonHeight:        10,
-		TotalVotingPower:    int64(nValidators) * validatorPower,
-		ByzantineValidators: conflictingVals.Validators,
-		Timestamp:           defaultEvidenceTime,
-	}
-
-	trustedBlockID := makeBlockID(trustedHeader.Hash(), 1000, []byte("partshash"))
-	trustedVoteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
-	trustedCommit, err := types.MakeCommit(
-		trustedBlockID,
-		height,
-		1,
-		trustedVoteSet,
-		conflictingPrivVals,
-		defaultEvidenceTime,
-	)
-	require.NoError(t, err)
+	ev, trusted, common := makeLunaticEvidence(t, height, commonHeight,
+		10, 5, 5, defaultEvidenceTime, defaultEvidenceTime.Add(1*time.Hour))
 
 	state := sm.State{
-		LastBlockTime:   defaultEvidenceTime.Add(1 * time.Minute),
-		LastBlockHeight: 11,
+		LastBlockTime:   defaultEvidenceTime.Add(2 * time.Hour),
+		LastBlockHeight: 110,
 		ConsensusParams: *types.DefaultConsensusParams(),
 	}
 
 	stateStore := &smmocks.Store{}
-	stateStore.On("LoadValidators", height).Return(conflictingVals, nil)
+	stateStore.On("LoadValidators", height).Return(trusted.ValidatorSet, nil)
+	stateStore.On("LoadValidators", commonHeight).Return(common.ValidatorSet, nil)
 	stateStore.On("Load").Return(state, nil)
 
 	blockStore := &mocks.BlockStore{}
-	blockStore.On("LoadBlockMeta", height).Return(&types.BlockMeta{Header: *trustedHeader})
-	blockStore.On("LoadBlockCommit", height).Return(trustedCommit)
+	blockStore.On("LoadBlockMeta", height).Return(&types.BlockMeta{Header: *trusted.Header})
+	blockStore.On("LoadBlockMeta", commonHeight).Return(&types.BlockMeta{Header: *common.Header})
+	blockStore.On("LoadBlockCommit", height).Return(trusted.Commit)
+	blockStore.On("LoadBlockCommit", commonHeight).Return(common.Commit)
 
 	pool, err := evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
 	require.NoError(t, err)
 
-	require.NoError(t, pool.AddEvidence(ev))
-	require.NoError(t, pool.CheckEvidence(types.EvidenceList{ev}))
+	hash := ev.Hash()
 
-	// Take away the last signature -> there are less validators then what we have detected,
-	// hence this should fail.
-	commit.Signatures = append(commit.Signatures[:nValidators-1], types.NewCommitSigAbsent())
+	require.NoError(t, pool.AddEvidence(ev))
+	require.NoError(t, pool.AddEvidence(ev))
+
+	pendingEv, _ := pool.PendingEvidence(state.ConsensusParams.Evidence.MaxBytes)
+	require.Equal(t, 1, len(pendingEv))
+	require.Equal(t, ev, pendingEv[0])
+
+	require.NoError(t, pool.CheckEvidence(pendingEv))
+	require.Equal(t, ev, pendingEv[0])
+
+	state.LastBlockHeight++
+	state.LastBlockTime = state.LastBlockTime.Add(1 * time.Minute)
+	pool.Update(state, pendingEv)
+	require.Equal(t, hash, pendingEv[0].Hash())
+
+	remaindingEv, _ := pool.PendingEvidence(state.ConsensusParams.Evidence.MaxBytes)
+	require.Empty(t, remaindingEv)
+
+	// evidence is already committed so it shouldn't pass
 	require.Error(t, pool.CheckEvidence(types.EvidenceList{ev}))
+	require.NoError(t, pool.AddEvidence(ev))
+
+	remaindingEv, _ = pool.PendingEvidence(state.ConsensusParams.Evidence.MaxBytes)
+	require.Empty(t, remaindingEv)
 }
 
 // Tests that restarting the evidence pool after a potential failure will recover the
@@ -389,7 +368,7 @@ func TestRecoverPendingEvidence(t *testing.T) {
 			Evidence: types.EvidenceParams{
 				MaxAgeNumBlocks: 20,
 				MaxAgeDuration:  20 * time.Minute,
-				MaxBytes:        1000,
+				MaxBytes:        defaultEvidenceMaxBytes,
 			},
 		},
 	}, nil)

evidence/verify.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
-	"sort"
 	"time"
 
 	"github.com/tendermint/tendermint/light"
@@ -129,56 +128,6 @@ func (evpool *Pool) verify(evidence types.Evidence) error {
 		if err != nil {
 			return types.NewErrInvalidEvidence(evidence, err)
 		}
-
-		// Find out what type of attack this was and thus extract the malicious
-		// validators. Note, in the case of an Amnesia attack we don't have any
-		// malicious validators.
-		validators := ev.GetByzantineValidators(commonVals, trustedHeader)
-
-		// Ensure this matches the validators that are listed in the evidence. They
-		// should be ordered based on power.
-		if validators == nil && ev.ByzantineValidators != nil {
-			return types.NewErrInvalidEvidence(
-				evidence,
-				fmt.Errorf(
-					"expected nil validators from an amnesia light client attack but got %d",
-					len(ev.ByzantineValidators),
-				),
-			)
-		}
-
-		if exp, got := len(validators), len(ev.ByzantineValidators); exp != got {
-			return types.NewErrInvalidEvidence(
-				evidence,
-				fmt.Errorf("expected %d byzantine validators from evidence but got %d", exp, got),
-			)
-		}
-
-		// ensure that both validator arrays are in the same order
-		sort.Sort(types.ValidatorsByVotingPower(ev.ByzantineValidators))
-
-		for idx, val := range validators {
-			if !bytes.Equal(ev.ByzantineValidators[idx].Address, val.Address) {
-				return types.NewErrInvalidEvidence(
-					evidence,
-					fmt.Errorf(
-						"evidence contained an unexpected byzantine validator address; expected: %v, got: %v",
-						val.Address, ev.ByzantineValidators[idx].Address,
-					),
-				)
-			}
-
-			if ev.ByzantineValidators[idx].VotingPower != val.VotingPower {
-				return types.NewErrInvalidEvidence(
-					evidence,
-					fmt.Errorf(
-						"evidence contained unexpected byzantine validator power; expected %d, got %d",
-						val.VotingPower, ev.ByzantineValidators[idx].VotingPower,
-					),
-				)
-			}
-		}
-
 		return nil
 
 	default:
@@ -206,7 +155,7 @@ func VerifyLightClientAttack(e *types.LightClientAttackEvidence, commonHeader, t
 		}
 
 		// In the case of equivocation and amnesia we expect all header hashes to be correctly derived
-	} else if isInvalidHeader(trustedHeader.Header, e.ConflictingBlock.Header) {
+	} else if e.ConflictingHeaderIsInvalid(trustedHeader.Header) {
 		return errors.New("common height is the same as conflicting block height so expected the conflicting" +
 			" block to be correctly derived yet it wasn't")
 	}
@@ -235,7 +184,7 @@ func VerifyLightClientAttack(e *types.LightClientAttackEvidence, commonHeader, t
 			trustedHeader.Hash())
 	}
 
-	return nil
+	return validateABCIEvidence(e, commonVals, trustedHeader)
 }
 
 // VerifyDuplicateVote verifies DuplicateVoteEvidence against the state of full node. This involves the
@@ -306,6 +255,55 @@ func VerifyDuplicateVote(e *types.DuplicateVoteEvidence, chainID string, valSet
 	return nil
 }
 
+// validateABCIEvidence validates the ABCI component of the light client attack
+// evidence i.e voting power and byzantine validators
+func validateABCIEvidence(
+	ev *types.LightClientAttackEvidence,
+	commonVals *types.ValidatorSet,
+	trustedHeader *types.SignedHeader,
+) error {
+	if evTotal, valsTotal := ev.TotalVotingPower, commonVals.TotalVotingPower(); evTotal != valsTotal {
+		return fmt.Errorf("total voting power from the evidence and our validator set does not match (%d != %d)",
+			evTotal, valsTotal)
+	}
+
+	// Find out what type of attack this was and thus extract the malicious
+	// validators. Note, in the case of an Amnesia attack we don't have any
+	// malicious validators.
+	validators := ev.GetByzantineValidators(commonVals, trustedHeader)
+
+	// Ensure this matches the validators that are listed in the evidence. They
+	// should be ordered based on power.
+	if validators == nil && ev.ByzantineValidators != nil {
+		return fmt.Errorf(
+			"expected nil validators from an amnesia light client attack but got %d",
+			len(ev.ByzantineValidators),
+		)
+	}
+
+	if exp, got := len(validators), len(ev.ByzantineValidators); exp != got {
+		return fmt.Errorf("expected %d byzantine validators from evidence but got %d", exp, got)
+	}
+
+	for idx, val := range validators {
+		if !bytes.Equal(ev.ByzantineValidators[idx].Address, val.Address) {
+			return fmt.Errorf(
+				"evidence contained an unexpected byzantine validator address; expected: %v, got: %v",
+				val.Address, ev.ByzantineValidators[idx].Address,
+			)
+		}
+
+		if ev.ByzantineValidators[idx].VotingPower != val.VotingPower {
+			return fmt.Errorf(
+				"evidence contained unexpected byzantine validator power; expected %d, got %d",
+				val.VotingPower, ev.ByzantineValidators[idx].VotingPower,
+			)
+		}
+	}
+
+	return nil
+}
+
 func getSignedHeader(blockStore BlockStore, height int64) (*types.SignedHeader, error) {
 	blockMeta := blockStore.LoadBlockMeta(height)
 	if blockMeta == nil {
@@ -320,15 +318,3 @@ func getSignedHeader(blockStore BlockStore, height int64) (*types.SignedHeader,
 		Commit: commit,
 	}, nil
 }
-
-// isInvalidHeader takes a trusted header and matches it againt a conflicting header
-// to determine whether the conflicting header was the product of a valid state transition
-// or not. If it is then all the deterministic fields of the header should be the same.
-// If not, it is an invalid header and constitutes a lunatic attack.
-func isInvalidHeader(trusted, conflicting *types.Header) bool {
-	return !bytes.Equal(trusted.ValidatorsHash, conflicting.ValidatorsHash) ||
-		!bytes.Equal(trusted.NextValidatorsHash, conflicting.NextValidatorsHash) ||
-		!bytes.Equal(trusted.ConsensusHash, conflicting.ConsensusHash) ||
-		!bytes.Equal(trusted.AppHash, conflicting.AppHash) ||
-		!bytes.Equal(trusted.LastResultsHash, conflicting.LastResultsHash)
-}

evidence/verify_test.go

@@ -1,6 +1,7 @@
 package evidence_test
 
 import (
+	"bytes"
 	"context"
 	"testing"
 	"time"
@@ -22,139 +23,170 @@ import (
 	"github.com/tendermint/tendermint/version"
 )
 
+const (
+	defaultVotingPower = 10
+)
+
 func TestVerifyLightClientAttack_Lunatic(t *testing.T) {
-	commonVals, commonPrivVals := types.RandValidatorSet(2, 10)
-
-	newVal, newPrivVal := types.RandValidator(false, 9)
-
-	conflictingVals, err := types.ValidatorSetFromExistingValidators(append(commonVals.Validators, newVal))
-	require.NoError(t, err)
-	conflictingPrivVals := append(commonPrivVals, newPrivVal)
-
-	commonHeader := makeHeaderRandom(4)
-	commonHeader.Time = defaultEvidenceTime
-	trustedHeader := makeHeaderRandom(10)
-	trustedHeader.Time = defaultEvidenceTime.Add(1 * time.Hour)
-
-	conflictingHeader := makeHeaderRandom(10)
-	conflictingHeader.Time = defaultEvidenceTime.Add(1 * time.Hour)
-	conflictingHeader.ValidatorsHash = conflictingVals.Hash()
-
-	// we are simulating a lunatic light client attack
-	blockID := makeBlockID(conflictingHeader.Hash(), 1000, []byte("partshash"))
-	voteSet := types.NewVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), conflictingVals)
-	commit, err := types.MakeCommit(blockID, 10, 1, voteSet, conflictingPrivVals, defaultEvidenceTime)
-	require.NoError(t, err)
-	ev := &types.LightClientAttackEvidence{
-		ConflictingBlock: &types.LightBlock{
-			SignedHeader: &types.SignedHeader{
-				Header: conflictingHeader,
-				Commit: commit,
-			},
-			ValidatorSet: conflictingVals,
-		},
-		CommonHeight:        4,
-		TotalVotingPower:    20,
-		ByzantineValidators: commonVals.Validators,
-		Timestamp:           defaultEvidenceTime,
-	}
-
-	commonSignedHeader := &types.SignedHeader{
-		Header: commonHeader,
-		Commit: &types.Commit{},
-	}
-	trustedBlockID := makeBlockID(trustedHeader.Hash(), 1000, []byte("partshash"))
-	vals, privVals := types.RandValidatorSet(3, 8)
-	trustedVoteSet := types.NewVoteSet(evidenceChainID, 10, 1, tmproto.SignedMsgType(2), vals)
-	trustedCommit, err := types.MakeCommit(trustedBlockID, 10, 1, trustedVoteSet, privVals, defaultEvidenceTime)
-	require.NoError(t, err)
-	trustedSignedHeader := &types.SignedHeader{
-		Header: trustedHeader,
-		Commit: trustedCommit,
-	}
+	const (
+		height       int64 = 10
+		commonHeight int64 = 4
+		totalVals          = 10
+		byzVals            = 4
+	)
+	attackTime := defaultEvidenceTime.Add(1 * time.Hour)
+	// create valid lunatic evidence
+	ev, trusted, common := makeLunaticEvidence(
+		t, height, commonHeight, totalVals, byzVals, totalVals-byzVals, defaultEvidenceTime, attackTime)
+	require.NoError(t, ev.ValidateBasic())
 
 	// good pass -> no error
-	err = evidence.VerifyLightClientAttack(ev, commonSignedHeader, trustedSignedHeader, commonVals,
+	err := evidence.VerifyLightClientAttack(ev, common.SignedHeader, trusted.SignedHeader, common.ValidatorSet,
 		defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour)
 	assert.NoError(t, err)
 
 	// trusted and conflicting hashes are the same -> an error should be returned
-	err = evidence.VerifyLightClientAttack(ev, commonSignedHeader, ev.ConflictingBlock.SignedHeader, commonVals,
+	err = evidence.VerifyLightClientAttack(ev, common.SignedHeader, ev.ConflictingBlock.SignedHeader, common.ValidatorSet,
 		defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour)
 	assert.Error(t, err)
 
 	// evidence with different total validator power should fail
-	ev.TotalVotingPower = 1
-	err = evidence.VerifyLightClientAttack(ev, commonSignedHeader, trustedSignedHeader, commonVals,
+	ev.TotalVotingPower = 1 * defaultVotingPower
+	err = evidence.VerifyLightClientAttack(ev, common.SignedHeader, trusted.SignedHeader, common.ValidatorSet,
 		defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour)
 	assert.Error(t, err)
-	ev.TotalVotingPower = 20
 
-	forwardConflictingHeader := makeHeaderRandom(11)
-	forwardConflictingHeader.Time = defaultEvidenceTime.Add(30 * time.Minute)
-	forwardConflictingHeader.ValidatorsHash = conflictingVals.Hash()
-	forwardBlockID := makeBlockID(forwardConflictingHeader.Hash(), 1000, []byte("partshash"))
-	forwardVoteSet := types.NewVoteSet(evidenceChainID, 11, 1, tmproto.SignedMsgType(2), conflictingVals)
-	forwardCommit, err := types.MakeCommit(forwardBlockID, 11, 1, forwardVoteSet, conflictingPrivVals, defaultEvidenceTime)
-	require.NoError(t, err)
-	forwardLunaticEv := &types.LightClientAttackEvidence{
-		ConflictingBlock: &types.LightBlock{
-			SignedHeader: &types.SignedHeader{
-				Header: forwardConflictingHeader,
-				Commit: forwardCommit,
-			},
-			ValidatorSet: conflictingVals,
-		},
-		CommonHeight:        4,
-		TotalVotingPower:    20,
-		ByzantineValidators: commonVals.Validators,
-		Timestamp:           defaultEvidenceTime,
-	}
-	err = evidence.VerifyLightClientAttack(forwardLunaticEv, commonSignedHeader, trustedSignedHeader, commonVals,
+	// evidence without enough malicious votes should fail
+	ev, trusted, common = makeLunaticEvidence(
+		t, height, commonHeight, totalVals, byzVals-1, totalVals-byzVals, defaultEvidenceTime, attackTime)
+	err = evidence.VerifyLightClientAttack(ev, common.SignedHeader, trusted.SignedHeader, common.ValidatorSet,
 		defaultEvidenceTime.Add(2*time.Hour), 3*time.Hour)
-	assert.NoError(t, err)
+	assert.Error(t, err)
+}
 
+func TestVerify_LunaticAttackAgainstState(t *testing.T) {
+	const (
+		height       int64 = 10
+		commonHeight int64 = 4
+		totalVals          = 10
+		byzVals            = 4
+	)
+	attackTime := defaultEvidenceTime.Add(1 * time.Hour)
+	// create valid lunatic evidence
+	ev, trusted, common := makeLunaticEvidence(
+		t, height, commonHeight, totalVals, byzVals, totalVals-byzVals, defaultEvidenceTime, attackTime)
+
+	// now we try to test verification against state
 	state := sm.State{
 		LastBlockTime:   defaultEvidenceTime.Add(2 * time.Hour),
-		LastBlockHeight: 11,
+		LastBlockHeight: height + 1,
 		ConsensusParams: *types.DefaultConsensusParams(),
 	}
 	stateStore := &smmocks.Store{}
-	stateStore.On("LoadValidators", int64(4)).Return(commonVals, nil)
+	stateStore.On("LoadValidators", commonHeight).Return(common.ValidatorSet, nil)
 	stateStore.On("Load").Return(state, nil)
 	blockStore := &mocks.BlockStore{}
-	blockStore.On("LoadBlockMeta", int64(4)).Return(&types.BlockMeta{Header: *commonHeader})
-	blockStore.On("LoadBlockMeta", int64(10)).Return(&types.BlockMeta{Header: *trustedHeader})
-	blockStore.On("LoadBlockMeta", int64(11)).Return(nil)
-	blockStore.On("LoadBlockCommit", int64(4)).Return(commit)
-	blockStore.On("LoadBlockCommit", int64(10)).Return(trustedCommit)
-	blockStore.On("Height").Return(int64(10))
-
+	blockStore.On("LoadBlockMeta", commonHeight).Return(&types.BlockMeta{Header: *common.Header})
+	blockStore.On("LoadBlockMeta", height).Return(&types.BlockMeta{Header: *trusted.Header})
+	blockStore.On("LoadBlockCommit", commonHeight).Return(common.Commit)
+	blockStore.On("LoadBlockCommit", height).Return(trusted.Commit)
 	pool, err := evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
 	require.NoError(t, err)
 
 	evList := types.EvidenceList{ev}
-	err = pool.CheckEvidence(evList)
-	assert.NoError(t, err)
+	// check that the evidence pool correctly verifies the evidence
+	assert.NoError(t, pool.CheckEvidence(evList))
 
+	// as it was not originally in the pending bucket, it should now have been added
 	pendingEvs, _ := pool.PendingEvidence(state.ConsensusParams.Evidence.MaxBytes)
 	assert.Equal(t, 1, len(pendingEvs))
+	assert.Equal(t, ev, pendingEvs[0])
 
 	// if we submit evidence only against a single byzantine validator when we see there are more validators then this
 	// should return an error
-	ev.ByzantineValidators = []*types.Validator{commonVals.Validators[0]}
-	err = pool.CheckEvidence(evList)
-	assert.Error(t, err)
-	ev.ByzantineValidators = commonVals.Validators // restore evidence
+	ev.ByzantineValidators = ev.ByzantineValidators[:1]
+	t.Log(evList)
+	assert.Error(t, pool.CheckEvidence(evList))
+	// restore original byz vals
+	ev.ByzantineValidators = ev.GetByzantineValidators(common.ValidatorSet, trusted.SignedHeader)
+
+	// duplicate evidence should be rejected
+	evList = types.EvidenceList{ev, ev}
+	pool, err = evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
+	require.NoError(t, err)
+	assert.Error(t, pool.CheckEvidence(evList))
 
 	// If evidence is submitted with an altered timestamp it should return an error
 	ev.Timestamp = defaultEvidenceTime.Add(1 * time.Minute)
-	err = pool.CheckEvidence(evList)
-	assert.Error(t, err)
+	pool, err = evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
+	require.NoError(t, err)
+	assert.Error(t, pool.AddEvidence(ev))
+	ev.Timestamp = defaultEvidenceTime
 
-	evList = types.EvidenceList{forwardLunaticEv}
-	err = pool.CheckEvidence(evList)
-	assert.NoError(t, err)
+	// Evidence submitted with a different validator power should fail
+	ev.TotalVotingPower = 1
+	pool, err = evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
+	require.NoError(t, err)
+	assert.Error(t, pool.AddEvidence(ev))
+	ev.TotalVotingPower = common.ValidatorSet.TotalVotingPower()
+}
+
+func TestVerify_ForwardLunaticAttack(t *testing.T) {
+	const (
+		nodeHeight   int64 = 8
+		attackHeight int64 = 10
+		commonHeight int64 = 4
+		totalVals          = 10
+		byzVals            = 5
+	)
+	attackTime := defaultEvidenceTime.Add(1 * time.Hour)
+
+	// create a forward lunatic attack
+	ev, trusted, common := makeLunaticEvidence(
+		t, attackHeight, commonHeight, totalVals, byzVals, totalVals-byzVals, defaultEvidenceTime, attackTime)
+
+	// now we try to test verification against state
+	state := sm.State{
+		LastBlockTime:   defaultEvidenceTime.Add(2 * time.Hour),
+		LastBlockHeight: nodeHeight,
+		ConsensusParams: *types.DefaultConsensusParams(),
+	}
+
+	// modify trusted light block so that it is of a height less than the conflicting one
+	trusted.Header.Height = state.LastBlockHeight
+	trusted.Header.Time = state.LastBlockTime
+
+	stateStore := &smmocks.Store{}
+	stateStore.On("LoadValidators", commonHeight).Return(common.ValidatorSet, nil)
+	stateStore.On("Load").Return(state, nil)
+	blockStore := &mocks.BlockStore{}
+	blockStore.On("LoadBlockMeta", commonHeight).Return(&types.BlockMeta{Header: *common.Header})
+	blockStore.On("LoadBlockMeta", nodeHeight).Return(&types.BlockMeta{Header: *trusted.Header})
+	blockStore.On("LoadBlockMeta", attackHeight).Return(nil)
+	blockStore.On("LoadBlockCommit", commonHeight).Return(common.Commit)
+	blockStore.On("LoadBlockCommit", nodeHeight).Return(trusted.Commit)
+	blockStore.On("Height").Return(nodeHeight)
+	pool, err := evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, blockStore)
+	require.NoError(t, err)
+
+	// check that the evidence pool correctly verifies the evidence
+	assert.NoError(t, pool.CheckEvidence(types.EvidenceList{ev}))
+
+	// now we use a time which isn't able to contradict the FLA - thus we can't verify the evidence
+	oldBlockStore := &mocks.BlockStore{}
+	oldHeader := trusted.Header
+	oldHeader.Time = defaultEvidenceTime
+	oldBlockStore.On("LoadBlockMeta", commonHeight).Return(&types.BlockMeta{Header: *common.Header})
+	oldBlockStore.On("LoadBlockMeta", nodeHeight).Return(&types.BlockMeta{Header: *oldHeader})
+	oldBlockStore.On("LoadBlockMeta", attackHeight).Return(nil)
+	oldBlockStore.On("LoadBlockCommit", commonHeight).Return(common.Commit)
+	oldBlockStore.On("LoadBlockCommit", nodeHeight).Return(trusted.Commit)
+	oldBlockStore.On("Height").Return(nodeHeight)
+	require.Equal(t, defaultEvidenceTime, oldBlockStore.LoadBlockMeta(nodeHeight).Header.Time)
+
+	pool, err = evidence.NewPool(log.TestingLogger(), dbm.NewMemDB(), stateStore, oldBlockStore)
+	require.NoError(t, err)
+	assert.Error(t, pool.CheckEvidence(types.EvidenceList{ev}))
 }
 
 func TestVerifyLightClientAttack_Equivocation(t *testing.T) {
@@ -414,6 +446,84 @@ func TestVerifyDuplicateVoteEvidence(t *testing.T) {
 	assert.Error(t, err)
 }
 
+func makeLunaticEvidence(
+	t *testing.T,
+	height, commonHeight int64,
+	totalVals, byzVals, phantomVals int,
+	commonTime, attackTime time.Time,
+) (ev *types.LightClientAttackEvidence, trusted *types.LightBlock, common *types.LightBlock) {
+	commonValSet, commonPrivVals := types.RandValidatorSet(totalVals, defaultVotingPower)
+
+	require.Greater(t, totalVals, byzVals)
+
+	// extract out the subset of byzantine validators in the common validator set
+	byzValSet, byzPrivVals := commonValSet.Validators[:byzVals], commonPrivVals[:byzVals]
+
+	phantomValSet, phantomPrivVals := types.RandValidatorSet(phantomVals, defaultVotingPower)
+
+	conflictingVals := phantomValSet.Copy()
+	require.NoError(t, conflictingVals.UpdateWithChangeSet(byzValSet))
+	conflictingPrivVals := append(phantomPrivVals, byzPrivVals...)
+
+	conflictingPrivVals = orderPrivValsByValSet(t, conflictingVals, conflictingPrivVals)
+
+	commonHeader := makeHeaderRandom(commonHeight)
+	commonHeader.Time = commonTime
+	trustedHeader := makeHeaderRandom(height)
+
+	conflictingHeader := makeHeaderRandom(height)
+	conflictingHeader.Time = attackTime
+	conflictingHeader.ValidatorsHash = conflictingVals.Hash()
+
+	blockID := makeBlockID(conflictingHeader.Hash(), 1000, []byte("partshash"))
+	voteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), conflictingVals)
+	commit, err := types.MakeCommit(blockID, height, 1, voteSet, conflictingPrivVals, defaultEvidenceTime)
+	require.NoError(t, err)
+	ev = &types.LightClientAttackEvidence{
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: &types.SignedHeader{
+				Header: conflictingHeader,
+				Commit: commit,
+			},
+			ValidatorSet: conflictingVals,
+		},
+		CommonHeight:        commonHeight,
+		TotalVotingPower:    commonValSet.TotalVotingPower(),
+		ByzantineValidators: byzValSet,
+		Timestamp:           commonTime,
+	}
+
+	common = &types.LightBlock{
+		SignedHeader: &types.SignedHeader{
+			Header: commonHeader,
+			// we can leave this empty because we shouldn't be checking this
+			Commit: &types.Commit{},
+		},
+		ValidatorSet: commonValSet,
+	}
+	trustedBlockID := makeBlockID(trustedHeader.Hash(), 1000, []byte("partshash"))
+	trustedVals, privVals := types.RandValidatorSet(totalVals, defaultVotingPower)
+	trustedVoteSet := types.NewVoteSet(evidenceChainID, height, 1, tmproto.SignedMsgType(2), trustedVals)
+	trustedCommit, err := types.MakeCommit(trustedBlockID, height, 1, trustedVoteSet, privVals, defaultEvidenceTime)
+	require.NoError(t, err)
+	trusted = &types.LightBlock{
+		SignedHeader: &types.SignedHeader{
+			Header: trustedHeader,
+			Commit: trustedCommit,
+		},
+		ValidatorSet: trustedVals,
+	}
+	return ev, trusted, common
+}
+
+// func makeEquivocationEvidence() *types.LightClientAttackEvidence {
+
+// }
+
+// func makeAmnesiaEvidence() *types.LightClientAttackEvidence {
+
+// }
+
 func makeVote(
 	t *testing.T, val types.PrivValidator, chainID string, valIndex int32, height int64,
 	round int32, step int, blockID types.BlockID, time time.Time) *types.Vote {
@@ -472,3 +582,20 @@ func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.Bloc
 		},
 	}
 }
+
+func orderPrivValsByValSet(
+	t *testing.T, vals *types.ValidatorSet, privVals []types.PrivValidator) []types.PrivValidator {
+	output := make([]types.PrivValidator, len(privVals))
+	for idx, v := range vals.Validators {
+		for _, p := range privVals {
+			pubKey, err := p.GetPubKey(context.Background())
+			require.NoError(t, err)
+			if bytes.Equal(v.Address, pubKey.Address()) {
+				output[idx] = p
+				break
+			}
+		}
+		require.NotEmpty(t, output[idx])
+	}
+	return output
+}

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Workiva/go-datastructures v1.0.53
 	github.com/btcsuite/btcd v0.21.0-beta
 	github.com/btcsuite/btcutil v1.0.2
-	github.com/confio/ics23/go v0.6.3
+	github.com/confio/ics23/go v0.6.6
 	github.com/cosmos/iavl v0.15.3
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/go-kit/kit v0.10.0
@@ -17,7 +17,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/google/orderedcode v0.0.1
 	github.com/gorilla/websocket v1.4.2
-	github.com/grpc-ecosystem/go-grpc-middleware v1.2.2
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/gtank/merlin v0.1.1
 	github.com/hdevalence/ed25519consensus v0.0.0-20210204194344-59a8610d2b87

go.sum

@@ -83,8 +83,9 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/confio/ics23/go v0.0.0-20200817220745-f173e6211efb/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg=
-github.com/confio/ics23/go v0.6.3 h1:PuGK2V1NJWZ8sSkNDq91jgT/cahFEW9RGp4Y5jxulf0=
 github.com/confio/ics23/go v0.6.3/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg=
+github.com/confio/ics23/go v0.6.6 h1:pkOy18YxxJ/r0XFDCnrl4Bjv6h4LkBSpLS6F38mrKL8=
+github.com/confio/ics23/go v0.6.6/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -228,8 +229,9 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.1/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
-github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=

libs/bits/bit_array.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
+	mrand "math/rand"
 	"regexp"
 	"strings"
 	"sync"
@@ -24,6 +25,8 @@ type BitArray struct {
 // NewBitArray returns a new bit array.
 // It returns nil if the number of bits is zero.
 func NewBitArray(bits int) *BitArray {
+	// Reseed non-deterministically.
+	tmrand.Reseed()
 	if bits <= 0 {
 		return nil
 	}
@@ -242,7 +245,7 @@ func (bA *BitArray) IsFull() bool {
 
 // PickRandom returns a random index for a set bit in the bit array.
 // If there is no such value, it returns 0, false.
-// It uses the global randomness in `random.go` to get this index.
+// It uses math/rand's global randomness Source to get this index.
 func (bA *BitArray) PickRandom() (int, bool) {
 	if bA == nil {
 		return 0, false
@@ -255,8 +258,8 @@ func (bA *BitArray) PickRandom() (int, bool) {
 	if len(trueIndices) == 0 { // no bits set to true
 		return 0, false
 	}
-
-	return trueIndices[tmrand.Intn(len(trueIndices))], true
+	// nolint:gosec // G404: Use of weak random number generator
+	return trueIndices[mrand.Intn(len(trueIndices))], true
 }
 
 func (bA *BitArray) getTrueIndices() []int {

libs/clist/clist_test.go

@@ -2,14 +2,13 @@ package clist
 
 import (
 	"fmt"
+	mrand "math/rand"
 	"runtime"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
-
-	tmrand "github.com/tendermint/tendermint/libs/rand"
 )
 
 func TestPanicOnMaxLength(t *testing.T) {
@@ -148,7 +147,7 @@ func _TestGCRandom(t *testing.T) {
 		els = append(els, el)
 	}
 
-	for _, i := range tmrand.Perm(numElements) {
+	for _, i := range mrand.Perm(numElements) {
 		el := els[i]
 		l.Remove(el)
 		_ = el.Next()
@@ -206,7 +205,7 @@ func TestScanRightDeleteRandom(t *testing.T) {
 	// Remove an element, push back an element.
 	for i := 0; i < numTimes; i++ {
 		// Pick an element to remove
-		rmElIdx := tmrand.Intn(len(els))
+		rmElIdx := mrand.Intn(len(els))
 		rmEl := els[rmElIdx]
 
 		// Remove it
@@ -260,7 +259,7 @@ func TestWaitChan(t *testing.T) {
 		for i := 1; i < 100; i++ {
 			l.PushBack(i)
 			pushed++
-			time.Sleep(time.Duration(tmrand.Intn(25)) * time.Millisecond)
+			time.Sleep(time.Duration(mrand.Intn(25)) * time.Millisecond)
 		}
 		// apply a deterministic pause so the counter has time to catch up
 		time.Sleep(25 * time.Millisecond)

libs/rand/random.go

@@ -2,161 +2,57 @@ package rand
 
 import (
 	crand "crypto/rand"
+	"encoding/binary"
+	"fmt"
 	mrand "math/rand"
-	"time"
-
-	tmsync "github.com/tendermint/tendermint/libs/sync"
 )
 
 const (
 	strChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" // 62 characters
 )
 
-// Rand is a prng, that is seeded with OS randomness.
-// The OS randomness is obtained from crypto/rand, however none of the provided
-// methods are suitable for cryptographic usage.
-// They all utilize math/rand's prng internally.
-//
-// All of the methods here are suitable for concurrent use.
-// This is achieved by using a mutex lock on all of the provided methods.
-type Rand struct {
-	tmsync.Mutex
-	rand *mrand.Rand
-}
-
-var grand *Rand
-
 func init() {
-	grand = NewRand()
-	grand.init()
+	Reseed()
 }
 
-func NewRand() *Rand {
-	rand := &Rand{}
-	rand.init()
-	return rand
+// NewRand returns a prng, that is seeded with OS randomness.
+// The OS randomness is obtained from crypto/rand, however, like with any math/rand.Rand
+// object none of the provided methods are suitable for cryptographic usage.
+//
+// Note that the returned instance of math/rand's Rand is not
+// suitable for concurrent use by multiple goroutines.
+//
+// For concurrent use, call Reseed to reseed math/rand's default source and
+// use math/rand's top-level convenience functions instead.
+func NewRand() *mrand.Rand {
+	seed := crandSeed()
+	// nolint:gosec // G404: Use of weak random number generator
+	return mrand.New(mrand.NewSource(seed))
 }
 
-func (r *Rand) init() {
-	bz := cRandBytes(8)
-	var seed uint64
-	for i := 0; i < 8; i++ {
-		seed |= uint64(bz[i])
-		seed <<= 8
-	}
-	r.reset(int64(seed))
-}
-
-func (r *Rand) reset(seed int64) {
-	r.rand = mrand.New(mrand.NewSource(seed)) // nolint:gosec // G404: Use of weak random number generator
-}
-
-//----------------------------------------
-// Global functions
-
-func Seed(seed int64) {
-	grand.Seed(seed)
+// Reseed conveniently re-seeds the default Source of math/rand with
+// randomness obtained from crypto/rand.
+//
+// Note that this does not make math/rand suitable for cryptographic usage.
+//
+// Use math/rand's top-level convenience functions remain suitable
+// for concurrent use by multiple goroutines.
+func Reseed() {
+	seed := crandSeed()
+	mrand.Seed(seed)
 }
 
+// Str constructs a random alphanumeric string of given length
+// from math/rand's global default Source.
 func Str(length int) string {
-	return grand.Str(length)
-}
-
-func Uint16() uint16 {
-	return grand.Uint16()
-}
-
-func Uint32() uint32 {
-	return grand.Uint32()
-}
-
-func Uint64() uint64 {
-	return grand.Uint64()
-}
-
-func Uint() uint {
-	return grand.Uint()
-}
-
-func Int16() int16 {
-	return grand.Int16()
-}
-
-func Int32() int32 {
-	return grand.Int32()
-}
-
-func Int64() int64 {
-	return grand.Int64()
-}
-
-func Int() int {
-	return grand.Int()
-}
-
-func Int31() int32 {
-	return grand.Int31()
-}
-
-func Int31n(n int32) int32 {
-	return grand.Int31n(n)
-}
-
-func Int63() int64 {
-	return grand.Int63()
-}
-
-func Int63n(n int64) int64 {
-	return grand.Int63n(n)
-}
-
-func Bool() bool {
-	return grand.Bool()
-}
-
-func Float32() float32 {
-	return grand.Float32()
-}
-
-func Float64() float64 {
-	return grand.Float64()
-}
-
-func Time() time.Time {
-	return grand.Time()
-}
-
-func Bytes(n int) []byte {
-	return grand.Bytes(n)
-}
-
-func Intn(n int) int {
-	return grand.Intn(n)
-}
-
-func Perm(n int) []int {
-	return grand.Perm(n)
-}
-
-//----------------------------------------
-// Rand methods
-
-func (r *Rand) Seed(seed int64) {
-	r.Lock()
-	r.reset(seed)
-	r.Unlock()
-}
-
-// Str constructs a random alphanumeric string of given length.
-func (r *Rand) Str(length int) string {
 	if length <= 0 {
 		return ""
 	}
 
-	chars := []byte{}
-MAIN_LOOP:
+	chars := make([]byte, 0, length)
 	for {
-		val := r.Int63()
+		// nolint:gosec // G404: Use of weak random number generator
+		val := mrand.Int63()
 		for i := 0; i < 10; i++ {
 			v := int(val & 0x3f) // rightmost 6 bits
 			if v >= 62 {         // only 62 characters in strChars
@@ -165,147 +61,29 @@ MAIN_LOOP:
 			} else {
 				chars = append(chars, strChars[v])
 				if len(chars) == length {
-					break MAIN_LOOP
+					return string(chars)
 				}
 				val >>= 6
 			}
 		}
 	}
-
-	return string(chars)
 }
 
-func (r *Rand) Uint16() uint16 {
-	return uint16(r.Uint32() & (1<<16 - 1))
-}
-
-func (r *Rand) Uint32() uint32 {
-	r.Lock()
-	u32 := r.rand.Uint32()
-	r.Unlock()
-	return u32
-}
-
-func (r *Rand) Uint64() uint64 {
-	return uint64(r.Uint32())<<32 + uint64(r.Uint32())
-}
-
-func (r *Rand) Uint() uint {
-	r.Lock()
-	i := r.rand.Int()
-	r.Unlock()
-	return uint(i)
-}
-
-func (r *Rand) Int16() int16 {
-	return int16(r.Uint32() & (1<<16 - 1))
-}
-
-func (r *Rand) Int32() int32 {
-	return int32(r.Uint32())
-}
-
-func (r *Rand) Int64() int64 {
-	return int64(r.Uint64())
-}
-
-func (r *Rand) Int() int {
-	r.Lock()
-	i := r.rand.Int()
-	r.Unlock()
-	return i
-}
-
-func (r *Rand) Int31() int32 {
-	r.Lock()
-	i31 := r.rand.Int31()
-	r.Unlock()
-	return i31
-}
-
-func (r *Rand) Int31n(n int32) int32 {
-	r.Lock()
-	i31n := r.rand.Int31n(n)
-	r.Unlock()
-	return i31n
-}
-
-func (r *Rand) Int63() int64 {
-	r.Lock()
-	i63 := r.rand.Int63()
-	r.Unlock()
-	return i63
-}
-
-func (r *Rand) Int63n(n int64) int64 {
-	r.Lock()
-	i63n := r.rand.Int63n(n)
-	r.Unlock()
-	return i63n
-}
-
-func (r *Rand) Float32() float32 {
-	r.Lock()
-	f32 := r.rand.Float32()
-	r.Unlock()
-	return f32
-}
-
-func (r *Rand) Float64() float64 {
-	r.Lock()
-	f64 := r.rand.Float64()
-	r.Unlock()
-	return f64
-}
-
-func (r *Rand) Time() time.Time {
-	return time.Unix(int64(r.Uint64()), 0)
-}
-
-// Bytes returns n random bytes generated from the internal
-// prng.
-func (r *Rand) Bytes(n int) []byte {
-	// cRandBytes isn't guaranteed to be fast so instead
-	// use random bytes generated from the internal PRNG
+// Bytes returns n random bytes generated from math/rand's global default Source.
+func Bytes(n int) []byte {
 	bs := make([]byte, n)
 	for i := 0; i < len(bs); i++ {
-		bs[i] = byte(r.Int() & 0xFF)
+		// nolint:gosec // G404: Use of weak random number generator
+		bs[i] = byte(mrand.Int() & 0xFF)
 	}
 	return bs
 }
 
-// Intn returns, as an int, a uniform pseudo-random number in the range [0, n).
-// It panics if n <= 0.
-func (r *Rand) Intn(n int) int {
-	r.Lock()
-	i := r.rand.Intn(n)
-	r.Unlock()
-	return i
-}
-
-// Bool returns a uniformly random boolean
-func (r *Rand) Bool() bool {
-	// See https://github.com/golang/go/issues/23804#issuecomment-365370418
-	// for reasoning behind computing like this
-	return r.Int63()%2 == 0
-}
-
-// Perm returns a pseudo-random permutation of n integers in [0, n).
-func (r *Rand) Perm(n int) []int {
-	r.Lock()
-	perm := r.rand.Perm(n)
-	r.Unlock()
-	return perm
-}
-
-// NOTE: This relies on the os's random number generator.
-// For real security, we should salt that with some seed.
-// See github.com/tendermint/tendermint/crypto for a more secure reader.
-func cRandBytes(numBytes int) []byte {
-	b := make([]byte, numBytes)
-	_, err := crand.Read(b)
+func crandSeed() int64 {
+	var seed int64
+	err := binary.Read(crand.Reader, binary.BigEndian, &seed)
 	if err != nil {
-		panic(err)
+		panic(fmt.Sprintf("could nor read random seed from crypto/rand: %v", err))
 	}
-	return b
+	return seed
 }

libs/rand/random_test.go

@@ -1,13 +1,7 @@
 package rand
 
 import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	mrand "math/rand"
-	"sync"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -24,71 +18,6 @@ func TestRandBytes(t *testing.T) {
 	assert.Equal(t, l, len(b))
 }
 
-func TestRandIntn(t *testing.T) {
-	n := 243
-	for i := 0; i < 100; i++ {
-		x := Intn(n)
-		assert.True(t, x < n)
-	}
-}
-
-// Test to make sure that we never call math.rand().
-// We do this by ensuring that outputs are deterministic.
-func TestDeterminism(t *testing.T) {
-	var firstOutput string
-
-	// Set math/rand's seed for the sake of debugging this test.
-	// (It isn't strictly necessary).
-	mrand.Seed(1)
-
-	for i := 0; i < 100; i++ {
-		output := testThemAll()
-		if i == 0 {
-			firstOutput = output
-		} else if firstOutput != output {
-			t.Errorf("run #%d's output was different from first run.\nfirst: %v\nlast: %v",
-				i, firstOutput, output)
-		}
-	}
-}
-
-func testThemAll() string {
-
-	// Such determinism.
-	grand.reset(1)
-
-	// Use it.
-	out := new(bytes.Buffer)
-	perm := Perm(10)
-	blob, _ := json.Marshal(perm)
-	fmt.Fprintf(out, "perm: %s\n", blob)
-	fmt.Fprintf(out, "randInt: %d\n", Int())
-	fmt.Fprintf(out, "randUint: %d\n", Uint())
-	fmt.Fprintf(out, "randIntn: %d\n", Intn(97))
-	fmt.Fprintf(out, "randInt31: %d\n", Int31())
-	fmt.Fprintf(out, "randInt32: %d\n", Int32())
-	fmt.Fprintf(out, "randInt63: %d\n", Int63())
-	fmt.Fprintf(out, "randInt64: %d\n", Int64())
-	fmt.Fprintf(out, "randUint32: %d\n", Uint32())
-	fmt.Fprintf(out, "randUint64: %d\n", Uint64())
-	return out.String()
-}
-
-func TestRngConcurrencySafety(t *testing.T) {
-	var wg sync.WaitGroup
-	for i := 0; i < 100; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			_ = Uint64()
-			<-time.After(time.Millisecond * time.Duration(Intn(100)))
-			_ = Perm(3)
-		}()
-	}
-	wg.Wait()
-}
-
 func BenchmarkRandBytes10B(b *testing.B) {
 	benchmarkRandBytes(b, 10)
 }

libs/tempfile/tempfile_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	mrand "math/rand"
 	"os"
 	testing "testing"
 
@@ -16,8 +17,8 @@ import (
 
 func TestWriteFileAtomic(t *testing.T) {
 	var (
-		data             = []byte(tmrand.Str(tmrand.Intn(2048)))
-		old              = tmrand.Bytes(tmrand.Intn(2048))
+		data             = []byte(tmrand.Str(mrand.Intn(2048)))
+		old              = tmrand.Bytes(mrand.Intn(2048))
 		perm os.FileMode = 0600
 	)
 

libs/test/mutate.go

@@ -1,7 +1,8 @@
+// nolint:gosec // G404: Use of weak random number generator
 package test
 
 import (
-	tmrand "github.com/tendermint/tendermint/libs/rand"
+	mrand "math/rand"
 )
 
 // Contract: !bytes.Equal(input, output) && len(input) >= len(output)
@@ -17,11 +18,11 @@ func MutateByteSlice(bytez []byte) []byte {
 	bytez = mBytez
 
 	// Try a random mutation
-	switch tmrand.Int() % 2 {
+	switch mrand.Int() % 2 {
 	case 0: // Mutate a single byte
-		bytez[tmrand.Int()%len(bytez)] += byte(tmrand.Int()%255 + 1)
+		bytez[mrand.Int()%len(bytez)] += byte(mrand.Int()%255 + 1)
 	case 1: // Remove an arbitrary byte
-		pos := tmrand.Int() % len(bytez)
+		pos := mrand.Int() % len(bytez)
 		bytez = append(bytez[:pos], bytez[pos+1:]...)
 	}
 	return bytez

light/proxy/routes.go

@@ -18,36 +18,36 @@ func RPCRoutes(c *lrpc.Client) map[string]*rpcserver.RPCFunc {
 		"unsubscribe_all": rpcserver.NewWSRPCFunc(c.UnsubscribeAllWS, ""),
 
 		// info API
-		"health":               rpcserver.NewRPCFunc(makeHealthFunc(c), ""),
-		"status":               rpcserver.NewRPCFunc(makeStatusFunc(c), ""),
-		"net_info":             rpcserver.NewRPCFunc(makeNetInfoFunc(c), ""),
-		"blockchain":           rpcserver.NewRPCFunc(makeBlockchainInfoFunc(c), "minHeight,maxHeight"),
-		"genesis":              rpcserver.NewRPCFunc(makeGenesisFunc(c), ""),
-		"block":                rpcserver.NewRPCFunc(makeBlockFunc(c), "height"),
-		"block_by_hash":        rpcserver.NewRPCFunc(makeBlockByHashFunc(c), "hash"),
-		"block_results":        rpcserver.NewRPCFunc(makeBlockResultsFunc(c), "height"),
-		"commit":               rpcserver.NewRPCFunc(makeCommitFunc(c), "height"),
-		"tx":                   rpcserver.NewRPCFunc(makeTxFunc(c), "hash,prove"),
-		"tx_search":            rpcserver.NewRPCFunc(makeTxSearchFunc(c), "query,prove,page,per_page,order_by"),
-		"block_search":         rpcserver.NewRPCFunc(makeBlockSearchFunc(c), "query,page,per_page,order_by"),
-		"validators":           rpcserver.NewRPCFunc(makeValidatorsFunc(c), "height,page,per_page"),
-		"dump_consensus_state": rpcserver.NewRPCFunc(makeDumpConsensusStateFunc(c), ""),
-		"consensus_state":      rpcserver.NewRPCFunc(makeConsensusStateFunc(c), ""),
-		"consensus_params":     rpcserver.NewRPCFunc(makeConsensusParamsFunc(c), "height"),
-		"unconfirmed_txs":      rpcserver.NewRPCFunc(makeUnconfirmedTxsFunc(c), "limit"),
-		"num_unconfirmed_txs":  rpcserver.NewRPCFunc(makeNumUnconfirmedTxsFunc(c), ""),
+		"health":               rpcserver.NewRPCFunc(makeHealthFunc(c), "", false),
+		"status":               rpcserver.NewRPCFunc(makeStatusFunc(c), "", false),
+		"net_info":             rpcserver.NewRPCFunc(makeNetInfoFunc(c), "", false),
+		"blockchain":           rpcserver.NewRPCFunc(makeBlockchainInfoFunc(c), "minHeight,maxHeight", true),
+		"genesis":              rpcserver.NewRPCFunc(makeGenesisFunc(c), "", true),
+		"block":                rpcserver.NewRPCFunc(makeBlockFunc(c), "height", true),
+		"block_by_hash":        rpcserver.NewRPCFunc(makeBlockByHashFunc(c), "hash", true),
+		"block_results":        rpcserver.NewRPCFunc(makeBlockResultsFunc(c), "height", true),
+		"commit":               rpcserver.NewRPCFunc(makeCommitFunc(c), "height", true),
+		"tx":                   rpcserver.NewRPCFunc(makeTxFunc(c), "hash,prove", true),
+		"tx_search":            rpcserver.NewRPCFunc(makeTxSearchFunc(c), "query,prove,page,per_page,order_by", false),
+		"block_search":         rpcserver.NewRPCFunc(makeBlockSearchFunc(c), "query,page,per_page,order_by", false),
+		"validators":           rpcserver.NewRPCFunc(makeValidatorsFunc(c), "height,page,per_page", true),
+		"dump_consensus_state": rpcserver.NewRPCFunc(makeDumpConsensusStateFunc(c), "", false),
+		"consensus_state":      rpcserver.NewRPCFunc(makeConsensusStateFunc(c), "", false),
+		"consensus_params":     rpcserver.NewRPCFunc(makeConsensusParamsFunc(c), "height", true),
+		"unconfirmed_txs":      rpcserver.NewRPCFunc(makeUnconfirmedTxsFunc(c), "limit", false),
+		"num_unconfirmed_txs":  rpcserver.NewRPCFunc(makeNumUnconfirmedTxsFunc(c), "", false),
 
 		// tx broadcast API
-		"broadcast_tx_commit": rpcserver.NewRPCFunc(makeBroadcastTxCommitFunc(c), "tx"),
-		"broadcast_tx_sync":   rpcserver.NewRPCFunc(makeBroadcastTxSyncFunc(c), "tx"),
-		"broadcast_tx_async":  rpcserver.NewRPCFunc(makeBroadcastTxAsyncFunc(c), "tx"),
+		"broadcast_tx_commit": rpcserver.NewRPCFunc(makeBroadcastTxCommitFunc(c), "tx", false),
+		"broadcast_tx_sync":   rpcserver.NewRPCFunc(makeBroadcastTxSyncFunc(c), "tx", false),
+		"broadcast_tx_async":  rpcserver.NewRPCFunc(makeBroadcastTxAsyncFunc(c), "tx", false),
 
 		// abci API
-		"abci_query": rpcserver.NewRPCFunc(makeABCIQueryFunc(c), "path,data,height,prove"),
-		"abci_info":  rpcserver.NewRPCFunc(makeABCIInfoFunc(c), ""),
+		"abci_query": rpcserver.NewRPCFunc(makeABCIQueryFunc(c), "path,data,height,prove", false),
+		"abci_info":  rpcserver.NewRPCFunc(makeABCIInfoFunc(c), "", true),
 
 		// evidence API
-		"broadcast_evidence": rpcserver.NewRPCFunc(makeBroadcastEvidenceFunc(c), "evidence"),
+		"broadcast_evidence": rpcserver.NewRPCFunc(makeBroadcastEvidenceFunc(c), "evidence", false),
 	}
 }
 

node/node.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math"
 	"net"
 	"net/http"
 	_ "net/http/pprof" // nolint: gosec // securely exposed on separate, optional port
@@ -43,9 +44,8 @@ import (
 	"github.com/tendermint/tendermint/state/indexer"
 	blockidxkv "github.com/tendermint/tendermint/state/indexer/block/kv"
 	blockidxnull "github.com/tendermint/tendermint/state/indexer/block/null"
-	"github.com/tendermint/tendermint/state/txindex"
-	"github.com/tendermint/tendermint/state/txindex/kv"
-	"github.com/tendermint/tendermint/state/txindex/null"
+	"github.com/tendermint/tendermint/state/indexer/tx/kv"
+	"github.com/tendermint/tendermint/state/indexer/tx/null"
 	"github.com/tendermint/tendermint/statesync"
 	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
@@ -95,6 +95,7 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) {
 	}
 	if config.Mode == cfg.ModeSeed {
 		return NewSeedNode(config,
+			DefaultDBProvider,
 			nodeKey,
 			DefaultGenesisDocProviderFunc(config),
 			logger,
@@ -224,9 +225,9 @@ type Node struct {
 	evidencePool      *evidence.Pool // tracking evidence
 	proxyApp          proxy.AppConns // connection to the application
 	rpcListeners      []net.Listener // rpc servers
-	txIndexer         txindex.TxIndexer
+	txIndexer         indexer.TxIndexer
 	blockIndexer      indexer.BlockIndexer
-	indexerService    *txindex.IndexerService
+	indexerService    *indexer.Service
 	prometheusSrv     *http.Server
 }
 
@@ -239,10 +240,6 @@ func initDBs(config *cfg.Config, dbProvider DBProvider) (blockStore *store.Block
 	blockStore = store.NewBlockStore(blockStoreDB)
 
 	stateDB, err = dbProvider(&DBContext{"state", config})
-	if err != nil {
-		return
-	}
-
 	return
 }
 
@@ -269,10 +266,10 @@ func createAndStartIndexerService(
 	dbProvider DBProvider,
 	eventBus *types.EventBus,
 	logger log.Logger,
-) (*txindex.IndexerService, txindex.TxIndexer, indexer.BlockIndexer, error) {
+) (*indexer.Service, indexer.TxIndexer, indexer.BlockIndexer, error) {
 
 	var (
-		txIndexer    txindex.TxIndexer
+		txIndexer    indexer.TxIndexer
 		blockIndexer indexer.BlockIndexer
 	)
 
@@ -290,7 +287,7 @@ func createAndStartIndexerService(
 		blockIndexer = &blockidxnull.BlockerIndexer{}
 	}
 
-	indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus)
+	indexerService := indexer.NewIndexerService(txIndexer, blockIndexer, eventBus)
 	indexerService.SetLogger(logger.With("module", "txindex"))
 
 	if err := indexerService.Start(); err != nil {
@@ -387,7 +384,7 @@ func createMempoolReactor(
 		peerUpdates *p2p.PeerUpdates
 	)
 
-	if config.P2P.UseNewP2P {
+	if config.P2P.DisableLegacy {
 		channels = makeChannelsFromShims(router, channelShims)
 		peerUpdates = peerManager.Subscribe()
 	} else {
@@ -438,7 +435,7 @@ func createEvidenceReactor(
 		peerUpdates *p2p.PeerUpdates
 	)
 
-	if config.P2P.UseNewP2P {
+	if config.P2P.DisableLegacy {
 		channels = makeChannelsFromShims(router, evidence.ChannelShims)
 		peerUpdates = peerManager.Subscribe()
 	} else {
@@ -479,7 +476,7 @@ func createBlockchainReactor(
 			peerUpdates *p2p.PeerUpdates
 		)
 
-		if config.P2P.UseNewP2P {
+		if config.P2P.DisableLegacy {
 			channels = makeChannelsFromShims(router, bcv0.ChannelShims)
 			peerUpdates = peerManager.Subscribe()
 		} else {
@@ -545,7 +542,7 @@ func createConsensusReactor(
 		peerUpdates *p2p.PeerUpdates
 	)
 
-	if config.P2P.UseNewP2P {
+	if config.P2P.DisableLegacy {
 		channels = makeChannelsFromShims(router, cs.ChannelShims)
 		peerUpdates = peerManager.Subscribe()
 	} else {
@@ -583,9 +580,35 @@ func createTransport(logger log.Logger, config *cfg.Config) *p2p.MConnTransport
 	)
 }
 
-func createPeerManager(config *cfg.Config, p2pLogger log.Logger, nodeID p2p.NodeID) (*p2p.PeerManager, error) {
+func createPeerManager(
+	config *cfg.Config,
+	dbProvider DBProvider,
+	p2pLogger log.Logger,
+	nodeID p2p.NodeID) (*p2p.PeerManager, error) {
+	var maxConns uint16
+	switch {
+	case config.P2P.MaxConnections > 0:
+		maxConns = config.P2P.MaxConnections
+
+	case config.P2P.MaxNumInboundPeers > 0 && config.P2P.MaxNumOutboundPeers > 0:
+		x := config.P2P.MaxNumInboundPeers + config.P2P.MaxNumOutboundPeers
+		if x > math.MaxUint16 {
+			return nil, fmt.Errorf(
+				"max inbound peers (%d) + max outbound peers (%d) exceeds maximum (%d)",
+				config.P2P.MaxNumInboundPeers,
+				config.P2P.MaxNumOutboundPeers,
+				math.MaxUint16,
+			)
+		}
+
+		maxConns = uint16(x)
+
+	default:
+		maxConns = 64
+	}
+
 	options := p2p.PeerManagerOptions{
-		MaxConnected:           64,
+		MaxConnected:           maxConns,
 		MaxConnectedUpgrade:    4,
 		MaxPeers:               1000,
 		MinRetryTime:           100 * time.Millisecond,
@@ -605,13 +628,25 @@ func createPeerManager(config *cfg.Config, p2pLogger log.Logger, nodeID p2p.Node
 		options.PersistentPeers = append(options.PersistentPeers, address.NodeID)
 	}
 
-	peerManager, err := p2p.NewPeerManager(nodeID, dbm.NewMemDB(), options)
+	for _, p := range splitAndTrimEmpty(config.P2P.BootstrapPeers, ",", " ") {
+		address, err := p2p.ParseNodeAddress(p)
+		if err != nil {
+			return nil, fmt.Errorf("invalid peer address %q: %w", p, err)
+		}
+		peers = append(peers, address)
+	}
+
+	peerDB, err := dbProvider(&DBContext{"peerstore", config})
+	if err != nil {
+		return nil, err
+	}
+	peerManager, err := p2p.NewPeerManager(nodeID, peerDB, options)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create peer manager: %w", err)
 	}
 
 	for _, peer := range peers {
-		if err := peerManager.Add(peer); err != nil {
+		if _, err := peerManager.Add(peer); err != nil {
 			return nil, fmt.Errorf("failed to add peer %q: %w", peer, err)
 		}
 	}
@@ -853,6 +888,7 @@ func startStateSync(ssR *statesync.Reactor, bcR fastSyncReactor, conR *cs.Reacto
 
 // NewSeedNode returns a new seed node, containing only p2p, pex reactor
 func NewSeedNode(config *cfg.Config,
+	dbProvider DBProvider,
 	nodeKey p2p.NodeKey,
 	genesisDocProvider GenesisDocProvider,
 	logger log.Logger,
@@ -897,7 +933,7 @@ func NewSeedNode(config *cfg.Config,
 		return nil, fmt.Errorf("could not create addrbook: %w", err)
 	}
 
-	peerManager, err := createPeerManager(config, p2pLogger, nodeKey.ID)
+	peerManager, err := createPeerManager(config, dbProvider, p2pLogger, nodeKey.ID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create peer manager: %w", err)
 	}
@@ -1058,7 +1094,7 @@ func NewNode(config *cfg.Config,
 	p2pLogger := logger.With("module", "p2p")
 	transport := createTransport(p2pLogger, config)
 
-	peerManager, err := createPeerManager(config, p2pLogger, nodeKey.ID)
+	peerManager, err := createPeerManager(config, dbProvider, p2pLogger, nodeKey.ID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create peer manager: %w", err)
 	}
@@ -1138,7 +1174,7 @@ func NewNode(config *cfg.Config,
 
 	stateSyncReactorShim = p2p.NewReactorShim(logger.With("module", "statesync"), "StateSyncShim", statesync.ChannelShims)
 
-	if config.P2P.UseNewP2P {
+	if config.P2P.DisableLegacy {
 		channels = makeChannelsFromShims(router, statesync.ChannelShims)
 		peerUpdates = peerManager.Subscribe()
 	} else {
@@ -1305,9 +1341,9 @@ func (n *Node) OnStart() error {
 
 	n.isListening = true
 
-	n.Logger.Info("p2p service", "legacy_enabled", !n.config.P2P.UseNewP2P)
+	n.Logger.Info("p2p service", "legacy_enabled", !n.config.P2P.DisableLegacy)
 
-	if n.config.P2P.UseNewP2P {
+	if n.config.P2P.DisableLegacy {
 		err = n.router.Start()
 	} else {
 		err = n.sw.Start()
@@ -1345,7 +1381,7 @@ func (n *Node) OnStart() error {
 		}
 	}
 
-	if n.config.P2P.UseNewP2P && n.pexReactorV2 != nil {
+	if n.config.P2P.DisableLegacy && n.pexReactorV2 != nil {
 		if err := n.pexReactorV2.Start(); err != nil {
 			return err
 		}
@@ -1388,7 +1424,6 @@ func (n *Node) OnStop() {
 	}
 
 	if n.config.Mode != cfg.ModeSeed {
-
 		// now stop the reactors
 		if n.config.FastSync.Version == "v0" {
 			// Stop the real blockchain reactor separately since the switch uses the shim.
@@ -1418,13 +1453,13 @@ func (n *Node) OnStop() {
 		}
 	}
 
-	if n.config.P2P.UseNewP2P && n.pexReactorV2 != nil {
+	if n.config.P2P.DisableLegacy && n.pexReactorV2 != nil {
 		if err := n.pexReactorV2.Stop(); err != nil {
 			n.Logger.Error("failed to stop the PEX v2 reactor", "err", err)
 		}
 	}
 
-	if n.config.P2P.UseNewP2P {
+	if n.config.P2P.DisableLegacy {
 		if err := n.router.Stop(); err != nil {
 			n.Logger.Error("failed to stop router", "err", err)
 		}
@@ -1708,7 +1743,7 @@ func (n *Node) Config() *cfg.Config {
 }
 
 // TxIndexer returns the Node's TxIndexer.
-func (n *Node) TxIndexer() txindex.TxIndexer {
+func (n *Node) TxIndexer() indexer.TxIndexer {
 	return n.txIndexer
 }
 
@@ -1732,7 +1767,7 @@ func (n *Node) NodeInfo() p2p.NodeInfo {
 func makeNodeInfo(
 	config *cfg.Config,
 	nodeKey p2p.NodeKey,
-	txIndexer txindex.TxIndexer,
+	txIndexer indexer.TxIndexer,
 	genDoc *types.GenesisDoc,
 	state sm.State,
 ) (p2p.NodeInfo, error) {
@@ -1960,7 +1995,7 @@ func getRouterConfig(conf *cfg.Config, proxyApp proxy.AppConns) p2p.RouterOption
 	}
 
 	if conf.P2P.MaxNumInboundPeers > 0 {
-		opts.MaxIncommingConnectionsPerIP = uint(conf.P2P.MaxNumInboundPeers)
+		opts.MaxIncomingConnectionAttempts = conf.P2P.MaxIncomingConnectionAttempts
 	}
 
 	if conf.FilterPeers && proxyApp != nil {

node/node_test.go

@@ -533,6 +533,7 @@ func TestNodeNewSeedNode(t *testing.T) {
 	require.NoError(t, err)
 
 	n, err := NewSeedNode(config,
+		DefaultDBProvider,
 		nodeKey,
 		DefaultGenesisDocProviderFunc(config),
 		log.TestingLogger(),

p2p/conn/secret_connection.go

@@ -167,9 +167,11 @@ func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (*
 	}
 
 	remPubKey, remSignature := authSigMsg.Key, authSigMsg.Sig
+
 	if _, ok := remPubKey.(ed25519.PubKey); !ok {
 		return nil, fmt.Errorf("expected ed25519 pubkey, got %T", remPubKey)
 	}
+
 	if !remPubKey.VerifySignature(challenge[:], remSignature) {
 		return nil, errors.New("challenge verification failed")
 	}

p2p/conn/secret_connection_test.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	mrand "math/rand"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -114,8 +115,8 @@ func TestSecretConnectionReadWrite(t *testing.T) {
 
 	// Pre-generate the things to write (for foo & bar)
 	for i := 0; i < 100; i++ {
-		fooWrites = append(fooWrites, tmrand.Str((tmrand.Int()%(dataMaxSize*5))+1))
-		barWrites = append(barWrites, tmrand.Str((tmrand.Int()%(dataMaxSize*5))+1))
+		fooWrites = append(fooWrites, tmrand.Str((mrand.Int()%(dataMaxSize*5))+1))
+		barWrites = append(barWrites, tmrand.Str((mrand.Int()%(dataMaxSize*5))+1))
 	}
 
 	// A helper that will run with (fooConn, fooWrites, fooReads) and vice versa
@@ -276,11 +277,10 @@ func TestNonEd25519Pubkey(t *testing.T) {
 	var fooPrvKey = ed25519.GenPrivKey()
 	var barPrvKey = sr25519.GenPrivKey()
 
-	go MakeSecretConnection(fooConn, fooPrvKey) //nolint:errcheck // ignore for tests
+	go MakeSecretConnection(barConn, barPrvKey) //nolint:errcheck // ignore for tests
 
-	_, err := MakeSecretConnection(barConn, barPrvKey)
+	_, err := MakeSecretConnection(fooConn, fooPrvKey)
 	require.Error(t, err)
-	assert.Contains(t, err.Error(), "is not supported")
 }
 
 func writeLots(t *testing.T, wg *sync.WaitGroup, conn io.Writer, txt string, n int) {
@@ -313,7 +313,7 @@ func createGoldenTestVectors(t *testing.T) string {
 		randSecret := new([32]byte)
 		copy((*randSecret)[:], randSecretVector)
 		data += hex.EncodeToString((*randSecret)[:]) + ","
-		locIsLeast := tmrand.Bool()
+		locIsLeast := mrand.Int63()%2 == 0
 		data += strconv.FormatBool(locIsLeast) + ","
 		recvSecret, sendSecret := deriveSecrets(randSecret, locIsLeast)
 		data += hex.EncodeToString((*recvSecret)[:]) + ","
@@ -413,7 +413,7 @@ func BenchmarkWriteSecretConnection(b *testing.B) {
 
 	b.StartTimer()
 	for i := 0; i < b.N; i++ {
-		idx := tmrand.Intn(len(fooWriteBytes))
+		idx := mrand.Intn(len(fooWriteBytes))
 		_, err := fooSecConn.Write(fooWriteBytes[idx])
 		if err != nil {
 			b.Errorf("failed to write to fooSecConn: %v", err)
@@ -447,7 +447,7 @@ func BenchmarkReadSecretConnection(b *testing.B) {
 	}
 	go func() {
 		for i := 0; i < b.N; i++ {
-			idx := tmrand.Intn(len(fooWriteBytes))
+			idx := mrand.Intn(len(fooWriteBytes))
 			_, err := fooSecConn.Write(fooWriteBytes[idx])
 			if err != nil {
 				b.Errorf("failed to write to fooSecConn: %v, %v,%v", err, i, b.N)

p2p/p2ptest/network.go

@@ -30,6 +30,12 @@ type Network struct {
 type NetworkOptions struct {
 	NumNodes   int
 	BufferSize int
+	NodeOpts   NodeOptions
+}
+
+type NodeOptions struct {
+	MaxPeers     uint16
+	MaxConnected uint16
 }
 
 func (opts *NetworkOptions) setDefaults() {
@@ -50,7 +56,7 @@ func MakeNetwork(t *testing.T, opts NetworkOptions) *Network {
 	}
 
 	for i := 0; i < opts.NumNodes; i++ {
-		node := network.MakeNode(t)
+		node := network.MakeNode(t, opts.NodeOpts)
 		network.Nodes[node.NodeID] = node
 	}
 
@@ -81,7 +87,9 @@ func (n *Network) Start(t *testing.T) {
 		for _, targetAddress := range dialQueue[i+1:] { // nodes <i already connected
 			targetNode := n.Nodes[targetAddress.NodeID]
 			targetSub := subs[targetAddress.NodeID]
-			require.NoError(t, sourceNode.PeerManager.Add(targetAddress))
+			added, err := sourceNode.PeerManager.Add(targetAddress)
+			require.NoError(t, err)
+			require.True(t, added)
 
 			select {
 			case peerUpdate := <-sourceSub.Updates():
@@ -107,7 +115,9 @@ func (n *Network) Start(t *testing.T) {
 
 			// Add the address to the target as well, so it's able to dial the
 			// source back if that's even necessary.
-			require.NoError(t, targetNode.PeerManager.Add(sourceAddress))
+			added, err = targetNode.PeerManager.Add(sourceAddress)
+			require.NoError(t, err)
+			require.True(t, added)
 		}
 	}
 }
@@ -214,7 +224,7 @@ type Node struct {
 // MakeNode creates a new Node configured for the network with a
 // running peer manager, but does not add it to the existing
 // network. Callers are responsible for updating peering relationships.
-func (n *Network) MakeNode(t *testing.T) *Node {
+func (n *Network) MakeNode(t *testing.T, opts NodeOptions) *Node {
 	privKey := ed25519.GenPrivKey()
 	nodeID := p2p.NodeIDFromPubKey(privKey.PubKey())
 	nodeInfo := p2p.NodeInfo{
@@ -230,6 +240,8 @@ func (n *Network) MakeNode(t *testing.T) *Node {
 		MinRetryTime:    10 * time.Millisecond,
 		MaxRetryTime:    100 * time.Millisecond,
 		RetryTimeJitter: time.Millisecond,
+		MaxPeers:        opts.MaxPeers,
+		MaxConnected:    opts.MaxConnected,
 	})
 	require.NoError(t, err)
 
@@ -294,11 +306,25 @@ func (n *Node) MakeChannelNoCleanup(
 // MakePeerUpdates opens a peer update subscription, with automatic cleanup.
 // It checks that all updates have been consumed during cleanup.
 func (n *Node) MakePeerUpdates(t *testing.T) *p2p.PeerUpdates {
+	t.Helper()
 	sub := n.PeerManager.Subscribe()
 	t.Cleanup(func() {
+		t.Helper()
 		RequireNoUpdates(t, sub)
 		sub.Close()
 	})
 
 	return sub
 }
+
+// MakePeerUpdatesNoRequireEmpty opens a peer update subscription, with automatic cleanup.
+// It does *not* check that all updates have been consumed, but will
+// close the update channel.
+func (n *Node) MakePeerUpdatesNoRequireEmpty(t *testing.T) *p2p.PeerUpdates {
+	sub := n.PeerManager.Subscribe()
+	t.Cleanup(func() {
+		sub.Close()
+	})
+
+	return sub
+}

p2p/p2ptest/require.go

@@ -93,6 +93,7 @@ func RequireSendReceive(
 
 // RequireNoUpdates requires that a PeerUpdates subscription is empty.
 func RequireNoUpdates(t *testing.T, peerUpdates *p2p.PeerUpdates) {
+	t.Helper()
 	select {
 	case update := <-peerUpdates.Updates():
 		require.Fail(t, "unexpected peer updates", "got %v", update)

p2p/peermanager.go

@@ -384,13 +384,14 @@ func (m *PeerManager) prunePeers() error {
 }
 
 // Add adds a peer to the manager, given as an address. If the peer already
-// exists, the address is added to it if not already present.
-func (m *PeerManager) Add(address NodeAddress) error {
+// exists, the address is added to it if it isn't already present. This will push
+// low scoring peers out of the address book if it exceeds the maximum size.
+func (m *PeerManager) Add(address NodeAddress) (bool, error) {
 	if err := address.Validate(); err != nil {
-		return err
+		return false, err
 	}
 	if address.NodeID == m.selfID {
-		return fmt.Errorf("can't add self (%v) to peer store", m.selfID)
+		return false, fmt.Errorf("can't add self (%v) to peer store", m.selfID)
 	}
 
 	m.mtx.Lock()
@@ -400,17 +401,32 @@ func (m *PeerManager) Add(address NodeAddress) error {
 	if !ok {
 		peer = m.newPeerInfo(address.NodeID)
 	}
-	if _, ok := peer.AddressInfo[address]; !ok {
-		peer.AddressInfo[address] = &peerAddressInfo{Address: address}
+	_, ok = peer.AddressInfo[address]
+	// if we already have the peer address, there's no need to continue
+	if ok {
+		return false, nil
 	}
+
+	// else add the new address
+	peer.AddressInfo[address] = &peerAddressInfo{Address: address}
 	if err := m.store.Set(peer); err != nil {
-		return err
+		return false, err
 	}
 	if err := m.prunePeers(); err != nil {
-		return err
+		return true, err
 	}
 	m.dialWaker.Wake()
-	return nil
+	return true, nil
+}
+
+// PeerRatio returns the ratio of peer addresses stored to the maximum size.
+func (m *PeerManager) PeerRatio() float64 {
+	if m.options.MaxPeers == 0 {
+		return 0
+	}
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	return float64(m.store.Size()) / float64(m.options.MaxPeers)
 }
 
 // DialNext finds an appropriate peer address to dial, and marks it as dialing.

p2p/peermanager_scoring_test.go

@@ -23,7 +23,9 @@ func TestPeerScoring(t *testing.T) {
 
 	// create a fake node
 	id := NodeID(strings.Repeat("a1", 20))
-	require.NoError(t, peerManager.Add(NodeAddress{NodeID: id, Protocol: "memory"}))
+	added, err := peerManager.Add(NodeAddress{NodeID: id, Protocol: "memory"})
+	require.NoError(t, err)
+	require.True(t, added)
 
 	t.Run("Synchronous", func(t *testing.T) {
 		// update the manager and make sure it's correct

p2p/peermanager_test.go

@@ -156,7 +156,9 @@ func TestNewPeerManager_Persistence(t *testing.T) {
 	defer peerManager.Close()
 
 	for _, addr := range append(append(aAddresses, bAddresses...), cAddresses...) {
-		require.NoError(t, peerManager.Add(addr))
+		added, err := peerManager.Add(addr)
+		require.NoError(t, err)
+		require.True(t, added)
 	}
 
 	require.ElementsMatch(t, aAddresses, peerManager.Addresses(aID))
@@ -198,8 +200,12 @@ func TestNewPeerManager_SelfIDChange(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, db, p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
-	require.NoError(t, peerManager.Add(b))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.ElementsMatch(t, []p2p.NodeID{a.NodeID, b.NodeID}, peerManager.Peers())
 	peerManager.Close()
 
@@ -228,32 +234,41 @@ func TestPeerManager_Add(t *testing.T) {
 		{Protocol: "memory", NodeID: aID},
 	}
 	for _, addr := range aAddresses {
-		err = peerManager.Add(addr)
+		added, err := peerManager.Add(addr)
 		require.NoError(t, err)
+		require.True(t, added)
 	}
 	require.ElementsMatch(t, aAddresses, peerManager.Addresses(aID))
 
 	// Adding a different peer should be fine.
 	bAddress := p2p.NodeAddress{Protocol: "tcp", NodeID: bID, Hostname: "localhost"}
-	require.NoError(t, peerManager.Add(bAddress))
+	added, err := peerManager.Add(bAddress)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Equal(t, []p2p.NodeAddress{bAddress}, peerManager.Addresses(bID))
 	require.ElementsMatch(t, aAddresses, peerManager.Addresses(aID))
 
 	// Adding an existing address again should be a noop.
-	require.NoError(t, peerManager.Add(aAddresses[0]))
+	added, err = peerManager.Add(aAddresses[0])
+	require.NoError(t, err)
+	require.False(t, added)
 	require.ElementsMatch(t, aAddresses, peerManager.Addresses(aID))
 
 	// Adding a third peer with MaxPeers=2 should cause bID, which is
 	// the lowest-scored peer (not in PersistentPeers), to be removed.
-	require.NoError(t, peerManager.Add(p2p.NodeAddress{
-		Protocol: "tcp", NodeID: cID, Hostname: "localhost"}))
+	added, err = peerManager.Add(p2p.NodeAddress{
+		Protocol: "tcp", NodeID: cID, Hostname: "localhost"})
+	require.NoError(t, err)
+	require.True(t, added)
 	require.ElementsMatch(t, []p2p.NodeID{aID, cID}, peerManager.Peers())
 
 	// Adding an invalid address should error.
-	require.Error(t, peerManager.Add(p2p.NodeAddress{Path: "foo"}))
+	_, err = peerManager.Add(p2p.NodeAddress{Path: "foo"})
+	require.Error(t, err)
 
 	// Adding self should error
-	require.Error(t, peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}))
+	_, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID})
+	require.Error(t, err)
 }
 
 func TestPeerManager_DialNext(t *testing.T) {
@@ -263,7 +278,9 @@ func TestPeerManager_DialNext(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add an address. DialNext should return it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	address, err := peerManager.DialNext(ctx)
 	require.NoError(t, err)
 	require.Equal(t, a, address)
@@ -287,7 +304,9 @@ func TestPeerManager_DialNext_Retry(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), options)
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 
 	// Do five dial retries (six dials total). The retry time should double for
 	// each failure. At the forth retry, MaxRetryTime should kick in.
@@ -330,7 +349,9 @@ func TestPeerManager_DialNext_WakeOnAdd(t *testing.T) {
 	// Spawn a goroutine to add a peer after a delay.
 	go func() {
 		time.Sleep(200 * time.Millisecond)
-		require.NoError(t, peerManager.Add(a))
+		added, err := peerManager.Add(a)
+		require.NoError(t, err)
+		require.True(t, added)
 	}()
 
 	// This will block until peer is added above.
@@ -351,13 +372,17 @@ func TestPeerManager_DialNext_WakeOnDialFailed(t *testing.T) {
 	b := p2p.NodeAddress{Protocol: "memory", NodeID: p2p.NodeID(strings.Repeat("b", 40))}
 
 	// Add and dial a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 
 	// Add b. We shouldn't be able to dial it, due to MaxConnected.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Zero(t, dial)
@@ -384,7 +409,9 @@ func TestPeerManager_DialNext_WakeOnDialFailedRetry(t *testing.T) {
 	a := p2p.NodeAddress{Protocol: "memory", NodeID: p2p.NodeID(strings.Repeat("a", 40))}
 
 	// Add a, dial it, and mark it a failure. This will start a retry timer.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
@@ -407,8 +434,9 @@ func TestPeerManager_DialNext_WakeOnDisconnected(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	err = peerManager.Add(a)
+	added, err := peerManager.Add(a)
 	require.NoError(t, err)
+	require.True(t, added)
 	err = peerManager.Accepted(a.NodeID)
 	require.NoError(t, err)
 
@@ -439,20 +467,26 @@ func TestPeerManager_TryDialNext_MaxConnected(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a and connect to it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Add b and start dialing it.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
 
 	// At this point, adding c will not allow dialing it.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Zero(t, dial)
@@ -480,28 +514,36 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a and connect to it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Add b and start dialing it.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
 
 	// Even though we are at capacity, we should be allowed to dial c for an
 	// upgrade of a, since it's higher-scored.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, c, dial)
 
 	// However, since we're using all upgrade slots now, we can't add and dial
 	// d, even though it's also higher-scored.
-	require.NoError(t, peerManager.Add(d))
+	added, err = peerManager.Add(d)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Zero(t, dial)
@@ -526,7 +568,9 @@ func TestPeerManager_TryDialNext_MaxConnectedUpgrade(t *testing.T) {
 	// should not be allowed to dial e even though there are upgrade slots,
 	// because there are no lower-scored nodes that can be upgraded.
 	require.NoError(t, peerManager.Disconnected(b.NodeID))
-	require.NoError(t, peerManager.Add(e))
+	added, err = peerManager.Add(e)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Zero(t, dial)
@@ -545,21 +589,27 @@ func TestPeerManager_TryDialNext_UpgradeReservesPeer(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a and connect to it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Add b and start dialing it. This will claim a for upgrading.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
 
 	// Adding c and dialing it will fail, because a is the only connected
 	// peer that can be upgraded, and b is already trying to upgrade it.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Empty(t, dial)
@@ -579,13 +629,17 @@ func TestPeerManager_TryDialNext_DialingConnected(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a and dial it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 
 	// Adding a's TCP address will not dispense a, since it's already dialing.
-	require.NoError(t, peerManager.Add(aTCP))
+	added, err = peerManager.Add(aTCP)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Zero(t, dial)
@@ -597,7 +651,9 @@ func TestPeerManager_TryDialNext_DialingConnected(t *testing.T) {
 	require.Zero(t, dial)
 
 	// Adding b and accepting a connection from it will not dispense it either.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(bID))
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
@@ -618,7 +674,9 @@ func TestPeerManager_TryDialNext_Multiple(t *testing.T) {
 	require.NoError(t, err)
 
 	for _, address := range addresses {
-		require.NoError(t, peerManager.Add(address))
+		added, err := peerManager.Add(address)
+		require.NoError(t, err)
+		require.True(t, added)
 	}
 
 	// All addresses should be dispensed as long as dialing them has failed.
@@ -648,7 +706,9 @@ func TestPeerManager_DialFailed(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 
 	// Dialing and then calling DialFailed with a different address (same
 	// NodeID) should unmark as dialing and allow us to dial the other address
@@ -686,14 +746,18 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) {
 	require.NoError(t, err)
 
 	// Add a and connect to it.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Add b and start dialing it. This will claim a for upgrading.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
@@ -701,7 +765,9 @@ func TestPeerManager_DialFailed_UnreservePeer(t *testing.T) {
 	// Adding c and dialing it will fail, even though it could upgrade a and we
 	// have free upgrade slots, because a is the only connected peer that can be
 	// upgraded and b is already trying to upgrade it.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Empty(t, dial)
@@ -721,7 +787,9 @@ func TestPeerManager_Dialed_Connected(t *testing.T) {
 	require.NoError(t, err)
 
 	// Marking a as dialed twice should error.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
@@ -730,7 +798,9 @@ func TestPeerManager_Dialed_Connected(t *testing.T) {
 	require.Error(t, peerManager.Dialed(a))
 
 	// Accepting a connection from b and then trying to mark it as dialed should fail.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
@@ -744,7 +814,8 @@ func TestPeerManager_Dialed_Self(t *testing.T) {
 	require.NoError(t, err)
 
 	// Dialing self should error.
-	require.Error(t, peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID}))
+	_, err = peerManager.Add(p2p.NodeAddress{Protocol: "memory", NodeID: selfID})
+	require.Error(t, err)
 }
 
 func TestPeerManager_Dialed_MaxConnected(t *testing.T) {
@@ -757,14 +828,18 @@ func TestPeerManager_Dialed_MaxConnected(t *testing.T) {
 	require.NoError(t, err)
 
 	// Start to dial a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
 
 	// Marking b as dialed in the meanwhile (even without TryDialNext)
 	// should be fine.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(b))
 
 	// Completing the dial for a should now error.
@@ -785,14 +860,20 @@ func TestPeerManager_Dialed_MaxConnectedUpgrade(t *testing.T) {
 	require.NoError(t, err)
 
 	// Dialing a and b is fine.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(b))
 
 	// Starting an upgrade of c should be fine.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, c, dial)
@@ -800,7 +881,9 @@ func TestPeerManager_Dialed_MaxConnectedUpgrade(t *testing.T) {
 
 	// Trying to mark d dialed should fail, since there are no more upgrade
 	// slots and a/b haven't been evicted yet.
-	require.NoError(t, peerManager.Add(d))
+	added, err = peerManager.Add(d)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Dialed(d))
 }
 
@@ -827,11 +910,15 @@ func TestPeerManager_Dialed_Upgrade(t *testing.T) {
 	require.NoError(t, err)
 
 	// Dialing a is fine.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Upgrading it with b should work, since b has a higher score.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
@@ -839,7 +926,9 @@ func TestPeerManager_Dialed_Upgrade(t *testing.T) {
 
 	// a hasn't been evicted yet, but c shouldn't be allowed to upgrade anyway
 	// since it's about to be evicted.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Empty(t, dial)
@@ -869,15 +958,21 @@ func TestPeerManager_Dialed_UpgradeEvenLower(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect to a and b.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(b))
 
 	// Start an upgrade with c, which should pick b to upgrade (since it
 	// has score 2).
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, c, dial)
@@ -885,7 +980,9 @@ func TestPeerManager_Dialed_UpgradeEvenLower(t *testing.T) {
 	// In the meanwhile, a disconnects and d connects. d is even lower-scored
 	// than b (1 vs 2), which is currently being upgraded.
 	require.NoError(t, peerManager.Disconnected(a.NodeID))
-	require.NoError(t, peerManager.Add(d))
+	added, err = peerManager.Add(d)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(d.NodeID))
 
 	// Once c completes the upgrade of b, it should instead evict d,
@@ -913,14 +1010,20 @@ func TestPeerManager_Dialed_UpgradeNoEvict(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect to a and b.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(b))
 
 	// Start an upgrade with c, which should pick a to upgrade.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, c, dial)
@@ -950,7 +1053,9 @@ func TestPeerManager_Accepted(t *testing.T) {
 	require.Error(t, peerManager.Accepted(selfID))
 
 	// Accepting a connection from a known peer should work.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 
 	// Accepting a connection from an already accepted peer should error.
@@ -962,7 +1067,9 @@ func TestPeerManager_Accepted(t *testing.T) {
 
 	// Accepting a connection from a peer that's being dialed should work, and
 	// should cause the dial to fail.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, c, dial)
@@ -970,7 +1077,9 @@ func TestPeerManager_Accepted(t *testing.T) {
 	require.Error(t, peerManager.Dialed(c))
 
 	// Accepting a connection from a peer that's been dialed should fail.
-	require.NoError(t, peerManager.Add(d))
+	added, err = peerManager.Add(d)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err = peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, d, dial)
@@ -989,14 +1098,20 @@ func TestPeerManager_Accepted_MaxConnected(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect to a and b.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(b.NodeID))
 
 	// Accepting c should now fail.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Accepted(c.NodeID))
 }
 
@@ -1017,23 +1132,32 @@ func TestPeerManager_Accepted_MaxConnectedUpgrade(t *testing.T) {
 	require.NoError(t, err)
 
 	// Dial a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Dialed(a))
 
 	// Accepting b should fail, since it's not an upgrade over a.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Accepted(b.NodeID))
 
 	// Accepting c should work, since it upgrades a.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(c.NodeID))
 
 	// a still hasn't been evicted, so accepting b should still fail.
-	require.NoError(t, peerManager.Add(b))
+	_, err = peerManager.Add(b)
+	require.NoError(t, err)
 	require.Error(t, peerManager.Accepted(b.NodeID))
 
 	// Also, accepting d should fail, since all upgrade slots are full.
-	require.NoError(t, peerManager.Add(d))
+	added, err = peerManager.Add(d)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Accepted(d.NodeID))
 }
 
@@ -1053,15 +1177,21 @@ func TestPeerManager_Accepted_Upgrade(t *testing.T) {
 	require.NoError(t, err)
 
 	// Accept a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 
 	// Accepting b should work, since it upgrades a.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(b.NodeID))
 
 	// c cannot get accepted, since a has been upgraded by b.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Accepted(c.NodeID))
 
 	// This should cause a to get evicted.
@@ -1090,18 +1220,24 @@ func TestPeerManager_Accepted_UpgradeDialing(t *testing.T) {
 	require.NoError(t, err)
 
 	// Accept a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 
 	// Start dial upgrade from a to b.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, b, dial)
 
 	// a has already been claimed as an upgrade of a, so accepting
 	// c should fail since there's noone else to upgrade.
-	require.NoError(t, peerManager.Add(c))
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Error(t, peerManager.Accepted(c.NodeID))
 
 	// However, if b connects to us while we're also trying to upgrade to it via
@@ -1126,7 +1262,9 @@ func TestPeerManager_Ready(t *testing.T) {
 	defer sub.Close()
 
 	// Connecting to a should still have it as status down.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.Equal(t, p2p.PeerStatusDown, peerManager.Status(a.NodeID))
 
@@ -1139,7 +1277,9 @@ func TestPeerManager_Ready(t *testing.T) {
 	}, <-sub.Updates())
 
 	// Marking an unconnected peer as ready should do nothing.
-	require.NoError(t, peerManager.Add(b))
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Equal(t, p2p.PeerStatusDown, peerManager.Status(b.NodeID))
 	require.NoError(t, peerManager.Ready(b.NodeID))
 	require.Equal(t, p2p.PeerStatusDown, peerManager.Status(b.NodeID))
@@ -1153,7 +1293,9 @@ func TestPeerManager_EvictNext(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 
@@ -1184,7 +1326,9 @@ func TestPeerManager_EvictNext_WakeOnError(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 
@@ -1214,14 +1358,18 @@ func TestPeerManager_EvictNext_WakeOnUpgradeDialed(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 
 	// Spawn a goroutine to upgrade to b with a delay.
 	go func() {
 		time.Sleep(200 * time.Millisecond)
-		require.NoError(t, peerManager.Add(b))
+		added, err := peerManager.Add(b)
+		require.NoError(t, err)
+		require.True(t, added)
 		dial, err := peerManager.TryDialNext()
 		require.NoError(t, err)
 		require.Equal(t, b, dial)
@@ -1248,7 +1396,9 @@ func TestPeerManager_EvictNext_WakeOnUpgradeAccepted(t *testing.T) {
 	require.NoError(t, err)
 
 	// Connect a.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 
@@ -1271,7 +1421,9 @@ func TestPeerManager_TryEvictNext(t *testing.T) {
 	peerManager, err := p2p.NewPeerManager(selfID, dbm.NewMemDB(), p2p.PeerManagerOptions{})
 	require.NoError(t, err)
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 
 	// Nothing is evicted with no peers connected.
 	evict, err := peerManager.TryEvictNext()
@@ -1314,13 +1466,16 @@ func TestPeerManager_Disconnected(t *testing.T) {
 	require.Empty(t, sub.Updates())
 
 	// Disconnecting an accepted non-ready peer does not send a status update.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Disconnected(a.NodeID))
 	require.Empty(t, sub.Updates())
 
 	// Disconnecting a ready peer sends a status update.
-	require.NoError(t, peerManager.Add(a))
+	_, err = peerManager.Add(a)
+	require.NoError(t, err)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 	require.Equal(t, p2p.PeerStatusUp, peerManager.Status(a.NodeID))
@@ -1365,7 +1520,9 @@ func TestPeerManager_Errored(t *testing.T) {
 
 	// Erroring a known peer does nothing, and won't evict it later,
 	// even when it connects.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Errored(a.NodeID, errors.New("foo")))
 	evict, err = peerManager.TryEvictNext()
 	require.NoError(t, err)
@@ -1394,7 +1551,9 @@ func TestPeerManager_Subscribe(t *testing.T) {
 	sub := peerManager.Subscribe()
 	defer sub.Close()
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.Empty(t, sub.Updates())
 
 	// Inbound connection.
@@ -1452,7 +1611,9 @@ func TestPeerManager_Subscribe_Close(t *testing.T) {
 	sub := peerManager.Subscribe()
 	defer sub.Close()
 
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.Empty(t, sub.Updates())
 
@@ -1482,7 +1643,9 @@ func TestPeerManager_Subscribe_Broadcast(t *testing.T) {
 	defer s3.Close()
 
 	// Connecting to a peer should send updates on all subscriptions.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.NoError(t, peerManager.Accepted(a.NodeID))
 	require.NoError(t, peerManager.Ready(a.NodeID))
 
@@ -1524,7 +1687,9 @@ func TestPeerManager_Close(t *testing.T) {
 
 	// This dial failure will start a retry timer for 10 seconds, which
 	// should be reaped.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	dial, err := peerManager.TryDialNext()
 	require.NoError(t, err)
 	require.Equal(t, a, dial)
@@ -1556,12 +1721,24 @@ func TestPeerManager_Advertise(t *testing.T) {
 	require.NoError(t, err)
 	defer peerManager.Close()
 
-	require.NoError(t, peerManager.Add(aTCP))
-	require.NoError(t, peerManager.Add(aMem))
-	require.NoError(t, peerManager.Add(bTCP))
-	require.NoError(t, peerManager.Add(bMem))
-	require.NoError(t, peerManager.Add(cTCP))
-	require.NoError(t, peerManager.Add(cMem))
+	added, err := peerManager.Add(aTCP)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(aMem)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(bTCP)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(bMem)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(cTCP)
+	require.NoError(t, err)
+	require.True(t, added)
+	added, err = peerManager.Add(cMem)
+	require.NoError(t, err)
+	require.True(t, added)
 
 	// d should get all addresses.
 	require.ElementsMatch(t, []p2p.NodeAddress{
@@ -1592,7 +1769,9 @@ func TestPeerManager_SetHeight_GetHeight(t *testing.T) {
 
 	// Getting a height should default to 0, for unknown peers and
 	// for known peers without height.
-	require.NoError(t, peerManager.Add(a))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
 	require.EqualValues(t, 0, peerManager.GetHeight(a.NodeID))
 	require.EqualValues(t, 0, peerManager.GetHeight(b.NodeID))
 

p2p/pex/addrbook.go

@@ -9,13 +9,12 @@ import (
 	"encoding/binary"
 	"fmt"
 	"math"
-	"math/rand"
+	mrand "math/rand"
 	"net"
 	"sync"
 	"time"
 
 	"github.com/minio/highwayhash"
-
 	"github.com/tendermint/tendermint/crypto"
 	tmmath "github.com/tendermint/tendermint/libs/math"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -89,7 +88,6 @@ type addrBook struct {
 
 	// accessed concurrently
 	mtx        tmsync.Mutex
-	rand       *tmrand.Rand
 	ourAddrs   map[string]struct{}
 	privateIDs map[p2p.NodeID]struct{}
 	addrLookup map[p2p.NodeID]*knownAddress // new & old
@@ -118,7 +116,6 @@ func newHashKey() []byte {
 // Use Start to begin processing asynchronous address updates.
 func NewAddrBook(filePath string, routabilityStrict bool) AddrBook {
 	am := &addrBook{
-		rand:              tmrand.NewRand(),
 		ourAddrs:          make(map[string]struct{}),
 		privateIDs:        make(map[p2p.NodeID]struct{}),
 		addrLookup:        make(map[p2p.NodeID]*knownAddress),
@@ -268,6 +265,7 @@ func (a *addrBook) Empty() bool {
 // and determines how biased we are to pick an address from a new bucket.
 // PickAddress returns nil if the AddrBook is empty or if we try to pick
 // from an empty bucket.
+// nolint:gosec // G404: Use of weak random number generator
 func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress {
 	a.mtx.Lock()
 	defer a.mtx.Unlock()
@@ -292,7 +290,7 @@ func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress {
 
 	// pick a random peer from a random bucket
 	var bucket map[string]*knownAddress
-	pickFromOldBucket := (newCorrelation+oldCorrelation)*a.rand.Float64() < oldCorrelation
+	pickFromOldBucket := (newCorrelation+oldCorrelation)*mrand.Float64() < oldCorrelation
 	if (pickFromOldBucket && a.nOld == 0) ||
 		(!pickFromOldBucket && a.nNew == 0) {
 		return nil
@@ -300,13 +298,13 @@ func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress {
 	// loop until we pick a random non-empty bucket
 	for len(bucket) == 0 {
 		if pickFromOldBucket {
-			bucket = a.bucketsOld[a.rand.Intn(len(a.bucketsOld))]
+			bucket = a.bucketsOld[mrand.Intn(len(a.bucketsOld))]
 		} else {
-			bucket = a.bucketsNew[a.rand.Intn(len(a.bucketsNew))]
+			bucket = a.bucketsNew[mrand.Intn(len(a.bucketsNew))]
 		}
 	}
 	// pick a random index and loop over the map to return that index
-	randIndex := a.rand.Intn(len(bucket))
+	randIndex := mrand.Intn(len(bucket))
 	for _, ka := range bucket {
 		if randIndex == 0 {
 			return ka.Addr
@@ -417,7 +415,8 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress {
 	// `numAddresses' since we are throwing the rest.
 	for i := 0; i < numAddresses; i++ {
 		// pick a number between current index and the end
-		j := tmrand.Intn(len(allAddr)-i) + i
+		// nolint:gosec // G404: Use of weak random number generator
+		j := mrand.Intn(len(allAddr)-i) + i
 		allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
 	}
 
@@ -681,7 +680,8 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error {
 		}
 		// The more entries we have, the less likely we are to add more.
 		factor := int32(2 * len(ka.Buckets))
-		if a.rand.Int31n(factor) != 0 {
+		// nolint:gosec // G404: Use of weak random number generator
+		if mrand.Int31n(factor) != 0 {
 			return nil
 		}
 	} else {
@@ -717,6 +717,7 @@ func (a *addrBook) randomPickAddresses(bucketType byte, num int) []*p2p.NetAddre
 	}
 	selection := make([]*p2p.NetAddress, 0, num)
 	chosenSet := make(map[string]bool, num)
+	rand := tmrand.NewRand()
 	rand.Shuffle(total, func(i, j int) {
 		addresses[i], addresses[j] = addresses[j], addresses[i]
 	})

p2p/pex/addrbook_test.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	mrand "math/rand"
 	"net"
 	"os"
 	"testing"
@@ -187,12 +188,12 @@ func randNetAddressPairs(t *testing.T, n int) []netAddressPair {
 func randIPv4Address(t *testing.T) *p2p.NetAddress {
 	for {
 		ip := fmt.Sprintf("%v.%v.%v.%v",
-			tmrand.Intn(254)+1,
-			tmrand.Intn(255),
-			tmrand.Intn(255),
-			tmrand.Intn(255),
+			mrand.Intn(254)+1,
+			mrand.Intn(255),
+			mrand.Intn(255),
+			mrand.Intn(255),
 		)
-		port := tmrand.Intn(65535-1) + 1
+		port := mrand.Intn(65535-1) + 1
 		id := p2p.NodeID(hex.EncodeToString(tmrand.Bytes(p2p.NodeIDByteLength)))
 		idAddr := p2p.IDAddressString(id, fmt.Sprintf("%v:%v", ip, port))
 		addr, err := p2p.NewNetAddressString(idAddr)
@@ -554,7 +555,7 @@ func TestMultipleAddrBookAddressSelection(t *testing.T) {
 	ranges := [...][]int{{33, 100}, {100, 175}}
 	bookSizes := make([]int, 0, len(ranges))
 	for _, r := range ranges {
-		bookSizes = append(bookSizes, tmrand.Intn(r[1]-r[0])+r[0])
+		bookSizes = append(bookSizes, mrand.Intn(r[1]-r[0])+r[0])
 	}
 	t.Logf("Testing address selection for the following book sizes %v\n", bookSizes)
 	for _, bookSize := range bookSizes {

p2p/pex/pex_reactor.go

@@ -525,7 +525,8 @@ func (r *Reactor) ensurePeers() {
 		peers := r.Switch.Peers().List()
 		peersCount := len(peers)
 		if peersCount > 0 {
-			peer := peers[tmrand.Int()%peersCount]
+			rand := tmrand.NewRand()
+			peer := peers[rand.Int()%peersCount]
 			r.Logger.Info("We need more addresses. Sending pexRequest to random peer", "peer", peer)
 			r.RequestAddrs(peer)
 		}
@@ -558,7 +559,8 @@ func (r *Reactor) dialPeer(addr *p2p.NetAddress) error {
 
 	// exponential backoff if it's not our first attempt to dial given address
 	if attempts > 0 {
-		jitter := time.Duration(tmrand.Float64() * float64(time.Second)) // 1s == (1e9 ns)
+		rand := tmrand.NewRand()
+		jitter := time.Duration(rand.Float64() * float64(time.Second)) // 1s == (1e9 ns)
 		backoffDuration := jitter + ((1 << uint(attempts)) * time.Second)
 		backoffDuration = r.maxBackoffDurationForPeer(addr, backoffDuration)
 		sinceLastDialed := time.Since(lastDialed)
@@ -624,7 +626,8 @@ func (r *Reactor) checkSeeds() (numOnline int, netAddrs []*p2p.NetAddress, err e
 
 // randomly dial seeds until we connect to one or exhaust them
 func (r *Reactor) dialSeeds() {
-	perm := tmrand.Perm(len(r.seedAddrs))
+	rand := tmrand.NewRand()
+	perm := rand.Perm(len(r.seedAddrs))
 	// perm := r.Switch.rng.Perm(lSeeds)
 	for _, i := range perm {
 		// dial a random seed

p2p/pex/reactor.go

@@ -3,9 +3,12 @@ package pex
 import (
 	"context"
 	"fmt"
+	"sync"
 	"time"
 
+	"github.com/tendermint/tendermint/libs/clist"
 	"github.com/tendermint/tendermint/libs/log"
+	tmmath "github.com/tendermint/tendermint/libs/math"
 	"github.com/tendermint/tendermint/libs/service"
 	"github.com/tendermint/tendermint/p2p"
 	protop2p "github.com/tendermint/tendermint/proto/tendermint/p2p"
@@ -16,15 +19,40 @@ var (
 	_ p2p.Wrapper     = (*protop2p.PexMessage)(nil)
 )
 
+// TODO: Consolidate with params file.
+// See https://github.com/tendermint/tendermint/issues/6371
 const (
-	maxAddresses   uint16 = 100
-	resolveTimeout        = 3 * time.Second
+	// the minimum time one peer can send another request to the same peer
+	minReceiveRequestInterval = 300 * time.Millisecond
+
+	// the maximum amount of addresses that can be included in a response
+	maxAddresses uint16 = 100
+
+	// allocated time to resolve a node address into a set of endpoints
+	resolveTimeout = 3 * time.Second
+
+	// How long to wait when there are no peers available before trying again
+	noAvailablePeersWaitPeriod = 1 * time.Second
+
+	// indicates the ping rate of the pex reactor when the peer store is full.
+	// The reactor should still look to add new peers in order to flush out low
+	// scoring peers that are still in the peer store
+	fullCapacityInterval = 10 * time.Minute
 )
 
 // ReactorV2 is a PEX reactor for the new P2P stack. The legacy reactor
 // is Reactor.
 //
 // FIXME: Rename this when Reactor is removed, and consider moving to p2p/.
+//
+// The peer exchange or PEX reactor supports the peer manager by sending
+// requests to other peers for addresses that can be given to the peer manager
+// and at the same time advertises addresses to peers that need more.
+//
+// The reactor is able to tweak the intensity of it's search by decreasing or
+// increasing the interval between each request. It tracks connected peers via
+// a linked list, sending a request to the node at the front of the list and
+// adding it to the back of the list once a response is received.
 type ReactorV2 struct {
 	service.BaseService
 
@@ -32,6 +60,33 @@ type ReactorV2 struct {
 	pexCh       *p2p.Channel
 	peerUpdates *p2p.PeerUpdates
 	closeCh     chan struct{}
+
+	// list of available peers to loop through and send peer requests to
+	availablePeers *clist.CList
+
+	mtx sync.RWMutex
+	// requestsSent keeps track of which peers the PEX reactor has sent requests
+	// to. This prevents the sending of spurious responses.
+	// NOTE: If a node never responds, they will remain in this map until a
+	// peer down status update is sent
+	requestsSent map[p2p.NodeID]struct{}
+
+	// lastReceivedRequests keeps track of when peers send a request to prevent
+	// peers from sending requests too often (as defined by
+	// minReceiveRequestInterval).
+	lastReceivedRequests map[p2p.NodeID]time.Time
+
+	// the time when another request will be sent
+	nextRequestTime time.Time
+
+	// keep track of how many new peers to existing peers we have received to
+	// extrapolate the size of the network
+	newPeers   uint32
+	totalPeers uint32
+	// discoveryRatio is the inverse ratio of new peers to old peers squared.
+	// This is multiplied by the minimum duration to calculate how long to wait
+	// between each request.
+	discoveryRatio float32
 }
 
 // NewReactor returns a reference to a new reactor.
@@ -42,10 +97,13 @@ func NewReactorV2(
 	peerUpdates *p2p.PeerUpdates,
 ) *ReactorV2 {
 	r := &ReactorV2{
-		peerManager: peerManager,
-		pexCh:       pexCh,
-		peerUpdates: peerUpdates,
-		closeCh:     make(chan struct{}),
+		peerManager:          peerManager,
+		pexCh:                pexCh,
+		peerUpdates:          peerUpdates,
+		closeCh:              make(chan struct{}),
+		availablePeers:       clist.New(),
+		requestsSent:         make(map[p2p.NodeID]struct{}),
+		lastReceivedRequests: make(map[p2p.NodeID]time.Time),
 	}
 
 	r.BaseService = *service.NewBaseService(logger, "PEX", r)
@@ -76,100 +134,6 @@ func (r *ReactorV2) OnStop() {
 	<-r.peerUpdates.Done()
 }
 
-// handlePexMessage handles envelopes sent from peers on the PexChannel.
-func (r *ReactorV2) handlePexMessage(envelope p2p.Envelope) error {
-	logger := r.Logger.With("peer", envelope.From)
-
-	// FIXME: We may want to add DoS protection here, by rate limiting peers and
-	// only processing addresses we actually requested.
-	switch msg := envelope.Message.(type) {
-	case *protop2p.PexRequest:
-		pexAddresses := r.resolve(r.peerManager.Advertise(envelope.From, maxAddresses), maxAddresses)
-		r.pexCh.Out <- p2p.Envelope{
-			To:      envelope.From,
-			Message: &protop2p.PexResponse{Addresses: pexAddresses},
-		}
-
-	case *protop2p.PexResponse:
-		for _, pexAddress := range msg.Addresses {
-			peerAddress, err := p2p.ParseNodeAddress(
-				fmt.Sprintf("%s@%s:%d", pexAddress.ID, pexAddress.IP, pexAddress.Port))
-			if err != nil {
-				logger.Debug("invalid PEX address", "address", pexAddress, "err", err)
-				continue
-			}
-			if err = r.peerManager.Add(peerAddress); err != nil {
-				logger.Debug("failed to register PEX address", "address", peerAddress, "err", err)
-			}
-		}
-
-	default:
-		return fmt.Errorf("received unknown message: %T", msg)
-	}
-
-	return nil
-}
-
-// resolve resolves a set of peer addresses into PEX addresses.
-//
-// FIXME: This is necessary because the current PEX protocol only supports
-// IP/port pairs, while the P2P stack uses NodeAddress URLs. The PEX protocol
-// should really use URLs too, to exchange DNS names instead of IPs and allow
-// different transport protocols (e.g. QUIC and MemoryTransport).
-//
-// FIXME: We may want to cache and parallelize this, but for now we'll just rely
-// on the operating system to cache it for us.
-func (r *ReactorV2) resolve(addresses []p2p.NodeAddress, limit uint16) []protop2p.PexAddress {
-	pexAddresses := make([]protop2p.PexAddress, 0, len(addresses))
-	for _, address := range addresses {
-		ctx, cancel := context.WithTimeout(context.Background(), resolveTimeout)
-		endpoints, err := address.Resolve(ctx)
-		cancel()
-		if err != nil {
-			r.Logger.Debug("failed to resolve address", "address", address, "err", err)
-			continue
-		}
-		for _, endpoint := range endpoints {
-			if len(pexAddresses) >= int(limit) {
-				return pexAddresses
-
-			} else if endpoint.IP != nil {
-				// PEX currently only supports IP-networked transports (as
-				// opposed to e.g. p2p.MemoryTransport).
-				pexAddresses = append(pexAddresses, protop2p.PexAddress{
-					ID:   string(address.NodeID),
-					IP:   endpoint.IP.String(),
-					Port: uint32(endpoint.Port),
-				})
-			}
-		}
-	}
-	return pexAddresses
-}
-
-// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
-// It will handle errors and any possible panics gracefully. A caller can handle
-// any error returned by sending a PeerError on the respective channel.
-func (r *ReactorV2) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
-	defer func() {
-		if e := recover(); e != nil {
-			err = fmt.Errorf("panic in processing message: %v", e)
-		}
-	}()
-
-	r.Logger.Debug("received message", "peer", envelope.From)
-
-	switch chID {
-	case p2p.ChannelID(PexChannel):
-		err = r.handlePexMessage(envelope)
-
-	default:
-		err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
-	}
-
-	return err
-}
-
 // processPexCh implements a blocking event loop where we listen for p2p
 // Envelope messages from the pexCh.
 func (r *ReactorV2) processPexCh() {
@@ -177,6 +141,16 @@ func (r *ReactorV2) processPexCh() {
 
 	for {
 		select {
+		case <-r.closeCh:
+			r.Logger.Debug("stopped listening on PEX channel; closing...")
+			return
+
+		// outbound requests for new peers
+		case <-r.waitUntilNextRequest():
+			r.sendRequestForPeers()
+
+		// inbound requests for new peers or responses to requests sent by this
+		// reactor
 		case envelope := <-r.pexCh.In:
 			if err := r.handleMessage(r.pexCh.ID, envelope); err != nil {
 				r.Logger.Error("failed to process message", "ch_id", r.pexCh.ID, "envelope", envelope, "err", err)
@@ -185,23 +159,6 @@ func (r *ReactorV2) processPexCh() {
 					Err:    err,
 				}
 			}
-
-		case <-r.closeCh:
-			r.Logger.Debug("stopped listening on PEX channel; closing...")
-			return
-		}
-	}
-}
-
-// processPeerUpdate processes a PeerUpdate. For added peers, PeerStatusUp, we
-// send a request for addresses.
-func (r *ReactorV2) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
-	r.Logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
-
-	if peerUpdate.Status == p2p.PeerStatusUp {
-		r.pexCh.Out <- p2p.Envelope{
-			To:      peerUpdate.NodeID,
-			Message: &protop2p.PexRequest{},
 		}
 	}
 }
@@ -223,3 +180,335 @@ func (r *ReactorV2) processPeerUpdates() {
 		}
 	}
 }
+
+// handlePexMessage handles envelopes sent from peers on the PexChannel.
+func (r *ReactorV2) handlePexMessage(envelope p2p.Envelope) error {
+	logger := r.Logger.With("peer", envelope.From)
+
+	switch msg := envelope.Message.(type) {
+
+	case *protop2p.PexRequest:
+		// check if the peer hasn't sent a prior request too close to this one
+		// in time
+		if err := r.markPeerRequest(envelope.From); err != nil {
+			return err
+		}
+
+		// parse and send the legacy PEX addresses
+		pexAddresses := r.resolve(r.peerManager.Advertise(envelope.From, maxAddresses))
+		r.pexCh.Out <- p2p.Envelope{
+			To:      envelope.From,
+			Message: &protop2p.PexResponse{Addresses: pexAddresses},
+		}
+
+	case *protop2p.PexResponse:
+		// check if the response matches a request that was made to that peer
+		if err := r.markPeerResponse(envelope.From); err != nil {
+			return err
+		}
+
+		// check the size of the response
+		if len(msg.Addresses) > int(maxAddresses) {
+			return fmt.Errorf("peer sent too many addresses (max: %d, got: %d)",
+				maxAddresses,
+				len(msg.Addresses),
+			)
+		}
+
+		for _, pexAddress := range msg.Addresses {
+			// no protocol is prefixed so we assume the default (mconn)
+			peerAddress, err := p2p.ParseNodeAddress(
+				fmt.Sprintf("%s@%s:%d", pexAddress.ID, pexAddress.IP, pexAddress.Port))
+			if err != nil {
+				continue
+			}
+			added, err := r.peerManager.Add(peerAddress)
+			if err != nil {
+				logger.Error("failed to add PEX address", "address", peerAddress, "err", err)
+			}
+			if added {
+				r.newPeers++
+				logger.Debug("added PEX address", "address", peerAddress)
+			}
+			r.totalPeers++
+		}
+
+	// V2 PEX MESSAGES
+	case *protop2p.PexRequestV2:
+		// check if the peer hasn't sent a prior request too close to this one
+		// in time
+		if err := r.markPeerRequest(envelope.From); err != nil {
+			return err
+		}
+
+		// request peers from the peer manager and parse the NodeAddresses into
+		// URL strings
+		nodeAddresses := r.peerManager.Advertise(envelope.From, maxAddresses)
+		pexAddressesV2 := make([]protop2p.PexAddressV2, len(nodeAddresses))
+		for idx, addr := range nodeAddresses {
+			pexAddressesV2[idx] = protop2p.PexAddressV2{
+				URL: addr.String(),
+			}
+		}
+		r.pexCh.Out <- p2p.Envelope{
+			To:      envelope.From,
+			Message: &protop2p.PexResponseV2{Addresses: pexAddressesV2},
+		}
+
+	case *protop2p.PexResponseV2:
+		// check if the response matches a request that was made to that peer
+		if err := r.markPeerResponse(envelope.From); err != nil {
+			return err
+		}
+
+		// check the size of the response
+		if len(msg.Addresses) > int(maxAddresses) {
+			return fmt.Errorf("peer sent too many addresses (max: %d, got: %d)",
+				maxAddresses,
+				len(msg.Addresses),
+			)
+		}
+
+		for _, pexAddress := range msg.Addresses {
+			peerAddress, err := p2p.ParseNodeAddress(pexAddress.URL)
+			if err != nil {
+				continue
+			}
+			added, err := r.peerManager.Add(peerAddress)
+			if err != nil {
+				logger.Error("failed to add V2 PEX address", "address", peerAddress, "err", err)
+			}
+			if added {
+				r.newPeers++
+				logger.Debug("added V2 PEX address", "address", peerAddress)
+			}
+			r.totalPeers++
+		}
+
+	default:
+		return fmt.Errorf("received unknown message: %T", msg)
+	}
+
+	return nil
+}
+
+// resolve resolves a set of peer addresses into PEX addresses.
+//
+// FIXME: This is necessary because the current PEX protocol only supports
+// IP/port pairs, while the P2P stack uses NodeAddress URLs. The PEX protocol
+// should really use URLs too, to exchange DNS names instead of IPs and allow
+// different transport protocols (e.g. QUIC and MemoryTransport).
+//
+// FIXME: We may want to cache and parallelize this, but for now we'll just rely
+// on the operating system to cache it for us.
+func (r *ReactorV2) resolve(addresses []p2p.NodeAddress) []protop2p.PexAddress {
+	limit := len(addresses)
+	pexAddresses := make([]protop2p.PexAddress, 0, limit)
+	for _, address := range addresses {
+		ctx, cancel := context.WithTimeout(context.Background(), resolveTimeout)
+		endpoints, err := address.Resolve(ctx)
+		r.Logger.Debug("resolved node address", "endpoints", endpoints)
+		cancel()
+		if err != nil {
+			r.Logger.Debug("failed to resolve address", "address", address, "err", err)
+			continue
+		}
+		for _, endpoint := range endpoints {
+			r.Logger.Debug("checking endpint", "IP", endpoint.IP, "Port", endpoint.Port)
+			if len(pexAddresses) >= limit {
+				return pexAddresses
+
+			} else if endpoint.IP != nil {
+				r.Logger.Debug("appending pex address")
+				// PEX currently only supports IP-networked transports (as
+				// opposed to e.g. p2p.MemoryTransport).
+				//
+				// FIXME: as the PEX address contains no information about the
+				// protocol, we jam this into the ID. We won't need to this once
+				// we support URLs
+				pexAddresses = append(pexAddresses, protop2p.PexAddress{
+					ID:   string(address.NodeID),
+					IP:   endpoint.IP.String(),
+					Port: uint32(endpoint.Port),
+				})
+			}
+		}
+	}
+	return pexAddresses
+}
+
+// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
+// It will handle errors and any possible panics gracefully. A caller can handle
+// any error returned by sending a PeerError on the respective channel.
+func (r *ReactorV2) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
+	defer func() {
+		if e := recover(); e != nil {
+			err = fmt.Errorf("panic in processing message: %v", e)
+		}
+	}()
+
+	r.Logger.Debug("received PEX message", "peer", envelope.From)
+
+	switch chID {
+	case p2p.ChannelID(PexChannel):
+		err = r.handlePexMessage(envelope)
+
+	default:
+		err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
+	}
+
+	return err
+}
+
+// processPeerUpdate processes a PeerUpdate. For added peers, PeerStatusUp, we
+// send a request for addresses.
+func (r *ReactorV2) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
+	r.Logger.Debug("received PEX peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
+	switch peerUpdate.Status {
+	case p2p.PeerStatusUp:
+		r.availablePeers.PushBack(peerUpdate.NodeID)
+	case p2p.PeerStatusDown:
+		r.removePeer(peerUpdate.NodeID)
+	default:
+	}
+}
+
+func (r *ReactorV2) waitUntilNextRequest() <-chan time.Time {
+	return time.After(time.Until(r.nextRequestTime))
+}
+
+// sendRequestForPeers pops the first peerID off the list and sends the
+// peer a request for more peer addresses. The function then moves the
+// peer into the requestsSent bucket and calculates when the next request
+// time should be
+func (r *ReactorV2) sendRequestForPeers() {
+	peer := r.availablePeers.Front()
+	if peer == nil {
+		// no peers are available
+		r.Logger.Debug("no available peers to send request to, waiting...")
+		r.nextRequestTime = time.Now().Add(noAvailablePeersWaitPeriod)
+		return
+	}
+	peerID := peer.Value.(p2p.NodeID)
+
+	// The node accommodates for both pex systems
+	if r.isLegacyPeer(peerID) {
+		r.pexCh.Out <- p2p.Envelope{
+			To:      peerID,
+			Message: &protop2p.PexRequest{},
+		}
+	} else {
+		r.pexCh.Out <- p2p.Envelope{
+			To:      peerID,
+			Message: &protop2p.PexRequestV2{},
+		}
+	}
+
+	// remove the peer from the available peers list and mark it in the requestsSent map
+	r.availablePeers.Remove(peer)
+	peer.DetachPrev()
+	r.mtx.Lock()
+	r.requestsSent[peerID] = struct{}{}
+	r.mtx.Unlock()
+
+	r.calculateNextRequestTime()
+	r.Logger.Debug("peer request sent", "next_request_time", r.nextRequestTime)
+}
+
+// calculateNextRequestTime implements something of a proportional controller
+// to estimate how often the reactor should be requesting new peer addresses.
+// The dependent variable in this calculation is the ratio of new peers to
+// all peers that the reactor receives. The interval is thus calculated as the
+// inverse squared. In the beginning, all peers should be new peers.
+// We  expect this ratio to be near 1 and thus the interval to be as short
+// as possible. As the node becomes more familiar with the network the ratio of
+// new nodes will plummet to a very small number, meaning the interval expands
+// to its upper bound.
+// CONTRACT: Must use a write lock as nextRequestTime is updated
+func (r *ReactorV2) calculateNextRequestTime() {
+	// check if the peer store is full. If so then there is no need
+	// to send peer requests too often
+	if ratio := r.peerManager.PeerRatio(); ratio >= 0.95 {
+		r.Logger.Debug("peer manager near full ratio, sleeping...",
+			"sleep_period", fullCapacityInterval, "ratio", ratio)
+		r.nextRequestTime = time.Now().Add(fullCapacityInterval)
+		return
+	}
+
+	// baseTime represents the shortest interval that we can send peer requests
+	// in. For example if we have 10 peers and we can't send a message to the
+	// same peer every 500ms, then we can send a request every 50ms. In practice
+	// we use a safety margin of 2, ergo 100ms
+	peers := tmmath.MinInt(r.availablePeers.Len(), 50)
+	baseTime := minReceiveRequestInterval
+	if peers > 0 {
+		baseTime = minReceiveRequestInterval * 2 / time.Duration(peers)
+	}
+
+	if r.totalPeers > 0 || r.discoveryRatio == 0 {
+		// find the ratio of new peers. NOTE: We add 1 to both sides to avoid
+		// divide by zero problems
+		ratio := float32(r.totalPeers+1) / float32(r.newPeers+1)
+		// square the ratio in order to get non linear time intervals
+		// NOTE: The longest possible interval for a network with 100 or more peers
+		// where a node is connected to 50 of them is 2 minutes.
+		r.discoveryRatio = ratio * ratio
+		r.newPeers = 0
+		r.totalPeers = 0
+	}
+	// NOTE: As ratio is always >= 1, discovery ratio is >= 1. Therefore we don't need to worry
+	// about the next request time being less than the minimum time
+	r.nextRequestTime = time.Now().Add(baseTime * time.Duration(r.discoveryRatio))
+}
+
+func (r *ReactorV2) removePeer(id p2p.NodeID) {
+	for e := r.availablePeers.Front(); e != nil; e = e.Next() {
+		if e.Value == id {
+			r.availablePeers.Remove(e)
+			e.DetachPrev()
+			break
+		}
+	}
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	delete(r.requestsSent, id)
+	delete(r.lastReceivedRequests, id)
+}
+
+func (r *ReactorV2) markPeerRequest(peer p2p.NodeID) error {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	if lastRequestTime, ok := r.lastReceivedRequests[peer]; ok {
+		if time.Now().Before(lastRequestTime.Add(minReceiveRequestInterval)) {
+			return fmt.Errorf("peer sent a request too close after a prior one. Minimum interval: %v",
+				minReceiveRequestInterval)
+		}
+	}
+	r.lastReceivedRequests[peer] = time.Now()
+	return nil
+}
+
+func (r *ReactorV2) markPeerResponse(peer p2p.NodeID) error {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	// check if a request to this peer was sent
+	if _, ok := r.requestsSent[peer]; !ok {
+		return fmt.Errorf("peer sent a PEX response when none was requested (%v)", peer)
+	}
+	delete(r.requestsSent, peer)
+	// attach to the back of the list so that the peer can be used again for
+	// future requests
+	r.availablePeers.PushBack(peer)
+	return nil
+}
+
+// all addresses must use a MCONN protocol for the peer to be considered part of the
+// legacy p2p pex system
+func (r *ReactorV2) isLegacyPeer(peer p2p.NodeID) bool {
+	for _, addr := range r.peerManager.Addresses(peer) {
+		if addr.Protocol != p2p.MConnProtocol {
+			return false
+		}
+	}
+	return true
+}

p2p/pex/reactor_test.go

@@ -0,0 +1,722 @@
+package pex_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/p2p/p2ptest"
+	"github.com/tendermint/tendermint/p2p/pex"
+	proto "github.com/tendermint/tendermint/proto/tendermint/p2p"
+)
+
+const (
+	checkFrequency    = 500 * time.Millisecond
+	defaultBufferSize = 2
+	shortWait         = 10 * time.Second
+	longWait          = 60 * time.Second
+
+	firstNode  = 0
+	secondNode = 1
+	thirdNode  = 2
+	fourthNode = 3
+)
+
+func TestReactorBasic(t *testing.T) {
+	// start a network with one mock reactor and one "real" reactor
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 2,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	// assert that the mock node receives a request from the real node
+	testNet.listenForRequest(t, secondNode, firstNode, shortWait)
+
+	// assert that when a mock node sends a request it receives a response (and
+	// the correct one)
+	testNet.sendRequest(t, firstNode, secondNode, true)
+	testNet.listenForResponse(t, secondNode, firstNode, shortWait, []proto.PexAddressV2(nil))
+}
+
+func TestReactorConnectFullNetwork(t *testing.T) {
+	testNet := setup(t, testOptions{
+		TotalNodes: 8,
+	})
+
+	// make every node be only connected with one other node (it actually ends up
+	// being two because of two way connections but oh well)
+	testNet.connectN(t, 1)
+	testNet.start(t)
+
+	// assert that all nodes add each other in the network
+	for idx := 0; idx < len(testNet.nodes); idx++ {
+		testNet.requireNumberOfPeers(t, idx, len(testNet.nodes)-1, longWait)
+	}
+}
+
+func TestReactorSendsRequestsTooOften(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 3,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	// firstNode sends two requests to the secondNode
+	testNet.sendRequest(t, firstNode, secondNode, true)
+	testNet.sendRequest(t, firstNode, secondNode, true)
+
+	// assert that the secondNode evicts the first node (although they reconnect
+	// straight away again)
+	testNet.listenForPeerUpdate(t, secondNode, firstNode, p2p.PeerStatusDown, shortWait)
+
+	// firstNode should still receive the address of the thirdNode by the secondNode
+	expectedAddrs := testNet.getV2AddressesFor([]int{thirdNode})
+	testNet.listenForResponse(t, secondNode, firstNode, shortWait, expectedAddrs)
+
+}
+
+func TestReactorSendsResponseWithoutRequest(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 3,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	// firstNode sends the secondNode an unrequested response
+	// NOTE: secondNode will send a request by default during startup so we send
+	// two responses to counter that.
+	testNet.sendResponse(t, firstNode, secondNode, []int{thirdNode}, true)
+	testNet.sendResponse(t, firstNode, secondNode, []int{thirdNode}, true)
+
+	// secondNode should evict the firstNode
+	testNet.listenForPeerUpdate(t, secondNode, firstNode, p2p.PeerStatusDown, shortWait)
+}
+
+func TestReactorNeverSendsTooManyPeers(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 2,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	testNet.addNodes(t, 110)
+	nodes := make([]int, 110)
+	for i := 0; i < len(nodes); i++ {
+		nodes[i] = i + 2
+	}
+	testNet.addAddresses(t, secondNode, nodes)
+
+	// first we check that even although we have 110 peers, honest pex reactors
+	// only send 100 (test if secondNode sends firstNode 100 addresses)
+	testNet.pingAndlistenForNAddresses(t, secondNode, firstNode, shortWait, 100)
+}
+
+func TestReactorErrorsOnReceivingTooManyPeers(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 2,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	testNet.addNodes(t, 110)
+	nodes := make([]int, 110)
+	for i := 0; i < len(nodes); i++ {
+		nodes[i] = i + 2
+	}
+
+	// now we send a response with more than 100 peers
+	testNet.sendResponse(t, firstNode, secondNode, nodes, true)
+	// secondNode should evict the firstNode
+	testNet.listenForPeerUpdate(t, secondNode, firstNode, p2p.PeerStatusDown, shortWait)
+}
+
+func TestReactorSmallPeerStoreInALargeNetwork(t *testing.T) {
+	testNet := setup(t, testOptions{
+		TotalNodes:   16,
+		MaxPeers:     8,
+		MaxConnected: 6,
+		BufferSize:   8,
+	})
+	testNet.connectN(t, 1)
+	testNet.start(t)
+
+	// test that all nodes reach full capacity
+	for _, nodeID := range testNet.nodes {
+		require.Eventually(t, func() bool {
+			// nolint:scopelint
+			return testNet.network.Nodes[nodeID].PeerManager.PeerRatio() >= 0.9
+		}, longWait, checkFrequency)
+	}
+}
+
+func TestReactorLargePeerStoreInASmallNetwork(t *testing.T) {
+	testNet := setup(t, testOptions{
+		TotalNodes:   10,
+		MaxPeers:     100,
+		MaxConnected: 100,
+		BufferSize:   10,
+	})
+	testNet.connectN(t, 1)
+	testNet.start(t)
+
+	// assert that all nodes add each other in the network
+	for idx := 0; idx < len(testNet.nodes); idx++ {
+		testNet.requireNumberOfPeers(t, idx, len(testNet.nodes)-1, longWait)
+	}
+}
+
+func TestReactorWithNetworkGrowth(t *testing.T) {
+	testNet := setup(t, testOptions{
+		TotalNodes: 5,
+		BufferSize: 5,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+
+	// assert that all nodes add each other in the network
+	for idx := 0; idx < len(testNet.nodes); idx++ {
+		testNet.requireNumberOfPeers(t, idx, len(testNet.nodes)-1, shortWait)
+	}
+
+	// now we inject 10 more nodes
+	testNet.addNodes(t, 10)
+	for i := 5; i < testNet.total; i++ {
+		node := testNet.nodes[i]
+		require.NoError(t, testNet.reactors[node].Start())
+		require.True(t, testNet.reactors[node].IsRunning())
+		// we connect all new nodes to a single entry point and check that the
+		// node can distribute the addresses to all the others
+		testNet.connectPeers(t, 0, i)
+	}
+	require.Len(t, testNet.reactors, 15)
+
+	// assert that all nodes add each other in the network
+	for idx := 0; idx < len(testNet.nodes); idx++ {
+		testNet.requireNumberOfPeers(t, idx, len(testNet.nodes)-1, longWait)
+	}
+}
+
+func TestReactorIntegrationWithLegacyHandleRequest(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 3,
+	})
+	testNet.connectAll(t)
+	testNet.start(t)
+	t.Log(testNet.nodes)
+
+	// mock node sends a V1 Pex message to the second node
+	testNet.sendRequest(t, firstNode, secondNode, false)
+	addrs := testNet.getAddressesFor(t, []int{thirdNode})
+	testNet.listenForLegacyResponse(t, secondNode, firstNode, shortWait, addrs)
+}
+
+func TestReactorIntegrationWithLegacyHandleResponse(t *testing.T) {
+	testNet := setup(t, testOptions{
+		MockNodes:  1,
+		TotalNodes: 4,
+		BufferSize: 4,
+	})
+	testNet.connectPeers(t, firstNode, secondNode)
+	testNet.connectPeers(t, firstNode, thirdNode)
+	testNet.connectPeers(t, firstNode, fourthNode)
+	testNet.start(t)
+
+	testNet.listenForRequest(t, secondNode, firstNode, shortWait)
+	// send a v1 response instead
+	testNet.sendResponse(t, firstNode, secondNode, []int{thirdNode, fourthNode}, false)
+	testNet.requireNumberOfPeers(t, secondNode, len(testNet.nodes)-1, shortWait)
+}
+
+type reactorTestSuite struct {
+	network *p2ptest.Network
+	logger  log.Logger
+
+	reactors    map[p2p.NodeID]*pex.ReactorV2
+	pexChannels map[p2p.NodeID]*p2p.Channel
+
+	peerChans   map[p2p.NodeID]chan p2p.PeerUpdate
+	peerUpdates map[p2p.NodeID]*p2p.PeerUpdates
+
+	nodes []p2p.NodeID
+	mocks []p2p.NodeID
+	total int
+	opts  testOptions
+}
+
+type testOptions struct {
+	MockNodes    int
+	TotalNodes   int
+	BufferSize   int
+	MaxPeers     uint16
+	MaxConnected uint16
+}
+
+// setup setups a test suite with a network of nodes. Mocknodes represent the
+// hollow nodes that the test can listen and send on
+func setup(t *testing.T, opts testOptions) *reactorTestSuite {
+	t.Helper()
+
+	require.Greater(t, opts.TotalNodes, opts.MockNodes)
+	if opts.BufferSize == 0 {
+		opts.BufferSize = defaultBufferSize
+	}
+	networkOpts := p2ptest.NetworkOptions{
+		NumNodes:   opts.TotalNodes,
+		BufferSize: opts.BufferSize,
+		NodeOpts: p2ptest.NodeOptions{
+			MaxPeers:     opts.MaxPeers,
+			MaxConnected: opts.MaxConnected,
+		},
+	}
+	chBuf := opts.BufferSize
+	realNodes := opts.TotalNodes - opts.MockNodes
+
+	rts := &reactorTestSuite{
+		logger:      log.TestingLogger().With("testCase", t.Name()),
+		network:     p2ptest.MakeNetwork(t, networkOpts),
+		reactors:    make(map[p2p.NodeID]*pex.ReactorV2, realNodes),
+		pexChannels: make(map[p2p.NodeID]*p2p.Channel, opts.TotalNodes),
+		peerChans:   make(map[p2p.NodeID]chan p2p.PeerUpdate, opts.TotalNodes),
+		peerUpdates: make(map[p2p.NodeID]*p2p.PeerUpdates, opts.TotalNodes),
+		total:       opts.TotalNodes,
+		opts:        opts,
+	}
+
+	// NOTE: we don't assert that the channels get drained after stopping the
+	// reactor
+	rts.pexChannels = rts.network.MakeChannelsNoCleanup(
+		t, p2p.ChannelID(pex.PexChannel), new(proto.PexMessage), chBuf,
+	)
+
+	idx := 0
+	for nodeID := range rts.network.Nodes {
+		rts.peerChans[nodeID] = make(chan p2p.PeerUpdate, chBuf)
+		rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], chBuf)
+		rts.network.Nodes[nodeID].PeerManager.Register(rts.peerUpdates[nodeID])
+
+		// the first nodes in the array are always mock nodes
+		if idx < opts.MockNodes {
+			rts.mocks = append(rts.mocks, nodeID)
+		} else {
+			rts.reactors[nodeID] = pex.NewReactorV2(
+				rts.logger.With("nodeID", nodeID),
+				rts.network.Nodes[nodeID].PeerManager,
+				rts.pexChannels[nodeID],
+				rts.peerUpdates[nodeID],
+			)
+		}
+		rts.nodes = append(rts.nodes, nodeID)
+
+		idx++
+	}
+
+	require.Len(t, rts.reactors, realNodes)
+
+	t.Cleanup(func() {
+		for nodeID, reactor := range rts.reactors {
+			if reactor.IsRunning() {
+				require.NoError(t, reactor.Stop())
+				require.False(t, reactor.IsRunning())
+			}
+			rts.pexChannels[nodeID].Close()
+			rts.peerUpdates[nodeID].Close()
+		}
+		for _, nodeID := range rts.mocks {
+			rts.pexChannels[nodeID].Close()
+			rts.peerUpdates[nodeID].Close()
+		}
+	})
+
+	return rts
+}
+
+// starts up the pex reactors for each node
+func (r *reactorTestSuite) start(t *testing.T) {
+	t.Helper()
+
+	for _, reactor := range r.reactors {
+		require.NoError(t, reactor.Start())
+		require.True(t, reactor.IsRunning())
+	}
+}
+
+func (r *reactorTestSuite) addNodes(t *testing.T, nodes int) {
+	t.Helper()
+
+	for i := 0; i < nodes; i++ {
+		node := r.network.MakeNode(t, p2ptest.NodeOptions{
+			MaxPeers:     r.opts.MaxPeers,
+			MaxConnected: r.opts.MaxConnected,
+		})
+		r.network.Nodes[node.NodeID] = node
+		nodeID := node.NodeID
+		r.pexChannels[nodeID] = node.MakeChannelNoCleanup(
+			t, p2p.ChannelID(pex.PexChannel), new(proto.PexMessage), r.opts.BufferSize,
+		)
+		r.peerChans[nodeID] = make(chan p2p.PeerUpdate, r.opts.BufferSize)
+		r.peerUpdates[nodeID] = p2p.NewPeerUpdates(r.peerChans[nodeID], r.opts.BufferSize)
+		r.network.Nodes[nodeID].PeerManager.Register(r.peerUpdates[nodeID])
+		r.reactors[nodeID] = pex.NewReactorV2(
+			r.logger.With("nodeID", nodeID),
+			r.network.Nodes[nodeID].PeerManager,
+			r.pexChannels[nodeID],
+			r.peerUpdates[nodeID],
+		)
+		r.nodes = append(r.nodes, nodeID)
+		r.total++
+	}
+}
+
+func (r *reactorTestSuite) listenFor(
+	t *testing.T,
+	node p2p.NodeID,
+	conditional func(msg p2p.Envelope) bool,
+	assertion func(t *testing.T, msg p2p.Envelope) bool,
+	waitPeriod time.Duration,
+) {
+	timesUp := time.After(waitPeriod)
+	for {
+		select {
+		case envelope := <-r.pexChannels[node].In:
+			if conditional(envelope) && assertion(t, envelope) {
+				return
+			}
+		case <-timesUp:
+			require.Fail(t, "timed out waiting for message",
+				"node=%v, waitPeriod=%s", node, waitPeriod)
+		}
+	}
+}
+
+func (r *reactorTestSuite) listenForRequest(t *testing.T, fromNode, toNode int, waitPeriod time.Duration) {
+	r.logger.Info("Listening for request", "from", fromNode, "to", toNode)
+	to, from := r.checkNodePair(t, toNode, fromNode)
+	conditional := func(msg p2p.Envelope) bool {
+		_, ok := msg.Message.(*proto.PexRequestV2)
+		return ok && msg.From == from
+	}
+	assertion := func(t *testing.T, msg p2p.Envelope) bool {
+		require.Equal(t, &proto.PexRequestV2{}, msg.Message)
+		return true
+	}
+	r.listenFor(t, to, conditional, assertion, waitPeriod)
+}
+
+func (r *reactorTestSuite) pingAndlistenForNAddresses(
+	t *testing.T,
+	fromNode, toNode int,
+	waitPeriod time.Duration,
+	addresses int,
+) {
+	r.logger.Info("Listening for addresses", "from", fromNode, "to", toNode)
+	to, from := r.checkNodePair(t, toNode, fromNode)
+	conditional := func(msg p2p.Envelope) bool {
+		_, ok := msg.Message.(*proto.PexResponseV2)
+		return ok && msg.From == from
+	}
+	assertion := func(t *testing.T, msg p2p.Envelope) bool {
+		m, ok := msg.Message.(*proto.PexResponseV2)
+		if !ok {
+			require.Fail(t, "expected pex response v2")
+			return true
+		}
+		// assert the same amount of addresses
+		if len(m.Addresses) == addresses {
+			return true
+		}
+		// if we didn't get the right length, we wait and send the
+		// request again
+		time.Sleep(300 * time.Millisecond)
+		r.sendRequest(t, toNode, fromNode, true)
+		return false
+	}
+	r.sendRequest(t, toNode, fromNode, true)
+	r.listenFor(t, to, conditional, assertion, waitPeriod)
+}
+
+func (r *reactorTestSuite) listenForResponse(
+	t *testing.T,
+	fromNode, toNode int,
+	waitPeriod time.Duration,
+	addresses []proto.PexAddressV2,
+) {
+	r.logger.Info("Listening for response", "from", fromNode, "to", toNode)
+	to, from := r.checkNodePair(t, toNode, fromNode)
+	conditional := func(msg p2p.Envelope) bool {
+		_, ok := msg.Message.(*proto.PexResponseV2)
+		r.logger.Info("message", msg, "ok", ok)
+		return ok && msg.From == from
+	}
+	assertion := func(t *testing.T, msg p2p.Envelope) bool {
+		require.Equal(t, &proto.PexResponseV2{Addresses: addresses}, msg.Message)
+		return true
+	}
+	r.listenFor(t, to, conditional, assertion, waitPeriod)
+}
+
+func (r *reactorTestSuite) listenForLegacyResponse(
+	t *testing.T,
+	fromNode, toNode int,
+	waitPeriod time.Duration,
+	addresses []proto.PexAddress,
+) {
+	r.logger.Info("Listening for response", "from", fromNode, "to", toNode)
+	to, from := r.checkNodePair(t, toNode, fromNode)
+	conditional := func(msg p2p.Envelope) bool {
+		_, ok := msg.Message.(*proto.PexResponse)
+		return ok && msg.From == from
+	}
+	assertion := func(t *testing.T, msg p2p.Envelope) bool {
+		require.Equal(t, &proto.PexResponse{Addresses: addresses}, msg.Message)
+		return true
+	}
+	r.listenFor(t, to, conditional, assertion, waitPeriod)
+}
+
+func (r *reactorTestSuite) listenForPeerUpdate(
+	t *testing.T,
+	onNode, withNode int,
+	status p2p.PeerStatus,
+	waitPeriod time.Duration,
+) {
+	on, with := r.checkNodePair(t, onNode, withNode)
+	sub := r.network.Nodes[on].PeerManager.Subscribe()
+	defer sub.Close()
+	timesUp := time.After(waitPeriod)
+	for {
+		select {
+		case peerUpdate := <-sub.Updates():
+			if peerUpdate.NodeID == with {
+				require.Equal(t, status, peerUpdate.Status)
+				return
+			}
+
+		case <-timesUp:
+			require.Fail(t, "timed out waiting for peer status", "%v with status %v",
+				with, status)
+			return
+		}
+	}
+}
+
+func (r *reactorTestSuite) getV2AddressesFor(nodes []int) []proto.PexAddressV2 {
+	addresses := make([]proto.PexAddressV2, len(nodes))
+	for idx, node := range nodes {
+		nodeID := r.nodes[node]
+		addresses[idx] = proto.PexAddressV2{
+			URL: r.network.Nodes[nodeID].NodeAddress.String(),
+		}
+	}
+	return addresses
+}
+
+func (r *reactorTestSuite) getAddressesFor(t *testing.T, nodes []int) []proto.PexAddress {
+	addresses := make([]proto.PexAddress, len(nodes))
+	for idx, node := range nodes {
+		nodeID := r.nodes[node]
+		nodeAddrs := r.network.Nodes[nodeID].NodeAddress
+		endpoints, err := nodeAddrs.Resolve(context.Background())
+		require.NoError(t, err)
+		require.Len(t, endpoints, 1)
+		addresses[idx] = proto.PexAddress{
+			ID:   string(nodeAddrs.NodeID),
+			IP:   endpoints[0].IP.String(),
+			Port: uint32(endpoints[0].Port),
+		}
+	}
+	return addresses
+}
+
+func (r *reactorTestSuite) sendRequest(t *testing.T, fromNode, toNode int, v2 bool) {
+	to, from := r.checkNodePair(t, toNode, fromNode)
+	if v2 {
+		r.pexChannels[from].Out <- p2p.Envelope{
+			To:      to,
+			Message: &proto.PexRequestV2{},
+		}
+	} else {
+		r.pexChannels[from].Out <- p2p.Envelope{
+			To:      to,
+			Message: &proto.PexRequest{},
+		}
+	}
+}
+
+func (r *reactorTestSuite) sendResponse(
+	t *testing.T,
+	fromNode, toNode int,
+	withNodes []int,
+	v2 bool,
+) {
+	from, to := r.checkNodePair(t, fromNode, toNode)
+	if v2 {
+		addrs := r.getV2AddressesFor(withNodes)
+		r.pexChannels[from].Out <- p2p.Envelope{
+			To: to,
+			Message: &proto.PexResponseV2{
+				Addresses: addrs,
+			},
+		}
+	} else {
+		addrs := r.getAddressesFor(t, withNodes)
+		r.pexChannels[from].Out <- p2p.Envelope{
+			To: to,
+			Message: &proto.PexResponse{
+				Addresses: addrs,
+			},
+		}
+	}
+}
+
+func (r *reactorTestSuite) requireNumberOfPeers(
+	t *testing.T,
+	nodeIndex, numPeers int,
+	waitPeriod time.Duration,
+) {
+	require.Eventuallyf(t, func() bool {
+		actualNumPeers := len(r.network.Nodes[r.nodes[nodeIndex]].PeerManager.Peers())
+		return actualNumPeers >= numPeers
+	}, waitPeriod, checkFrequency, "peer failed to connect with the asserted amount of peers "+
+		"index=%d, node=%q, waitPeriod=%s expected=%d actual=%d",
+		nodeIndex, r.nodes[nodeIndex], waitPeriod, numPeers,
+		len(r.network.Nodes[r.nodes[nodeIndex]].PeerManager.Peers()),
+	)
+}
+
+func (r *reactorTestSuite) connectAll(t *testing.T) {
+	r.connectN(t, r.total-1)
+}
+
+// connects all nodes with n other nodes
+func (r *reactorTestSuite) connectN(t *testing.T, n int) {
+	if n >= r.total {
+		require.Fail(t, "connectN: n must be less than the size of the network - 1")
+	}
+
+	for i := 0; i < r.total; i++ {
+		for j := 0; j < n; j++ {
+			r.connectPeers(t, i, (i+j+1)%r.total)
+		}
+	}
+}
+
+// connects node1 to node2
+func (r *reactorTestSuite) connectPeers(t *testing.T, sourceNode, targetNode int) {
+	t.Helper()
+	node1, node2 := r.checkNodePair(t, sourceNode, targetNode)
+	r.logger.Info("connecting peers", "sourceNode", sourceNode, "targetNode", targetNode)
+
+	n1 := r.network.Nodes[node1]
+	if n1 == nil {
+		require.Fail(t, "connectPeers: source node %v is not part of the testnet", node1)
+		return
+	}
+
+	n2 := r.network.Nodes[node2]
+	if n2 == nil {
+		require.Fail(t, "connectPeers: target node %v is not part of the testnet", node2)
+		return
+	}
+
+	sourceSub := n1.PeerManager.Subscribe()
+	defer sourceSub.Close()
+	targetSub := n2.PeerManager.Subscribe()
+	defer targetSub.Close()
+
+	sourceAddress := n1.NodeAddress
+	r.logger.Debug("source address", "address", sourceAddress)
+	targetAddress := n2.NodeAddress
+	r.logger.Debug("target address", "address", targetAddress)
+
+	added, err := n1.PeerManager.Add(targetAddress)
+	require.NoError(t, err)
+
+	if !added {
+		r.logger.Debug("nodes already know about one another",
+			"sourceNode", sourceNode, "targetNode", targetNode)
+		return
+	}
+
+	select {
+	case peerUpdate := <-targetSub.Updates():
+		require.Equal(t, p2p.PeerUpdate{
+			NodeID: node1,
+			Status: p2p.PeerStatusUp,
+		}, peerUpdate)
+		r.logger.Debug("target connected with source")
+	case <-time.After(time.Second):
+		require.Fail(t, "timed out waiting for peer", "%v accepting %v",
+			targetNode, sourceNode)
+	}
+
+	select {
+	case peerUpdate := <-sourceSub.Updates():
+		require.Equal(t, p2p.PeerUpdate{
+			NodeID: node2,
+			Status: p2p.PeerStatusUp,
+		}, peerUpdate)
+		r.logger.Debug("source connected with target")
+	case <-time.After(time.Second):
+		require.Fail(t, "timed out waiting for peer", "%v dialing %v",
+			sourceNode, targetNode)
+	}
+
+	added, err = n2.PeerManager.Add(sourceAddress)
+	require.NoError(t, err)
+	require.True(t, added)
+}
+
+// nolint: unused
+func (r *reactorTestSuite) pexAddresses(t *testing.T, nodeIndices []int) []proto.PexAddress {
+	var addresses []proto.PexAddress
+	for _, i := range nodeIndices {
+		if i < len(r.nodes) {
+			require.Fail(t, "index for pex address is greater than number of nodes")
+		}
+		nodeAddrs := r.network.Nodes[r.nodes[i]].NodeAddress
+		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+		endpoints, err := nodeAddrs.Resolve(ctx)
+		cancel()
+		require.NoError(t, err)
+		for _, endpoint := range endpoints {
+			if endpoint.IP != nil {
+				addresses = append(addresses, proto.PexAddress{
+					ID:   string(nodeAddrs.NodeID),
+					IP:   endpoint.IP.String(),
+					Port: uint32(endpoint.Port),
+				})
+			}
+		}
+
+	}
+	return addresses
+}
+
+func (r *reactorTestSuite) checkNodePair(t *testing.T, first, second int) (p2p.NodeID, p2p.NodeID) {
+	require.NotEqual(t, first, second)
+	require.Less(t, first, r.total)
+	require.Less(t, second, r.total)
+	return r.nodes[first], r.nodes[second]
+}
+
+func (r *reactorTestSuite) addAddresses(t *testing.T, node int, addrs []int) {
+	peerManager := r.network.Nodes[r.nodes[node]].PeerManager
+	for _, addr := range addrs {
+		require.Less(t, addr, r.total)
+		address := r.network.Nodes[r.nodes[addr]].NodeAddress
+		added, err := peerManager.Add(address)
+		require.NoError(t, err)
+		require.True(t, added)
+	}
+}

p2p/router.go

@@ -128,13 +128,13 @@ type RouterOptions struct {
 	// no timeout.
 	HandshakeTimeout time.Duration
 
-	// QueueType must be "wdrr" (Weighed Deficit Round Robin),
-	// "priority", or FIFO. Defaults to FIFO.
+	// QueueType must be "wdrr" (Weighed Deficit Round Robin), "priority", or
+	// "fifo". Defaults to "fifo".
 	QueueType string
 
-	// MaxIncommingConnectionsPerIP limits the number of incoming
-	// connections per IP address. Defaults to 100.
-	MaxIncommingConnectionsPerIP uint
+	// MaxIncomingConnectionAttempts rate limits the number of incoming connection
+	// attempts per IP address. Defaults to 100.
+	MaxIncomingConnectionAttempts uint
 
 	// IncomingConnectionWindow describes how often an IP address
 	// can attempt to create a new connection. Defaults to 10
@@ -182,8 +182,8 @@ func (o *RouterOptions) Validate() error {
 			o.IncomingConnectionWindow)
 	}
 
-	if o.MaxIncommingConnectionsPerIP == 0 {
-		o.MaxIncommingConnectionsPerIP = 100
+	if o.MaxIncomingConnectionAttempts == 0 {
+		o.MaxIncomingConnectionAttempts = 100
 	}
 
 	return nil
@@ -278,8 +278,9 @@ func NewRouter(
 		nodeInfo: nodeInfo,
 		privKey:  privKey,
 		connTracker: newConnTracker(
-			options.MaxIncommingConnectionsPerIP,
-			options.IncomingConnectionWindow),
+			options.MaxIncomingConnectionAttempts,
+			options.IncomingConnectionWindow,
+		),
 		chDescs:            make([]ChannelDescriptor, 0),
 		transports:         transports,
 		protocolTransports: map[Protocol]Transport{},
@@ -526,7 +527,8 @@ func (r *Router) acceptPeers(transport Transport) {
 			r.logger.Debug("rate limiting incoming peer",
 				"err", err,
 				"ip", incomingIP.String(),
-				"closeErr", closeErr)
+				"close_err", closeErr,
+			)
 
 			return
 		}
@@ -545,9 +547,7 @@ func (r *Router) openConnection(ctx context.Context, conn Connection) {
 	incomingIP := re.IP
 
 	if err := r.filterPeersIP(ctx, incomingIP, re.Port); err != nil {
-		r.logger.Debug("peer filtered by IP",
-			"ip", incomingIP.String(),
-			"err", err)
+		r.logger.Debug("peer filtered by IP", "ip", incomingIP.String(), "err", err)
 		return
 	}
 
@@ -576,9 +576,7 @@ func (r *Router) openConnection(ctx context.Context, conn Connection) {
 	}
 
 	if err := r.filterPeersID(ctx, peerInfo.NodeID); err != nil {
-		r.logger.Debug("peer filtered by node ID",
-			"node", peerInfo.NodeID,
-			"err", err)
+		r.logger.Debug("peer filtered by node ID", "node", peerInfo.NodeID, "err", err)
 		return
 	}
 

p2p/router_test.go

@@ -80,7 +80,7 @@ func TestRouter_Network(t *testing.T) {
 
 	// We then submit an error for a peer, and watch it get disconnected and
 	// then reconnected as the router retries it.
-	peerUpdates := local.MakePeerUpdates(t)
+	peerUpdates := local.MakePeerUpdatesNoRequireEmpty(t)
 	channel.Error <- p2p.PeerError{
 		NodeID: peers[0].NodeID,
 		Err:    errors.New("boom"),
@@ -573,7 +573,9 @@ func TestRouter_DialPeers(t *testing.T) {
 			require.NoError(t, err)
 			defer peerManager.Close()
 
-			require.NoError(t, peerManager.Add(address))
+			added, err := peerManager.Add(address)
+			require.NoError(t, err)
+			require.True(t, added)
 			sub := peerManager.Subscribe()
 			defer sub.Close()
 
@@ -648,9 +650,17 @@ func TestRouter_DialPeers_Parallel(t *testing.T) {
 	require.NoError(t, err)
 	defer peerManager.Close()
 
-	require.NoError(t, peerManager.Add(a))
-	require.NoError(t, peerManager.Add(b))
-	require.NoError(t, peerManager.Add(c))
+	added, err := peerManager.Add(a)
+	require.NoError(t, err)
+	require.True(t, added)
+
+	added, err = peerManager.Add(b)
+	require.NoError(t, err)
+	require.True(t, added)
+
+	added, err = peerManager.Add(c)
+	require.NoError(t, err)
+	require.True(t, added)
 
 	router, err := p2p.NewRouter(
 		log.TestingLogger(),

p2p/shim.go

@@ -144,7 +144,10 @@ func (rs *ReactorShim) proxyPeerEnvelopes() {
 					}
 
 					if !src.Send(cs.Descriptor.ID, bz) {
-						rs.Logger.Error(
+						// This usually happens when we try to send across a channel
+						// that the peer doesn't have open. To avoid bloating the
+						// logs we set this to be Debug
+						rs.Logger.Debug(
 							"failed to proxy message to peer",
 							"ch_id", cs.Descriptor.ID,
 							"peer", e.To,

p2p/switch.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	mrand "math/rand"
 	"net"
 	"sync"
 	"time"
@@ -12,7 +13,7 @@ import (
 	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/libs/cmap"
-	"github.com/tendermint/tendermint/libs/rand"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
 	"github.com/tendermint/tendermint/libs/service"
 	"github.com/tendermint/tendermint/p2p/conn"
 )
@@ -115,8 +116,6 @@ type Switch struct {
 	connFilters   []ConnFilterFunc
 	conns         ConnSet
 
-	rng *rand.Rand // seed for randomizing dial times and orders
-
 	metrics *Metrics
 }
 
@@ -159,8 +158,8 @@ func NewSwitch(
 		conns:                NewConnSet(),
 	}
 
-	// Ensure we have a completely undeterministic PRNG.
-	sw.rng = rand.NewRand()
+	// Ensure PRNG is reseeded.
+	tmrand.Reseed()
 
 	sw.BaseService = *service.NewBaseService(nil, "P2P Switch", sw)
 
@@ -554,7 +553,7 @@ func (sw *Switch) dialPeersAsync(netAddrs []*NetAddress) {
 	}
 
 	// permute the list, dial them in random order.
-	perm := sw.rng.Perm(len(netAddrs))
+	perm := mrand.Perm(len(netAddrs))
 	for i := 0; i < len(perm); i++ {
 		go func(i int) {
 			j := perm[i]
@@ -597,7 +596,8 @@ func (sw *Switch) DialPeerWithAddress(addr *NetAddress) error {
 
 // sleep for interval plus some random amount of ms on [0, dialRandomizerIntervalMilliseconds]
 func (sw *Switch) randomSleep(interval time.Duration) {
-	r := time.Duration(sw.rng.Int63n(dialRandomizerIntervalMilliseconds)) * time.Millisecond
+	// nolint:gosec // G404: Use of weak random number generator
+	r := time.Duration(mrand.Int63n(dialRandomizerIntervalMilliseconds)) * time.Millisecond
 	time.Sleep(r + interval)
 }
 

p2p/test_util.go

@@ -3,6 +3,7 @@ package p2p
 import (
 	"context"
 	"fmt"
+	mrand "math/rand"
 	"net"
 
 	"github.com/tendermint/tendermint/libs/log"
@@ -35,15 +36,16 @@ func CreateRandomPeer(outbound bool) Peer {
 	return p
 }
 
+// nolint:gosec // G404: Use of weak random number generator
 func CreateRoutableAddr() (addr string, netAddr *NetAddress) {
 	for {
 		var err error
 		addr = fmt.Sprintf("%X@%v.%v.%v.%v:26656",
 			tmrand.Bytes(20),
-			tmrand.Int()%256,
-			tmrand.Int()%256,
-			tmrand.Int()%256,
-			tmrand.Int()%256)
+			mrand.Int()%256,
+			mrand.Int()%256,
+			mrand.Int()%256,
+			mrand.Int()%256)
 		netAddr, err = NewNetAddressString(addr)
 		if err != nil {
 			panic(err)

p2p/transport_memory.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"sync"
 
 	"github.com/tendermint/tendermint/crypto"
@@ -130,6 +131,10 @@ func (t *MemoryTransport) Endpoints() []Endpoint {
 		return []Endpoint{{
 			Protocol: MemoryProtocol,
 			Path:     string(t.nodeID),
+			// An arbitrary IP and port is used in order for the pex
+			// reactor to be able to send addresses to one another.
+			IP:   net.IPv4zero,
+			Port: 0,
 		}}
 	}
 }
@@ -153,6 +158,10 @@ func (t *MemoryTransport) Dial(ctx context.Context, endpoint Endpoint) (Connecti
 	if endpoint.Path == "" {
 		return nil, errors.New("no path")
 	}
+	if err := endpoint.Validate(); err != nil {
+		return nil, err
+	}
+
 	nodeID, err := NewNodeID(endpoint.Path)
 	if err != nil {
 		return nil, err

p2p/transport_test.go

@@ -113,7 +113,7 @@ func TestTransport_DialEndpoints(t *testing.T) {
 		require.Error(t, err)
 
 		// Tests for networked endpoints (with IP).
-		if len(endpoint.IP) > 0 {
+		if len(endpoint.IP) > 0 && endpoint.Protocol != p2p.MemoryProtocol {
 			for _, tc := range ipTestCases {
 				tc := tc
 				t.Run(tc.ip.String(), func(t *testing.T) {
@@ -124,7 +124,7 @@ func TestTransport_DialEndpoints(t *testing.T) {
 						require.NoError(t, conn.Close())
 						require.NoError(t, err)
 					} else {
-						require.Error(t, err)
+						require.Error(t, err, "endpoint=%s", e)
 					}
 				})
 			}

proto/tendermint/crypto/keys.pb.go

@@ -29,6 +29,7 @@ type PublicKey struct {
 	// Types that are valid to be assigned to Sum:
 	//	*PublicKey_Ed25519
 	//	*PublicKey_Secp256K1
+	//	*PublicKey_Sr25519
 	Sum isPublicKey_Sum `protobuf_oneof:"sum"`
 }
 
@@ -79,9 +80,13 @@ type PublicKey_Ed25519 struct {
 type PublicKey_Secp256K1 struct {
 	Secp256K1 []byte `protobuf:"bytes,2,opt,name=secp256k1,proto3,oneof" json:"secp256k1,omitempty"`
 }
+type PublicKey_Sr25519 struct {
+	Sr25519 []byte `protobuf:"bytes,3,opt,name=sr25519,proto3,oneof" json:"sr25519,omitempty"`
+}
 
 func (*PublicKey_Ed25519) isPublicKey_Sum()   {}
 func (*PublicKey_Secp256K1) isPublicKey_Sum() {}
+func (*PublicKey_Sr25519) isPublicKey_Sum()   {}
 
 func (m *PublicKey) GetSum() isPublicKey_Sum {
 	if m != nil {
@@ -104,11 +109,19 @@ func (m *PublicKey) GetSecp256K1() []byte {
 	return nil
 }
 
+func (m *PublicKey) GetSr25519() []byte {
+	if x, ok := m.GetSum().(*PublicKey_Sr25519); ok {
+		return x.Sr25519
+	}
+	return nil
+}
+
 // XXX_OneofWrappers is for the internal use of the proto package.
 func (*PublicKey) XXX_OneofWrappers() []interface{} {
 	return []interface{}{
 		(*PublicKey_Ed25519)(nil),
 		(*PublicKey_Secp256K1)(nil),
+		(*PublicKey_Sr25519)(nil),
 	}
 }
 
@@ -119,20 +132,21 @@ func init() {
 func init() { proto.RegisterFile("tendermint/crypto/keys.proto", fileDescriptor_cb048658b234868c) }
 
 var fileDescriptor_cb048658b234868c = []byte{
-	// 199 bytes of a gzipped FileDescriptorProto
+	// 210 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x29, 0x49, 0xcd, 0x4b,
 	0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x4f, 0x2e, 0xaa, 0x2c, 0x28, 0xc9, 0xd7, 0xcf, 0x4e,
 	0xad, 0x2c, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x44, 0xc8, 0xea, 0x41, 0x64, 0xa5,
-	0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0xb2, 0xfa, 0x20, 0x16, 0x44, 0xa1, 0x52, 0x04, 0x17, 0x67,
+	0x44, 0xd2, 0xf3, 0xd3, 0xf3, 0xc1, 0xb2, 0xfa, 0x20, 0x16, 0x44, 0xa1, 0x52, 0x09, 0x17, 0x67,
 	0x40, 0x69, 0x52, 0x4e, 0x66, 0xb2, 0x77, 0x6a, 0xa5, 0x90, 0x14, 0x17, 0x7b, 0x6a, 0x8a, 0x91,
 	0xa9, 0xa9, 0xa1, 0xa5, 0x04, 0xa3, 0x02, 0xa3, 0x06, 0x8f, 0x07, 0x43, 0x10, 0x4c, 0x40, 0x48,
 	0x8e, 0x8b, 0xb3, 0x38, 0x35, 0xb9, 0xc0, 0xc8, 0xd4, 0x2c, 0xdb, 0x50, 0x82, 0x09, 0x2a, 0x8b,
-	0x10, 0xb2, 0xe2, 0x78, 0xb1, 0x40, 0x9e, 0xf1, 0xc5, 0x42, 0x79, 0x46, 0x27, 0x56, 0x2e, 0xe6,
-	0xe2, 0xd2, 0x5c, 0xa7, 0xa0, 0x13, 0x8f, 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48,
-	0x8e, 0x71, 0xc2, 0x63, 0x39, 0x86, 0x0b, 0x8f, 0xe5, 0x18, 0x6e, 0x3c, 0x96, 0x63, 0x88, 0xb2,
-	0x48, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x47, 0xf2, 0x05, 0x12, 0x13,
-	0xe2, 0x4c, 0x0c, 0x1f, 0x26, 0xb1, 0x81, 0x25, 0x8c, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe8,
-	0x1d, 0x1e, 0xe2, 0xfd, 0x00, 0x00, 0x00,
+	0x10, 0x02, 0xe9, 0x2d, 0x2e, 0x82, 0xe8, 0x65, 0x86, 0xe9, 0x85, 0x0a, 0x58, 0x71, 0xbc, 0x58,
+	0x20, 0xcf, 0xf8, 0x62, 0xa1, 0x3c, 0xa3, 0x13, 0x2b, 0x17, 0x73, 0x71, 0x69, 0xae, 0x53, 0xd0,
+	0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c,
+	0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0x59, 0xa4, 0x67, 0x96, 0x64, 0x94,
+	0x26, 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0x23, 0xf9, 0x10, 0x89, 0x09, 0xf1, 0x02, 0x86, 0xef, 0x93,
+	0xd8, 0xc0, 0x12, 0xc6, 0x80, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaa, 0x12, 0x2c, 0x35, 0x19, 0x01,
+	0x00, 0x00,
 }
 
 func (this *PublicKey) Compare(that interface{}) int {
@@ -173,6 +187,8 @@ func (this *PublicKey) Compare(that interface{}) int {
 			thisType = 0
 		case *PublicKey_Secp256K1:
 			thisType = 1
+		case *PublicKey_Sr25519:
+			thisType = 2
 		default:
 			panic(fmt.Sprintf("compare: unexpected type %T in oneof", this.Sum))
 		}
@@ -182,6 +198,8 @@ func (this *PublicKey) Compare(that interface{}) int {
 			that1Type = 0
 		case *PublicKey_Secp256K1:
 			that1Type = 1
+		case *PublicKey_Sr25519:
+			that1Type = 2
 		default:
 			panic(fmt.Sprintf("compare: unexpected type %T in oneof", that1.Sum))
 		}
@@ -257,6 +275,36 @@ func (this *PublicKey_Secp256K1) Compare(that interface{}) int {
 	}
 	return 0
 }
+func (this *PublicKey_Sr25519) Compare(that interface{}) int {
+	if that == nil {
+		if this == nil {
+			return 0
+		}
+		return 1
+	}
+
+	that1, ok := that.(*PublicKey_Sr25519)
+	if !ok {
+		that2, ok := that.(PublicKey_Sr25519)
+		if ok {
+			that1 = &that2
+		} else {
+			return 1
+		}
+	}
+	if that1 == nil {
+		if this == nil {
+			return 0
+		}
+		return 1
+	} else if this == nil {
+		return -1
+	}
+	if c := bytes.Compare(this.Sr25519, that1.Sr25519); c != 0 {
+		return c
+	}
+	return 0
+}
 func (this *PublicKey) Equal(that interface{}) bool {
 	if that == nil {
 		return this == nil
@@ -335,6 +383,30 @@ func (this *PublicKey_Secp256K1) Equal(that interface{}) bool {
 	}
 	return true
 }
+func (this *PublicKey_Sr25519) Equal(that interface{}) bool {
+	if that == nil {
+		return this == nil
+	}
+
+	that1, ok := that.(*PublicKey_Sr25519)
+	if !ok {
+		that2, ok := that.(PublicKey_Sr25519)
+		if ok {
+			that1 = &that2
+		} else {
+			return false
+		}
+	}
+	if that1 == nil {
+		return this == nil
+	} else if this == nil {
+		return false
+	}
+	if !bytes.Equal(this.Sr25519, that1.Sr25519) {
+		return false
+	}
+	return true
+}
 func (m *PublicKey) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -399,6 +471,22 @@ func (m *PublicKey_Secp256K1) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	}
 	return len(dAtA) - i, nil
 }
+func (m *PublicKey_Sr25519) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PublicKey_Sr25519) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.Sr25519 != nil {
+		i -= len(m.Sr25519)
+		copy(dAtA[i:], m.Sr25519)
+		i = encodeVarintKeys(dAtA, i, uint64(len(m.Sr25519)))
+		i--
+		dAtA[i] = 0x1a
+	}
+	return len(dAtA) - i, nil
+}
 func encodeVarintKeys(dAtA []byte, offset int, v uint64) int {
 	offset -= sovKeys(v)
 	base := offset
@@ -446,6 +534,18 @@ func (m *PublicKey_Secp256K1) Size() (n int) {
 	}
 	return n
 }
+func (m *PublicKey_Sr25519) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.Sr25519 != nil {
+		l = len(m.Sr25519)
+		n += 1 + l + sovKeys(uint64(l))
+	}
+	return n
+}
 
 func sovKeys(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
@@ -548,6 +648,39 @@ func (m *PublicKey) Unmarshal(dAtA []byte) error {
 			copy(v, dAtA[iNdEx:postIndex])
 			m.Sum = &PublicKey_Secp256K1{v}
 			iNdEx = postIndex
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Sr25519", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowKeys
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthKeys
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthKeys
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			v := make([]byte, postIndex-iNdEx)
+			copy(v, dAtA[iNdEx:postIndex])
+			m.Sum = &PublicKey_Sr25519{v}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipKeys(dAtA[iNdEx:])

proto/tendermint/crypto/keys.proto

@@ -13,5 +13,6 @@ message PublicKey {
   oneof sum {
     bytes ed25519   = 1;
     bytes secp256k1 = 2;
+    bytes sr25519   = 3;
   }
 }

proto/tendermint/p2p/pex.go

@@ -13,8 +13,12 @@ func (m *PexMessage) Wrap(pb proto.Message) error {
 		m.Sum = &PexMessage_PexRequest{PexRequest: msg}
 	case *PexResponse:
 		m.Sum = &PexMessage_PexResponse{PexResponse: msg}
+	case *PexRequestV2:
+		m.Sum = &PexMessage_PexRequestV2{PexRequestV2: msg}
+	case *PexResponseV2:
+		m.Sum = &PexMessage_PexResponseV2{PexResponseV2: msg}
 	default:
-		return fmt.Errorf("unknown message: %T", msg)
+		return fmt.Errorf("unknown pex message: %T", msg)
 	}
 	return nil
 }
@@ -27,7 +31,11 @@ func (m *PexMessage) Unwrap() (proto.Message, error) {
 		return msg.PexRequest, nil
 	case *PexMessage_PexResponse:
 		return msg.PexResponse, nil
+	case *PexMessage_PexRequestV2:
+		return msg.PexRequestV2, nil
+	case *PexMessage_PexResponseV2:
+		return msg.PexResponseV2, nil
 	default:
-		return nil, fmt.Errorf("unknown message: %T", msg)
+		return nil, fmt.Errorf("unknown pex message: %T", msg)
 	}
 }

proto/tendermint/p2p/pex.pb.go

@@ -163,10 +163,136 @@ func (m *PexResponse) GetAddresses() []PexAddress {
 	return nil
 }
 
+type PexAddressV2 struct {
+	URL string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"`
+}
+
+func (m *PexAddressV2) Reset()         { *m = PexAddressV2{} }
+func (m *PexAddressV2) String() string { return proto.CompactTextString(m) }
+func (*PexAddressV2) ProtoMessage()    {}
+func (*PexAddressV2) Descriptor() ([]byte, []int) {
+	return fileDescriptor_81c2f011fd13be57, []int{3}
+}
+func (m *PexAddressV2) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *PexAddressV2) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_PexAddressV2.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *PexAddressV2) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PexAddressV2.Merge(m, src)
+}
+func (m *PexAddressV2) XXX_Size() int {
+	return m.Size()
+}
+func (m *PexAddressV2) XXX_DiscardUnknown() {
+	xxx_messageInfo_PexAddressV2.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PexAddressV2 proto.InternalMessageInfo
+
+func (m *PexAddressV2) GetURL() string {
+	if m != nil {
+		return m.URL
+	}
+	return ""
+}
+
+type PexRequestV2 struct {
+}
+
+func (m *PexRequestV2) Reset()         { *m = PexRequestV2{} }
+func (m *PexRequestV2) String() string { return proto.CompactTextString(m) }
+func (*PexRequestV2) ProtoMessage()    {}
+func (*PexRequestV2) Descriptor() ([]byte, []int) {
+	return fileDescriptor_81c2f011fd13be57, []int{4}
+}
+func (m *PexRequestV2) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *PexRequestV2) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_PexRequestV2.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *PexRequestV2) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PexRequestV2.Merge(m, src)
+}
+func (m *PexRequestV2) XXX_Size() int {
+	return m.Size()
+}
+func (m *PexRequestV2) XXX_DiscardUnknown() {
+	xxx_messageInfo_PexRequestV2.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PexRequestV2 proto.InternalMessageInfo
+
+type PexResponseV2 struct {
+	Addresses []PexAddressV2 `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
+}
+
+func (m *PexResponseV2) Reset()         { *m = PexResponseV2{} }
+func (m *PexResponseV2) String() string { return proto.CompactTextString(m) }
+func (*PexResponseV2) ProtoMessage()    {}
+func (*PexResponseV2) Descriptor() ([]byte, []int) {
+	return fileDescriptor_81c2f011fd13be57, []int{5}
+}
+func (m *PexResponseV2) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *PexResponseV2) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_PexResponseV2.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *PexResponseV2) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PexResponseV2.Merge(m, src)
+}
+func (m *PexResponseV2) XXX_Size() int {
+	return m.Size()
+}
+func (m *PexResponseV2) XXX_DiscardUnknown() {
+	xxx_messageInfo_PexResponseV2.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PexResponseV2 proto.InternalMessageInfo
+
+func (m *PexResponseV2) GetAddresses() []PexAddressV2 {
+	if m != nil {
+		return m.Addresses
+	}
+	return nil
+}
+
 type PexMessage struct {
 	// Types that are valid to be assigned to Sum:
 	//	*PexMessage_PexRequest
 	//	*PexMessage_PexResponse
+	//	*PexMessage_PexRequestV2
+	//	*PexMessage_PexResponseV2
 	Sum isPexMessage_Sum `protobuf_oneof:"sum"`
 }
 
@@ -174,7 +300,7 @@ func (m *PexMessage) Reset()         { *m = PexMessage{} }
 func (m *PexMessage) String() string { return proto.CompactTextString(m) }
 func (*PexMessage) ProtoMessage()    {}
 func (*PexMessage) Descriptor() ([]byte, []int) {
-	return fileDescriptor_81c2f011fd13be57, []int{3}
+	return fileDescriptor_81c2f011fd13be57, []int{6}
 }
 func (m *PexMessage) XXX_Unmarshal(b []byte) error {
 	return m.Unmarshal(b)
@@ -215,9 +341,17 @@ type PexMessage_PexRequest struct {
 type PexMessage_PexResponse struct {
 	PexResponse *PexResponse `protobuf:"bytes,2,opt,name=pex_response,json=pexResponse,proto3,oneof" json:"pex_response,omitempty"`
 }
+type PexMessage_PexRequestV2 struct {
+	PexRequestV2 *PexRequestV2 `protobuf:"bytes,3,opt,name=pex_request_v2,json=pexRequestV2,proto3,oneof" json:"pex_request_v2,omitempty"`
+}
+type PexMessage_PexResponseV2 struct {
+	PexResponseV2 *PexResponseV2 `protobuf:"bytes,4,opt,name=pex_response_v2,json=pexResponseV2,proto3,oneof" json:"pex_response_v2,omitempty"`
+}
 
-func (*PexMessage_PexRequest) isPexMessage_Sum()  {}
-func (*PexMessage_PexResponse) isPexMessage_Sum() {}
+func (*PexMessage_PexRequest) isPexMessage_Sum()    {}
+func (*PexMessage_PexResponse) isPexMessage_Sum()   {}
+func (*PexMessage_PexRequestV2) isPexMessage_Sum()  {}
+func (*PexMessage_PexResponseV2) isPexMessage_Sum() {}
 
 func (m *PexMessage) GetSum() isPexMessage_Sum {
 	if m != nil {
@@ -240,11 +374,27 @@ func (m *PexMessage) GetPexResponse() *PexResponse {
 	return nil
 }
 
+func (m *PexMessage) GetPexRequestV2() *PexRequestV2 {
+	if x, ok := m.GetSum().(*PexMessage_PexRequestV2); ok {
+		return x.PexRequestV2
+	}
+	return nil
+}
+
+func (m *PexMessage) GetPexResponseV2() *PexResponseV2 {
+	if x, ok := m.GetSum().(*PexMessage_PexResponseV2); ok {
+		return x.PexResponseV2
+	}
+	return nil
+}
+
 // XXX_OneofWrappers is for the internal use of the proto package.
 func (*PexMessage) XXX_OneofWrappers() []interface{} {
 	return []interface{}{
 		(*PexMessage_PexRequest)(nil),
 		(*PexMessage_PexResponse)(nil),
+		(*PexMessage_PexRequestV2)(nil),
+		(*PexMessage_PexResponseV2)(nil),
 	}
 }
 
@@ -252,33 +402,42 @@ func init() {
 	proto.RegisterType((*PexAddress)(nil), "tendermint.p2p.PexAddress")
 	proto.RegisterType((*PexRequest)(nil), "tendermint.p2p.PexRequest")
 	proto.RegisterType((*PexResponse)(nil), "tendermint.p2p.PexResponse")
+	proto.RegisterType((*PexAddressV2)(nil), "tendermint.p2p.PexAddressV2")
+	proto.RegisterType((*PexRequestV2)(nil), "tendermint.p2p.PexRequestV2")
+	proto.RegisterType((*PexResponseV2)(nil), "tendermint.p2p.PexResponseV2")
 	proto.RegisterType((*PexMessage)(nil), "tendermint.p2p.PexMessage")
 }
 
 func init() { proto.RegisterFile("tendermint/p2p/pex.proto", fileDescriptor_81c2f011fd13be57) }
 
 var fileDescriptor_81c2f011fd13be57 = []byte{
-	// 310 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x51, 0x31, 0x4b, 0xc3, 0x40,
-	0x18, 0xbd, 0x4b, 0x6b, 0xa1, 0x97, 0xea, 0x70, 0x88, 0x84, 0x0a, 0xd7, 0x92, 0xa9, 0x53, 0x02,
-	0x11, 0x47, 0x45, 0x83, 0x43, 0x1d, 0x8a, 0xe5, 0x46, 0x17, 0x69, 0xcd, 0x47, 0xcc, 0xd0, 0xde,
-	0x67, 0xee, 0x0a, 0xfd, 0x19, 0x0e, 0xfe, 0xa8, 0x8e, 0x1d, 0x9d, 0x8a, 0xa4, 0x7f, 0x44, 0xbc,
-	0x13, 0x93, 0x42, 0xb7, 0x7b, 0xef, 0xfb, 0xde, 0xfb, 0xde, 0xf1, 0x58, 0x60, 0x60, 0x99, 0x41,
-	0xb9, 0x28, 0x96, 0x26, 0xc6, 0x04, 0x63, 0x84, 0x75, 0x84, 0xa5, 0x32, 0x8a, 0x9f, 0xd5, 0x93,
-	0x08, 0x13, 0xec, 0x9f, 0xe7, 0x2a, 0x57, 0x76, 0x14, 0xff, 0xbe, 0xdc, 0x56, 0x38, 0x65, 0x6c,
-	0x0a, 0xeb, 0xfb, 0x2c, 0x2b, 0x41, 0x6b, 0x7e, 0xc1, 0xbc, 0x22, 0x0b, 0xe8, 0x90, 0x8e, 0xba,
-	0x69, 0xa7, 0xda, 0x0d, 0xbc, 0xc7, 0x07, 0xe9, 0x15, 0x99, 0xe5, 0x31, 0xf0, 0x1a, 0xfc, 0x54,
-	0x7a, 0x05, 0x72, 0xce, 0xda, 0xa8, 0x4a, 0x13, 0xb4, 0x86, 0x74, 0x74, 0x2a, 0xed, 0x3b, 0xec,
-	0x59, 0x47, 0x09, 0xef, 0x2b, 0xd0, 0x26, 0x9c, 0x30, 0xdf, 0x22, 0x8d, 0x6a, 0xa9, 0x81, 0xdf,
-	0xb2, 0xee, 0xcc, 0xdd, 0x02, 0x1d, 0xd0, 0x61, 0x6b, 0xe4, 0x27, 0xfd, 0xe8, 0x30, 0x68, 0x54,
-	0xe7, 0x49, 0xdb, 0x9b, 0xdd, 0x80, 0xc8, 0x5a, 0x12, 0x7e, 0x52, 0xeb, 0x3e, 0x01, 0xad, 0x67,
-	0x39, 0xf0, 0x1b, 0xe6, 0x23, 0xac, 0x5f, 0x4a, 0x77, 0xcc, 0x06, 0x3f, 0x6e, 0xf8, 0x17, 0x67,
-	0x4c, 0x24, 0xc3, 0x7f, 0xc4, 0xef, 0x58, 0xcf, 0xc9, 0x5d, 0x3a, 0xfb, 0x41, 0x3f, 0xb9, 0x3c,
-	0xaa, 0x77, 0x2b, 0x63, 0x22, 0x7d, 0xac, 0x61, 0x7a, 0xc2, 0x5a, 0x7a, 0xb5, 0x48, 0x9f, 0x36,
-	0x95, 0xa0, 0xdb, 0x4a, 0xd0, 0xef, 0x4a, 0xd0, 0x8f, 0xbd, 0x20, 0xdb, 0xbd, 0x20, 0x5f, 0x7b,
-	0x41, 0x9e, 0xaf, 0xf3, 0xc2, 0xbc, 0xad, 0xe6, 0xd1, 0xab, 0x5a, 0xc4, 0x8d, 0xaa, 0x9a, 0xad,
-	0xd9, 0x4a, 0x0e, 0x6b, 0x9c, 0x77, 0x2c, 0x7b, 0xf5, 0x13, 0x00, 0x00, 0xff, 0xff, 0xa6, 0xa1,
-	0x59, 0x3c, 0xdf, 0x01, 0x00, 0x00,
+	// 407 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xdd, 0x8a, 0xda, 0x40,
+	0x14, 0xc7, 0xf3, 0x61, 0x2d, 0x9e, 0x44, 0x0b, 0x43, 0x29, 0xa9, 0x6d, 0xa3, 0xe4, 0xca, 0xde,
+	0x24, 0x30, 0xa5, 0x97, 0x2d, 0x36, 0x08, 0xb5, 0x50, 0xa9, 0x1d, 0xd8, 0x5c, 0xec, 0x8d, 0xe8,
+	0x66, 0xc8, 0x06, 0x56, 0x33, 0x9b, 0x49, 0x16, 0x1f, 0x63, 0xdf, 0x61, 0x5f, 0xc6, 0x4b, 0x2f,
+	0xf7, 0x4a, 0x96, 0xf8, 0x22, 0x8b, 0x13, 0x31, 0x23, 0xba, 0x7b, 0x37, 0xe7, 0x7f, 0xbe, 0x7e,
+	0xe7, 0xcc, 0x01, 0x2b, 0xa3, 0x8b, 0x90, 0xa6, 0xf3, 0x78, 0x91, 0x79, 0x0c, 0x33, 0x8f, 0xd1,
+	0xa5, 0xcb, 0xd2, 0x24, 0x4b, 0x50, 0xab, 0xf2, 0xb8, 0x0c, 0xb3, 0xf6, 0xfb, 0x28, 0x89, 0x12,
+	0xe1, 0xf2, 0x76, 0xaf, 0x32, 0xca, 0x19, 0x03, 0x8c, 0xe9, 0xf2, 0x57, 0x18, 0xa6, 0x94, 0x73,
+	0xf4, 0x01, 0xb4, 0x38, 0xb4, 0xd4, 0xae, 0xda, 0x6b, 0xf8, 0xf5, 0x62, 0xd3, 0xd1, 0xfe, 0x0c,
+	0x88, 0x16, 0x87, 0x42, 0x67, 0x96, 0x26, 0xe9, 0x63, 0xa2, 0xc5, 0x0c, 0x21, 0xa8, 0xb1, 0x24,
+	0xcd, 0x2c, 0xbd, 0xab, 0xf6, 0x9a, 0x44, 0xbc, 0x1d, 0x53, 0x54, 0x24, 0xf4, 0x36, 0xa7, 0x3c,
+	0x73, 0x46, 0x60, 0x08, 0x8b, 0xb3, 0x64, 0xc1, 0x29, 0xfa, 0x09, 0x8d, 0x69, 0xd9, 0x8b, 0x72,
+	0x4b, 0xed, 0xea, 0x3d, 0x03, 0xb7, 0xdd, 0x63, 0x50, 0xb7, 0xe2, 0xf1, 0x6b, 0xab, 0x4d, 0x47,
+	0x21, 0x55, 0x8a, 0xf3, 0x15, 0xcc, 0xca, 0x1d, 0x60, 0xf4, 0x11, 0xf4, 0x3c, 0xbd, 0xd9, 0x13,
+	0xbf, 0x2d, 0x36, 0x1d, 0xfd, 0x82, 0xfc, 0x25, 0x3b, 0xcd, 0x69, 0x89, 0xd0, 0x3d, 0x47, 0x80,
+	0x9d, 0xff, 0xd0, 0x94, 0x48, 0x02, 0x8c, 0xfa, 0xa7, 0x2c, 0x9f, 0x5f, 0x66, 0x09, 0xf0, 0x29,
+	0xcd, 0x83, 0x26, 0x66, 0x1d, 0x51, 0xce, 0xa7, 0x11, 0x45, 0x3f, 0xc0, 0x60, 0x74, 0x39, 0x49,
+	0xcb, 0x96, 0x02, 0xea, 0xfc, 0x78, 0x7b, 0xa8, 0xa1, 0x42, 0x80, 0x1d, 0x2c, 0xd4, 0x07, 0xb3,
+	0x4c, 0x2f, 0x09, 0xc5, 0xba, 0x0d, 0xfc, 0xe9, 0x6c, 0x7e, 0x19, 0x32, 0x54, 0x88, 0xc1, 0xa4,
+	0xed, 0x0e, 0xa0, 0x25, 0x01, 0x4c, 0xee, 0xb0, 0xf8, 0x98, 0xf3, 0x63, 0x1d, 0x16, 0x33, 0x54,
+	0x88, 0xc9, 0x24, 0x1b, 0xfd, 0x86, 0x77, 0x32, 0xc7, 0xae, 0x4c, 0x4d, 0x94, 0xf9, 0xf2, 0x0a,
+	0x8a, 0xa8, 0xd3, 0x64, 0xb2, 0xe0, 0xbf, 0x01, 0x9d, 0xe7, 0x73, 0xff, 0xdf, 0xaa, 0xb0, 0xd5,
+	0x75, 0x61, 0xab, 0x4f, 0x85, 0xad, 0xde, 0x6f, 0x6d, 0x65, 0xbd, 0xb5, 0x95, 0xc7, 0xad, 0xad,
+	0x5c, 0x7e, 0x8f, 0xe2, 0xec, 0x3a, 0x9f, 0xb9, 0x57, 0xc9, 0xdc, 0x93, 0xee, 0x58, 0x3e, 0x69,
+	0x71, 0xaf, 0xc7, 0x37, 0x3e, 0xab, 0x0b, 0xf5, 0xdb, 0x73, 0x00, 0x00, 0x00, 0xff, 0xff, 0x9f,
+	0x9b, 0xfd, 0x75, 0xfc, 0x02, 0x00, 0x00,
 }
 
 func (m *PexAddress) Marshal() (dAtA []byte, err error) {
@@ -383,6 +542,96 @@ func (m *PexResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) {
 	return len(dAtA) - i, nil
 }
 
+func (m *PexAddressV2) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PexAddressV2) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PexAddressV2) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.URL) > 0 {
+		i -= len(m.URL)
+		copy(dAtA[i:], m.URL)
+		i = encodeVarintPex(dAtA, i, uint64(len(m.URL)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *PexRequestV2) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PexRequestV2) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PexRequestV2) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	return len(dAtA) - i, nil
+}
+
+func (m *PexResponseV2) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *PexResponseV2) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PexResponseV2) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for iNdEx := len(m.Addresses) - 1; iNdEx >= 0; iNdEx-- {
+			{
+				size, err := m.Addresses[iNdEx].MarshalToSizedBuffer(dAtA[:i])
+				if err != nil {
+					return 0, err
+				}
+				i -= size
+				i = encodeVarintPex(dAtA, i, uint64(size))
+			}
+			i--
+			dAtA[i] = 0xa
+		}
+	}
+	return len(dAtA) - i, nil
+}
+
 func (m *PexMessage) Marshal() (dAtA []byte, err error) {
 	size := m.Size()
 	dAtA = make([]byte, size)
@@ -457,6 +706,48 @@ func (m *PexMessage_PexResponse) MarshalToSizedBuffer(dAtA []byte) (int, error)
 	}
 	return len(dAtA) - i, nil
 }
+func (m *PexMessage_PexRequestV2) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PexMessage_PexRequestV2) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.PexRequestV2 != nil {
+		{
+			size, err := m.PexRequestV2.MarshalToSizedBuffer(dAtA[:i])
+			if err != nil {
+				return 0, err
+			}
+			i -= size
+			i = encodeVarintPex(dAtA, i, uint64(size))
+		}
+		i--
+		dAtA[i] = 0x1a
+	}
+	return len(dAtA) - i, nil
+}
+func (m *PexMessage_PexResponseV2) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *PexMessage_PexResponseV2) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	if m.PexResponseV2 != nil {
+		{
+			size, err := m.PexResponseV2.MarshalToSizedBuffer(dAtA[:i])
+			if err != nil {
+				return 0, err
+			}
+			i -= size
+			i = encodeVarintPex(dAtA, i, uint64(size))
+		}
+		i--
+		dAtA[i] = 0x22
+	}
+	return len(dAtA) - i, nil
+}
 func encodeVarintPex(dAtA []byte, offset int, v uint64) int {
 	offset -= sovPex(v)
 	base := offset
@@ -512,6 +803,43 @@ func (m *PexResponse) Size() (n int) {
 	return n
 }
 
+func (m *PexAddressV2) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.URL)
+	if l > 0 {
+		n += 1 + l + sovPex(uint64(l))
+	}
+	return n
+}
+
+func (m *PexRequestV2) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	return n
+}
+
+func (m *PexResponseV2) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if len(m.Addresses) > 0 {
+		for _, e := range m.Addresses {
+			l = e.Size()
+			n += 1 + l + sovPex(uint64(l))
+		}
+	}
+	return n
+}
+
 func (m *PexMessage) Size() (n int) {
 	if m == nil {
 		return 0
@@ -548,6 +876,30 @@ func (m *PexMessage_PexResponse) Size() (n int) {
 	}
 	return n
 }
+func (m *PexMessage_PexRequestV2) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.PexRequestV2 != nil {
+		l = m.PexRequestV2.Size()
+		n += 1 + l + sovPex(uint64(l))
+	}
+	return n
+}
+func (m *PexMessage_PexResponseV2) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.PexResponseV2 != nil {
+		l = m.PexResponseV2.Size()
+		n += 1 + l + sovPex(uint64(l))
+	}
+	return n
+}
 
 func sovPex(x uint64) (n int) {
 	return (math_bits.Len64(x|1) + 6) / 7
@@ -822,6 +1174,222 @@ func (m *PexResponse) Unmarshal(dAtA []byte) error {
 	}
 	return nil
 }
+func (m *PexAddressV2) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPex
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PexAddressV2: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PexAddressV2: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field URL", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPex
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				stringLen |= uint64(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthPex
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex < 0 {
+				return ErrInvalidLengthPex
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.URL = string(dAtA[iNdEx:postIndex])
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPex(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPex
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *PexRequestV2) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPex
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PexRequestV2: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PexRequestV2: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPex(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPex
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *PexResponseV2) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowPex
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: PexResponseV2: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: PexResponseV2: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPex
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPex
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPex
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, PexAddressV2{})
+			if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipPex(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthPex
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
 func (m *PexMessage) Unmarshal(dAtA []byte) error {
 	l := len(dAtA)
 	iNdEx := 0
@@ -921,6 +1489,76 @@ func (m *PexMessage) Unmarshal(dAtA []byte) error {
 			}
 			m.Sum = &PexMessage_PexResponse{v}
 			iNdEx = postIndex
+		case 3:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field PexRequestV2", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPex
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPex
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPex
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			v := &PexRequestV2{}
+			if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			m.Sum = &PexMessage_PexRequestV2{v}
+			iNdEx = postIndex
+		case 4:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field PexResponseV2", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowPex
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthPex
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthPex
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			v := &PexResponseV2{}
+			if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			m.Sum = &PexMessage_PexResponseV2{v}
+			iNdEx = postIndex
 		default:
 			iNdEx = preIndex
 			skippy, err := skipPex(dAtA[iNdEx:])

proto/tendermint/p2p/pex.proto

@@ -17,9 +17,22 @@ message PexResponse {
   repeated PexAddress addresses = 1 [(gogoproto.nullable) = false];
 }
 
+message PexAddressV2 {
+  string url = 1 [(gogoproto.customname) = "URL"];
+}
+
+message PexRequestV2 {}
+
+message PexResponseV2 {
+  repeated PexAddressV2 addresses = 1 [(gogoproto.nullable) = false];
+}
+
+
 message PexMessage {
   oneof sum {
-    PexRequest  pex_request  = 1;
-    PexResponse pex_response = 2;
+    PexRequest    pex_request     = 1;
+    PexResponse   pex_response    = 2;
+    PexRequestV2  pex_request_v2  = 3;
+    PexResponseV2 pex_response_v2 = 4;
   }
 }

rpc/core/env.go

@@ -14,7 +14,6 @@ import (
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/state/indexer"
-	"github.com/tendermint/tendermint/state/txindex"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -83,7 +82,7 @@ type Environment struct {
 	// objects
 	PubKey           crypto.PubKey
 	GenDoc           *types.GenesisDoc // cache the genesis structure
-	TxIndexer        txindex.TxIndexer
+	TxIndexer        indexer.TxIndexer
 	BlockIndexer     indexer.BlockIndexer
 	ConsensusReactor *consensus.Reactor
 	EventBus         *types.EventBus // thread safe

rpc/core/routes.go

@@ -14,43 +14,43 @@ var Routes = map[string]*rpc.RPCFunc{
 	"unsubscribe_all": rpc.NewWSRPCFunc(UnsubscribeAll, ""),
 
 	// info API
-	"health":               rpc.NewRPCFunc(Health, ""),
-	"status":               rpc.NewRPCFunc(Status, ""),
-	"net_info":             rpc.NewRPCFunc(NetInfo, ""),
-	"blockchain":           rpc.NewRPCFunc(BlockchainInfo, "minHeight,maxHeight"),
-	"genesis":              rpc.NewRPCFunc(Genesis, ""),
-	"block":                rpc.NewRPCFunc(Block, "height"),
-	"block_by_hash":        rpc.NewRPCFunc(BlockByHash, "hash"),
-	"block_results":        rpc.NewRPCFunc(BlockResults, "height"),
-	"commit":               rpc.NewRPCFunc(Commit, "height"),
-	"check_tx":             rpc.NewRPCFunc(CheckTx, "tx"),
-	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove"),
-	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page,order_by"),
-	"block_search":         rpc.NewRPCFunc(BlockSearch, "query,page,per_page,order_by"),
-	"validators":           rpc.NewRPCFunc(Validators, "height,page,per_page"),
-	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""),
-	"consensus_state":      rpc.NewRPCFunc(ConsensusState, ""),
-	"consensus_params":     rpc.NewRPCFunc(ConsensusParams, "height"),
-	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, "limit"),
-	"num_unconfirmed_txs":  rpc.NewRPCFunc(NumUnconfirmedTxs, ""),
+	"health":               rpc.NewRPCFunc(Health, "", false),
+	"status":               rpc.NewRPCFunc(Status, "", false),
+	"net_info":             rpc.NewRPCFunc(NetInfo, "", false),
+	"blockchain":           rpc.NewRPCFunc(BlockchainInfo, "minHeight,maxHeight", true),
+	"genesis":              rpc.NewRPCFunc(Genesis, "", true),
+	"block":                rpc.NewRPCFunc(Block, "height", true),
+	"block_by_hash":        rpc.NewRPCFunc(BlockByHash, "hash", true),
+	"block_results":        rpc.NewRPCFunc(BlockResults, "height", true),
+	"commit":               rpc.NewRPCFunc(Commit, "height", true),
+	"check_tx":             rpc.NewRPCFunc(CheckTx, "tx", true),
+	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove", true),
+	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page,order_by", false),
+	"block_search":         rpc.NewRPCFunc(BlockSearch, "query,page,per_page,order_by", false),
+	"validators":           rpc.NewRPCFunc(Validators, "height,page,per_page", true),
+	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, "", false),
+	"consensus_state":      rpc.NewRPCFunc(ConsensusState, "", false),
+	"consensus_params":     rpc.NewRPCFunc(ConsensusParams, "height", true),
+	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, "limit", false),
+	"num_unconfirmed_txs":  rpc.NewRPCFunc(NumUnconfirmedTxs, "", false),
 
 	// tx broadcast API
-	"broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommit, "tx"),
-	"broadcast_tx_sync":   rpc.NewRPCFunc(BroadcastTxSync, "tx"),
-	"broadcast_tx_async":  rpc.NewRPCFunc(BroadcastTxAsync, "tx"),
+	"broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommit, "tx", false),
+	"broadcast_tx_sync":   rpc.NewRPCFunc(BroadcastTxSync, "tx", false),
+	"broadcast_tx_async":  rpc.NewRPCFunc(BroadcastTxAsync, "tx", false),
 
 	// abci API
-	"abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"),
-	"abci_info":  rpc.NewRPCFunc(ABCIInfo, ""),
+	"abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove", false),
+	"abci_info":  rpc.NewRPCFunc(ABCIInfo, "", true),
 
 	// evidence API
-	"broadcast_evidence": rpc.NewRPCFunc(BroadcastEvidence, "evidence"),
+	"broadcast_evidence": rpc.NewRPCFunc(BroadcastEvidence, "evidence", false),
 }
 
 // AddUnsafeRoutes adds unsafe routes.
 func AddUnsafeRoutes() {
 	// control API
-	Routes["dial_seeds"] = rpc.NewRPCFunc(UnsafeDialSeeds, "seeds")
-	Routes["dial_peers"] = rpc.NewRPCFunc(UnsafeDialPeers, "peers,persistent,unconditional,private")
-	Routes["unsafe_flush_mempool"] = rpc.NewRPCFunc(UnsafeFlushMempool, "")
+	Routes["dial_seeds"] = rpc.NewRPCFunc(UnsafeDialSeeds, "seeds", false)
+	Routes["dial_peers"] = rpc.NewRPCFunc(UnsafeDialPeers, "peers,persistent,unconditional,private", false)
+	Routes["unsafe_flush_mempool"] = rpc.NewRPCFunc(UnsafeFlushMempool, "", false)
 }

rpc/core/tx.go

@@ -9,7 +9,7 @@ import (
 	tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
 	ctypes "github.com/tendermint/tendermint/rpc/core/types"
 	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
-	"github.com/tendermint/tendermint/state/txindex/null"
+	"github.com/tendermint/tendermint/state/indexer/tx/null"
 	"github.com/tendermint/tendermint/types"
 )
 

rpc/jsonrpc/client/ws_client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	mrand "math/rand"
 	"net"
 	"net/http"
 	"sync"
@@ -12,7 +13,6 @@ import (
 	"github.com/gorilla/websocket"
 	metrics "github.com/rcrowley/go-metrics"
 
-	tmrand "github.com/tendermint/tendermint/libs/rand"
 	"github.com/tendermint/tendermint/libs/service"
 	tmsync "github.com/tendermint/tendermint/libs/sync"
 	types "github.com/tendermint/tendermint/rpc/jsonrpc/types"
@@ -266,7 +266,8 @@ func (c *WSClient) reconnect() error {
 	}()
 
 	for {
-		jitter := time.Duration(tmrand.Float64() * float64(time.Second)) // 1s == (1e9 ns)
+		// nolint:gosec // G404: Use of weak random number generator
+		jitter := time.Duration(mrand.Float64() * float64(time.Second)) // 1s == (1e9 ns)
 		backoffDuration := jitter + ((1 << attempt) * time.Second)
 
 		c.Logger.Info("reconnecting", "attempt", attempt+1, "backoff_duration", backoffDuration)

rpc/jsonrpc/doc.go

@@ -55,7 +55,7 @@
 // Define some routes
 //
 //   var Routes = map[string]*rpcserver.RPCFunc{
-//	    "status": rpcserver.NewRPCFunc(Status, "arg"),
+//	    "status": rpcserver.NewRPCFunc(Status, "arg", false),
 //   }
 //
 // An rpc function:

rpc/jsonrpc/jsonrpc_test.go

@@ -6,6 +6,7 @@ import (
 	crand "crypto/rand"
 	"encoding/json"
 	"fmt"
+	mrand "math/rand"
 	"net/http"
 	"os"
 	"os/exec"
@@ -18,8 +19,6 @@ import (
 
 	tmbytes "github.com/tendermint/tendermint/libs/bytes"
 	"github.com/tendermint/tendermint/libs/log"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
-
 	client "github.com/tendermint/tendermint/rpc/jsonrpc/client"
 	server "github.com/tendermint/tendermint/rpc/jsonrpc/server"
 	types "github.com/tendermint/tendermint/rpc/jsonrpc/types"
@@ -59,11 +58,11 @@ type ResultEchoDataBytes struct {
 
 // Define some routes
 var Routes = map[string]*server.RPCFunc{
-	"echo":            server.NewRPCFunc(EchoResult, "arg"),
+	"echo":            server.NewRPCFunc(EchoResult, "arg", false),
 	"echo_ws":         server.NewWSRPCFunc(EchoWSResult, "arg"),
-	"echo_bytes":      server.NewRPCFunc(EchoBytesResult, "arg"),
-	"echo_data_bytes": server.NewRPCFunc(EchoDataBytesResult, "arg"),
-	"echo_int":        server.NewRPCFunc(EchoIntResult, "arg"),
+	"echo_bytes":      server.NewRPCFunc(EchoBytesResult, "arg", false),
+	"echo_data_bytes": server.NewRPCFunc(EchoDataBytesResult, "arg", false),
+	"echo_int":        server.NewRPCFunc(EchoIntResult, "arg", false),
 }
 
 func EchoResult(ctx *types.Context, v string) (*ResultEcho, error) {
@@ -215,7 +214,7 @@ func testWithHTTPClient(t *testing.T, cl client.HTTPClient) {
 	require.Nil(t, err)
 	assert.Equal(t, got3, val3)
 
-	val4 := tmrand.Intn(10000)
+	val4 := mrand.Intn(10000)
 	got4, err := echoIntViaHTTP(cl, val4)
 	require.Nil(t, err)
 	assert.Equal(t, got4, val4)
@@ -400,7 +399,7 @@ func TestWSClientPingPong(t *testing.T) {
 }
 
 func randBytes(t *testing.T) []byte {
-	n := tmrand.Intn(10) + 2
+	n := mrand.Intn(10) + 2
 	buf := make([]byte, n)
 	_, err := crand.Read(buf)
 	require.Nil(t, err)

rpc/jsonrpc/server/http_json_handler.go

@@ -57,6 +57,11 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han
 			requests = []types.RPCRequest{request}
 		}
 
+		// Set the default response cache to true unless
+		// 1. Any RPC request rrror.
+		// 2. Any RPC request doesn't allow to be cached.
+		// 3. Any RPC request has the height argument and the value is 0 (the default).
+		var c = true
 		for _, request := range requests {
 			request := request
 
@@ -74,11 +79,13 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han
 					responses,
 					types.RPCInvalidRequestError(request.ID, fmt.Errorf("path %s is invalid", r.URL.Path)),
 				)
+				c = false
 				continue
 			}
 			rpcFunc, ok := funcMap[request.Method]
 			if !ok || rpcFunc.ws {
 				responses = append(responses, types.RPCMethodNotFoundError(request.ID))
+				c = false
 				continue
 			}
 			ctx := &types.Context{JSONReq: &request, HTTPReq: r}
@@ -90,9 +97,15 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han
 						responses,
 						types.RPCInvalidParamsError(request.ID, fmt.Errorf("error converting json params to arguments: %w", err)),
 					)
+					c = false
 					continue
 				}
 				args = append(args, fnArgs...)
+
+			}
+
+			if hasDefaultHeight(request, args) {
+				c = false
 			}
 
 			returns := rpcFunc.f.Call(args)
@@ -106,23 +119,28 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, logger log.Logger) http.Han
 			// if this already of type RPC error then forward that error
 			case *types.RPCError:
 				responses = append(responses, types.NewRPCErrorResponse(request.ID, e.Code, e.Message, e.Data))
-
+				c = false
 			default: // we need to unwrap the error and parse it accordingly
 				switch errors.Unwrap(err) {
 				// check if the error was due to an invald request
 				case ctypes.ErrZeroOrNegativeHeight, ctypes.ErrZeroOrNegativePerPage,
 					ctypes.ErrPageOutOfRange, ctypes.ErrInvalidRequest:
 					responses = append(responses, types.RPCInvalidRequestError(request.ID, err))
-
+					c = false
 				// lastly default all remaining errors as internal errors
 				default: // includes ctypes.ErrHeightNotAvailable and ctypes.ErrHeightExceedsChainHead
 					responses = append(responses, types.RPCInternalError(request.ID, err))
+					c = false
 				}
 			}
+
+			if c && !rpcFunc.cache {
+				c = false
+			}
 		}
 
 		if len(responses) > 0 {
-			if wErr := WriteRPCResponseHTTP(w, responses...); wErr != nil {
+			if wErr := WriteRPCResponseHTTP(w, c, responses...); wErr != nil {
 				logger.Error("failed to write responses", "res", responses, "err", wErr)
 			}
 		}
@@ -258,3 +276,12 @@ func writeListOfEndpoints(w http.ResponseWriter, r *http.Request, funcMap map[st
 	w.WriteHeader(200)
 	w.Write(buf.Bytes()) // nolint: errcheck
 }
+
+func hasDefaultHeight(r types.RPCRequest, h []reflect.Value) bool {
+	switch r.Method {
+	case "block", "block_results", "commit", "consensus_params", "validators":
+		return len(h) < 2 || h[1].IsZero()
+	default:
+		return false
+	}
+}

rpc/jsonrpc/server/http_json_handler_test.go

@@ -18,7 +18,8 @@ import (
 
 func testMux() *http.ServeMux {
 	funcMap := map[string]*RPCFunc{
-		"c": NewRPCFunc(func(ctx *types.Context, s string, i int) (string, error) { return "foo", nil }, "s,i"),
+		"c":     NewRPCFunc(func(ctx *types.Context, s string, i int) (string, error) { return "foo", nil }, "s,i", false),
+		"block": NewRPCFunc(func(ctx *types.Context, h int) (string, error) { return "block", nil }, "height", true),
 	}
 	mux := http.NewServeMux()
 	buf := new(bytes.Buffer)
@@ -227,3 +228,35 @@ func TestUnknownRPCPath(t *testing.T) {
 	require.Equal(t, http.StatusNotFound, res.StatusCode, "should always return 404")
 	res.Body.Close()
 }
+
+func TestRPCResponseCache(t *testing.T) {
+	mux := testMux()
+	body := strings.NewReader(`{"jsonrpc": "2.0","method":"block","id": 0, "params": ["1"]}`)
+	req, _ := http.NewRequest("Get", "http://localhost/", body)
+	rec := httptest.NewRecorder()
+	mux.ServeHTTP(rec, req)
+	res := rec.Result()
+
+	// Always expecting back a JSONRPCResponse
+	require.True(t, statusOK(res.StatusCode), "should always return 2XX")
+	require.Equal(t, "max-age=31536000", res.Header.Get("Cache-control"))
+
+	_, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	require.Nil(t, err, "reading from the body should not give back an error")
+
+	// send a request with default height.
+	body = strings.NewReader(`{"jsonrpc": "2.0","method":"block","id": 0, "params": ["0"]}`)
+	req, _ = http.NewRequest("Get", "http://localhost/", body)
+	rec = httptest.NewRecorder()
+	mux.ServeHTTP(rec, req)
+	res = rec.Result()
+
+	// Always expecting back a JSONRPCResponse
+	require.True(t, statusOK(res.StatusCode), "should always return 2XX")
+	require.Equal(t, "", res.Header.Get("Cache-control"))
+
+	_, err = ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	require.Nil(t, err, "reading from the body should not give back an error")
+}

rpc/jsonrpc/server/http_server.go

@@ -133,7 +133,8 @@ func WriteRPCResponseHTTPError(
 }
 
 // WriteRPCResponseHTTP marshals res as JSON (with indent) and writes it to w.
-func WriteRPCResponseHTTP(w http.ResponseWriter, res ...types.RPCResponse) error {
+// If the rpc response can be cached, add cache-control to the response header.
+func WriteRPCResponseHTTP(w http.ResponseWriter, c bool, res ...types.RPCResponse) error {
 	var v interface{}
 	if len(res) == 1 {
 		v = res[0]
@@ -146,6 +147,9 @@ func WriteRPCResponseHTTP(w http.ResponseWriter, res ...types.RPCResponse) error
 		return fmt.Errorf("json marshal: %w", err)
 	}
 	w.Header().Set("Content-Type", "application/json")
+	if c {
+		w.Header().Set("Cache-Control", "max-age=31536000") // expired after one year
+	}
 	w.WriteHeader(200)
 	_, err = w.Write(jsonBytes)
 	return err
@@ -186,7 +190,7 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler
 
 				// If RPCResponse
 				if res, ok := e.(types.RPCResponse); ok {
-					if wErr := WriteRPCResponseHTTP(rww, res); wErr != nil {
+					if wErr := WriteRPCResponseHTTP(rww, false, res); wErr != nil {
 						logger.Error("failed to write response", "res", res, "err", wErr)
 					}
 				} else {

rpc/jsonrpc/server/http_server_test.go

@@ -112,7 +112,7 @@ func TestWriteRPCResponseHTTP(t *testing.T) {
 
 	// one argument
 	w := httptest.NewRecorder()
-	err := WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(id, &sampleResult{"hello"}))
+	err := WriteRPCResponseHTTP(w, true, types.NewRPCSuccessResponse(id, &sampleResult{"hello"}))
 	require.NoError(t, err)
 	resp := w.Result()
 	body, err := ioutil.ReadAll(resp.Body)
@@ -120,6 +120,7 @@ func TestWriteRPCResponseHTTP(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 200, resp.StatusCode)
 	assert.Equal(t, "application/json", resp.Header.Get("Content-Type"))
+	assert.Equal(t, "max-age=31536000", resp.Header.Get("Cache-control"))
 	assert.Equal(t, `{
   "jsonrpc": "2.0",
   "id": -1,
@@ -131,6 +132,7 @@ func TestWriteRPCResponseHTTP(t *testing.T) {
 	// multiple arguments
 	w = httptest.NewRecorder()
 	err = WriteRPCResponseHTTP(w,
+		false,
 		types.NewRPCSuccessResponse(id, &sampleResult{"hello"}),
 		types.NewRPCSuccessResponse(id, &sampleResult{"world"}))
 	require.NoError(t, err)

rpc/jsonrpc/server/http_uri_handler.go

@@ -61,7 +61,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, logger log.Logger) func(http.ResponseWrit
 		// if no error then return a success response
 		case nil:
 			res := types.NewRPCSuccessResponse(dummyID, result)
-			if wErr := WriteRPCResponseHTTP(w, res); wErr != nil {
+			if wErr := WriteRPCResponseHTTP(w, rpcFunc.cache, res); wErr != nil {
 				logger.Error("failed to write response", "res", res, "err", wErr)
 			}
 

rpc/jsonrpc/server/parse_test.go

@@ -135,7 +135,7 @@ func TestParseJSONArray(t *testing.T) {
 
 func TestParseJSONRPC(t *testing.T) {
 	demo := func(ctx *types.Context, height int, name string) {}
-	call := NewRPCFunc(demo, "height,name")
+	call := NewRPCFunc(demo, "height,name", false)
 
 	cases := []struct {
 		raw    string
@@ -172,7 +172,7 @@ func TestParseJSONRPC(t *testing.T) {
 
 func TestParseURI(t *testing.T) {
 	demo := func(ctx *types.Context, height int, name string) {}
-	call := NewRPCFunc(demo, "height,name")
+	call := NewRPCFunc(demo, "height,name", false)
 
 	cases := []struct {
 		raw    []string

rpc/jsonrpc/server/rpc_func.go

@@ -31,20 +31,22 @@ type RPCFunc struct {
 	returns  []reflect.Type // type of each return arg
 	argNames []string       // name of each argument
 	ws       bool           // websocket only
+	cache    bool           // allow the RPC response can be cached by the proxy cache server
 }
 
 // NewRPCFunc wraps a function for introspection.
 // f is the function, args are comma separated argument names
-func NewRPCFunc(f interface{}, args string) *RPCFunc {
-	return newRPCFunc(f, args, false)
+// cache is a bool value to allow the client proxy server to cache the RPC results
+func NewRPCFunc(f interface{}, args string, cache bool) *RPCFunc {
+	return newRPCFunc(f, args, false, cache)
 }
 
 // NewWSRPCFunc wraps a function for introspection and use in the websockets.
 func NewWSRPCFunc(f interface{}, args string) *RPCFunc {
-	return newRPCFunc(f, args, true)
+	return newRPCFunc(f, args, true, false)
 }
 
-func newRPCFunc(f interface{}, args string, ws bool) *RPCFunc {
+func newRPCFunc(f interface{}, args string, ws bool, c bool) *RPCFunc {
 	var argNames []string
 	if args != "" {
 		argNames = strings.Split(args, ",")
@@ -55,6 +57,7 @@ func newRPCFunc(f interface{}, args string, ws bool) *RPCFunc {
 		returns:  funcReturnTypes(f),
 		argNames: argNames,
 		ws:       ws,
+		cache:    c,
 	}
 }
 

rpc/jsonrpc/test/main.go

@@ -12,7 +12,7 @@ import (
 )
 
 var routes = map[string]*rpcserver.RPCFunc{
-	"hello_world": rpcserver.NewRPCFunc(HelloWorld, "name,num"),
+	"hello_world": rpcserver.NewRPCFunc(HelloWorld, "name,num", false),
 }
 
 func HelloWorld(ctx *rpctypes.Context, name string, num int) (Result, error) {

scripts/keymigrate/migrate.go

@@ -0,0 +1,402 @@
+// Package keymigrate translates all legacy formatted keys to their
+// new components.
+//
+// The key migration operation as implemented provides a potential
+// model for database migration operations. Crucially, the migration
+// as implemented does not depend on any tendermint code.
+package keymigrate
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"math/rand"
+	"runtime"
+	"strconv"
+	"sync"
+
+	"github.com/google/orderedcode"
+	"github.com/pkg/errors"
+	dbm "github.com/tendermint/tm-db"
+)
+
+type (
+	keyID       []byte
+	migrateFunc func(keyID) (keyID, error)
+)
+
+func getAllLegacyKeys(db dbm.DB) ([]keyID, error) {
+	out := []keyID{}
+
+	iter, err := db.Iterator(nil, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	for ; iter.Valid(); iter.Next() {
+		k := iter.Key()
+
+		// make sure it's a key with a legacy format, and skip
+		// all other keys, to make it safe to resume the migration.
+		if !keyIsLegacy(k) {
+			continue
+		}
+
+		// there's inconsistency around tm-db's handling of
+		// key copies.
+		nk := make([]byte, len(k))
+		copy(nk, k)
+		out = append(out, nk)
+	}
+
+	if err = iter.Error(); err != nil {
+		return nil, err
+	}
+
+	if err = iter.Close(); err != nil {
+		return nil, err
+	}
+
+	return out, nil
+}
+
+func makeKeyChan(keys []keyID) <-chan keyID {
+	out := make(chan keyID, len(keys))
+	defer close(out)
+
+	for _, key := range keys {
+		out <- key
+	}
+
+	return out
+}
+
+func keyIsLegacy(key keyID) bool {
+	for _, prefix := range []keyID{
+		// core "store"
+		keyID("consensusParamsKey:"),
+		keyID("abciResponsesKey:"),
+		keyID("validatorsKey:"),
+		keyID("stateKey"),
+		keyID("H:"),
+		keyID("P:"),
+		keyID("C:"),
+		keyID("SC:"),
+		keyID("BH:"),
+		// light
+		keyID("size"),
+		keyID("lb/"),
+		// evidence
+		keyID([]byte{0x00}),
+		keyID([]byte{0x01}),
+		// tx index
+		keyID("tx.height/"),
+		keyID("tx.hash/"),
+	} {
+		if bytes.HasPrefix(key, prefix) {
+			return true
+		}
+	}
+
+	// this means it's a tx index...
+	if bytes.Count(key, []byte("/")) >= 3 {
+		return true
+	}
+
+	return keyIsHash(key)
+}
+
+func keyIsHash(key keyID) bool {
+	return len(key) == 32 && !bytes.Contains(key, []byte("/"))
+}
+
+func migarateKey(key keyID) (keyID, error) {
+	switch {
+	case bytes.HasPrefix(key, keyID("H:")):
+		val, err := strconv.Atoi(string(key[2:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(0), int64(val))
+	case bytes.HasPrefix(key, keyID("P:")):
+		parts := bytes.Split(key[2:], []byte(":"))
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("block parts key has %d rather than 2 components",
+				len(parts))
+		}
+		valOne, err := strconv.Atoi(string(parts[0]))
+		if err != nil {
+			return nil, err
+		}
+
+		valTwo, err := strconv.Atoi(string(parts[1]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(1), int64(valOne), int64(valTwo))
+	case bytes.HasPrefix(key, keyID("C:")):
+		val, err := strconv.Atoi(string(key[2:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(2), int64(val))
+	case bytes.HasPrefix(key, keyID("SC:")):
+		val, err := strconv.Atoi(string(key[3:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(3), int64(val))
+	case bytes.HasPrefix(key, keyID("BH:")):
+		val, err := strconv.Atoi(string(key[3:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(4), int64(val))
+	case bytes.HasPrefix(key, keyID("validatorsKey:")):
+		val, err := strconv.Atoi(string(key[14:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(5), int64(val))
+	case bytes.HasPrefix(key, keyID("consensusParamsKey:")):
+		val, err := strconv.Atoi(string(key[19:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(6), int64(val))
+	case bytes.HasPrefix(key, keyID("abciResponsesKey:")):
+		val, err := strconv.Atoi(string(key[17:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(7), int64(val))
+	case bytes.HasPrefix(key, keyID("stateKey")):
+		return orderedcode.Append(nil, int64(8))
+	case bytes.HasPrefix(key, []byte{0x00}): // committed evidence
+		return convertEvidence(key, 9)
+	case bytes.HasPrefix(key, []byte{0x01}): // pending evidence
+		return convertEvidence(key, 10)
+	case bytes.HasPrefix(key, keyID("lb/")):
+		if len(key) < 24 {
+			return nil, fmt.Errorf("light block evidence %q in invalid format", string(key))
+		}
+
+		val, err := strconv.Atoi(string(key[len(key)-20:]))
+		if err != nil {
+			return nil, err
+		}
+
+		return orderedcode.Append(nil, int64(11), int64(val))
+	case bytes.HasPrefix(key, keyID("size")):
+		return orderedcode.Append(nil, int64(12))
+	case bytes.HasPrefix(key, keyID("tx.height")):
+		parts := bytes.Split(key, []byte("/"))
+		if len(parts) != 4 {
+			return nil, fmt.Errorf("key has %d parts rather than 4", len(parts))
+		}
+		parts = parts[1:] // drop prefix
+
+		elems := make([]interface{}, 0, len(parts)+1)
+		elems = append(elems, "tx.height")
+
+		for idx, pt := range parts {
+			val, err := strconv.Atoi(string(pt))
+			if err != nil {
+				return nil, err
+			}
+			if idx == 0 {
+				elems = append(elems, fmt.Sprintf("%d", val))
+			} else {
+				elems = append(elems, int64(val))
+			}
+		}
+
+		return orderedcode.Append(nil, elems...)
+	case bytes.Count(key, []byte("/")) >= 3: // tx indexer
+		parts := bytes.Split(key, []byte("/"))
+
+		elems := make([]interface{}, 0, 4)
+		if len(parts) == 4 {
+			elems = append(elems, string(parts[0]), string(parts[1]))
+
+			val, err := strconv.Atoi(string(parts[2]))
+			if err != nil {
+				return nil, err
+			}
+			elems = append(elems, int64(val))
+
+			val2, err := strconv.Atoi(string(parts[3]))
+			if err != nil {
+				return nil, err
+			}
+			elems = append(elems, int64(val2))
+		} else {
+			elems = append(elems, string(parts[0]))
+			parts = parts[1:]
+
+			val, err := strconv.Atoi(string(parts[len(parts)-1]))
+			if err != nil {
+				return nil, err
+			}
+
+			val2, err := strconv.Atoi(string(parts[len(parts)-2]))
+			if err != nil {
+				return nil, err
+			}
+
+			appKey := bytes.Join(parts[:len(parts)-3], []byte("/"))
+			elems = append(elems, string(appKey), int64(val), int64(val2))
+		}
+		return orderedcode.Append(nil, elems...)
+	case keyIsHash(key):
+		return orderedcode.Append(nil, "tx.hash", string(key))
+	default:
+		return nil, fmt.Errorf("key %q is in the wrong format", string(key))
+	}
+}
+
+func convertEvidence(key keyID, newPrefix int64) ([]byte, error) {
+	parts := bytes.Split(key[1:], []byte("/"))
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("evidence key is malformed with %d parts not 2",
+			len(parts))
+	}
+
+	hb, err := hex.DecodeString(string(parts[0]))
+	if err != nil {
+		return nil, err
+	}
+
+	evidenceHash, err := hex.DecodeString(string(parts[1]))
+	if err != nil {
+		return nil, err
+	}
+
+	return orderedcode.Append(nil, newPrefix, binary.BigEndian.Uint64(hb), string(evidenceHash))
+}
+
+func replaceKey(db dbm.DB, key keyID, gooseFn migrateFunc) error {
+	exists, err := db.Has(key)
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return nil
+	}
+
+	newKey, err := gooseFn(key)
+	if err != nil {
+		return err
+	}
+
+	val, err := db.Get(key)
+	if err != nil {
+		return err
+	}
+
+	batch := db.NewBatch()
+
+	if err = batch.Set(newKey, val); err != nil {
+		return err
+	}
+	if err = batch.Delete(key); err != nil {
+		return err
+	}
+
+	// 10% of the time, force a write to disk, but mostly don't,
+	// because it's faster.
+	if rand.Intn(100)%10 == 0 { // nolint:gosec
+		if err = batch.WriteSync(); err != nil {
+			return err
+		}
+	} else {
+		if err = batch.Write(); err != nil {
+			return err
+		}
+	}
+
+	if err = batch.Close(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Migrate converts all legacy key formats to new key formats. The
+// operation is idempotent, so it's safe to resume a failed
+// operation. The operation is somewhat parallelized, relying on the
+// concurrency safety of the underlying databases.
+//
+// Migrate has "continue on error" semantics and will iterate through
+// all legacy keys attempt to migrate them, and will collect all
+// errors and will return only at the end of the operation.
+//
+// The context allows for a safe termination of the operation
+// (e.g connected to a singal handler,) to abort the operation
+// in-between migration operations.
+func Migrate(ctx context.Context, db dbm.DB) error {
+	keys, err := getAllLegacyKeys(db)
+	if err != nil {
+		return err
+	}
+
+	numWorkers := runtime.NumCPU()
+	wg := &sync.WaitGroup{}
+
+	errs := make(chan error, numWorkers)
+
+	keyCh := makeKeyChan(keys)
+
+	// run migrations.
+	for i := 0; i < numWorkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for key := range keyCh {
+				err := replaceKey(db, key, migarateKey)
+				if err != nil {
+					errs <- err
+				}
+
+				if ctx.Err() != nil {
+					return
+				}
+			}
+		}()
+	}
+
+	// collect and process the errors.
+	errStrs := []string{}
+	signal := make(chan struct{})
+	go func() {
+		defer close(signal)
+		for err := range errs {
+			if err == nil {
+				continue
+			}
+			errStrs = append(errStrs, err.Error())
+		}
+	}()
+
+	// Wait for everything to be done.
+	wg.Wait()
+	close(errs)
+	<-signal
+
+	// check the error results
+	if len(errs) != 0 {
+		return errors.Errorf("encountered errors during migration: %v", errStrs)
+	}
+
+	return nil
+}

scripts/keymigrate/migrate_test.go

@@ -0,0 +1,241 @@
+package keymigrate
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/google/orderedcode"
+	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+)
+
+func makeKey(t *testing.T, elems ...interface{}) []byte {
+	t.Helper()
+	out, err := orderedcode.Append([]byte{}, elems...)
+	require.NoError(t, err)
+	return out
+}
+
+func getLegacyPrefixKeys(val int) map[string][]byte {
+	return map[string][]byte{
+		"Height":            []byte(fmt.Sprintf("H:%d", val)),
+		"BlockPart":         []byte(fmt.Sprintf("P:%d:%d", val, val)),
+		"BlockPartTwo":      []byte(fmt.Sprintf("P:%d:%d", val+2, val+val)),
+		"BlockCommit":       []byte(fmt.Sprintf("C:%d", val)),
+		"SeenCommit":        []byte(fmt.Sprintf("SC:%d", val)),
+		"BlockHeight":       []byte(fmt.Sprintf("BH:%d", val)),
+		"Validators":        []byte(fmt.Sprintf("validatorsKey:%d", val)),
+		"ConsensusParams":   []byte(fmt.Sprintf("consensusParamsKey:%d", val)),
+		"ABCIResponse":      []byte(fmt.Sprintf("abciResponsesKey:%d", val)),
+		"State":             []byte("stateKey"),
+		"CommittedEvidence": append([]byte{0x00}, []byte(fmt.Sprintf("%0.16X/%X", int64(val), []byte("committed")))...),
+		"PendingEvidence":   append([]byte{0x01}, []byte(fmt.Sprintf("%0.16X/%X", int64(val), []byte("pending")))...),
+		"LightBLock":        []byte(fmt.Sprintf("lb/foo/%020d", val)),
+		"Size":              []byte("size"),
+		"UserKey0":          []byte(fmt.Sprintf("foo/bar/%d/%d", val, val)),
+		"UserKey1":          []byte(fmt.Sprintf("foo/bar/baz/%d/%d", val, val)),
+		"TxHeight":          []byte(fmt.Sprintf("tx.height/%s/%d/%d", fmt.Sprint(val), val, val)),
+		"TxHash": append(
+			bytes.Repeat([]byte{fmt.Sprint(val)[0]}, 16),
+			bytes.Repeat([]byte{fmt.Sprint(val)[len([]byte(fmt.Sprint(val)))-1]}, 16)...,
+		),
+	}
+}
+
+func getNewPrefixKeys(t *testing.T, val int) map[string][]byte {
+	t.Helper()
+	return map[string][]byte{
+		"Height":            makeKey(t, int64(0), int64(val)),
+		"BlockPart":         makeKey(t, int64(1), int64(val), int64(val)),
+		"BlockPartTwo":      makeKey(t, int64(1), int64(val+2), int64(val+val)),
+		"BlockCommit":       makeKey(t, int64(2), int64(val)),
+		"SeenCommit":        makeKey(t, int64(3), int64(val)),
+		"BlockHeight":       makeKey(t, int64(4), int64(val)),
+		"Validators":        makeKey(t, int64(5), int64(val)),
+		"ConsensusParams":   makeKey(t, int64(6), int64(val)),
+		"ABCIResponse":      makeKey(t, int64(7), int64(val)),
+		"State":             makeKey(t, int64(8)),
+		"CommittedEvidence": makeKey(t, int64(9), int64(val)),
+		"PendingEvidence":   makeKey(t, int64(10), int64(val)),
+		"LightBLock":        makeKey(t, int64(11), int64(val)),
+		"Size":              makeKey(t, int64(12)),
+		"UserKey0":          makeKey(t, "foo", "bar", int64(val), int64(val)),
+		"UserKey1":          makeKey(t, "foo", "bar/baz", int64(val), int64(val)),
+		"TxHeight":          makeKey(t, "tx.height", fmt.Sprint(val), int64(val), int64(val+2), int64(val+val)),
+		"TxHash":            makeKey(t, "tx.hash", string(bytes.Repeat([]byte{[]byte(fmt.Sprint(val))[0]}, 32))),
+	}
+}
+
+func getLegacyDatabase(t *testing.T) (int, dbm.DB) {
+	db := dbm.NewMemDB()
+	batch := db.NewBatch()
+	ct := 0
+
+	generated := []map[string][]byte{
+		getLegacyPrefixKeys(8),
+		getLegacyPrefixKeys(9001),
+		getLegacyPrefixKeys(math.MaxInt32 << 1),
+		getLegacyPrefixKeys(math.MaxInt64 - 8),
+	}
+
+	// populate database
+	for _, km := range generated {
+		for _, key := range km {
+			ct++
+			require.NoError(t, batch.Set(key, []byte(fmt.Sprintf(`{"value": %d}`, ct))))
+		}
+	}
+	require.NoError(t, batch.WriteSync())
+	require.NoError(t, batch.Close())
+	return ct - (2 * len(generated)) + 2, db
+}
+
+func TestMigration(t *testing.T) {
+	t.Run("Idempotency", func(t *testing.T) {
+		// we want to make sure that the key space for new and
+		// legacy keys are entirely non-overlapping.
+
+		legacyPrefixes := getLegacyPrefixKeys(42)
+
+		newPrefixes := getNewPrefixKeys(t, 42)
+
+		require.Equal(t, len(legacyPrefixes), len(newPrefixes))
+
+		t.Run("Legacy", func(t *testing.T) {
+			for kind, le := range legacyPrefixes {
+				require.True(t, keyIsLegacy(le), kind)
+			}
+		})
+		t.Run("New", func(t *testing.T) {
+			for kind, ne := range newPrefixes {
+				require.False(t, keyIsLegacy(ne), kind)
+			}
+		})
+		t.Run("Conversion", func(t *testing.T) {
+			for kind, le := range legacyPrefixes {
+				nk, err := migarateKey(le)
+				require.NoError(t, err, kind)
+				require.False(t, keyIsLegacy(nk), kind)
+			}
+		})
+		t.Run("Hashes", func(t *testing.T) {
+			t.Run("NewKeysAreNotHashes", func(t *testing.T) {
+				for _, key := range getNewPrefixKeys(t, 9001) {
+					require.True(t, len(key) != 32)
+				}
+			})
+			t.Run("ContrivedLegacyKeyDetection", func(t *testing.T) {
+				require.True(t, keyIsLegacy([]byte("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")))
+				require.False(t, keyIsLegacy([]byte("xxxxxxxxxxxxxxx/xxxxxxxxxxxxxxxx")))
+			})
+		})
+	})
+	t.Run("Migrations", func(t *testing.T) {
+		t.Run("Errors", func(t *testing.T) {
+			table := map[string][]byte{
+				"Height":          []byte(fmt.Sprintf("H:%f", 4.22222)),
+				"BlockPart":       []byte(fmt.Sprintf("P:%f", 4.22222)),
+				"BlockPartTwo":    []byte(fmt.Sprintf("P:%d", 42)),
+				"BlockPartThree":  []byte(fmt.Sprintf("P:%f:%f", 4.222, 8.444)),
+				"BlockPartFour":   []byte(fmt.Sprintf("P:%d:%f", 4222, 8.444)),
+				"BlockCommit":     []byte(fmt.Sprintf("C:%f", 4.22222)),
+				"SeenCommit":      []byte(fmt.Sprintf("SC:%f", 4.22222)),
+				"BlockHeight":     []byte(fmt.Sprintf("BH:%f", 4.22222)),
+				"Validators":      []byte(fmt.Sprintf("validatorsKey:%f", 4.22222)),
+				"ConsensusParams": []byte(fmt.Sprintf("consensusParamsKey:%f", 4.22222)),
+				"ABCIResponse":    []byte(fmt.Sprintf("abciResponsesKey:%f", 4.22222)),
+				"LightBlockShort": []byte(fmt.Sprintf("lb/foo/%010d", 42)),
+				"LightBlockLong":  []byte("lb/foo/12345678910.1234567890"),
+				"Invalid":         {0x03},
+				"BadTXHeight0":    []byte(fmt.Sprintf("tx.height/%s/%f/%f", "boop", 4.4, 4.5)),
+				"BadTXHeight1":    []byte(fmt.Sprintf("tx.height/%s/%f", "boop", 4.4)),
+				"UserKey0":        []byte("foo/bar/1.3/3.4"),
+				"UserKey1":        []byte("foo/bar/1/3.4"),
+				"UserKey2":        []byte("foo/bar/baz/1/3.4"),
+				"UserKey3":        []byte("foo/bar/baz/1.2/4"),
+			}
+			for kind, key := range table {
+				out, err := migarateKey(key)
+				require.Error(t, err, kind)
+				require.Nil(t, out, kind)
+			}
+		})
+		t.Run("Replacement", func(t *testing.T) {
+			t.Run("MissingKey", func(t *testing.T) {
+				db := dbm.NewMemDB()
+				require.NoError(t, replaceKey(db, keyID("hi"), nil))
+			})
+			t.Run("ReplacementFails", func(t *testing.T) {
+				db := dbm.NewMemDB()
+				key := keyID("hi")
+				require.NoError(t, db.Set(key, []byte("world")))
+				require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
+					return nil, errors.New("hi")
+				}))
+			})
+			t.Run("KeyDisapears", func(t *testing.T) {
+				db := dbm.NewMemDB()
+				key := keyID("hi")
+				require.NoError(t, db.Set(key, []byte("world")))
+				require.Error(t, replaceKey(db, key, func(k keyID) (keyID, error) {
+					require.NoError(t, db.Delete(key))
+					return keyID("wat"), nil
+				}))
+
+				exists, err := db.Has(key)
+				require.NoError(t, err)
+				require.False(t, exists)
+
+				exists, err = db.Has(keyID("wat"))
+				require.NoError(t, err)
+				require.False(t, exists)
+			})
+		})
+	})
+	t.Run("Integration", func(t *testing.T) {
+		t.Run("KeyDiscovery", func(t *testing.T) {
+			size, db := getLegacyDatabase(t)
+			keys, err := getAllLegacyKeys(db)
+			require.NoError(t, err)
+			require.Equal(t, size, len(keys))
+			legacyKeys := 0
+			for _, k := range keys {
+				if keyIsLegacy(k) {
+					legacyKeys++
+				}
+			}
+			require.Equal(t, size, legacyKeys)
+		})
+		t.Run("KeyIdempotency", func(t *testing.T) {
+			for _, key := range getNewPrefixKeys(t, 84) {
+				require.False(t, keyIsLegacy(key))
+			}
+		})
+		t.Run("ChannelConversion", func(t *testing.T) {
+			ch := makeKeyChan([]keyID{
+				makeKey(t, "abc", int64(2), int64(42)),
+				makeKey(t, int64(42)),
+			})
+			count := 0
+			for range ch {
+				count++
+			}
+			require.Equal(t, 2, count)
+		})
+		t.Run("Migrate", func(t *testing.T) {
+			_, db := getLegacyDatabase(t)
+
+			ctx := context.Background()
+			err := Migrate(ctx, db)
+			require.NoError(t, err)
+			keys, err := getAllLegacyKeys(db)
+			require.NoError(t, err)
+			require.Equal(t, 0, len(keys))
+
+		})
+	})
+}

state/indexer/block.go

@@ -1,22 +0,0 @@
-package indexer
-
-import (
-	"context"
-
-	"github.com/tendermint/tendermint/libs/pubsub/query"
-	"github.com/tendermint/tendermint/types"
-)
-
-// BlockIndexer defines an interface contract for indexing block events.
-type BlockIndexer interface {
-	// Has returns true if the given height has been indexed. An error is returned
-	// upon database query failure.
-	Has(height int64) (bool, error)
-
-	// Index indexes BeginBlock and EndBlock events for a given block by its height.
-	Index(types.EventDataNewBlockHeader) error
-
-	// Search performs a query for block heights that match a given BeginBlock
-	// and Endblock event search criteria.
-	Search(ctx context.Context, q *query.Query) ([]int64, error)
-}

state/indexer/indexer.go

@@ -1,4 +1,4 @@
-package txindex
+package indexer
 
 import (
 	"context"
@@ -6,10 +6,9 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
+	"github.com/tendermint/tendermint/types"
 )
 
-// XXX/TODO: These types should be moved to the indexer package.
-
 // TxIndexer interface defines methods to index and search transactions.
 type TxIndexer interface {
 	// AddBatch analyzes, indexes and stores a batch of transactions.
@@ -26,6 +25,20 @@ type TxIndexer interface {
 	Search(ctx context.Context, q *query.Query) ([]*abci.TxResult, error)
 }
 
+// BlockIndexer defines an interface contract for indexing block events.
+type BlockIndexer interface {
+	// Has returns true if the given height has been indexed. An error is returned
+	// upon database query failure.
+	Has(height int64) (bool, error)
+
+	// Index indexes BeginBlock and EndBlock events for a given block by its height.
+	Index(types.EventDataNewBlockHeader) error
+
+	// Search performs a query for block heights that match a given BeginBlock
+	// and Endblock event search criteria.
+	Search(ctx context.Context, q *query.Query) ([]int64, error)
+}
+
 // Batch groups together multiple Index operations to be performed at the same time.
 // NOTE: Batch is NOT thread-safe and must not be modified after starting its execution.
 type Batch struct {

state/indexer/indexer_service.go

@@ -1,10 +1,9 @@
-package txindex
+package indexer
 
 import (
 	"context"
 
 	"github.com/tendermint/tendermint/libs/service"
-	"github.com/tendermint/tendermint/state/indexer"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -14,31 +13,31 @@ const (
 	subscriber = "IndexerService"
 )
 
-// IndexerService connects event bus, transaction and block indexers together in
+// Service connects event bus, transaction and block indexers together in
 // order to index transactions and blocks coming from the event bus.
-type IndexerService struct {
+type Service struct {
 	service.BaseService
 
 	txIdxr    TxIndexer
-	blockIdxr indexer.BlockIndexer
+	blockIdxr BlockIndexer
 	eventBus  *types.EventBus
 }
 
 // NewIndexerService returns a new service instance.
 func NewIndexerService(
 	txIdxr TxIndexer,
-	blockIdxr indexer.BlockIndexer,
+	blockIdxr BlockIndexer,
 	eventBus *types.EventBus,
-) *IndexerService {
+) *Service {
 
-	is := &IndexerService{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus}
+	is := &Service{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus}
 	is.BaseService = *service.NewBaseService(nil, "IndexerService", is)
 	return is
 }
 
 // OnStart implements service.Service by subscribing for all transactions
 // and indexing them by events.
-func (is *IndexerService) OnStart() error {
+func (is *Service) OnStart() error {
 	// Use SubscribeUnbuffered here to ensure both subscriptions does not get
 	// canceled due to not pulling messages fast enough. Cause this might
 	// sometimes happen when there are no other subscribers.
@@ -93,7 +92,7 @@ func (is *IndexerService) OnStart() error {
 }
 
 // OnStop implements service.Service by unsubscribing from all transactions.
-func (is *IndexerService) OnStop() {
+func (is *Service) OnStop() {
 	if is.eventBus.IsRunning() {
 		_ = is.eventBus.UnsubscribeAll(context.Background(), subscriber)
 	}

state/indexer/indexer_service_test.go

@@ -1,4 +1,4 @@
-package txindex_test
+package indexer_test
 
 import (
 	"testing"
@@ -9,9 +9,9 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/log"
+	indexer "github.com/tendermint/tendermint/state/indexer"
 	blockidxkv "github.com/tendermint/tendermint/state/indexer/block/kv"
-	"github.com/tendermint/tendermint/state/txindex"
-	"github.com/tendermint/tendermint/state/txindex/kv"
+	"github.com/tendermint/tendermint/state/indexer/tx/kv"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -32,7 +32,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) {
 	txIndexer := kv.NewTxIndex(store)
 	blockIndexer := blockidxkv.New(db.NewPrefixDB(store, []byte("block_events")))
 
-	service := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus)
+	service := indexer.NewIndexerService(txIndexer, blockIndexer, eventBus)
 	service.SetLogger(log.TestingLogger())
 	err = service.Start()
 	require.NoError(t, err)

state/indexer/tx/kv/kv.go

@@ -13,12 +13,11 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
-	"github.com/tendermint/tendermint/state/indexer"
-	"github.com/tendermint/tendermint/state/txindex"
+	indexer "github.com/tendermint/tendermint/state/indexer"
 	"github.com/tendermint/tendermint/types"
 )
 
-var _ txindex.TxIndexer = (*TxIndex)(nil)
+var _ indexer.TxIndexer = (*TxIndex)(nil)
 
 // TxIndex is the simplest possible indexer
 // It is backed by two kv stores:
@@ -39,7 +38,7 @@ func NewTxIndex(store dbm.DB) *TxIndex {
 // transaction is not found.
 func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) {
 	if len(hash) == 0 {
-		return nil, txindex.ErrorEmptyHash
+		return nil, indexer.ErrorEmptyHash
 	}
 
 	rawBytes, err := txi.store.Get(primaryKey(hash))
@@ -63,7 +62,7 @@ func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) {
 // key that indexed from the tx's events is a composite of the event type and
 // the respective attribute's key delimited by a "." (eg. "account.number").
 // Any event with an empty type is not indexed.
-func (txi *TxIndex) AddBatch(b *txindex.Batch) error {
+func (txi *TxIndex) AddBatch(b *indexer.Batch) error {
 	storeBatch := txi.store.NewBatch()
 	defer storeBatch.Close()
 

state/indexer/tx/kv/kv_test.go

@@ -16,12 +16,12 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
-	"github.com/tendermint/tendermint/state/txindex"
+	indexer "github.com/tendermint/tendermint/state/indexer"
 	"github.com/tendermint/tendermint/types"
 )
 
 func TestTxIndex(t *testing.T) {
-	indexer := NewTxIndex(db.NewMemDB())
+	txIndexer := NewTxIndex(db.NewMemDB())
 
 	tx := types.Tx("HELLO WORLD")
 	txResult := &abci.TxResult{
@@ -35,14 +35,14 @@ func TestTxIndex(t *testing.T) {
 	}
 	hash := tx.Hash()
 
-	batch := txindex.NewBatch(1)
+	batch := indexer.NewBatch(1)
 	if err := batch.Add(txResult); err != nil {
 		t.Error(err)
 	}
-	err := indexer.AddBatch(batch)
+	err := txIndexer.AddBatch(batch)
 	require.NoError(t, err)
 
-	loadedTxResult, err := indexer.Get(hash)
+	loadedTxResult, err := txIndexer.Get(hash)
 	require.NoError(t, err)
 	assert.True(t, proto.Equal(txResult, loadedTxResult))
 
@@ -58,10 +58,10 @@ func TestTxIndex(t *testing.T) {
 	}
 	hash2 := tx2.Hash()
 
-	err = indexer.Index(txResult2)
+	err = txIndexer.Index(txResult2)
 	require.NoError(t, err)
 
-	loadedTxResult2, err := indexer.Get(hash2)
+	loadedTxResult2, err := txIndexer.Get(hash2)
 	require.NoError(t, err)
 	assert.True(t, proto.Equal(txResult2, loadedTxResult2))
 }
@@ -341,9 +341,9 @@ func benchmarkTxIndex(txsCount int64, b *testing.B) {
 
 	store, err := db.NewDB("tx_index", "goleveldb", dir)
 	require.NoError(b, err)
-	indexer := NewTxIndex(store)
+	txIndexer := NewTxIndex(store)
 
-	batch := txindex.NewBatch(txsCount)
+	batch := indexer.NewBatch(txsCount)
 	txIndex := uint32(0)
 	for i := int64(0); i < txsCount; i++ {
 		tx := tmrand.Bytes(250)
@@ -367,7 +367,7 @@ func benchmarkTxIndex(txsCount int64, b *testing.B) {
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		err = indexer.AddBatch(batch)
+		err = txIndexer.AddBatch(batch)
 	}
 	if err != nil {
 		b.Fatal(err)

state/indexer/tx/null/null.go

@@ -6,10 +6,10 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/libs/pubsub/query"
-	"github.com/tendermint/tendermint/state/txindex"
+	"github.com/tendermint/tendermint/state/indexer"
 )
 
-var _ txindex.TxIndexer = (*TxIndex)(nil)
+var _ indexer.TxIndexer = (*TxIndex)(nil)
 
 // TxIndex acts as a /dev/null.
 type TxIndex struct{}
@@ -20,7 +20,7 @@ func (txi *TxIndex) Get(hash []byte) (*abci.TxResult, error) {
 }
 
 // AddBatch is a noop and always returns nil.
-func (txi *TxIndex) AddBatch(batch *txindex.Batch) error {
+func (txi *TxIndex) AddBatch(batch *indexer.Batch) error {
 	return nil
 }
 

state/state_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"math"
 	"math/big"
+	mrand "math/rand"
 	"os"
 	"testing"
 
@@ -18,7 +19,6 @@ import (
 	cfg "github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
 	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
 	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
@@ -356,18 +356,18 @@ func TestProposerFrequency(t *testing.T) {
 	maxPower := 1000
 	nTestCases := 5
 	for i := 0; i < nTestCases; i++ {
-		N := tmrand.Int()%maxVals + 1
+		N := mrand.Int()%maxVals + 1
 		vals := make([]*types.Validator, N)
 		totalVotePower := int64(0)
 		for j := 0; j < N; j++ {
 			// make sure votePower > 0
-			votePower := int64(tmrand.Int()%maxPower) + 1
+			votePower := int64(mrand.Int()%maxPower) + 1
 			totalVotePower += votePower
 			privVal := types.NewMockPV()
 			pubKey, err := privVal.GetPubKey(context.Background())
 			require.NoError(t, err)
 			val := types.NewValidator(pubKey, votePower)
-			val.ProposerPriority = tmrand.Int64()
+			val.ProposerPriority = mrand.Int63()
 			vals[j] = val
 		}
 		valSet := types.NewValidatorSet(vals)
@@ -384,7 +384,7 @@ func genValSetWithPowers(powers []int64) *types.ValidatorSet {
 	for i := 0; i < size; i++ {
 		totalVotePower += powers[i]
 		val := types.NewValidator(ed25519.GenPrivKey().PubKey(), powers[i])
-		val.ProposerPriority = tmrand.Int64()
+		val.ProposerPriority = mrand.Int63()
 		vals[i] = val
 	}
 	valSet := types.NewValidatorSet(vals)

statesync/reactor.go

@@ -477,14 +477,18 @@ func (r *Reactor) Sync(stateProvider StateProvider, discoveryTime time.Duration)
 	r.syncer = newSyncer(r.Logger, r.conn, r.connQuery, stateProvider, r.snapshotCh.Out, r.chunkCh.Out, r.tempDir)
 	r.mtx.Unlock()
 
-	// request snapshots from all currently connected peers
-	r.Logger.Debug("requesting snapshots from known peers")
-	r.snapshotCh.Out <- p2p.Envelope{
-		Broadcast: true,
-		Message:   &ssproto.SnapshotsRequest{},
+	hook := func() {
+		// request snapshots from all currently connected peers
+		r.Logger.Debug("requesting snapshots from known peers")
+		r.snapshotCh.Out <- p2p.Envelope{
+			Broadcast: true,
+			Message:   &ssproto.SnapshotsRequest{},
+		}
 	}
 
-	state, commit, err := r.syncer.SyncAny(discoveryTime)
+	hook()
+
+	state, commit, err := r.syncer.SyncAny(discoveryTime, hook)
 
 	r.mtx.Lock()
 	r.syncer = nil

statesync/snapshots.go

@@ -177,12 +177,41 @@ func (p *snapshotPool) Ranked() []*snapshot {
 	p.Lock()
 	defer p.Unlock()
 
-	candidates := make([]*snapshot, 0, len(p.snapshots))
-	for _, snapshot := range p.snapshots {
-		candidates = append(candidates, snapshot)
+	if len(p.snapshots) == 0 {
+		return []*snapshot{}
 	}
 
-	sort.Slice(candidates, func(i, j int) bool {
+	numPeers := make([]int, 0, len(p.snapshots))
+	for key := range p.snapshots {
+		numPeers = append(numPeers, len(p.snapshotPeers[key]))
+	}
+	sort.Ints(numPeers)
+	median := len(numPeers) / 2
+	if len(numPeers)%2 == 0 {
+		median = (numPeers[median-1] + numPeers[median]) / 2
+	} else {
+		median = numPeers[median]
+	}
+
+	commonCandidates := make([]*snapshot, 0, len(p.snapshots)/2)
+	uncommonCandidates := make([]*snapshot, 0, len(p.snapshots)/2)
+	for key := range p.snapshots {
+		if len(p.snapshotPeers[key]) > median {
+			commonCandidates = append(commonCandidates, p.snapshots[key])
+			continue
+		}
+
+		uncommonCandidates = append(uncommonCandidates, p.snapshots[key])
+	}
+
+	sort.Slice(commonCandidates, p.sorterFactory(commonCandidates))
+	sort.Slice(uncommonCandidates, p.sorterFactory(uncommonCandidates))
+
+	return append(commonCandidates, uncommonCandidates...)
+}
+
+func (p *snapshotPool) sorterFactory(candidates []*snapshot) func(int, int) bool {
+	return func(i, j int) bool {
 		a := candidates[i]
 		b := candidates[j]
 
@@ -191,18 +220,16 @@ func (p *snapshotPool) Ranked() []*snapshot {
 			return true
 		case a.Height < b.Height:
 			return false
+		case len(p.snapshotPeers[a.Key()]) > len(p.snapshotPeers[b.Key()]):
+			return true
 		case a.Format > b.Format:
 			return true
 		case a.Format < b.Format:
 			return false
-		case len(p.snapshotPeers[a.Key()]) > len(p.snapshotPeers[b.Key()]):
-			return true
 		default:
 			return false
 		}
-	})
-
-	return candidates
+	}
 }
 
 // Reject rejects a snapshot. Rejected snapshots will never be used again.

statesync/snapshots_test.go

@@ -148,11 +148,11 @@ func TestSnapshotPool_Ranked_Best(t *testing.T) {
 		snapshot *snapshot
 		peers    []p2p.NodeID
 	}{
-		{&snapshot{Height: 2, Format: 2, Chunks: 4, Hash: []byte{1, 3}}, []p2p.NodeID{"AA", "BB", "CC"}},
-		{&snapshot{Height: 2, Format: 2, Chunks: 5, Hash: []byte{1, 2}}, []p2p.NodeID{"AA"}},
-		{&snapshot{Height: 2, Format: 1, Chunks: 3, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB"}},
-		{&snapshot{Height: 1, Format: 2, Chunks: 5, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB"}},
-		{&snapshot{Height: 1, Format: 1, Chunks: 4, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB", "CC"}},
+		{&snapshot{Height: 2, Format: 2, Chunks: 4, Hash: []byte{1, 3}}, []p2p.NodeID{"AA", "BB", "CC", "DD"}},
+		{&snapshot{Height: 1, Format: 1, Chunks: 4, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB", "CC", "DD"}},
+		{&snapshot{Height: 2, Format: 2, Chunks: 5, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB"}},
+		{&snapshot{Height: 2, Format: 1, Chunks: 3, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB", "CC"}},
+		{&snapshot{Height: 1, Format: 2, Chunks: 5, Hash: []byte{1, 2}}, []p2p.NodeID{"AA", "BB", "CC"}},
 	}
 
 	// Add snapshots in reverse order, to make sure the pool enforces some order.

statesync/syncer.go

@@ -24,6 +24,9 @@ const (
 	chunkTimeout = 2 * time.Minute
 	// requestTimeout is the timeout before rerequesting a chunk, possibly from a different peer.
 	chunkRequestTimeout = 10 * time.Second
+	// minimumDiscoveryTime is the lowest allowable time for a
+	// SyncAny discovery time.
+	minimumDiscoveryTime = 5 * time.Second
 )
 
 var (
@@ -138,7 +141,11 @@ func (s *syncer) RemovePeer(peerID p2p.NodeID) {
 // SyncAny tries to sync any of the snapshots in the snapshot pool, waiting to discover further
 // snapshots if none were found and discoveryTime > 0. It returns the latest state and block commit
 // which the caller must use to bootstrap the node.
-func (s *syncer) SyncAny(discoveryTime time.Duration) (sm.State, *types.Commit, error) {
+func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.State, *types.Commit, error) {
+	if discoveryTime != 0 && discoveryTime < minimumDiscoveryTime {
+		discoveryTime = 5 * minimumDiscoveryTime
+	}
+
 	if discoveryTime > 0 {
 		s.logger.Info(fmt.Sprintf("Discovering snapshots for %v", discoveryTime))
 		time.Sleep(discoveryTime)
@@ -161,6 +168,7 @@ func (s *syncer) SyncAny(discoveryTime time.Duration) (sm.State, *types.Commit,
 			if discoveryTime == 0 {
 				return sm.State{}, nil, errNoSnapshots
 			}
+			retryHook()
 			s.logger.Info(fmt.Sprintf("Discovering snapshots for %v", discoveryTime))
 			time.Sleep(discoveryTime)
 			continue

statesync/syncer_test.go

@@ -68,7 +68,7 @@ func TestSyncer_SyncAny(t *testing.T) {
 
 	peerAID := p2p.NodeID("aa")
 	peerBID := p2p.NodeID("bb")
-
+	peerCID := p2p.NodeID("cc")
 	rts := setup(t, connSnapshot, connQuery, stateProvider, 3)
 
 	// Adding a chunk should error when no sync is in progress
@@ -96,10 +96,15 @@ func TestSyncer_SyncAny(t *testing.T) {
 	require.NoError(t, err)
 	require.False(t, new)
 
-	new, err = rts.syncer.AddSnapshot(peerBID, &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1}})
+	s2 := &snapshot{Height: 2, Format: 2, Chunks: 3, Hash: []byte{1}}
+	new, err = rts.syncer.AddSnapshot(peerBID, s2)
 	require.NoError(t, err)
 	require.True(t, new)
 
+	new, err = rts.syncer.AddSnapshot(peerCID, s2)
+	require.NoError(t, err)
+	require.False(t, new)
+
 	// We start a sync, with peers sending back chunks when requested. We first reject the snapshot
 	// with height 2 format 2, and accept the snapshot at height 1.
 	connSnapshot.On("OfferSnapshotSync", ctx, abci.RequestOfferSnapshot{
@@ -175,7 +180,7 @@ func TestSyncer_SyncAny(t *testing.T) {
 		LastBlockAppHash: []byte("app_hash"),
 	}, nil)
 
-	newState, lastCommit, err := rts.syncer.SyncAny(0)
+	newState, lastCommit, err := rts.syncer.SyncAny(0, func() {})
 	require.NoError(t, err)
 
 	wg.Wait()
@@ -201,7 +206,7 @@ func TestSyncer_SyncAny_noSnapshots(t *testing.T) {
 
 	rts := setup(t, nil, nil, stateProvider, 2)
 
-	_, _, err := rts.syncer.SyncAny(0)
+	_, _, err := rts.syncer.SyncAny(0, func() {})
 	require.Equal(t, errNoSnapshots, err)
 }
 
@@ -221,7 +226,7 @@ func TestSyncer_SyncAny_abort(t *testing.T) {
 		Snapshot: toABCI(s), AppHash: []byte("app_hash"),
 	}).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil)
 
-	_, _, err = rts.syncer.SyncAny(0)
+	_, _, err = rts.syncer.SyncAny(0, func() {})
 	require.Equal(t, errAbort, err)
 	rts.conn.AssertExpectations(t)
 }
@@ -260,7 +265,7 @@ func TestSyncer_SyncAny_reject(t *testing.T) {
 		Snapshot: toABCI(s11), AppHash: []byte("app_hash"),
 	}).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
 
-	_, _, err = rts.syncer.SyncAny(0)
+	_, _, err = rts.syncer.SyncAny(0, func() {})
 	require.Equal(t, errNoSnapshots, err)
 	rts.conn.AssertExpectations(t)
 }
@@ -295,7 +300,7 @@ func TestSyncer_SyncAny_reject_format(t *testing.T) {
 		Snapshot: toABCI(s11), AppHash: []byte("app_hash"),
 	}).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ABORT}, nil)
 
-	_, _, err = rts.syncer.SyncAny(0)
+	_, _, err = rts.syncer.SyncAny(0, func() {})
 	require.Equal(t, errAbort, err)
 	rts.conn.AssertExpectations(t)
 }
@@ -341,7 +346,7 @@ func TestSyncer_SyncAny_reject_sender(t *testing.T) {
 		Snapshot: toABCI(sa), AppHash: []byte("app_hash"),
 	}).Once().Return(&abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT}, nil)
 
-	_, _, err = rts.syncer.SyncAny(0)
+	_, _, err = rts.syncer.SyncAny(0, func() {})
 	require.Equal(t, errNoSnapshots, err)
 	rts.conn.AssertExpectations(t)
 }
@@ -364,7 +369,7 @@ func TestSyncer_SyncAny_abciError(t *testing.T) {
 		Snapshot: toABCI(s), AppHash: []byte("app_hash"),
 	}).Once().Return(nil, errBoom)
 
-	_, _, err = rts.syncer.SyncAny(0)
+	_, _, err = rts.syncer.SyncAny(0, func() {})
 	require.True(t, errors.Is(err, errBoom))
 	rts.conn.AssertExpectations(t)
 }

test/e2e/app/main.go

@@ -156,6 +156,7 @@ func startSeedNode(cfg *Config) error {
 
 	n, err := node.NewSeedNode(
 		tmcfg,
+		node.DefaultDBProvider,
 		*nodeKey,
 		node.DefaultGenesisDocProviderFunc(tmcfg),
 		nodeLogger,

test/e2e/app/snapshots.go

@@ -2,7 +2,6 @@
 package main
 
 import (
-	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -88,11 +87,10 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
 	if err != nil {
 		return abci.Snapshot{}, err
 	}
-	hash := sha256.Sum256(bz)
 	snapshot := abci.Snapshot{
 		Height: state.Height,
 		Format: 1,
-		Hash:   hash[:],
+		Hash:   hashItems(state.Values),
 		Chunks: byteChunks(bz),
 	}
 	err = ioutil.WriteFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", state.Height)), bz, 0644)
@@ -111,10 +109,9 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
 func (s *SnapshotStore) List() ([]*abci.Snapshot, error) {
 	s.RLock()
 	defer s.RUnlock()
-	snapshots := []*abci.Snapshot{}
-	for _, snapshot := range s.metadata {
-		s := snapshot // copy to avoid pointer to range variable
-		snapshots = append(snapshots, &s)
+	snapshots := make([]*abci.Snapshot, len(s.metadata))
+	for idx := range s.metadata {
+		snapshots[idx] = &s.metadata[idx]
 	}
 	return snapshots, nil
 }

test/e2e/generator/generate.go

@@ -78,9 +78,9 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 
 	switch p2pInfo := opt["useNewP2P"].(type) {
 	case bool:
-		manifest.UseNewP2P = p2pInfo
+		manifest.DisableLegacyP2P = p2pInfo
 	case int:
-		manifest.UseNewP2P = false
+		manifest.DisableLegacyP2P = false
 		p2pNodeFactor = p2pInfo
 	}
 
@@ -104,11 +104,13 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 	for i := 1; i <= numSeeds; i++ {
 		node := generateNode(r, e2e.ModeSeed, 0, manifest.InitialHeight, false)
 		node.QueueType = manifest.QueueType
+
 		if p2pNodeFactor == 0 {
-			node.UseNewP2P = manifest.UseNewP2P
+			node.DisableLegacyP2P = manifest.DisableLegacyP2P
 		} else if p2pNodeFactor%i == 0 {
-			node.UseNewP2P = !manifest.UseNewP2P
+			node.DisableLegacyP2P = !manifest.DisableLegacyP2P
 		}
+
 		manifest.Nodes[fmt.Sprintf("seed%02d", i)] = node
 	}
 
@@ -129,10 +131,11 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 
 		node.QueueType = manifest.QueueType
 		if p2pNodeFactor == 0 {
-			node.UseNewP2P = manifest.UseNewP2P
+			node.DisableLegacyP2P = manifest.DisableLegacyP2P
 		} else if p2pNodeFactor%i == 0 {
-			node.UseNewP2P = !manifest.UseNewP2P
+			node.DisableLegacyP2P = !manifest.DisableLegacyP2P
 		}
+
 		manifest.Nodes[name] = node
 
 		if startAt == 0 {
@@ -164,9 +167,9 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
 		node := generateNode(r, e2e.ModeFull, startAt, manifest.InitialHeight, false)
 		node.QueueType = manifest.QueueType
 		if p2pNodeFactor == 0 {
-			node.UseNewP2P = manifest.UseNewP2P
+			node.DisableLegacyP2P = manifest.DisableLegacyP2P
 		} else if p2pNodeFactor%i == 0 {
-			node.UseNewP2P = !manifest.UseNewP2P
+			node.DisableLegacyP2P = !manifest.DisableLegacyP2P
 		}
 		manifest.Nodes[fmt.Sprintf("full%02d", i)] = node
 	}

test/e2e/networks/ci.toml

@@ -1,10 +1,10 @@
 # This testnet is run by CI, and attempts to cover a broad range of
 # functionality with a single network.
 
+disable_legacy_p2p = false
+evidence = 5
 initial_height = 1000
-evidence = 0
-initial_state = { initial01 = "a", initial02 = "b", initial03 = "c" }
-use_new_p2p = false
+initial_state = {initial01 = "a", initial02 = "b", initial03 = "c"}
 queue_type = "priority"
 
 [validators]
@@ -29,71 +29,71 @@ validator05 = 50
 
 [node.seed01]
 mode = "seed"
-seeds = ["seed02"]
 perturb = ["restart"]
+seeds = ["seed02"]
 
 [node.seed02]
 mode = "seed"
 seeds = ["seed01"]
 
 [node.validator01]
+perturb = ["disconnect"]
 seeds = ["seed01"]
 snapshot_interval = 5
-perturb = ["disconnect"]
 
 [node.validator02]
-seeds = ["seed02"]
-database = "boltdb"
 abci_protocol = "tcp"
-privval_protocol = "tcp"
+database = "boltdb"
 persist_interval = 0
 perturb = ["restart"]
+privval_protocol = "tcp"
+seeds = ["seed02"]
 
 [node.validator03]
-seeds = ["seed01"]
 database = "badgerdb"
+seeds = ["seed01"]
 # FIXME: should be grpc, disabled due to https://github.com/tendermint/tendermint/issues/5439
 #abci_protocol = "grpc"
-privval_protocol = "grpc"
 persist_interval = 3
-retain_blocks = 3
 perturb = ["kill"]
+privval_protocol = "grpc"
+retain_blocks = 5
 
 [node.validator04]
-persistent_peers = ["validator01"]
-database = "rocksdb"
 abci_protocol = "builtin"
+database = "rocksdb"
+persistent_peers = ["validator01"]
 perturb = ["pause"]
 
 [node.validator05]
+database = "cleveldb" 
+fast_sync = "v0" 
+seeds = ["seed02"] 
 start_at = 1005 # Becomes part of the validator set at 1010
-seeds = ["seed02"]
-database = "cleveldb"
-fast_sync = "v0"
 # FIXME: should be grpc, disabled due to https://github.com/tendermint/tendermint/issues/5439
 #abci_protocol = "grpc"
-privval_protocol = "tcp"
 perturb = ["kill", "pause", "disconnect", "restart"]
+privval_protocol = "tcp"
 
 [node.full01]
-start_at = 1010
 mode = "full"
+start_at = 1010
 # FIXME: should be v2, disabled due to flake
 fast_sync = "v0"
 persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
-retain_blocks = 3
 perturb = ["restart"]
+retain_blocks = 5
 
 [node.full02]
-start_at = 1015
 mode = "full"
+start_at = 1015
 # FIXME: should be v2, disabled due to flake
 fast_sync = "v0"
-state_sync = true
-seeds = ["seed01"]
 perturb = ["restart"]
+seeds = ["seed01"]
+state_sync = true
 
 [node.light01]
-mode= "light"
-start_at= 1010
-persistent_peers = ["validator01", "validator02", "validator03"]
+mode = "light"
+persistent_peers = ["validator01", "validator02", "validator03"]
+start_at = 1010

test/e2e/pkg/manifest.go

@@ -59,9 +59,8 @@ type Manifest struct {
 	// by individual nodes.
 	LogLevel string `toml:"log_level"`
 
-	// UseNewP2P enables use of the new p2p layer for all nodes in
-	// a test.
-	UseNewP2P bool `toml:"use_new_p2p"`
+	// DisableLegacyP2P enables use of the new p2p layer for all nodes in a test.
+	DisableLegacyP2P bool `toml:"disable_legacy_p2p"`
 
 	// QueueType describes the type of queue that the system uses internally
 	QueueType string `toml:"queue_type"`
@@ -143,7 +142,7 @@ type ManifestNode struct {
 	LogLevel string `toml:"log_level"`
 
 	// UseNewP2P enables use of the new p2p layer for this node.
-	UseNewP2P bool `toml:"use_new_p2p"`
+	DisableLegacyP2P bool `toml:"disable_legacy_p2p"`
 
 	// QueueType describes the type of queue that the p2p layer
 	// uses internally.

test/e2e/pkg/testnet.go

@@ -48,7 +48,7 @@ const (
 	PerturbationPause      Perturbation = "pause"
 	PerturbationRestart    Perturbation = "restart"
 
-	EvidenceAgeHeight int64         = 3
+	EvidenceAgeHeight int64         = 5
 	EvidenceAgeTime   time.Duration = 10 * time.Second
 )
 
@@ -90,7 +90,7 @@ type Node struct {
 	PersistentPeers  []*Node
 	Perturbations    []Perturbation
 	LogLevel         string
-	UseNewP2P        bool
+	DisableLegacyP2P bool
 	QueueType        string
 }
 
@@ -169,9 +169,10 @@ func LoadTestnet(file string) (*Testnet, error) {
 			RetainBlocks:     nodeManifest.RetainBlocks,
 			Perturbations:    []Perturbation{},
 			LogLevel:         manifest.LogLevel,
-			UseNewP2P:        manifest.UseNewP2P,
+			DisableLegacyP2P: manifest.DisableLegacyP2P,
 			QueueType:        manifest.QueueType,
 		}
+
 		if node.StartAt == testnet.InitialHeight {
 			node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
 		}

test/e2e/runner/benchmark.go

@@ -2,8 +2,10 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math"
+	"path/filepath"
 	"time"
 
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -26,7 +28,7 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	}
 
 	logger.Info("Beginning benchmark period...", "height", block.Height)
-
+	startAt := time.Now()
 	// wait for the length of the benchmark period in blocks to pass. We allow 5 seconds for each block
 	// which should be sufficient.
 	waitingTime := time.Duration(benchmarkLength*5) * time.Second
@@ -34,6 +36,7 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	if err != nil {
 		return err
 	}
+	dur := time.Since(startAt)
 
 	logger.Info("Ending benchmark period", "height", endHeight)
 
@@ -46,18 +49,34 @@ func Benchmark(testnet *e2e.Testnet, benchmarkLength int64) error {
 	// slice into time intervals and collate data
 	timeIntervals := splitIntoBlockIntervals(blocks)
 	testnetStats := extractTestnetStats(timeIntervals)
+
+	// populate data
+	testnetStats.populateTxns(blocks)
+	testnetStats.totalTime = dur
+	testnetStats.benchmarkLength = benchmarkLength
 	testnetStats.startHeight = blocks[0].Header.Height
 	testnetStats.endHeight = blocks[len(blocks)-1].Header.Height
 
 	// print and return
 	logger.Info(testnetStats.String())
+	logger.Info(testnetStats.getReportJSON(testnet))
 	return nil
 }
 
+func (t *testnetStats) populateTxns(blocks []*types.BlockMeta) {
+	t.numtxns = 0
+	for _, b := range blocks {
+		t.numtxns += int64(b.NumTxs)
+	}
+}
+
 type testnetStats struct {
 	startHeight int64
 	endHeight   int64
 
+	benchmarkLength int64
+	numtxns         int64
+	totalTime       time.Duration
 	// average time to produce a block
 	mean time.Duration
 	// standard deviation of block production
@@ -68,6 +87,27 @@ type testnetStats struct {
 	min time.Duration
 }
 
+func (t *testnetStats) getReportJSON(net *e2e.Testnet) string {
+	jsn, err := json.Marshal(map[string]interface{}{
+		"case":   filepath.Base(net.File),
+		"blocks": t.endHeight - t.startHeight,
+		"stddev": t.std,
+		"mean":   t.mean.Seconds(),
+		"max":    t.max.Seconds(),
+		"min":    t.min.Seconds(),
+		"size":   len(net.Nodes),
+		"txns":   t.numtxns,
+		"dur":    t.totalTime.Seconds(),
+		"length": t.benchmarkLength,
+	})
+
+	if err != nil {
+		return ""
+	}
+
+	return string(jsn)
+}
+
 func (t *testnetStats) String() string {
 	return fmt.Sprintf(`Benchmarked from height %v to %v
 	Mean Block Interval: %v

test/e2e/runner/evidence.go

@@ -19,11 +19,8 @@ import (
 	"github.com/tendermint/tendermint/version"
 )
 
-// 1 in 11 evidence is light client evidence, the rest is duplicate vote
-// FIXME: Setting to 11 disables light client attack evidence since nodes
-// don't follow a minimum retention height invariant. When we fix this we
-// should use a ratio of 4.
-const lightClientEvidenceRatio = 11
+// 1 in 4 evidence is light client evidence, the rest is duplicate vote evidence
+const lightClientEvidenceRatio = 4
 
 // InjectEvidence takes a running testnet and generates an amount of valid
 // evidence and broadcasts it to a random node through the rpc endpoint `/broadcast_evidence`.
@@ -74,7 +71,7 @@ func InjectEvidence(testnet *e2e.Testnet, amount int) error {
 	duplicateVoteTime := status.SyncInfo.LatestBlockTime
 
 	var ev types.Evidence
-	for i := 0; i < amount; i++ {
+	for i := 1; i <= amount; i++ {
 		if i%lightClientEvidenceRatio == 0 {
 			ev, err = generateLightClientAttackEvidence(
 				privVals, lightEvidenceCommonHeight, valSet, testnet.Name, blockRes.Block.Time,

test/e2e/runner/setup.go

@@ -239,7 +239,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
 	cfg.RPC.PprofListenAddress = ":6060"
 	cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false))
 	cfg.P2P.AddrBookStrict = false
-	cfg.P2P.UseNewP2P = node.UseNewP2P
+	cfg.P2P.DisableLegacy = node.DisableLegacyP2P
 	cfg.P2P.QueueType = node.QueueType
 	cfg.DBBackend = node.Database
 	cfg.StateSync.DiscoveryTime = 5 * time.Second

test/e2e/tests/block_test.go

@@ -37,8 +37,12 @@ func TestBlock_Header(t *testing.T) {
 			}
 			resp, err := client.Block(ctx, &block.Header.Height)
 			require.NoError(t, err)
+
 			require.Equal(t, block, resp.Block,
-				"block mismatch for height %v", block.Header.Height)
+				"block mismatch for height %d", block.Header.Height)
+
+			require.NoError(t, resp.Block.ValidateBasic(),
+				"block at height %d is invalid", block.Header.Height)
 		}
 	})
 }

test/fuzz/rpc/jsonrpc/server/handler.go

@@ -14,7 +14,7 @@ import (
 )
 
 var rpcFuncMap = map[string]*rs.RPCFunc{
-	"c": rs.NewRPCFunc(func(s string, i int) (string, int) { return "foo", 200 }, "s,i"),
+	"c": rs.NewRPCFunc(func(s string, i int) (string, int) { return "foo", 200 }, "s,i", false),
 }
 var mux *http.ServeMux