Merge branch 'master' into callum/handshake

internal/blocksync/doc.go

@@ -13,14 +13,9 @@ will no longer blocksync and thus no longer run the blocksync process.
 Note, the blocksync reactor Service gossips entire block and relevant data such
 that each receiving peer may construct the entire view of the blocksync state.
 
-There are currently two versions of the blocksync reactor Service:
-
-- v0: The initial implementation that is battle-tested, but whose test coverage
-	is lacking and is not formally verifiable.
-- v2: The latest implementation that has much higher test coverage and is formally
-	verified. However, the current implementation of v2 is not as battle-tested and
-	is known to have various bugs that could make it unreliable in production
-	environments.
+There is currently only one version of the blocksync reactor Service
+that is battle-tested, but whose test coverage is lacking and is not
+formally verified.
 
 The v0 blocksync reactor Service has one p2p channel, BlockchainChannel. This
 channel is responsible for handling messages that both request blocks and respond

internal/blocksync/pool.go

@@ -1,4 +1,4 @@
-package v0
+package blocksync
 
 import (
 	"errors"

internal/blocksync/pool_test.go

@@ -1,4 +1,4 @@
-package v0
+package blocksync
 
 import (
 	"fmt"

internal/blocksync/reactor.go

@@ -1,4 +1,4 @@
-package v0
+package blocksync
 
 import (
 	"fmt"
@@ -6,46 +6,22 @@ import (
 	"sync"
 	"time"
 
-	bc "github.com/tendermint/tendermint/internal/blocksync"
-	cons "github.com/tendermint/tendermint/internal/consensus"
+	"github.com/tendermint/tendermint/internal/consensus"
 	"github.com/tendermint/tendermint/internal/p2p"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/libs/service"
-	tmSync "github.com/tendermint/tendermint/libs/sync"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
 	bcproto "github.com/tendermint/tendermint/proto/tendermint/blocksync"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
 
-var (
-	_ service.Service = (*Reactor)(nil)
-
-	// ChannelShims contains a map of ChannelDescriptorShim objects, where each
-	// object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
-	// p2p proto.Message the new p2p Channel is responsible for handling.
-	//
-	//
-	// TODO: Remove once p2p refactor is complete.
-	// ref: https://github.com/tendermint/tendermint/issues/5670
-	ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
-		BlockchainChannel: {
-			MsgType: new(bcproto.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(BlockchainChannel),
-				Priority:            5,
-				SendQueueCapacity:   1000,
-				RecvBufferCapacity:  1024,
-				RecvMessageCapacity: bc.MaxMsgSize,
-				MaxSendBytes:        100,
-			},
-		},
-	}
-)
+var _ service.Service = (*Reactor)(nil)
 
 const (
-	// BlockchainChannel is a channel for blocks and status updates
-	BlockchainChannel = p2p.ChannelID(0x40)
+	// BlockSyncChannel is a channel for blocks and status updates
+	BlockSyncChannel = p2p.ChannelID(0x40)
 
 	trySyncIntervalMS = 10
 
@@ -59,8 +35,19 @@ const (
 	syncTimeout = 60 * time.Second
 )
 
+func GetChannelDescriptor() *p2p.ChannelDescriptor {
+	return &p2p.ChannelDescriptor{
+		ID:                  BlockSyncChannel,
+		MessageType:         new(bcproto.Message),
+		Priority:            5,
+		SendQueueCapacity:   1000,
+		RecvBufferCapacity:  1024,
+		RecvMessageCapacity: MaxMsgSize,
+	}
+}
+
 type consensusReactor interface {
-	// For when we switch from blockchain reactor and block sync to the consensus
+	// For when we switch from block sync reactor to the consensus
 	// machine.
 	SwitchToConsensus(state sm.State, skipWAL bool)
 }
@@ -85,19 +72,19 @@ type Reactor struct {
 	store       *store.BlockStore
 	pool        *BlockPool
 	consReactor consensusReactor
-	blockSync   *tmSync.AtomicBool
+	blockSync   *tmsync.AtomicBool
 
-	blockchainCh *p2p.Channel
-	// blockchainOutBridgeCh defines a channel that acts as a bridge between sending Envelope
-	// messages that the reactor will consume in processBlockchainCh and receiving messages
+	blockSyncCh *p2p.Channel
+	// blockSyncOutBridgeCh defines a channel that acts as a bridge between sending Envelope
+	// messages that the reactor will consume in processBlockSyncCh and receiving messages
 	// from the peer updates channel and other goroutines. We do this instead of directly
-	// sending on blockchainCh.Out to avoid race conditions in the case where other goroutines
-	// send Envelopes directly to the to blockchainCh.Out channel, since processBlockchainCh
-	// may close the blockchainCh.Out channel at the same time that other goroutines send to
-	// blockchainCh.Out.
-	blockchainOutBridgeCh chan p2p.Envelope
-	peerUpdates           *p2p.PeerUpdates
-	closeCh               chan struct{}
+	// sending on blockSyncCh.Out to avoid race conditions in the case where other goroutines
+	// send Envelopes directly to the to blockSyncCh.Out channel, since processBlockSyncCh
+	// may close the blockSyncCh.Out channel at the same time that other goroutines send to
+	// blockSyncCh.Out.
+	blockSyncOutBridgeCh chan p2p.Envelope
+	peerUpdates          *p2p.PeerUpdates
+	closeCh              chan struct{}
 
 	requestsCh <-chan BlockRequest
 	errorsCh   <-chan peerError
@@ -107,7 +94,7 @@ type Reactor struct {
 	// stopping the p2p Channel(s).
 	poolWG sync.WaitGroup
 
-	metrics *cons.Metrics
+	metrics *consensus.Metrics
 
 	syncStartTime time.Time
 }
@@ -119,10 +106,10 @@ func NewReactor(
 	blockExec *sm.BlockExecutor,
 	store *store.BlockStore,
 	consReactor consensusReactor,
-	blockchainCh *p2p.Channel,
+	blockSyncCh *p2p.Channel,
 	peerUpdates *p2p.PeerUpdates,
 	blockSync bool,
-	metrics *cons.Metrics,
+	metrics *consensus.Metrics,
 ) (*Reactor, error) {
 	if state.LastBlockHeight != store.Height() {
 		return nil, fmt.Errorf("state (%v) and store (%v) height mismatch", state.LastBlockHeight, store.Height())
@@ -137,23 +124,23 @@ func NewReactor(
 	errorsCh := make(chan peerError, maxPeerErrBuffer) // NOTE: The capacity should be larger than the peer count.
 
 	r := &Reactor{
-		initialState:          state,
-		blockExec:             blockExec,
-		store:                 store,
-		pool:                  NewBlockPool(startHeight, requestsCh, errorsCh),
-		consReactor:           consReactor,
-		blockSync:             tmSync.NewBool(blockSync),
-		requestsCh:            requestsCh,
-		errorsCh:              errorsCh,
-		blockchainCh:          blockchainCh,
-		blockchainOutBridgeCh: make(chan p2p.Envelope),
-		peerUpdates:           peerUpdates,
-		closeCh:               make(chan struct{}),
-		metrics:               metrics,
-		syncStartTime:         time.Time{},
+		initialState:         state,
+		blockExec:            blockExec,
+		store:                store,
+		pool:                 NewBlockPool(startHeight, requestsCh, errorsCh),
+		consReactor:          consReactor,
+		blockSync:            tmsync.NewBool(blockSync),
+		requestsCh:           requestsCh,
+		errorsCh:             errorsCh,
+		blockSyncCh:          blockSyncCh,
+		blockSyncOutBridgeCh: make(chan p2p.Envelope),
+		peerUpdates:          peerUpdates,
+		closeCh:              make(chan struct{}),
+		metrics:              metrics,
+		syncStartTime:        time.Time{},
 	}
 
-	r.BaseService = *service.NewBaseService(logger, "Blockchain", r)
+	r.BaseService = *service.NewBaseService(logger, "BlockSync", r)
 	return r, nil
 }
 
@@ -169,12 +156,14 @@ func (r *Reactor) OnStart() error {
 		if err := r.pool.Start(); err != nil {
 			return err
 		}
+		r.poolWG.Add(1)
+		go r.requestRoutine()
 
 		r.poolWG.Add(1)
 		go r.poolRoutine(false)
 	}
 
-	go r.processBlockchainCh()
+	go r.processBlockSyncCh()
 	go r.processPeerUpdates()
 
 	return nil
@@ -199,7 +188,7 @@ func (r *Reactor) OnStop() {
 	// Wait for all p2p Channels to be closed before returning. This ensures we
 	// can easily reason about synchronization of all p2p Channels and ensure no
 	// panics will occur.
-	<-r.blockchainCh.Done()
+	<-r.blockSyncCh.Done()
 	<-r.peerUpdates.Done()
 }
 
@@ -214,7 +203,7 @@ func (r *Reactor) respondToPeer(msg *bcproto.BlockRequest, peerID types.NodeID)
 			return
 		}
 
-		r.blockchainCh.Out <- p2p.Envelope{
+		r.blockSyncCh.Out <- p2p.Envelope{
 			To:      peerID,
 			Message: &bcproto.BlockResponse{Block: blockProto},
 		}
@@ -223,16 +212,16 @@ func (r *Reactor) respondToPeer(msg *bcproto.BlockRequest, peerID types.NodeID)
 	}
 
 	r.Logger.Info("peer requesting a block we do not have", "peer", peerID, "height", msg.Height)
-	r.blockchainCh.Out <- p2p.Envelope{
+	r.blockSyncCh.Out <- p2p.Envelope{
 		To:      peerID,
 		Message: &bcproto.NoBlockResponse{Height: msg.Height},
 	}
 }
 
-// handleBlockchainMessage handles envelopes sent from peers on the
-// BlockchainChannel. It returns an error only if the Envelope.Message is unknown
+// handleBlockSyncMessage handles envelopes sent from peers on the
+// BlockSyncChannel. It returns an error only if the Envelope.Message is unknown
 // for this channel. This should never be called outside of handleMessage.
-func (r *Reactor) handleBlockchainMessage(envelope p2p.Envelope) error {
+func (r *Reactor) handleBlockSyncMessage(envelope p2p.Envelope) error {
 	logger := r.Logger.With("peer", envelope.From)
 
 	switch msg := envelope.Message.(type) {
@@ -249,7 +238,7 @@ func (r *Reactor) handleBlockchainMessage(envelope p2p.Envelope) error {
 		r.pool.AddBlock(envelope.From, block, block.Size())
 
 	case *bcproto.StatusRequest:
-		r.blockchainCh.Out <- p2p.Envelope{
+		r.blockSyncCh.Out <- p2p.Envelope{
 			To: envelope.From,
 			Message: &bcproto.StatusResponse{
 				Height: r.store.Height(),
@@ -288,8 +277,8 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err
 	r.Logger.Debug("received message", "message", envelope.Message, "peer", envelope.From)
 
 	switch chID {
-	case BlockchainChannel:
-		err = r.handleBlockchainMessage(envelope)
+	case BlockSyncChannel:
+		err = r.handleBlockSyncMessage(envelope)
 
 	default:
 		err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
@@ -298,30 +287,30 @@ func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err
 	return err
 }
 
-// processBlockchainCh initiates a blocking process where we listen for and handle
-// envelopes on the BlockchainChannel and blockchainOutBridgeCh. Any error encountered during
-// message execution will result in a PeerError being sent on the BlockchainChannel.
+// processBlockSyncCh initiates a blocking process where we listen for and handle
+// envelopes on the BlockSyncChannel and blockSyncOutBridgeCh. Any error encountered during
+// message execution will result in a PeerError being sent on the BlockSyncChannel.
 // When the reactor is stopped, we will catch the signal and close the p2p Channel
 // gracefully.
-func (r *Reactor) processBlockchainCh() {
-	defer r.blockchainCh.Close()
+func (r *Reactor) processBlockSyncCh() {
+	defer r.blockSyncCh.Close()
 
 	for {
 		select {
-		case envelope := <-r.blockchainCh.In:
-			if err := r.handleMessage(r.blockchainCh.ID, envelope); err != nil {
-				r.Logger.Error("failed to process message", "ch_id", r.blockchainCh.ID, "envelope", envelope, "err", err)
-				r.blockchainCh.Error <- p2p.PeerError{
+		case envelope := <-r.blockSyncCh.In:
+			if err := r.handleMessage(r.blockSyncCh.ID, envelope); err != nil {
+				r.Logger.Error("failed to process message", "ch_id", r.blockSyncCh.ID, "envelope", envelope, "err", err)
+				r.blockSyncCh.Error <- p2p.PeerError{
 					NodeID: envelope.From,
 					Err:    err,
 				}
 			}
 
-		case envelope := <-r.blockchainOutBridgeCh:
-			r.blockchainCh.Out <- envelope
+		case envelope := <-r.blockSyncOutBridgeCh:
+			r.blockSyncCh.Out <- envelope
 
 		case <-r.closeCh:
-			r.Logger.Debug("stopped listening on blockchain channel; closing...")
+			r.Logger.Debug("stopped listening on block sync channel; closing...")
 			return
 
 		}
@@ -340,7 +329,7 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
 	switch peerUpdate.Status {
 	case p2p.PeerStatusUp:
 		// send a status update the newly added peer
-		r.blockchainOutBridgeCh <- p2p.Envelope{
+		r.blockSyncOutBridgeCh <- p2p.Envelope{
 			To: peerUpdate.NodeID,
 			Message: &bcproto.StatusResponse{
 				Base:   r.store.Base(),
@@ -384,6 +373,9 @@ func (r *Reactor) SwitchToBlockSync(state sm.State) error {
 
 	r.syncStartTime = time.Now()
 
+	r.poolWG.Add(1)
+	go r.requestRoutine()
+
 	r.poolWG.Add(1)
 	go r.poolRoutine(true)
 
@@ -394,7 +386,6 @@ func (r *Reactor) requestRoutine() {
 	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
 	defer statusUpdateTicker.Stop()
 
-	r.poolWG.Add(1)
 	defer r.poolWG.Done()
 
 	for {
@@ -406,13 +397,13 @@ func (r *Reactor) requestRoutine() {
 			return
 
 		case request := <-r.requestsCh:
-			r.blockchainOutBridgeCh <- p2p.Envelope{
+			r.blockSyncOutBridgeCh <- p2p.Envelope{
 				To:      request.PeerID,
 				Message: &bcproto.BlockRequest{Height: request.Height},
 			}
 
 		case pErr := <-r.errorsCh:
-			r.blockchainCh.Error <- p2p.PeerError{
+			r.blockSyncCh.Error <- p2p.PeerError{
 				NodeID: pErr.peerID,
 				Err:    pErr.err,
 			}
@@ -423,7 +414,7 @@ func (r *Reactor) requestRoutine() {
 			go func() {
 				defer r.poolWG.Done()
 
-				r.blockchainOutBridgeCh <- p2p.Envelope{
+				r.blockSyncOutBridgeCh <- p2p.Envelope{
 					Broadcast: true,
 					Message:   &bcproto.StatusRequest{},
 				}
@@ -455,8 +446,6 @@ func (r *Reactor) poolRoutine(stateSynced bool) {
 	defer trySyncTicker.Stop()
 	defer switchToConsensusTicker.Stop()
 
-	go r.requestRoutine()
-
 	defer r.poolWG.Done()
 
 FOR_LOOP:
@@ -554,14 +543,14 @@ FOR_LOOP:
 				// NOTE: We've already removed the peer's request, but we still need
 				// to clean up the rest.
 				peerID := r.pool.RedoRequest(first.Height)
-				r.blockchainCh.Error <- p2p.PeerError{
+				r.blockSyncCh.Error <- p2p.PeerError{
 					NodeID: peerID,
 					Err:    err,
 				}
 
 				peerID2 := r.pool.RedoRequest(second.Height)
 				if peerID2 != peerID {
-					r.blockchainCh.Error <- p2p.PeerError{
+					r.blockSyncCh.Error <- p2p.PeerError{
 						NodeID: peerID2,
 						Err:    err,
 					}
@@ -605,6 +594,8 @@ FOR_LOOP:
 
 		case <-r.closeCh:
 			break FOR_LOOP
+		case <-r.pool.Quit():
+			break FOR_LOOP
 		}
 	}
 }

internal/blocksync/reactor_test.go

@@ -1,4 +1,4 @@
-package v0
+package blocksync
 
 import (
 	"os"
@@ -6,22 +6,23 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
-	cons "github.com/tendermint/tendermint/internal/consensus"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/consensus"
 	"github.com/tendermint/tendermint/internal/mempool/mock"
 	"github.com/tendermint/tendermint/internal/p2p"
 	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
+	"github.com/tendermint/tendermint/internal/proxy"
+	sm "github.com/tendermint/tendermint/internal/state"
+	sf "github.com/tendermint/tendermint/internal/state/test/factory"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	bcproto "github.com/tendermint/tendermint/proto/tendermint/blocksync"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
-	sf "github.com/tendermint/tendermint/state/test/factory"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
-	dbm "github.com/tendermint/tm-db"
 )
 
 type reactorTestSuite struct {
@@ -32,9 +33,9 @@ type reactorTestSuite struct {
 	reactors map[types.NodeID]*Reactor
 	app      map[types.NodeID]proxy.AppConns
 
-	blockchainChannels map[types.NodeID]*p2p.Channel
-	peerChans          map[types.NodeID]chan p2p.PeerUpdate
-	peerUpdates        map[types.NodeID]*p2p.PeerUpdates
+	blockSyncChannels map[types.NodeID]*p2p.Channel
+	peerChans         map[types.NodeID]chan p2p.PeerUpdate
+	peerUpdates       map[types.NodeID]*p2p.PeerUpdates
 
 	blockSync bool
 }
@@ -53,19 +54,19 @@ func setup(
 		"must specify at least one block height (nodes)")
 
 	rts := &reactorTestSuite{
-		logger:             log.TestingLogger().With("module", "blockchain", "testCase", t.Name()),
-		network:            p2ptest.MakeNetwork(t, p2ptest.NetworkOptions{NumNodes: numNodes}),
-		nodes:              make([]types.NodeID, 0, numNodes),
-		reactors:           make(map[types.NodeID]*Reactor, numNodes),
-		app:                make(map[types.NodeID]proxy.AppConns, numNodes),
-		blockchainChannels: make(map[types.NodeID]*p2p.Channel, numNodes),
-		peerChans:          make(map[types.NodeID]chan p2p.PeerUpdate, numNodes),
-		peerUpdates:        make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
-		blockSync:          true,
+		logger:            log.TestingLogger().With("module", "block_sync", "testCase", t.Name()),
+		network:           p2ptest.MakeNetwork(t, p2ptest.NetworkOptions{NumNodes: numNodes}),
+		nodes:             make([]types.NodeID, 0, numNodes),
+		reactors:          make(map[types.NodeID]*Reactor, numNodes),
+		app:               make(map[types.NodeID]proxy.AppConns, numNodes),
+		blockSyncChannels: make(map[types.NodeID]*p2p.Channel, numNodes),
+		peerChans:         make(map[types.NodeID]chan p2p.PeerUpdate, numNodes),
+		peerUpdates:       make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
+		blockSync:         true,
 	}
 
-	chDesc := p2p.ChannelDescriptor{ID: byte(BlockchainChannel)}
-	rts.blockchainChannels = rts.network.MakeChannelsNoCleanup(t, chDesc, new(bcproto.Message), int(chBuf))
+	chDesc := &p2p.ChannelDescriptor{ID: BlockSyncChannel, MessageType: new(bcproto.Message)}
+	rts.blockSyncChannels = rts.network.MakeChannelsNoCleanup(t, chDesc)
 
 	i := 0
 	for nodeID := range rts.network.Nodes {
@@ -97,7 +98,7 @@ func (rts *reactorTestSuite) addNode(t *testing.T,
 	t.Helper()
 
 	rts.nodes = append(rts.nodes, nodeID)
-	rts.app[nodeID] = proxy.NewAppConns(proxy.NewLocalClientCreator(&abci.BaseApplication{}))
+	rts.app[nodeID] = proxy.NewAppConns(abciclient.NewLocalCreator(&abci.BaseApplication{}), proxy.NopMetrics())
 	require.NoError(t, rts.app[nodeID].Start())
 
 	blockDB := dbm.NewMemDB()
@@ -161,10 +162,10 @@ func (rts *reactorTestSuite) addNode(t *testing.T,
 		blockExec,
 		blockStore,
 		nil,
-		rts.blockchainChannels[nodeID],
+		rts.blockSyncChannels[nodeID],
 		rts.peerUpdates[nodeID],
 		rts.blockSync,
-		cons.NopMetrics())
+		consensus.NopMetrics())
 	require.NoError(t, err)
 
 	require.NoError(t, rts.reactors[nodeID].Start())
@@ -181,10 +182,10 @@ func (rts *reactorTestSuite) start(t *testing.T) {
 }
 
 func TestReactor_AbruptDisconnect(t *testing.T) {
-	config := cfg.ResetTestRoot("blockchain_reactor_test")
-	defer os.RemoveAll(config.RootDir)
+	cfg := config.ResetTestRoot("block_sync_reactor_test")
+	defer os.RemoveAll(cfg.RootDir)
 
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, 1, false, 30)
 	maxBlockHeight := int64(64)
 
 	rts := setup(t, genDoc, privVals[0], []int64{maxBlockHeight, 0}, 0)
@@ -216,10 +217,10 @@ func TestReactor_AbruptDisconnect(t *testing.T) {
 }
 
 func TestReactor_SyncTime(t *testing.T) {
-	config := cfg.ResetTestRoot("blockchain_reactor_test")
-	defer os.RemoveAll(config.RootDir)
+	cfg := config.ResetTestRoot("block_sync_reactor_test")
+	defer os.RemoveAll(cfg.RootDir)
 
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, 1, false, 30)
 	maxBlockHeight := int64(101)
 
 	rts := setup(t, genDoc, privVals[0], []int64{maxBlockHeight, 0}, 0)
@@ -239,10 +240,10 @@ func TestReactor_SyncTime(t *testing.T) {
 }
 
 func TestReactor_NoBlockResponse(t *testing.T) {
-	config := cfg.ResetTestRoot("blockchain_reactor_test")
-	defer os.RemoveAll(config.RootDir)
+	cfg := config.ResetTestRoot("block_sync_reactor_test")
+	defer os.RemoveAll(cfg.RootDir)
 
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, 1, false, 30)
 	maxBlockHeight := int64(65)
 
 	rts := setup(t, genDoc, privVals[0], []int64{maxBlockHeight, 0}, 0)
@@ -286,11 +287,11 @@ func TestReactor_BadBlockStopsPeer(t *testing.T) {
 	// See: https://github.com/tendermint/tendermint/issues/6005
 	t.SkipNow()
 
-	config := cfg.ResetTestRoot("blockchain_reactor_test")
-	defer os.RemoveAll(config.RootDir)
+	cfg := config.ResetTestRoot("block_sync_reactor_test")
+	defer os.RemoveAll(cfg.RootDir)
 
 	maxBlockHeight := int64(48)
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, 1, false, 30)
 
 	rts := setup(t, genDoc, privVals[0], []int64{maxBlockHeight, 0, 0, 0, 0}, 1000)
 
@@ -324,7 +325,7 @@ func TestReactor_BadBlockStopsPeer(t *testing.T) {
 	//
 	// XXX: This causes a potential race condition.
 	// See: https://github.com/tendermint/tendermint/issues/6005
-	otherGenDoc, otherPrivVals := factory.RandGenesisDoc(config, 1, false, 30)
+	otherGenDoc, otherPrivVals := factory.RandGenesisDoc(cfg, 1, false, 30)
 	newNode := rts.network.MakeNode(t, p2ptest.NodeOptions{
 		MaxPeers:     uint16(len(rts.nodes) + 1),
 		MaxConnected: uint16(len(rts.nodes) + 1),

internal/blocksync/v2/internal/behavior/doc.go

@@ -1,42 +0,0 @@
-/*
-Package Behavior provides a mechanism for reactors to report behavior of peers.
-
-Instead of a reactor calling the switch directly it will call the behavior module which will
-handle the stoping and marking peer as good on behalf of the reactor.
-
-There are four different behaviors a reactor can report.
-
-1. bad message
-
-type badMessage struct {
-	explanation string
-}
-
-This message will request the peer be stopped for an error
-
-2. message out of order
-
-type messageOutOfOrder struct {
-	explanation string
-}
-
-This message will request the peer be stopped for an error
-
-3. consesnsus Vote
-
-type consensusVote struct {
-	explanation string
-}
-
-This message will request the peer be marked as good
-
-4. block part
-
-type blockPart struct {
-	explanation string
-}
-
-This message will request the peer be marked as good
-
-*/
-package behavior

internal/blocksync/v2/internal/behavior/peer_behaviour.go

@@ -1,47 +0,0 @@
-package behavior
-
-import "github.com/tendermint/tendermint/types"
-
-// PeerBehavior is a struct describing a behavior a peer performed.
-// `peerID` identifies the peer and reason characterizes the specific
-// behavior performed by the peer.
-type PeerBehavior struct {
-	peerID types.NodeID
-	reason interface{}
-}
-
-type badMessage struct {
-	explanation string
-}
-
-// BadMessage returns a badMessage PeerBehavior.
-func BadMessage(peerID types.NodeID, explanation string) PeerBehavior {
-	return PeerBehavior{peerID: peerID, reason: badMessage{explanation}}
-}
-
-type messageOutOfOrder struct {
-	explanation string
-}
-
-// MessageOutOfOrder returns a messagOutOfOrder PeerBehavior.
-func MessageOutOfOrder(peerID types.NodeID, explanation string) PeerBehavior {
-	return PeerBehavior{peerID: peerID, reason: messageOutOfOrder{explanation}}
-}
-
-type consensusVote struct {
-	explanation string
-}
-
-// ConsensusVote returns a consensusVote PeerBehavior.
-func ConsensusVote(peerID types.NodeID, explanation string) PeerBehavior {
-	return PeerBehavior{peerID: peerID, reason: consensusVote{explanation}}
-}
-
-type blockPart struct {
-	explanation string
-}
-
-// BlockPart returns blockPart PeerBehavior.
-func BlockPart(peerID types.NodeID, explanation string) PeerBehavior {
-	return PeerBehavior{peerID: peerID, reason: blockPart{explanation}}
-}

internal/blocksync/v2/internal/behavior/reporter.go

@@ -1,87 +0,0 @@
-package behavior
-
-import (
-	"errors"
-
-	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/types"
-)
-
-// Reporter provides an interface for reactors to report the behavior
-// of peers synchronously to other components.
-type Reporter interface {
-	Report(behavior PeerBehavior) error
-}
-
-// SwitchReporter reports peer behavior to an internal Switch.
-type SwitchReporter struct {
-	sw *p2p.Switch
-}
-
-// NewSwitchReporter return a new SwitchReporter instance which wraps the Switch.
-func NewSwitchReporter(sw *p2p.Switch) *SwitchReporter {
-	return &SwitchReporter{
-		sw: sw,
-	}
-}
-
-// Report reports the behavior of a peer to the Switch.
-func (spbr *SwitchReporter) Report(behavior PeerBehavior) error {
-	peer := spbr.sw.Peers().Get(behavior.peerID)
-	if peer == nil {
-		return errors.New("peer not found")
-	}
-
-	switch reason := behavior.reason.(type) {
-	case consensusVote, blockPart:
-		spbr.sw.MarkPeerAsGood(peer)
-	case badMessage:
-		spbr.sw.StopPeerForError(peer, reason.explanation)
-	case messageOutOfOrder:
-		spbr.sw.StopPeerForError(peer, reason.explanation)
-	default:
-		return errors.New("unknown reason reported")
-	}
-
-	return nil
-}
-
-// MockReporter is a concrete implementation of the Reporter
-// interface used in reactor tests to ensure reactors report the correct
-// behavior in manufactured scenarios.
-type MockReporter struct {
-	mtx tmsync.RWMutex
-	pb  map[types.NodeID][]PeerBehavior
-}
-
-// NewMockReporter returns a Reporter which records all reported
-// behaviors in memory.
-func NewMockReporter() *MockReporter {
-	return &MockReporter{
-		pb: map[types.NodeID][]PeerBehavior{},
-	}
-}
-
-// Report stores the PeerBehavior produced by the peer identified by peerID.
-func (mpbr *MockReporter) Report(behavior PeerBehavior) error {
-	mpbr.mtx.Lock()
-	defer mpbr.mtx.Unlock()
-	mpbr.pb[behavior.peerID] = append(mpbr.pb[behavior.peerID], behavior)
-
-	return nil
-}
-
-// GetBehaviors returns all behaviors reported on the peer identified by peerID.
-func (mpbr *MockReporter) GetBehaviors(peerID types.NodeID) []PeerBehavior {
-	mpbr.mtx.RLock()
-	defer mpbr.mtx.RUnlock()
-	if items, ok := mpbr.pb[peerID]; ok {
-		result := make([]PeerBehavior, len(items))
-		copy(result, items)
-
-		return result
-	}
-
-	return []PeerBehavior{}
-}

internal/blocksync/v2/internal/behavior/reporter_test.go

@@ -1,205 +0,0 @@
-package behavior_test
-
-import (
-	"sync"
-	"testing"
-
-	bh "github.com/tendermint/tendermint/internal/blocksync/v2/internal/behavior"
-	"github.com/tendermint/tendermint/types"
-)
-
-// TestMockReporter tests the MockReporter's ability to store reported
-// peer behavior in memory indexed by the peerID.
-func TestMockReporter(t *testing.T) {
-	var peerID types.NodeID = "MockPeer"
-	pr := bh.NewMockReporter()
-
-	behaviors := pr.GetBehaviors(peerID)
-	if len(behaviors) != 0 {
-		t.Error("Expected to have no behaviors reported")
-	}
-
-	badMessage := bh.BadMessage(peerID, "bad message")
-	if err := pr.Report(badMessage); err != nil {
-		t.Error(err)
-	}
-	behaviors = pr.GetBehaviors(peerID)
-	if len(behaviors) != 1 {
-		t.Error("Expected the peer have one reported behavior")
-	}
-
-	if behaviors[0] != badMessage {
-		t.Error("Expected Bad Message to have been reported")
-	}
-}
-
-type scriptItem struct {
-	peerID   types.NodeID
-	behavior bh.PeerBehavior
-}
-
-// equalBehaviors returns true if a and b contain the same PeerBehaviors with
-// the same freequencies and otherwise false.
-func equalBehaviors(a []bh.PeerBehavior, b []bh.PeerBehavior) bool {
-	aHistogram := map[bh.PeerBehavior]int{}
-	bHistogram := map[bh.PeerBehavior]int{}
-
-	for _, behavior := range a {
-		aHistogram[behavior]++
-	}
-
-	for _, behavior := range b {
-		bHistogram[behavior]++
-	}
-
-	if len(aHistogram) != len(bHistogram) {
-		return false
-	}
-
-	for _, behavior := range a {
-		if aHistogram[behavior] != bHistogram[behavior] {
-			return false
-		}
-	}
-
-	for _, behavior := range b {
-		if bHistogram[behavior] != aHistogram[behavior] {
-			return false
-		}
-	}
-
-	return true
-}
-
-// TestEqualPeerBehaviors tests that equalBehaviors can tell that two slices
-// of peer behaviors can be compared for the behaviors they contain and the
-// freequencies that those behaviors occur.
-func TestEqualPeerBehaviors(t *testing.T) {
-	var (
-		peerID        types.NodeID = "MockPeer"
-		consensusVote              = bh.ConsensusVote(peerID, "voted")
-		blockPart                  = bh.BlockPart(peerID, "blocked")
-		equals                     = []struct {
-			left  []bh.PeerBehavior
-			right []bh.PeerBehavior
-		}{
-			// Empty sets
-			{[]bh.PeerBehavior{}, []bh.PeerBehavior{}},
-			// Single behaviors
-			{[]bh.PeerBehavior{consensusVote}, []bh.PeerBehavior{consensusVote}},
-			// Equal Frequencies
-			{[]bh.PeerBehavior{consensusVote, consensusVote},
-				[]bh.PeerBehavior{consensusVote, consensusVote}},
-			// Equal frequencies different orders
-			{[]bh.PeerBehavior{consensusVote, blockPart},
-				[]bh.PeerBehavior{blockPart, consensusVote}},
-		}
-		unequals = []struct {
-			left  []bh.PeerBehavior
-			right []bh.PeerBehavior
-		}{
-			// Comparing empty sets to non empty sets
-			{[]bh.PeerBehavior{}, []bh.PeerBehavior{consensusVote}},
-			// Different behaviors
-			{[]bh.PeerBehavior{consensusVote}, []bh.PeerBehavior{blockPart}},
-			// Same behavior with different frequencies
-			{[]bh.PeerBehavior{consensusVote},
-				[]bh.PeerBehavior{consensusVote, consensusVote}},
-		}
-	)
-
-	for _, test := range equals {
-		if !equalBehaviors(test.left, test.right) {
-			t.Errorf("expected %#v and %#v to be equal", test.left, test.right)
-		}
-	}
-
-	for _, test := range unequals {
-		if equalBehaviors(test.left, test.right) {
-			t.Errorf("expected %#v and %#v to be unequal", test.left, test.right)
-		}
-	}
-}
-
-// TestPeerBehaviorConcurrency constructs a scenario in which
-// multiple goroutines are using the same MockReporter instance.
-// This test reproduces the conditions in which MockReporter will
-// be used within a Reactor `Receive` method tests to ensure thread safety.
-func TestMockPeerBehaviorReporterConcurrency(t *testing.T) {
-	var (
-		behaviorScript = []struct {
-			peerID    types.NodeID
-			behaviors []bh.PeerBehavior
-		}{
-			{"1", []bh.PeerBehavior{bh.ConsensusVote("1", "")}},
-			{"2", []bh.PeerBehavior{bh.ConsensusVote("2", ""), bh.ConsensusVote("2", ""), bh.ConsensusVote("2", "")}},
-			{
-				"3",
-				[]bh.PeerBehavior{bh.BlockPart("3", ""),
-					bh.ConsensusVote("3", ""),
-					bh.BlockPart("3", ""),
-					bh.ConsensusVote("3", "")}},
-			{
-				"4",
-				[]bh.PeerBehavior{bh.ConsensusVote("4", ""),
-					bh.ConsensusVote("4", ""),
-					bh.ConsensusVote("4", ""),
-					bh.ConsensusVote("4", "")}},
-			{
-				"5",
-				[]bh.PeerBehavior{bh.BlockPart("5", ""),
-					bh.ConsensusVote("5", ""),
-					bh.BlockPart("5", ""),
-					bh.ConsensusVote("5", "")}},
-		}
-	)
-
-	var receiveWg sync.WaitGroup
-	pr := bh.NewMockReporter()
-	scriptItems := make(chan scriptItem)
-	done := make(chan int)
-	numConsumers := 3
-	for i := 0; i < numConsumers; i++ {
-		receiveWg.Add(1)
-		go func() {
-			defer receiveWg.Done()
-			for {
-				select {
-				case pb := <-scriptItems:
-					if err := pr.Report(pb.behavior); err != nil {
-						t.Error(err)
-					}
-				case <-done:
-					return
-				}
-			}
-		}()
-	}
-
-	var sendingWg sync.WaitGroup
-	sendingWg.Add(1)
-	go func() {
-		defer sendingWg.Done()
-		for _, item := range behaviorScript {
-			for _, reason := range item.behaviors {
-				scriptItems <- scriptItem{item.peerID, reason}
-			}
-		}
-	}()
-
-	sendingWg.Wait()
-
-	for i := 0; i < numConsumers; i++ {
-		done <- 1
-	}
-
-	receiveWg.Wait()
-
-	for _, items := range behaviorScript {
-		reported := pr.GetBehaviors(items.peerID)
-		if !equalBehaviors(reported, items.behaviors) {
-			t.Errorf("expected peer %s to have behaved \nExpected: %#v \nGot %#v \n",
-				items.peerID, items.behaviors, reported)
-		}
-	}
-}

internal/blocksync/v2/io.go

@@ -1,187 +0,0 @@
-package v2
-
-import (
-	"errors"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/tendermint/tendermint/internal/p2p"
-	bcproto "github.com/tendermint/tendermint/proto/tendermint/blocksync"
-	"github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-var (
-	errPeerQueueFull = errors.New("peer queue full")
-)
-
-type iIO interface {
-	sendBlockRequest(peer p2p.Peer, height int64) error
-	sendBlockToPeer(block *types.Block, peer p2p.Peer) error
-	sendBlockNotFound(height int64, peer p2p.Peer) error
-	sendStatusResponse(base, height int64, peer p2p.Peer) error
-
-	sendStatusRequest(peer p2p.Peer) error
-	broadcastStatusRequest() error
-
-	trySwitchToConsensus(state state.State, skipWAL bool) bool
-}
-
-type switchIO struct {
-	sw *p2p.Switch
-}
-
-func newSwitchIo(sw *p2p.Switch) *switchIO {
-	return &switchIO{
-		sw: sw,
-	}
-}
-
-const (
-	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
-	BlockchainChannel = byte(0x40)
-)
-
-type consensusReactor interface {
-	// for when we switch from blockchain reactor and block sync to
-	// the consensus machine
-	SwitchToConsensus(state state.State, skipWAL bool)
-}
-
-func (sio *switchIO) sendBlockRequest(peer p2p.Peer, height int64) error {
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_BlockRequest{
-			BlockRequest: &bcproto.BlockRequest{
-				Height: height,
-			},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	queued := peer.TrySend(BlockchainChannel, msgBytes)
-	if !queued {
-		return errPeerQueueFull
-	}
-	return nil
-}
-
-func (sio *switchIO) sendStatusResponse(base int64, height int64, peer p2p.Peer) error {
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_StatusResponse{
-			StatusResponse: &bcproto.StatusResponse{
-				Height: height,
-				Base:   base,
-			},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
-		return errPeerQueueFull
-	}
-
-	return nil
-}
-
-func (sio *switchIO) sendBlockToPeer(block *types.Block, peer p2p.Peer) error {
-	if block == nil {
-		panic("trying to send nil block")
-	}
-
-	bpb, err := block.ToProto()
-	if err != nil {
-		return err
-	}
-
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_BlockResponse{
-			BlockResponse: &bcproto.BlockResponse{
-				Block: bpb,
-			},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
-		return errPeerQueueFull
-	}
-
-	return nil
-}
-
-func (sio *switchIO) sendBlockNotFound(height int64, peer p2p.Peer) error {
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_NoBlockResponse{
-			NoBlockResponse: &bcproto.NoBlockResponse{
-				Height: height,
-			},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
-		return errPeerQueueFull
-	}
-
-	return nil
-}
-
-func (sio *switchIO) trySwitchToConsensus(state state.State, skipWAL bool) bool {
-	conR, ok := sio.sw.Reactor("CONSENSUS").(consensusReactor)
-	if ok {
-		conR.SwitchToConsensus(state, skipWAL)
-	}
-	return ok
-}
-
-func (sio *switchIO) sendStatusRequest(peer p2p.Peer) error {
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_StatusRequest{
-			StatusRequest: &bcproto.StatusRequest{},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
-		return errPeerQueueFull
-	}
-
-	return nil
-}
-
-func (sio *switchIO) broadcastStatusRequest() error {
-	msgProto := &bcproto.Message{
-		Sum: &bcproto.Message_StatusRequest{
-			StatusRequest: &bcproto.StatusRequest{},
-		},
-	}
-
-	msgBytes, err := proto.Marshal(msgProto)
-	if err != nil {
-		return err
-	}
-
-	// XXX: maybe we should use an io specific peer list here
-	sio.sw.Broadcast(BlockchainChannel, msgBytes)
-
-	return nil
-}

internal/blocksync/v2/metrics.go

@@ -1,125 +0,0 @@
-package v2
-
-import (
-	"github.com/go-kit/kit/metrics"
-	"github.com/go-kit/kit/metrics/discard"
-	"github.com/go-kit/kit/metrics/prometheus"
-	stdprometheus "github.com/prometheus/client_golang/prometheus"
-)
-
-const (
-	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
-	// package.
-	MetricsSubsystem = "blockchain"
-)
-
-// Metrics contains metrics exposed by this package.
-type Metrics struct {
-	// events_in
-	EventsIn metrics.Counter
-	// events_in
-	EventsHandled metrics.Counter
-	// events_out
-	EventsOut metrics.Counter
-	// errors_in
-	ErrorsIn metrics.Counter
-	// errors_handled
-	ErrorsHandled metrics.Counter
-	// errors_out
-	ErrorsOut metrics.Counter
-	// events_shed
-	EventsShed metrics.Counter
-	// events_sent
-	EventsSent metrics.Counter
-	// errors_sent
-	ErrorsSent metrics.Counter
-	// errors_shed
-	ErrorsShed metrics.Counter
-}
-
-// PrometheusMetrics returns metrics for in and out events, errors, etc. handled by routines.
-// Can we burn in the routine name here?
-func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
-	labels := []string{}
-	for i := 0; i < len(labelsAndValues); i += 2 {
-		labels = append(labels, labelsAndValues[i])
-	}
-	return &Metrics{
-		EventsIn: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "events_in",
-			Help:      "Events read from the channel.",
-		}, labels).With(labelsAndValues...),
-		EventsHandled: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "events_handled",
-			Help:      "Events handled",
-		}, labels).With(labelsAndValues...),
-		EventsOut: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "events_out",
-			Help:      "Events output from routine.",
-		}, labels).With(labelsAndValues...),
-		ErrorsIn: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "errors_in",
-			Help:      "Errors read from the channel.",
-		}, labels).With(labelsAndValues...),
-		ErrorsHandled: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "errors_handled",
-			Help:      "Errors handled.",
-		}, labels).With(labelsAndValues...),
-		ErrorsOut: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "errors_out",
-			Help:      "Errors output from routine.",
-		}, labels).With(labelsAndValues...),
-		ErrorsSent: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "errors_sent",
-			Help:      "Errors sent to routine.",
-		}, labels).With(labelsAndValues...),
-		ErrorsShed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "errors_shed",
-			Help:      "Errors dropped from sending.",
-		}, labels).With(labelsAndValues...),
-		EventsSent: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "events_sent",
-			Help:      "Events sent to routine.",
-		}, labels).With(labelsAndValues...),
-		EventsShed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: MetricsSubsystem,
-			Name:      "events_shed",
-			Help:      "Events dropped from sending.",
-		}, labels).With(labelsAndValues...),
-	}
-}
-
-// NopMetrics returns no-op Metrics.
-func NopMetrics() *Metrics {
-	return &Metrics{
-		EventsIn:      discard.NewCounter(),
-		EventsHandled: discard.NewCounter(),
-		EventsOut:     discard.NewCounter(),
-		ErrorsIn:      discard.NewCounter(),
-		ErrorsHandled: discard.NewCounter(),
-		ErrorsOut:     discard.NewCounter(),
-		EventsShed:    discard.NewCounter(),
-		EventsSent:    discard.NewCounter(),
-		ErrorsSent:    discard.NewCounter(),
-		ErrorsShed:    discard.NewCounter(),
-	}
-}

internal/blocksync/v2/processor.go

@@ -1,193 +0,0 @@
-package v2
-
-import (
-	"fmt"
-
-	tmState "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-// Events generated by the processor:
-// block execution failure, event will indicate the peer(s) that caused the error
-type pcBlockVerificationFailure struct {
-	priorityNormal
-	height       int64
-	firstPeerID  types.NodeID
-	secondPeerID types.NodeID
-}
-
-func (e pcBlockVerificationFailure) String() string {
-	return fmt.Sprintf("pcBlockVerificationFailure{%d 1st peer: %v, 2nd peer: %v}",
-		e.height, e.firstPeerID, e.secondPeerID)
-}
-
-// successful block execution
-type pcBlockProcessed struct {
-	priorityNormal
-	height int64
-	peerID types.NodeID
-}
-
-func (e pcBlockProcessed) String() string {
-	return fmt.Sprintf("pcBlockProcessed{%d peer: %v}", e.height, e.peerID)
-}
-
-// processor has finished
-type pcFinished struct {
-	priorityNormal
-	blocksSynced int
-	tmState      tmState.State
-}
-
-func (p pcFinished) Error() string {
-	return "finished"
-}
-
-type queueItem struct {
-	block  *types.Block
-	peerID types.NodeID
-}
-
-type blockQueue map[int64]queueItem
-
-type pcState struct {
-	// blocks waiting to be processed
-	queue blockQueue
-
-	// draining indicates that the next rProcessBlock event with a queue miss constitutes completion
-	draining bool
-
-	// the number of blocks successfully synced by the processor
-	blocksSynced int
-
-	// the processorContext which contains the processor dependencies
-	context processorContext
-}
-
-func (state *pcState) String() string {
-	return fmt.Sprintf("height: %d queue length: %d draining: %v blocks synced: %d",
-		state.height(), len(state.queue), state.draining, state.blocksSynced)
-}
-
-// newPcState returns a pcState initialized with the last verified block enqueued
-func newPcState(context processorContext) *pcState {
-	return &pcState{
-		queue:        blockQueue{},
-		draining:     false,
-		blocksSynced: 0,
-		context:      context,
-	}
-}
-
-// nextTwo returns the next two unverified blocks
-func (state *pcState) nextTwo() (queueItem, queueItem, error) {
-	if first, ok := state.queue[state.height()+1]; ok {
-		if second, ok := state.queue[state.height()+2]; ok {
-			return first, second, nil
-		}
-	}
-	return queueItem{}, queueItem{}, fmt.Errorf("not found")
-}
-
-// synced returns true when at most the last verified block remains in the queue
-func (state *pcState) synced() bool {
-	return len(state.queue) <= 1
-}
-
-func (state *pcState) enqueue(peerID types.NodeID, block *types.Block, height int64) {
-	if item, ok := state.queue[height]; ok {
-		panic(fmt.Sprintf(
-			"duplicate block %d (%X) enqueued by processor (sent by %v; existing block %X from %v)",
-			height, block.Hash(), peerID, item.block.Hash(), item.peerID))
-	}
-
-	state.queue[height] = queueItem{block: block, peerID: peerID}
-}
-
-func (state *pcState) height() int64 {
-	return state.context.tmState().LastBlockHeight
-}
-
-// purgePeer moves all unprocessed blocks from the queue
-func (state *pcState) purgePeer(peerID types.NodeID) {
-	// what if height is less than state.height?
-	for height, item := range state.queue {
-		if item.peerID == peerID {
-			delete(state.queue, height)
-		}
-	}
-}
-
-// handle processes FSM events
-func (state *pcState) handle(event Event) (Event, error) {
-	switch event := event.(type) {
-	case bcResetState:
-		state.context.setState(event.state)
-		return noOp, nil
-
-	case scFinishedEv:
-		if state.synced() {
-			return pcFinished{tmState: state.context.tmState(), blocksSynced: state.blocksSynced}, nil
-		}
-		state.draining = true
-		return noOp, nil
-
-	case scPeerError:
-		state.purgePeer(event.peerID)
-		return noOp, nil
-
-	case scBlockReceived:
-		if event.block == nil {
-			return noOp, nil
-		}
-
-		// enqueue block if height is higher than state height, else ignore it
-		if event.block.Height > state.height() {
-			state.enqueue(event.peerID, event.block, event.block.Height)
-		}
-		return noOp, nil
-
-	case rProcessBlock:
-		tmState := state.context.tmState()
-		firstItem, secondItem, err := state.nextTwo()
-		if err != nil {
-			if state.draining {
-				return pcFinished{tmState: tmState, blocksSynced: state.blocksSynced}, nil
-			}
-			return noOp, nil
-		}
-
-		var (
-			first, second = firstItem.block, secondItem.block
-			firstParts    = first.MakePartSet(types.BlockPartSizeBytes)
-			firstID       = types.BlockID{Hash: first.Hash(), PartSetHeader: firstParts.Header()}
-		)
-
-		// verify if +second+ last commit "confirms" +first+ block
-		err = state.context.verifyCommit(tmState.ChainID, firstID, first.Height, second.LastCommit)
-		if err != nil {
-			state.purgePeer(firstItem.peerID)
-			if firstItem.peerID != secondItem.peerID {
-				state.purgePeer(secondItem.peerID)
-			}
-			return pcBlockVerificationFailure{
-					height: first.Height, firstPeerID: firstItem.peerID, secondPeerID: secondItem.peerID},
-				nil
-		}
-
-		state.context.saveBlock(first, firstParts, second.LastCommit)
-
-		if err := state.context.applyBlock(firstID, first); err != nil {
-			panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
-		}
-
-		state.context.recordConsMetrics(first)
-
-		delete(state.queue, first.Height)
-		state.blocksSynced++
-
-		return pcBlockProcessed{height: first.Height, peerID: firstItem.peerID}, nil
-	}
-
-	return noOp, nil
-}

internal/blocksync/v2/processor_context.go

@@ -1,112 +0,0 @@
-package v2
-
-import (
-	"fmt"
-
-	cons "github.com/tendermint/tendermint/internal/consensus"
-	"github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-type processorContext interface {
-	applyBlock(blockID types.BlockID, block *types.Block) error
-	verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error
-	saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
-	tmState() state.State
-	setState(state.State)
-	recordConsMetrics(block *types.Block)
-}
-
-type pContext struct {
-	store   blockStore
-	applier blockApplier
-	state   state.State
-	metrics *cons.Metrics
-}
-
-func newProcessorContext(st blockStore, ex blockApplier, s state.State, m *cons.Metrics) *pContext {
-	return &pContext{
-		store:   st,
-		applier: ex,
-		state:   s,
-		metrics: m,
-	}
-}
-
-func (pc *pContext) applyBlock(blockID types.BlockID, block *types.Block) error {
-	newState, err := pc.applier.ApplyBlock(pc.state, blockID, block)
-	pc.state = newState
-	return err
-}
-
-func (pc pContext) tmState() state.State {
-	return pc.state
-}
-
-func (pc *pContext) setState(state state.State) {
-	pc.state = state
-}
-
-func (pc pContext) verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error {
-	return pc.state.Validators.VerifyCommitLight(chainID, blockID, height, commit)
-}
-
-func (pc *pContext) saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
-	pc.store.SaveBlock(block, blockParts, seenCommit)
-}
-
-func (pc *pContext) recordConsMetrics(block *types.Block) {
-	pc.metrics.RecordConsMetrics(block)
-}
-
-type mockPContext struct {
-	applicationBL  []int64
-	verificationBL []int64
-	state          state.State
-}
-
-func newMockProcessorContext(
-	state state.State,
-	verificationBlackList []int64,
-	applicationBlackList []int64) *mockPContext {
-	return &mockPContext{
-		applicationBL:  applicationBlackList,
-		verificationBL: verificationBlackList,
-		state:          state,
-	}
-}
-
-func (mpc *mockPContext) applyBlock(blockID types.BlockID, block *types.Block) error {
-	for _, h := range mpc.applicationBL {
-		if h == block.Height {
-			return fmt.Errorf("generic application error")
-		}
-	}
-	mpc.state.LastBlockHeight = block.Height
-	return nil
-}
-
-func (mpc *mockPContext) verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error {
-	for _, h := range mpc.verificationBL {
-		if h == height {
-			return fmt.Errorf("generic verification error")
-		}
-	}
-	return nil
-}
-
-func (mpc *mockPContext) saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
-
-}
-
-func (mpc *mockPContext) setState(state state.State) {
-	mpc.state = state
-}
-
-func (mpc *mockPContext) tmState() state.State {
-	return mpc.state
-}
-
-func (mpc *mockPContext) recordConsMetrics(block *types.Block) {
-
-}

internal/blocksync/v2/processor_test.go

@@ -1,305 +0,0 @@
-package v2
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	tmState "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-// pcBlock is a test helper structure with simple types. Its purpose is to help with test readability.
-type pcBlock struct {
-	pid    string
-	height int64
-}
-
-// params is a test structure used to create processor state.
-type params struct {
-	height       int64
-	items        []pcBlock
-	blocksSynced int
-	verBL        []int64
-	appBL        []int64
-	draining     bool
-}
-
-// makePcBlock makes an empty block.
-func makePcBlock(height int64) *types.Block {
-	return &types.Block{Header: types.Header{Height: height}}
-}
-
-// makeState takes test parameters and creates a specific processor state.
-func makeState(p *params) *pcState {
-	var (
-		tmState = tmState.State{LastBlockHeight: p.height}
-		context = newMockProcessorContext(tmState, p.verBL, p.appBL)
-	)
-	state := newPcState(context)
-
-	for _, item := range p.items {
-		state.enqueue(types.NodeID(item.pid), makePcBlock(item.height), item.height)
-	}
-
-	state.blocksSynced = p.blocksSynced
-	state.draining = p.draining
-	return state
-}
-
-func mBlockResponse(peerID types.NodeID, height int64) scBlockReceived {
-	return scBlockReceived{
-		peerID: peerID,
-		block:  makePcBlock(height),
-	}
-}
-
-type pcFsmMakeStateValues struct {
-	currentState  *params
-	event         Event
-	wantState     *params
-	wantNextEvent Event
-	wantErr       error
-	wantPanic     bool
-}
-
-type testFields struct {
-	name  string
-	steps []pcFsmMakeStateValues
-}
-
-func executeProcessorTests(t *testing.T, tests []testFields) {
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			var state *pcState
-			for _, step := range tt.steps {
-				defer func() {
-					r := recover()
-					if (r != nil) != step.wantPanic {
-						t.Errorf("recover = %v, wantPanic = %v", r, step.wantPanic)
-					}
-				}()
-
-				// First step must always initialize the currentState as state.
-				if step.currentState != nil {
-					state = makeState(step.currentState)
-				}
-				if state == nil {
-					panic("Bad (initial?) step")
-				}
-
-				nextEvent, err := state.handle(step.event)
-				t.Log(state)
-				assert.Equal(t, step.wantErr, err)
-				assert.Equal(t, makeState(step.wantState), state)
-				assert.Equal(t, step.wantNextEvent, nextEvent)
-				// Next step may use the wantedState as their currentState.
-				state = makeState(step.wantState)
-			}
-		})
-	}
-}
-
-func TestRProcessPeerError(t *testing.T) {
-	tests := []testFields{
-		{
-			name: "error for existing peer",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
-					event:         scPeerError{peerID: "P2"},
-					wantState:     &params{items: []pcBlock{{"P1", 1}}},
-					wantNextEvent: noOp,
-				},
-			},
-		},
-		{
-			name: "error for unknown peer",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
-					event:         scPeerError{peerID: "P3"},
-					wantState:     &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
-					wantNextEvent: noOp,
-				},
-			},
-		},
-	}
-
-	executeProcessorTests(t, tests)
-}
-
-func TestPcBlockResponse(t *testing.T) {
-	tests := []testFields{
-		{
-			name: "add one block",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{}, event: mBlockResponse("P1", 1),
-					wantState: &params{items: []pcBlock{{"P1", 1}}}, wantNextEvent: noOp,
-				},
-			},
-		},
-
-		{
-			name: "add two blocks",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{}, event: mBlockResponse("P1", 3),
-					wantState: &params{items: []pcBlock{{"P1", 3}}}, wantNextEvent: noOp,
-				},
-				{ // use previous wantState as currentState,
-					event:     mBlockResponse("P1", 4),
-					wantState: &params{items: []pcBlock{{"P1", 3}, {"P1", 4}}}, wantNextEvent: noOp,
-				},
-			},
-		},
-	}
-
-	executeProcessorTests(t, tests)
-}
-
-func TestRProcessBlockSuccess(t *testing.T) {
-	tests := []testFields{
-		{
-			name: "noop - no blocks over current height",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{}, event: rProcessBlock{},
-					wantState: &params{}, wantNextEvent: noOp,
-				},
-			},
-		},
-		{
-			name: "noop - high new blocks",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{height: 5, items: []pcBlock{{"P1", 30}, {"P2", 31}}}, event: rProcessBlock{},
-					wantState: &params{height: 5, items: []pcBlock{{"P1", 30}, {"P2", 31}}}, wantNextEvent: noOp,
-				},
-			},
-		},
-		{
-			name: "blocks H+1 and H+2 present",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}}, event: rProcessBlock{},
-					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}}, blocksSynced: 1},
-					wantNextEvent: pcBlockProcessed{height: 1, peerID: "P1"},
-				},
-			},
-		},
-		{
-			name: "blocks H+1 and H+2 present after draining",
-			steps: []pcFsmMakeStateValues{
-				{ // some contiguous blocks - on stop check draining is set
-					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P1", 4}}},
-					event:         scFinishedEv{},
-					wantState:     &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P1", 4}}, draining: true},
-					wantNextEvent: noOp,
-				},
-				{
-					event:         rProcessBlock{},
-					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}, {"P1", 4}}, blocksSynced: 1, draining: true},
-					wantNextEvent: pcBlockProcessed{height: 1, peerID: "P1"},
-				},
-				{ // finish when H+1 or/and H+2 are missing
-					event:         rProcessBlock{},
-					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}, {"P1", 4}}, blocksSynced: 1, draining: true},
-					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 1}, blocksSynced: 1},
-				},
-			},
-		},
-	}
-
-	executeProcessorTests(t, tests)
-}
-
-func TestRProcessBlockFailures(t *testing.T) {
-	tests := []testFields{
-		{
-			name: "blocks H+1 and H+2 present from different peers - H+1 verification fails ",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}, verBL: []int64{1}}, event: rProcessBlock{},
-					wantState:     &params{items: []pcBlock{}, verBL: []int64{1}},
-					wantNextEvent: pcBlockVerificationFailure{height: 1, firstPeerID: "P1", secondPeerID: "P2"},
-				},
-			},
-		},
-		{
-			name: "blocks H+1 and H+2 present from same peer - H+1 applyBlock fails ",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}, appBL: []int64{1}}, event: rProcessBlock{},
-					wantState: &params{items: []pcBlock{}, appBL: []int64{1}}, wantPanic: true,
-				},
-			},
-		},
-		{
-			name: "blocks H+1 and H+2 present from same peers - H+1 verification fails ",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{height: 0, items: []pcBlock{{"P1", 1}, {"P1", 2}, {"P2", 3}},
-						verBL: []int64{1}}, event: rProcessBlock{},
-					wantState:     &params{height: 0, items: []pcBlock{{"P2", 3}}, verBL: []int64{1}},
-					wantNextEvent: pcBlockVerificationFailure{height: 1, firstPeerID: "P1", secondPeerID: "P1"},
-				},
-			},
-		},
-		{
-			name: "blocks H+1 and H+2 present from different peers - H+1 applyBlock fails ",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P2", 3}}, appBL: []int64{1}},
-					event:        rProcessBlock{},
-					wantState:    &params{items: []pcBlock{{"P2", 3}}, appBL: []int64{1}}, wantPanic: true,
-				},
-			},
-		},
-	}
-
-	executeProcessorTests(t, tests)
-}
-
-func TestScFinishedEv(t *testing.T) {
-	tests := []testFields{
-		{
-			name: "no blocks",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{height: 100, items: []pcBlock{}, blocksSynced: 100}, event: scFinishedEv{},
-					wantState:     &params{height: 100, items: []pcBlock{}, blocksSynced: 100},
-					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 100}, blocksSynced: 100},
-				},
-			},
-		},
-		{
-			name: "maxHeight+1 block present",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{height: 100, items: []pcBlock{
-						{"P1", 101}}, blocksSynced: 100}, event: scFinishedEv{},
-					wantState:     &params{height: 100, items: []pcBlock{{"P1", 101}}, blocksSynced: 100},
-					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 100}, blocksSynced: 100},
-				},
-			},
-		},
-		{
-			name: "more blocks present",
-			steps: []pcFsmMakeStateValues{
-				{
-					currentState: &params{height: 100, items: []pcBlock{
-						{"P1", 101}, {"P1", 102}}, blocksSynced: 100}, event: scFinishedEv{},
-					wantState: &params{height: 100, items: []pcBlock{
-						{"P1", 101}, {"P1", 102}}, blocksSynced: 100, draining: true},
-					wantNextEvent: noOp,
-					wantErr:       nil,
-				},
-			},
-		},
-	}
-
-	executeProcessorTests(t, tests)
-}

internal/blocksync/v2/reactor.go

@@ -1,643 +0,0 @@
-package v2
-
-import (
-	"errors"
-	"fmt"
-	"time"
-
-	proto "github.com/gogo/protobuf/proto"
-
-	bc "github.com/tendermint/tendermint/internal/blocksync"
-	"github.com/tendermint/tendermint/internal/blocksync/v2/internal/behavior"
-	cons "github.com/tendermint/tendermint/internal/consensus"
-	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/libs/sync"
-	bcproto "github.com/tendermint/tendermint/proto/tendermint/blocksync"
-	"github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/types"
-)
-
-const (
-	// chBufferSize is the buffer size of all event channels.
-	chBufferSize int = 1000
-)
-
-type blockStore interface {
-	LoadBlock(height int64) *types.Block
-	SaveBlock(*types.Block, *types.PartSet, *types.Commit)
-	Base() int64
-	Height() int64
-}
-
-// BlockchainReactor handles block sync protocol.
-type BlockchainReactor struct {
-	p2p.BaseReactor
-
-	blockSync   *sync.AtomicBool // enable block sync on start when it's been Set
-	stateSynced bool             // set to true when SwitchToBlockSync is called by state sync
-	scheduler   *Routine
-	processor   *Routine
-	logger      log.Logger
-
-	mtx           tmsync.RWMutex
-	maxPeerHeight int64
-	syncHeight    int64
-	events        chan Event // non-nil during a block sync
-
-	reporter behavior.Reporter
-	io       iIO
-	store    blockStore
-
-	syncStartTime   time.Time
-	syncStartHeight int64
-	lastSyncRate    float64 // # blocks sync per sec base on the last 100 blocks
-}
-
-type blockApplier interface {
-	ApplyBlock(state state.State, blockID types.BlockID, block *types.Block) (state.State, error)
-}
-
-// XXX: unify naming in this package around tmState
-func newReactor(state state.State, store blockStore, reporter behavior.Reporter,
-	blockApplier blockApplier, blockSync bool, metrics *cons.Metrics) *BlockchainReactor {
-	initHeight := state.LastBlockHeight + 1
-	if initHeight == 1 {
-		initHeight = state.InitialHeight
-	}
-	scheduler := newScheduler(initHeight, time.Now())
-	pContext := newProcessorContext(store, blockApplier, state, metrics)
-	// TODO: Fix naming to just newProcesssor
-	// newPcState requires a processorContext
-	processor := newPcState(pContext)
-
-	return &BlockchainReactor{
-		scheduler:       newRoutine("scheduler", scheduler.handle, chBufferSize),
-		processor:       newRoutine("processor", processor.handle, chBufferSize),
-		store:           store,
-		reporter:        reporter,
-		logger:          log.NewNopLogger(),
-		blockSync:       sync.NewBool(blockSync),
-		syncStartHeight: initHeight,
-		syncStartTime:   time.Time{},
-		lastSyncRate:    0,
-	}
-}
-
-// NewBlockchainReactor creates a new reactor instance.
-func NewBlockchainReactor(
-	state state.State,
-	blockApplier blockApplier,
-	store blockStore,
-	blockSync bool,
-	metrics *cons.Metrics) *BlockchainReactor {
-	reporter := behavior.NewMockReporter()
-	return newReactor(state, store, reporter, blockApplier, blockSync, metrics)
-}
-
-// SetSwitch implements Reactor interface.
-func (r *BlockchainReactor) SetSwitch(sw *p2p.Switch) {
-	r.Switch = sw
-	if sw != nil {
-		r.io = newSwitchIo(sw)
-	} else {
-		r.io = nil
-	}
-}
-
-func (r *BlockchainReactor) setMaxPeerHeight(height int64) {
-	r.mtx.Lock()
-	defer r.mtx.Unlock()
-	if height > r.maxPeerHeight {
-		r.maxPeerHeight = height
-	}
-}
-
-func (r *BlockchainReactor) setSyncHeight(height int64) {
-	r.mtx.Lock()
-	defer r.mtx.Unlock()
-	r.syncHeight = height
-}
-
-// SyncHeight returns the height to which the BlockchainReactor has synced.
-func (r *BlockchainReactor) SyncHeight() int64 {
-	r.mtx.RLock()
-	defer r.mtx.RUnlock()
-	return r.syncHeight
-}
-
-// SetLogger sets the logger of the reactor.
-func (r *BlockchainReactor) SetLogger(logger log.Logger) {
-	r.logger = logger
-	r.scheduler.setLogger(logger)
-	r.processor.setLogger(logger)
-}
-
-// Start implements cmn.Service interface
-func (r *BlockchainReactor) Start() error {
-	r.reporter = behavior.NewSwitchReporter(r.BaseReactor.Switch)
-	if r.blockSync.IsSet() {
-		err := r.startSync(nil)
-		if err != nil {
-			return fmt.Errorf("failed to start block sync: %w", err)
-		}
-	}
-	return nil
-}
-
-// startSync begins a block sync, signaled by r.events being non-nil. If state is non-nil,
-// the scheduler and processor is updated with this state on startup.
-func (r *BlockchainReactor) startSync(state *state.State) error {
-	r.mtx.Lock()
-	defer r.mtx.Unlock()
-	if r.events != nil {
-		return errors.New("block sync already in progress")
-	}
-	r.events = make(chan Event, chBufferSize)
-	go r.scheduler.start()
-	go r.processor.start()
-	if state != nil {
-		<-r.scheduler.ready()
-		<-r.processor.ready()
-		r.scheduler.send(bcResetState{state: *state})
-		r.processor.send(bcResetState{state: *state})
-	}
-	go r.demux(r.events)
-	return nil
-}
-
-// endSync ends a block sync
-func (r *BlockchainReactor) endSync() {
-	r.mtx.Lock()
-	defer r.mtx.Unlock()
-	if r.events != nil {
-		close(r.events)
-	}
-	r.events = nil
-	r.scheduler.stop()
-	r.processor.stop()
-}
-
-// SwitchToBlockSync is called by the state sync reactor when switching to block sync.
-func (r *BlockchainReactor) SwitchToBlockSync(state state.State) error {
-	r.stateSynced = true
-	state = state.Copy()
-
-	err := r.startSync(&state)
-	if err == nil {
-		r.syncStartTime = time.Now()
-	}
-
-	return err
-}
-
-// reactor generated ticker events:
-// ticker for cleaning peers
-type rTryPrunePeer struct {
-	priorityHigh
-	time time.Time
-}
-
-func (e rTryPrunePeer) String() string {
-	return fmt.Sprintf("rTryPrunePeer{%v}", e.time)
-}
-
-// ticker event for scheduling block requests
-type rTrySchedule struct {
-	priorityHigh
-	time time.Time
-}
-
-func (e rTrySchedule) String() string {
-	return fmt.Sprintf("rTrySchedule{%v}", e.time)
-}
-
-// ticker for block processing
-type rProcessBlock struct {
-	priorityNormal
-}
-
-func (e rProcessBlock) String() string {
-	return "rProcessBlock"
-}
-
-// reactor generated events based on blockchain related messages from peers:
-// blockResponse message received from a peer
-type bcBlockResponse struct {
-	priorityNormal
-	time   time.Time
-	peerID types.NodeID
-	size   int64
-	block  *types.Block
-}
-
-func (resp bcBlockResponse) String() string {
-	return fmt.Sprintf("bcBlockResponse{%d#%X (size: %d bytes) from %v at %v}",
-		resp.block.Height, resp.block.Hash(), resp.size, resp.peerID, resp.time)
-}
-
-// blockNoResponse message received from a peer
-type bcNoBlockResponse struct {
-	priorityNormal
-	time   time.Time
-	peerID types.NodeID
-	height int64
-}
-
-func (resp bcNoBlockResponse) String() string {
-	return fmt.Sprintf("bcNoBlockResponse{%v has no block at height %d at %v}",
-		resp.peerID, resp.height, resp.time)
-}
-
-// statusResponse message received from a peer
-type bcStatusResponse struct {
-	priorityNormal
-	time   time.Time
-	peerID types.NodeID
-	base   int64
-	height int64
-}
-
-func (resp bcStatusResponse) String() string {
-	return fmt.Sprintf("bcStatusResponse{%v is at height %d (base: %d) at %v}",
-		resp.peerID, resp.height, resp.base, resp.time)
-}
-
-// new peer is connected
-type bcAddNewPeer struct {
-	priorityNormal
-	peerID types.NodeID
-}
-
-func (resp bcAddNewPeer) String() string {
-	return fmt.Sprintf("bcAddNewPeer{%v}", resp.peerID)
-}
-
-// existing peer is removed
-type bcRemovePeer struct {
-	priorityHigh
-	peerID types.NodeID
-	reason interface{}
-}
-
-func (resp bcRemovePeer) String() string {
-	return fmt.Sprintf("bcRemovePeer{%v due to %v}", resp.peerID, resp.reason)
-}
-
-// resets the scheduler and processor state, e.g. following a switch from state syncing
-type bcResetState struct {
-	priorityHigh
-	state state.State
-}
-
-func (e bcResetState) String() string {
-	return fmt.Sprintf("bcResetState{%v}", e.state)
-}
-
-// Takes the channel as a parameter to avoid race conditions on r.events.
-func (r *BlockchainReactor) demux(events <-chan Event) {
-	var lastHundred = time.Now()
-
-	var (
-		processBlockFreq = 20 * time.Millisecond
-		doProcessBlockCh = make(chan struct{}, 1)
-		doProcessBlockTk = time.NewTicker(processBlockFreq)
-	)
-	defer doProcessBlockTk.Stop()
-
-	var (
-		prunePeerFreq = 1 * time.Second
-		doPrunePeerCh = make(chan struct{}, 1)
-		doPrunePeerTk = time.NewTicker(prunePeerFreq)
-	)
-	defer doPrunePeerTk.Stop()
-
-	var (
-		scheduleFreq = 20 * time.Millisecond
-		doScheduleCh = make(chan struct{}, 1)
-		doScheduleTk = time.NewTicker(scheduleFreq)
-	)
-	defer doScheduleTk.Stop()
-
-	var (
-		statusFreq = 10 * time.Second
-		doStatusCh = make(chan struct{}, 1)
-		doStatusTk = time.NewTicker(statusFreq)
-	)
-	defer doStatusTk.Stop()
-	doStatusCh <- struct{}{} // immediately broadcast to get status of existing peers
-
-	// Memoize the scSchedulerFail error to avoid printing it every scheduleFreq.
-	var scSchedulerFailErr error
-
-	// XXX: Extract timers to make testing atemporal
-	for {
-		select {
-		// Pacers: send at most per frequency but don't saturate
-		case <-doProcessBlockTk.C:
-			select {
-			case doProcessBlockCh <- struct{}{}:
-			default:
-			}
-		case <-doPrunePeerTk.C:
-			select {
-			case doPrunePeerCh <- struct{}{}:
-			default:
-			}
-		case <-doScheduleTk.C:
-			select {
-			case doScheduleCh <- struct{}{}:
-			default:
-			}
-		case <-doStatusTk.C:
-			select {
-			case doStatusCh <- struct{}{}:
-			default:
-			}
-
-		// Tickers: perform tasks periodically
-		case <-doScheduleCh:
-			r.scheduler.send(rTrySchedule{time: time.Now()})
-		case <-doPrunePeerCh:
-			r.scheduler.send(rTryPrunePeer{time: time.Now()})
-		case <-doProcessBlockCh:
-			r.processor.send(rProcessBlock{})
-		case <-doStatusCh:
-			if err := r.io.broadcastStatusRequest(); err != nil {
-				r.logger.Error("Error broadcasting status request", "err", err)
-			}
-
-		// Events from peers. Closing the channel signals event loop termination.
-		case event, ok := <-events:
-			if !ok {
-				r.logger.Info("Stopping event processing")
-				return
-			}
-			switch event := event.(type) {
-			case bcStatusResponse:
-				r.setMaxPeerHeight(event.height)
-				r.scheduler.send(event)
-			case bcAddNewPeer, bcRemovePeer, bcBlockResponse, bcNoBlockResponse:
-				r.scheduler.send(event)
-			default:
-				r.logger.Error("Received unexpected event", "event", fmt.Sprintf("%T", event))
-			}
-
-		// Incremental events from scheduler
-		case event := <-r.scheduler.next():
-			switch event := event.(type) {
-			case scBlockReceived:
-				r.processor.send(event)
-			case scPeerError:
-				r.processor.send(event)
-				if err := r.reporter.Report(behavior.BadMessage(event.peerID, "scPeerError")); err != nil {
-					r.logger.Error("Error reporting peer", "err", err)
-				}
-			case scBlockRequest:
-				peer := r.Switch.Peers().Get(event.peerID)
-				if peer == nil {
-					r.logger.Error("Wanted to send block request, but no such peer", "peerID", event.peerID)
-					continue
-				}
-				if err := r.io.sendBlockRequest(peer, event.height); err != nil {
-					r.logger.Error("Error sending block request", "err", err)
-				}
-			case scFinishedEv:
-				r.processor.send(event)
-				r.scheduler.stop()
-			case scSchedulerFail:
-				if scSchedulerFailErr != event.reason {
-					r.logger.Error("Scheduler failure", "err", event.reason.Error())
-					scSchedulerFailErr = event.reason
-				}
-			case scPeersPruned:
-				// Remove peers from the processor.
-				for _, peerID := range event.peers {
-					r.processor.send(scPeerError{peerID: peerID, reason: errors.New("peer was pruned")})
-				}
-				r.logger.Debug("Pruned peers", "count", len(event.peers))
-			case noOpEvent:
-			default:
-				r.logger.Error("Received unexpected scheduler event", "event", fmt.Sprintf("%T", event))
-			}
-
-		// Incremental events from processor
-		case event := <-r.processor.next():
-			switch event := event.(type) {
-			case pcBlockProcessed:
-				r.setSyncHeight(event.height)
-				if (r.syncHeight-r.syncStartHeight)%100 == 0 {
-					newSyncRate := 100 / time.Since(lastHundred).Seconds()
-					if r.lastSyncRate == 0 {
-						r.lastSyncRate = newSyncRate
-					} else {
-						r.lastSyncRate = 0.9*r.lastSyncRate + 0.1*newSyncRate
-					}
-					r.logger.Info("block sync Rate", "height", r.syncHeight,
-						"max_peer_height", r.maxPeerHeight, "blocks/s", r.lastSyncRate)
-					lastHundred = time.Now()
-				}
-				r.scheduler.send(event)
-			case pcBlockVerificationFailure:
-				r.scheduler.send(event)
-			case pcFinished:
-				r.logger.Info("block sync complete, switching to consensus")
-				if !r.io.trySwitchToConsensus(event.tmState, event.blocksSynced > 0 || r.stateSynced) {
-					r.logger.Error("Failed to switch to consensus reactor")
-				}
-				r.endSync()
-				r.blockSync.UnSet()
-				return
-			case noOpEvent:
-			default:
-				r.logger.Error("Received unexpected processor event", "event", fmt.Sprintf("%T", event))
-			}
-
-		// Terminal event from scheduler
-		case err := <-r.scheduler.final():
-			switch err {
-			case nil:
-				r.logger.Info("Scheduler stopped")
-			default:
-				r.logger.Error("Scheduler aborted with error", "err", err)
-			}
-
-		// Terminal event from processor
-		case err := <-r.processor.final():
-			switch err {
-			case nil:
-				r.logger.Info("Processor stopped")
-			default:
-				r.logger.Error("Processor aborted with error", "err", err)
-			}
-		}
-	}
-}
-
-// Stop implements cmn.Service interface.
-func (r *BlockchainReactor) Stop() error {
-	r.logger.Info("reactor stopping")
-	r.endSync()
-	r.logger.Info("reactor stopped")
-	return nil
-}
-
-// Receive implements Reactor by handling different message types.
-// XXX: do not call any methods that can block or incur heavy processing.
-// https://github.com/tendermint/tendermint/issues/2888
-func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
-	logger := r.logger.With("src", src.ID(), "chID", chID)
-
-	msgProto := new(bcproto.Message)
-
-	if err := proto.Unmarshal(msgBytes, msgProto); err != nil {
-		logger.Error("error decoding message", "err", err)
-		_ = r.reporter.Report(behavior.BadMessage(src.ID(), err.Error()))
-		return
-	}
-
-	if err := msgProto.Validate(); err != nil {
-		logger.Error("peer sent us an invalid msg", "msg", msgProto, "err", err)
-		_ = r.reporter.Report(behavior.BadMessage(src.ID(), err.Error()))
-		return
-	}
-
-	r.logger.Debug("received", "msg", msgProto)
-
-	switch msg := msgProto.Sum.(type) {
-	case *bcproto.Message_StatusRequest:
-		if err := r.io.sendStatusResponse(r.store.Base(), r.store.Height(), src); err != nil {
-			logger.Error("Could not send status message to src peer")
-		}
-
-	case *bcproto.Message_BlockRequest:
-		block := r.store.LoadBlock(msg.BlockRequest.Height)
-		if block != nil {
-			if err := r.io.sendBlockToPeer(block, src); err != nil {
-				logger.Error("Could not send block message to src peer", "err", err)
-			}
-		} else {
-			logger.Info("peer asking for a block we don't have", "height", msg.BlockRequest.Height)
-			if err := r.io.sendBlockNotFound(msg.BlockRequest.Height, src); err != nil {
-				logger.Error("Couldn't send block not found msg", "err", err)
-			}
-		}
-
-	case *bcproto.Message_StatusResponse:
-		r.mtx.RLock()
-		if r.events != nil {
-			r.events <- bcStatusResponse{
-				peerID: src.ID(),
-				base:   msg.StatusResponse.Base,
-				height: msg.StatusResponse.Height,
-			}
-		}
-		r.mtx.RUnlock()
-
-	case *bcproto.Message_BlockResponse:
-		bi, err := types.BlockFromProto(msg.BlockResponse.Block)
-		if err != nil {
-			logger.Error("error transitioning block from protobuf", "err", err)
-			_ = r.reporter.Report(behavior.BadMessage(src.ID(), err.Error()))
-			return
-		}
-		r.mtx.RLock()
-		if r.events != nil {
-			r.events <- bcBlockResponse{
-				peerID: src.ID(),
-				block:  bi,
-				size:   int64(len(msgBytes)),
-				time:   time.Now(),
-			}
-		}
-		r.mtx.RUnlock()
-
-	case *bcproto.Message_NoBlockResponse:
-		r.mtx.RLock()
-		if r.events != nil {
-			r.events <- bcNoBlockResponse{
-				peerID: src.ID(),
-				height: msg.NoBlockResponse.Height,
-				time:   time.Now(),
-			}
-		}
-		r.mtx.RUnlock()
-	}
-}
-
-// AddPeer implements Reactor interface
-func (r *BlockchainReactor) AddPeer(peer p2p.Peer) {
-	err := r.io.sendStatusResponse(r.store.Base(), r.store.Height(), peer)
-	if err != nil {
-		r.logger.Error("could not send our status to the new peer", "peer", peer.ID, "err", err)
-	}
-
-	err = r.io.sendStatusRequest(peer)
-	if err != nil {
-		r.logger.Error("could not send status request to the new peer", "peer", peer.ID, "err", err)
-	}
-
-	r.mtx.RLock()
-	defer r.mtx.RUnlock()
-	if r.events != nil {
-		r.events <- bcAddNewPeer{peerID: peer.ID()}
-	}
-}
-
-// RemovePeer implements Reactor interface.
-func (r *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
-	r.mtx.RLock()
-	defer r.mtx.RUnlock()
-	if r.events != nil {
-		r.events <- bcRemovePeer{
-			peerID: peer.ID(),
-			reason: reason,
-		}
-	}
-}
-
-// GetChannels implements Reactor
-func (r *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
-	return []*p2p.ChannelDescriptor{
-		{
-			ID:                  BlockchainChannel,
-			Priority:            5,
-			SendQueueCapacity:   2000,
-			RecvBufferCapacity:  1024,
-			RecvMessageCapacity: bc.MaxMsgSize,
-		},
-	}
-}
-
-func (r *BlockchainReactor) GetMaxPeerBlockHeight() int64 {
-	r.mtx.RLock()
-	defer r.mtx.RUnlock()
-	return r.maxPeerHeight
-}
-
-func (r *BlockchainReactor) GetTotalSyncedTime() time.Duration {
-	if !r.blockSync.IsSet() || r.syncStartTime.IsZero() {
-		return time.Duration(0)
-	}
-	return time.Since(r.syncStartTime)
-}
-
-func (r *BlockchainReactor) GetRemainingSyncTime() time.Duration {
-	if !r.blockSync.IsSet() {
-		return time.Duration(0)
-	}
-
-	r.mtx.RLock()
-	defer r.mtx.RUnlock()
-
-	targetSyncs := r.maxPeerHeight - r.syncStartHeight
-	currentSyncs := r.syncHeight - r.syncStartHeight + 1
-	if currentSyncs < 0 || r.lastSyncRate < 0.001 {
-		return time.Duration(0)
-	}
-
-	remain := float64(targetSyncs-currentSyncs) / r.lastSyncRate
-
-	return time.Duration(int64(remain * float64(time.Second)))
-}

internal/blocksync/v2/reactor_test.go

@@ -1,532 +0,0 @@
-package v2
-
-import (
-	"fmt"
-	"net"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/gogo/protobuf/proto"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	dbm "github.com/tendermint/tm-db"
-
-	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
-	"github.com/tendermint/tendermint/internal/blocksync/v2/internal/behavior"
-	cons "github.com/tendermint/tendermint/internal/consensus"
-	"github.com/tendermint/tendermint/internal/mempool/mock"
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/internal/p2p/conn"
-	"github.com/tendermint/tendermint/internal/test/factory"
-	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/libs/service"
-	bcproto "github.com/tendermint/tendermint/proto/tendermint/blocksync"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
-	sf "github.com/tendermint/tendermint/state/test/factory"
-	tmstore "github.com/tendermint/tendermint/store"
-	"github.com/tendermint/tendermint/types"
-)
-
-type mockPeer struct {
-	service.Service
-	id types.NodeID
-}
-
-func (mp mockPeer) FlushStop()           {}
-func (mp mockPeer) ID() types.NodeID     { return mp.id }
-func (mp mockPeer) RemoteIP() net.IP     { return net.IP{} }
-func (mp mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.RemoteIP(), Port: 8800} }
-
-func (mp mockPeer) IsOutbound() bool   { return true }
-func (mp mockPeer) IsPersistent() bool { return true }
-func (mp mockPeer) CloseConn() error   { return nil }
-
-func (mp mockPeer) NodeInfo() types.NodeInfo {
-	return types.NodeInfo{
-		NodeID:     "",
-		ListenAddr: "",
-	}
-}
-func (mp mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} }
-func (mp mockPeer) SocketAddr() *p2p.NetAddress   { return &p2p.NetAddress{} }
-
-func (mp mockPeer) Send(byte, []byte) bool    { return true }
-func (mp mockPeer) TrySend(byte, []byte) bool { return true }
-
-func (mp mockPeer) Set(string, interface{}) {}
-func (mp mockPeer) Get(string) interface{}  { return struct{}{} }
-
-//nolint:unused
-type mockBlockStore struct {
-	blocks map[int64]*types.Block
-}
-
-//nolint:unused
-func (ml *mockBlockStore) Height() int64 {
-	return int64(len(ml.blocks))
-}
-
-//nolint:unused
-func (ml *mockBlockStore) LoadBlock(height int64) *types.Block {
-	return ml.blocks[height]
-}
-
-//nolint:unused
-func (ml *mockBlockStore) SaveBlock(block *types.Block, part *types.PartSet, commit *types.Commit) {
-	ml.blocks[block.Height] = block
-}
-
-type mockBlockApplier struct {
-}
-
-// XXX: Add whitelist/blacklist?
-func (mba *mockBlockApplier) ApplyBlock(
-	state sm.State, blockID types.BlockID, block *types.Block,
-) (sm.State, error) {
-	state.LastBlockHeight++
-	return state, nil
-}
-
-type mockSwitchIo struct {
-	mtx                 sync.Mutex
-	switchedToConsensus bool
-	numStatusResponse   int
-	numBlockResponse    int
-	numNoBlockResponse  int
-	numStatusRequest    int
-}
-
-var _ iIO = (*mockSwitchIo)(nil)
-
-func (sio *mockSwitchIo) sendBlockRequest(_ p2p.Peer, _ int64) error {
-	return nil
-}
-
-func (sio *mockSwitchIo) sendStatusResponse(_, _ int64, _ p2p.Peer) error {
-	sio.mtx.Lock()
-	defer sio.mtx.Unlock()
-	sio.numStatusResponse++
-	return nil
-}
-
-func (sio *mockSwitchIo) sendBlockToPeer(_ *types.Block, _ p2p.Peer) error {
-	sio.mtx.Lock()
-	defer sio.mtx.Unlock()
-	sio.numBlockResponse++
-	return nil
-}
-
-func (sio *mockSwitchIo) sendBlockNotFound(_ int64, _ p2p.Peer) error {
-	sio.mtx.Lock()
-	defer sio.mtx.Unlock()
-	sio.numNoBlockResponse++
-	return nil
-}
-
-func (sio *mockSwitchIo) trySwitchToConsensus(_ sm.State, _ bool) bool {
-	sio.mtx.Lock()
-	defer sio.mtx.Unlock()
-	sio.switchedToConsensus = true
-	return true
-}
-
-func (sio *mockSwitchIo) broadcastStatusRequest() error {
-	return nil
-}
-
-func (sio *mockSwitchIo) sendStatusRequest(_ p2p.Peer) error {
-	sio.mtx.Lock()
-	defer sio.mtx.Unlock()
-	sio.numStatusRequest++
-	return nil
-}
-
-type testReactorParams struct {
-	logger      log.Logger
-	genDoc      *types.GenesisDoc
-	privVals    []types.PrivValidator
-	startHeight int64
-	mockA       bool
-}
-
-func newTestReactor(t *testing.T, p testReactorParams) *BlockchainReactor {
-	store, state, _ := newReactorStore(t, p.genDoc, p.privVals, p.startHeight)
-	reporter := behavior.NewMockReporter()
-
-	var appl blockApplier
-
-	if p.mockA {
-		appl = &mockBlockApplier{}
-	} else {
-		app := &testApp{}
-		cc := proxy.NewLocalClientCreator(app)
-		proxyApp := proxy.NewAppConns(cc)
-		err := proxyApp.Start()
-		require.NoError(t, err)
-		db := dbm.NewMemDB()
-		stateStore := sm.NewStore(db)
-		blockStore := tmstore.NewBlockStore(dbm.NewMemDB())
-		appl = sm.NewBlockExecutor(
-			stateStore, p.logger, proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{}, blockStore)
-		err = stateStore.Save(state)
-		require.NoError(t, err)
-	}
-
-	r := newReactor(state, store, reporter, appl, true, cons.NopMetrics())
-	logger := log.TestingLogger()
-	r.SetLogger(logger.With("module", "blockchain"))
-
-	return r
-}
-
-// This test is left here and not deleted to retain the termination cases for
-// future improvement in [#4482](https://github.com/tendermint/tendermint/issues/4482).
-// func TestReactorTerminationScenarios(t *testing.T) {
-
-// 	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
-// 	defer os.RemoveAll(config.RootDir)
-// 	genDoc, privVals := randGenesisDoc(config.ChainID(), 1, false, 30)
-// 	refStore, _, _ := newReactorStore(genDoc, privVals, 20)
-
-// 	params := testReactorParams{
-// 		logger:      log.TestingLogger(),
-// 		genDoc:      genDoc,
-// 		privVals:    privVals,
-// 		startHeight: 10,
-// 		bufferSize:  100,
-// 		mockA:       true,
-// 	}
-
-// 	type testEvent struct {
-// 		evType string
-// 		peer   string
-// 		height int64
-// 	}
-
-// 	tests := []struct {
-// 		name   string
-// 		params testReactorParams
-// 		msgs   []testEvent
-// 	}{
-// 		{
-// 			name:   "simple termination on max peer height - one peer",
-// 			params: params,
-// 			msgs: []testEvent{
-// 				{evType: "AddPeer", peer: "P1"},
-// 				{evType: "ReceiveS", peer: "P1", height: 13},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P1", height: 11},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P1", height: 12},
-// 				{evType: "Process"},
-// 				{evType: "ReceiveB", peer: "P1", height: 13},
-// 				{evType: "Process"},
-// 			},
-// 		},
-// 		{
-// 			name:   "simple termination on max peer height - two peers",
-// 			params: params,
-// 			msgs: []testEvent{
-// 				{evType: "AddPeer", peer: "P1"},
-// 				{evType: "AddPeer", peer: "P2"},
-// 				{evType: "ReceiveS", peer: "P1", height: 13},
-// 				{evType: "ReceiveS", peer: "P2", height: 15},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P1", height: 11},
-// 				{evType: "ReceiveB", peer: "P2", height: 12},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P1", height: 13},
-// 				{evType: "Process"},
-// 				{evType: "ReceiveB", peer: "P2", height: 14},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 15},
-// 				{evType: "Process"},
-// 			},
-// 		},
-// 		{
-// 			name:   "termination on max peer height - two peers, noBlock error",
-// 			params: params,
-// 			msgs: []testEvent{
-// 				{evType: "AddPeer", peer: "P1"},
-// 				{evType: "AddPeer", peer: "P2"},
-// 				{evType: "ReceiveS", peer: "P1", height: 13},
-// 				{evType: "ReceiveS", peer: "P2", height: 15},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveNB", peer: "P1", height: 11},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 12},
-// 				{evType: "ReceiveB", peer: "P2", height: 11},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 13},
-// 				{evType: "Process"},
-// 				{evType: "ReceiveB", peer: "P2", height: 14},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 15},
-// 				{evType: "Process"},
-// 			},
-// 		},
-// 		{
-// 			name:   "termination on max peer height - two peers, remove one peer",
-// 			params: params,
-// 			msgs: []testEvent{
-// 				{evType: "AddPeer", peer: "P1"},
-// 				{evType: "AddPeer", peer: "P2"},
-// 				{evType: "ReceiveS", peer: "P1", height: 13},
-// 				{evType: "ReceiveS", peer: "P2", height: 15},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "RemovePeer", peer: "P1"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 12},
-// 				{evType: "ReceiveB", peer: "P2", height: 11},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 13},
-// 				{evType: "Process"},
-// 				{evType: "ReceiveB", peer: "P2", height: 14},
-// 				{evType: "Process"},
-// 				{evType: "BlockReq"},
-// 				{evType: "ReceiveB", peer: "P2", height: 15},
-// 				{evType: "Process"},
-// 			},
-// 		},
-// 	}
-
-// 	for _, tt := range tests {
-// 		tt := tt
-// 		t.Run(tt.name, func(t *testing.T) {
-// 			reactor := newTestReactor(params)
-// 			reactor.Start()
-// 			reactor.reporter = behavior.NewMockReporter()
-// 			mockSwitch := &mockSwitchIo{switchedToConsensus: false}
-// 			reactor.io = mockSwitch
-// 			// time for go routines to start
-// 			time.Sleep(time.Millisecond)
-
-// 			for _, step := range tt.msgs {
-// 				switch step.evType {
-// 				case "AddPeer":
-// 					reactor.scheduler.send(bcAddNewPeer{peerID: p2p.ID(step.peer)})
-// 				case "RemovePeer":
-// 					reactor.scheduler.send(bcRemovePeer{peerID: p2p.ID(step.peer)})
-// 				case "ReceiveS":
-// 					reactor.scheduler.send(bcStatusResponse{
-// 						peerID: p2p.ID(step.peer),
-// 						height: step.height,
-// 						time:   time.Now(),
-// 					})
-// 				case "ReceiveB":
-// 					reactor.scheduler.send(bcBlockResponse{
-// 						peerID: p2p.ID(step.peer),
-// 						block:  refStore.LoadBlock(step.height),
-// 						size:   10,
-// 						time:   time.Now(),
-// 					})
-// 				case "ReceiveNB":
-// 					reactor.scheduler.send(bcNoBlockResponse{
-// 						peerID: p2p.ID(step.peer),
-// 						height: step.height,
-// 						time:   time.Now(),
-// 					})
-// 				case "BlockReq":
-// 					reactor.scheduler.send(rTrySchedule{time: time.Now()})
-// 				case "Process":
-// 					reactor.processor.send(rProcessBlock{})
-// 				}
-// 				// give time for messages to propagate between routines
-// 				time.Sleep(time.Millisecond)
-// 			}
-
-// 			// time for processor to finish and reactor to switch to consensus
-// 			time.Sleep(20 * time.Millisecond)
-// 			assert.True(t, mockSwitch.hasSwitchedToConsensus())
-// 			reactor.Stop()
-// 		})
-// 	}
-// }
-
-func TestReactorHelperMode(t *testing.T) {
-	var (
-		channelID = byte(0x40)
-	)
-
-	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
-	defer os.RemoveAll(config.RootDir)
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
-
-	params := testReactorParams{
-		logger:      log.TestingLogger(),
-		genDoc:      genDoc,
-		privVals:    privVals,
-		startHeight: 20,
-		mockA:       true,
-	}
-
-	type testEvent struct {
-		peer  string
-		event interface{}
-	}
-
-	tests := []struct {
-		name   string
-		params testReactorParams
-		msgs   []testEvent
-	}{
-		{
-			name:   "status request",
-			params: params,
-			msgs: []testEvent{
-				{"P1", bcproto.StatusRequest{}},
-				{"P1", bcproto.BlockRequest{Height: 13}},
-				{"P1", bcproto.BlockRequest{Height: 20}},
-				{"P1", bcproto.BlockRequest{Height: 22}},
-			},
-		},
-	}
-
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			reactor := newTestReactor(t, params)
-			mockSwitch := &mockSwitchIo{switchedToConsensus: false}
-			reactor.io = mockSwitch
-			err := reactor.Start()
-			require.NoError(t, err)
-
-			for i := 0; i < len(tt.msgs); i++ {
-				step := tt.msgs[i]
-				switch ev := step.event.(type) {
-				case bcproto.StatusRequest:
-					old := mockSwitch.numStatusResponse
-
-					msgProto := new(bcproto.Message)
-					require.NoError(t, msgProto.Wrap(&ev))
-
-					msgBz, err := proto.Marshal(msgProto)
-					require.NoError(t, err)
-
-					reactor.Receive(channelID, mockPeer{id: types.NodeID(step.peer)}, msgBz)
-					assert.Equal(t, old+1, mockSwitch.numStatusResponse)
-				case bcproto.BlockRequest:
-					if ev.Height > params.startHeight {
-						old := mockSwitch.numNoBlockResponse
-
-						msgProto := new(bcproto.Message)
-						require.NoError(t, msgProto.Wrap(&ev))
-
-						msgBz, err := proto.Marshal(msgProto)
-						require.NoError(t, err)
-
-						reactor.Receive(channelID, mockPeer{id: types.NodeID(step.peer)}, msgBz)
-						assert.Equal(t, old+1, mockSwitch.numNoBlockResponse)
-					} else {
-						old := mockSwitch.numBlockResponse
-
-						msgProto := new(bcproto.Message)
-						require.NoError(t, msgProto.Wrap(&ev))
-
-						msgBz, err := proto.Marshal(msgProto)
-						require.NoError(t, err)
-
-						reactor.Receive(channelID, mockPeer{id: types.NodeID(step.peer)}, msgBz)
-						assert.Equal(t, old+1, mockSwitch.numBlockResponse)
-					}
-				}
-			}
-			err = reactor.Stop()
-			require.NoError(t, err)
-		})
-	}
-}
-
-func TestReactorSetSwitchNil(t *testing.T) {
-	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
-	defer os.RemoveAll(config.RootDir)
-	genDoc, privVals := factory.RandGenesisDoc(config, 1, false, 30)
-
-	reactor := newTestReactor(t, testReactorParams{
-		logger:   log.TestingLogger(),
-		genDoc:   genDoc,
-		privVals: privVals,
-	})
-	reactor.SetSwitch(nil)
-
-	assert.Nil(t, reactor.Switch)
-	assert.Nil(t, reactor.io)
-}
-
-type testApp struct {
-	abci.BaseApplication
-}
-
-func newReactorStore(
-	t *testing.T,
-	genDoc *types.GenesisDoc,
-	privVals []types.PrivValidator,
-	maxBlockHeight int64) (*tmstore.BlockStore, sm.State, *sm.BlockExecutor) {
-	t.Helper()
-
-	require.Len(t, privVals, 1)
-	app := &testApp{}
-	cc := proxy.NewLocalClientCreator(app)
-	proxyApp := proxy.NewAppConns(cc)
-	err := proxyApp.Start()
-	if err != nil {
-		panic(fmt.Errorf("error start app: %w", err))
-	}
-
-	stateDB := dbm.NewMemDB()
-	blockStore := tmstore.NewBlockStore(dbm.NewMemDB())
-	stateStore := sm.NewStore(stateDB)
-	state, err := sm.MakeGenesisState(genDoc)
-	require.NoError(t, err)
-
-	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
-		mock.Mempool{}, sm.EmptyEvidencePool{}, blockStore)
-	err = stateStore.Save(state)
-	require.NoError(t, err)
-
-	// add blocks in
-	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
-		if blockHeight > 1 {
-			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
-			lastBlock := blockStore.LoadBlock(blockHeight - 1)
-			vote, err := factory.MakeVote(
-				privVals[0],
-				lastBlock.Header.ChainID, 0,
-				lastBlock.Header.Height, 0, 2,
-				lastBlockMeta.BlockID,
-				time.Now(),
-			)
-			require.NoError(t, err)
-			lastCommit = types.NewCommit(vote.Height, vote.Round,
-				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
-		}
-
-		thisBlock := sf.MakeBlock(state, blockHeight, lastCommit)
-
-		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
-		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
-
-		state, err = blockExec.ApplyBlock(state, blockID, thisBlock)
-		require.NoError(t, err)
-
-		blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
-	}
-	return blockStore, state, blockExec
-}

internal/blocksync/v2/routine.go

@@ -1,166 +0,0 @@
-package v2
-
-import (
-	"fmt"
-	"strings"
-	"sync/atomic"
-
-	"github.com/Workiva/go-datastructures/queue"
-
-	"github.com/tendermint/tendermint/libs/log"
-)
-
-type handleFunc = func(event Event) (Event, error)
-
-const historySize = 25
-
-// Routine is a structure that models a finite state machine as serialized
-// stream of events processed by a handle function. This Routine structure
-// handles the concurrency and messaging guarantees. Events are sent via
-// `send` are handled by the `handle` function to produce an iterator
-// `next()`. Calling `stop()` on a routine will conclude processing of all
-// sent events and produce `final()` event representing the terminal state.
-type Routine struct {
-	name    string
-	handle  handleFunc
-	queue   *queue.PriorityQueue
-	history []Event
-	out     chan Event
-	fin     chan error
-	rdy     chan struct{}
-	running *uint32
-	logger  log.Logger
-	metrics *Metrics
-}
-
-func newRoutine(name string, handleFunc handleFunc, bufferSize int) *Routine {
-	return &Routine{
-		name:    name,
-		handle:  handleFunc,
-		queue:   queue.NewPriorityQueue(bufferSize, true),
-		history: make([]Event, 0, historySize),
-		out:     make(chan Event, bufferSize),
-		rdy:     make(chan struct{}, 1),
-		fin:     make(chan error, 1),
-		running: new(uint32),
-		logger:  log.NewNopLogger(),
-		metrics: NopMetrics(),
-	}
-}
-
-func (rt *Routine) setLogger(logger log.Logger) {
-	rt.logger = logger
-}
-
-// nolint:unused
-func (rt *Routine) setMetrics(metrics *Metrics) {
-	rt.metrics = metrics
-}
-
-func (rt *Routine) start() {
-	rt.logger.Info(fmt.Sprintf("%s: run", rt.name))
-	running := atomic.CompareAndSwapUint32(rt.running, uint32(0), uint32(1))
-	if !running {
-		panic(fmt.Sprintf("%s is already running", rt.name))
-	}
-	close(rt.rdy)
-	defer func() {
-		if r := recover(); r != nil {
-			var (
-				b strings.Builder
-				j int
-			)
-			for i := len(rt.history) - 1; i >= 0; i-- {
-				fmt.Fprintf(&b, "%d: %+v\n", j, rt.history[i])
-				j++
-			}
-			panic(fmt.Sprintf("%v\nlast events:\n%v", r, b.String()))
-		}
-		stopped := atomic.CompareAndSwapUint32(rt.running, uint32(1), uint32(0))
-		if !stopped {
-			panic(fmt.Sprintf("%s is failed to stop", rt.name))
-		}
-	}()
-
-	for {
-		events, err := rt.queue.Get(1)
-		if err == queue.ErrDisposed {
-			rt.terminate(nil)
-			return
-		} else if err != nil {
-			rt.terminate(err)
-			return
-		}
-		oEvent, err := rt.handle(events[0].(Event))
-		rt.metrics.EventsHandled.With("routine", rt.name).Add(1)
-		if err != nil {
-			rt.terminate(err)
-			return
-		}
-		rt.metrics.EventsOut.With("routine", rt.name).Add(1)
-		rt.logger.Debug(fmt.Sprintf("%s: produced %T %+v", rt.name, oEvent, oEvent))
-
-		// Skip rTrySchedule and rProcessBlock events as they clutter the history
-		// due to their frequency.
-		switch events[0].(type) {
-		case rTrySchedule:
-		case rProcessBlock:
-		default:
-			rt.history = append(rt.history, events[0].(Event))
-			if len(rt.history) > historySize {
-				rt.history = rt.history[1:]
-			}
-		}
-
-		rt.out <- oEvent
-	}
-}
-
-// XXX: look into returning OpError in the net package
-func (rt *Routine) send(event Event) bool {
-	rt.logger.Debug(fmt.Sprintf("%s: received %T %+v", rt.name, event, event))
-	if !rt.isRunning() {
-		return false
-	}
-	err := rt.queue.Put(event)
-	if err != nil {
-		rt.metrics.EventsShed.With("routine", rt.name).Add(1)
-		rt.logger.Error(fmt.Sprintf("%s: send failed, queue was full/stopped", rt.name))
-		return false
-	}
-
-	rt.metrics.EventsSent.With("routine", rt.name).Add(1)
-	return true
-}
-
-func (rt *Routine) isRunning() bool {
-	return atomic.LoadUint32(rt.running) == 1
-}
-
-func (rt *Routine) next() chan Event {
-	return rt.out
-}
-
-func (rt *Routine) ready() chan struct{} {
-	return rt.rdy
-}
-
-func (rt *Routine) stop() {
-	if !rt.isRunning() { // XXX: this should check rt.queue.Disposed()
-		return
-	}
-
-	rt.logger.Info(fmt.Sprintf("%s: stop", rt.name))
-	rt.queue.Dispose() // this should block until all queue items are free?
-}
-
-func (rt *Routine) final() chan error {
-	return rt.fin
-}
-
-// XXX: Maybe get rid of this
-func (rt *Routine) terminate(reason error) {
-	// We don't close the rt.out channel here, to avoid spinning on the closed channel
-	// in the event loop.
-	rt.fin <- reason
-}

internal/blocksync/v2/routine_test.go

@@ -1,163 +0,0 @@
-package v2
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-type eventA struct {
-	priorityNormal
-}
-
-var errDone = fmt.Errorf("done")
-
-func simpleHandler(event Event) (Event, error) {
-	if _, ok := event.(eventA); ok {
-		return noOp, errDone
-	}
-	return noOp, nil
-}
-
-func TestRoutineFinal(t *testing.T) {
-	var (
-		bufferSize = 10
-		routine    = newRoutine("simpleRoutine", simpleHandler, bufferSize)
-	)
-
-	assert.False(t, routine.isRunning(),
-		"expected an initialized routine to not be running")
-	go routine.start()
-	<-routine.ready()
-	assert.True(t, routine.isRunning(),
-		"expected an started routine")
-
-	assert.True(t, routine.send(eventA{}),
-		"expected sending to a ready routine to succeed")
-
-	assert.Equal(t, errDone, <-routine.final(),
-		"expected the final event to be done")
-
-	assert.False(t, routine.isRunning(),
-		"expected an completed routine to no longer be running")
-}
-
-func TestRoutineStop(t *testing.T) {
-	var (
-		bufferSize = 10
-		routine    = newRoutine("simpleRoutine", simpleHandler, bufferSize)
-	)
-
-	assert.False(t, routine.send(eventA{}),
-		"expected sending to an unstarted routine to fail")
-
-	go routine.start()
-	<-routine.ready()
-
-	assert.True(t, routine.send(eventA{}),
-		"expected sending to a running routine to succeed")
-
-	routine.stop()
-
-	assert.False(t, routine.send(eventA{}),
-		"expected sending to a stopped routine to fail")
-}
-
-type finalCount struct {
-	count int
-}
-
-func (f finalCount) Error() string {
-	return "end"
-}
-
-func genStatefulHandler(maxCount int) handleFunc {
-	counter := 0
-	return func(event Event) (Event, error) {
-		if _, ok := event.(eventA); ok {
-			counter++
-			if counter >= maxCount {
-				return noOp, finalCount{counter}
-			}
-
-			return eventA{}, nil
-		}
-		return noOp, nil
-	}
-}
-
-func feedback(r *Routine) {
-	for event := range r.next() {
-		r.send(event)
-	}
-}
-
-func TestStatefulRoutine(t *testing.T) {
-	var (
-		count      = 10
-		handler    = genStatefulHandler(count)
-		bufferSize = 20
-		routine    = newRoutine("statefulRoutine", handler, bufferSize)
-	)
-
-	go routine.start()
-	go feedback(routine)
-	<-routine.ready()
-
-	assert.True(t, routine.send(eventA{}),
-		"expected sending to a started routine to succeed")
-
-	final := <-routine.final()
-	if fnl, ok := final.(finalCount); ok {
-		assert.Equal(t, count, fnl.count,
-			"expected the routine to count to 10")
-	} else {
-		t.Fail()
-	}
-}
-
-type lowPriorityEvent struct {
-	priorityLow
-}
-
-type highPriorityEvent struct {
-	priorityHigh
-}
-
-func handleWithPriority(event Event) (Event, error) {
-	switch event.(type) {
-	case lowPriorityEvent:
-		return noOp, nil
-	case highPriorityEvent:
-		return noOp, errDone
-	}
-	return noOp, nil
-}
-
-func TestPriority(t *testing.T) {
-	var (
-		bufferSize = 20
-		routine    = newRoutine("priorityRoutine", handleWithPriority, bufferSize)
-	)
-
-	go routine.start()
-	<-routine.ready()
-	go func() {
-		for {
-			routine.send(lowPriorityEvent{})
-			time.Sleep(1 * time.Millisecond)
-		}
-	}()
-	time.Sleep(10 * time.Millisecond)
-
-	assert.True(t, routine.isRunning(),
-		"expected an started routine")
-	assert.True(t, routine.send(highPriorityEvent{}),
-		"expected send to succeed even when saturated")
-
-	assert.Equal(t, errDone, <-routine.final())
-	assert.False(t, routine.isRunning(),
-		"expected an started routine")
-}

internal/blocksync/v2/scheduler.go

@@ -1,711 +0,0 @@
-package v2
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"math"
-	"sort"
-	"time"
-
-	"github.com/tendermint/tendermint/types"
-)
-
-// Events generated by the scheduler:
-// all blocks have been processed
-type scFinishedEv struct {
-	priorityNormal
-	reason string
-}
-
-func (e scFinishedEv) String() string {
-	return fmt.Sprintf("scFinishedEv{%v}", e.reason)
-}
-
-// send a blockRequest message
-type scBlockRequest struct {
-	priorityNormal
-	peerID types.NodeID
-	height int64
-}
-
-func (e scBlockRequest) String() string {
-	return fmt.Sprintf("scBlockRequest{%d from %v}", e.height, e.peerID)
-}
-
-// a block has been received and validated by the scheduler
-type scBlockReceived struct {
-	priorityNormal
-	peerID types.NodeID
-	block  *types.Block
-}
-
-func (e scBlockReceived) String() string {
-	return fmt.Sprintf("scBlockReceived{%d#%X from %v}", e.block.Height, e.block.Hash(), e.peerID)
-}
-
-// scheduler detected a peer error
-type scPeerError struct {
-	priorityHigh
-	peerID types.NodeID
-	reason error
-}
-
-func (e scPeerError) String() string {
-	return fmt.Sprintf("scPeerError{%v errored with %v}", e.peerID, e.reason)
-}
-
-// scheduler removed a set of peers (timed out or slow peer)
-type scPeersPruned struct {
-	priorityHigh
-	peers []types.NodeID
-}
-
-func (e scPeersPruned) String() string {
-	return fmt.Sprintf("scPeersPruned{%v}", e.peers)
-}
-
-// XXX: make this fatal?
-// scheduler encountered a fatal error
-type scSchedulerFail struct {
-	priorityHigh
-	reason error
-}
-
-func (e scSchedulerFail) String() string {
-	return fmt.Sprintf("scSchedulerFail{%v}", e.reason)
-}
-
-type blockState int
-
-const (
-	blockStateUnknown   blockState = iota + 1 // no known peer has this block
-	blockStateNew                             // indicates that a peer has reported having this block
-	blockStatePending                         // indicates that this block has been requested from a peer
-	blockStateReceived                        // indicates that this block has been received by a peer
-	blockStateProcessed                       // indicates that this block has been applied
-)
-
-func (e blockState) String() string {
-	switch e {
-	case blockStateUnknown:
-		return "Unknown"
-	case blockStateNew:
-		return "New"
-	case blockStatePending:
-		return "Pending"
-	case blockStateReceived:
-		return "Received"
-	case blockStateProcessed:
-		return "Processed"
-	default:
-		return fmt.Sprintf("invalid blockState: %d", e)
-	}
-}
-
-type peerState int
-
-const (
-	peerStateNew = iota + 1
-	peerStateReady
-	peerStateRemoved
-)
-
-func (e peerState) String() string {
-	switch e {
-	case peerStateNew:
-		return "New"
-	case peerStateReady:
-		return "Ready"
-	case peerStateRemoved:
-		return "Removed"
-	default:
-		panic(fmt.Sprintf("unknown peerState: %d", e))
-	}
-}
-
-type scPeer struct {
-	peerID types.NodeID
-
-	// initialized as New when peer is added, updated to Ready when statusUpdate is received,
-	// updated to Removed when peer is removed
-	state peerState
-
-	base        int64 // updated when statusResponse is received
-	height      int64 // updated when statusResponse is received
-	lastTouched time.Time
-	lastRate    int64 // last receive rate in bytes
-}
-
-func (p scPeer) String() string {
-	return fmt.Sprintf("{state %v, base %d, height %d, lastTouched %v, lastRate %d, id %v}",
-		p.state, p.base, p.height, p.lastTouched, p.lastRate, p.peerID)
-}
-
-func newScPeer(peerID types.NodeID) *scPeer {
-	return &scPeer{
-		peerID:      peerID,
-		state:       peerStateNew,
-		base:        -1,
-		height:      -1,
-		lastTouched: time.Time{},
-	}
-}
-
-// The scheduler keep track of the state of each block and each peer. The
-// scheduler will attempt to schedule new block requests with `trySchedule`
-// events and remove slow peers with `tryPrune` events.
-type scheduler struct {
-	initHeight int64
-
-	// next block that needs to be processed. All blocks with smaller height are
-	// in Processed state.
-	height int64
-
-	// lastAdvance tracks the last time a block execution happened.
-	// syncTimeout is the maximum time the scheduler waits to advance in the block sync process before finishing.
-	// This covers the cases where there are no peers or all peers have a lower height.
-	lastAdvance time.Time
-	syncTimeout time.Duration
-
-	// a map of peerID to scheduler specific peer struct `scPeer` used to keep
-	// track of peer specific state
-	peers       map[types.NodeID]*scPeer
-	peerTimeout time.Duration // maximum response time from a peer otherwise prune
-	minRecvRate int64         // minimum receive rate from peer otherwise prune
-
-	// the maximum number of blocks that should be New, Received or Pending at any point
-	// in time. This is used to enforce a limit on the blockStates map.
-	targetPending int
-	// a list of blocks to be scheduled (New), Pending or Received. Its length should be
-	// smaller than targetPending.
-	blockStates map[int64]blockState
-
-	// a map of heights to the peer we are waiting a response from
-	pendingBlocks map[int64]types.NodeID
-
-	// the time at which a block was put in blockStatePending
-	pendingTime map[int64]time.Time
-
-	// a map of heights to the peers that put the block in blockStateReceived
-	receivedBlocks map[int64]types.NodeID
-}
-
-func (sc scheduler) String() string {
-	return fmt.Sprintf("ih: %d, bst: %v, peers: %v, pblks: %v, ptm %v, rblks: %v",
-		sc.initHeight, sc.blockStates, sc.peers, sc.pendingBlocks, sc.pendingTime, sc.receivedBlocks)
-}
-
-func newScheduler(initHeight int64, startTime time.Time) *scheduler {
-	sc := scheduler{
-		initHeight:     initHeight,
-		lastAdvance:    startTime,
-		syncTimeout:    60 * time.Second,
-		height:         initHeight,
-		blockStates:    make(map[int64]blockState),
-		peers:          make(map[types.NodeID]*scPeer),
-		pendingBlocks:  make(map[int64]types.NodeID),
-		pendingTime:    make(map[int64]time.Time),
-		receivedBlocks: make(map[int64]types.NodeID),
-		targetPending:  10,               // TODO - pass as param
-		peerTimeout:    15 * time.Second, // TODO - pass as param
-		minRecvRate:    0,                // int64(7680), TODO - pass as param
-	}
-
-	return &sc
-}
-
-func (sc *scheduler) ensurePeer(peerID types.NodeID) *scPeer {
-	if _, ok := sc.peers[peerID]; !ok {
-		sc.peers[peerID] = newScPeer(peerID)
-	}
-	return sc.peers[peerID]
-}
-
-func (sc *scheduler) touchPeer(peerID types.NodeID, time time.Time) error {
-	peer, ok := sc.peers[peerID]
-	if !ok {
-		return fmt.Errorf("couldn't find peer %s", peerID)
-	}
-
-	if peer.state != peerStateReady {
-		return fmt.Errorf("tried to touch peer in state %s, must be Ready", peer.state)
-	}
-
-	peer.lastTouched = time
-
-	return nil
-}
-
-func (sc *scheduler) removePeer(peerID types.NodeID) {
-	peer, ok := sc.peers[peerID]
-	if !ok {
-		return
-	}
-	if peer.state == peerStateRemoved {
-		return
-	}
-
-	for height, pendingPeerID := range sc.pendingBlocks {
-		if pendingPeerID == peerID {
-			sc.setStateAtHeight(height, blockStateNew)
-			delete(sc.pendingTime, height)
-			delete(sc.pendingBlocks, height)
-		}
-	}
-
-	for height, rcvPeerID := range sc.receivedBlocks {
-		if rcvPeerID == peerID {
-			sc.setStateAtHeight(height, blockStateNew)
-			delete(sc.receivedBlocks, height)
-		}
-	}
-
-	// remove the blocks from blockStates if the peer removal causes the max peer height to be lower.
-	peer.state = peerStateRemoved
-	maxPeerHeight := int64(0)
-	for _, otherPeer := range sc.peers {
-		if otherPeer.state != peerStateReady {
-			continue
-		}
-		if otherPeer.peerID != peer.peerID && otherPeer.height > maxPeerHeight {
-			maxPeerHeight = otherPeer.height
-		}
-	}
-	for h := range sc.blockStates {
-		if h > maxPeerHeight {
-			delete(sc.blockStates, h)
-		}
-	}
-}
-
-// check if the blockPool is running low and add new blocks in New state to be requested.
-// This function is called when there is an increase in the maximum peer height or when
-// blocks are processed.
-func (sc *scheduler) addNewBlocks() {
-	if len(sc.blockStates) >= sc.targetPending {
-		return
-	}
-
-	for i := sc.height; i < int64(sc.targetPending)+sc.height; i++ {
-		if i > sc.maxHeight() {
-			break
-		}
-		if sc.getStateAtHeight(i) == blockStateUnknown {
-			sc.setStateAtHeight(i, blockStateNew)
-		}
-	}
-}
-
-func (sc *scheduler) setPeerRange(peerID types.NodeID, base int64, height int64) error {
-	peer := sc.ensurePeer(peerID)
-
-	if peer.state == peerStateRemoved {
-		return nil // noop
-	}
-
-	if height < peer.height {
-		sc.removePeer(peerID)
-		return fmt.Errorf("cannot move peer height lower. from %d to %d", peer.height, height)
-	}
-
-	if base > height {
-		sc.removePeer(peerID)
-		return fmt.Errorf("cannot set peer base higher than its height")
-	}
-
-	peer.base = base
-	peer.height = height
-	peer.state = peerStateReady
-
-	sc.addNewBlocks()
-	return nil
-}
-
-func (sc *scheduler) getStateAtHeight(height int64) blockState {
-	if height < sc.height {
-		return blockStateProcessed
-	} else if state, ok := sc.blockStates[height]; ok {
-		return state
-	} else {
-		return blockStateUnknown
-	}
-}
-
-func (sc *scheduler) getPeersWithHeight(height int64) []types.NodeID {
-	peers := make([]types.NodeID, 0)
-	for _, peer := range sc.peers {
-		if peer.state != peerStateReady {
-			continue
-		}
-		if peer.base <= height && peer.height >= height {
-			peers = append(peers, peer.peerID)
-		}
-	}
-	return peers
-}
-
-func (sc *scheduler) prunablePeers(peerTimout time.Duration, minRecvRate int64, now time.Time) []types.NodeID {
-	prunable := make([]types.NodeID, 0)
-	for peerID, peer := range sc.peers {
-		if peer.state != peerStateReady {
-			continue
-		}
-		if now.Sub(peer.lastTouched) > peerTimout || peer.lastRate < minRecvRate {
-			prunable = append(prunable, peerID)
-		}
-	}
-	// Tests for handleTryPrunePeer() may fail without sort due to range non-determinism
-	sort.Sort(PeerByID(prunable))
-	return prunable
-}
-
-func (sc *scheduler) setStateAtHeight(height int64, state blockState) {
-	sc.blockStates[height] = state
-}
-
-// CONTRACT: peer exists and in Ready state.
-func (sc *scheduler) markReceived(peerID types.NodeID, height int64, size int64, now time.Time) error {
-	peer := sc.peers[peerID]
-
-	if state := sc.getStateAtHeight(height); state != blockStatePending || sc.pendingBlocks[height] != peerID {
-		return fmt.Errorf("received block %d from peer %s without being requested", height, peerID)
-	}
-
-	pendingTime, ok := sc.pendingTime[height]
-	if !ok || now.Sub(pendingTime) <= 0 {
-		return fmt.Errorf("clock error: block %d received at %s but requested at %s",
-			height, pendingTime, now)
-	}
-
-	peer.lastRate = size / now.Sub(pendingTime).Nanoseconds()
-
-	sc.setStateAtHeight(height, blockStateReceived)
-	delete(sc.pendingBlocks, height)
-	delete(sc.pendingTime, height)
-
-	sc.receivedBlocks[height] = peerID
-
-	return nil
-}
-
-func (sc *scheduler) markPending(peerID types.NodeID, height int64, time time.Time) error {
-	state := sc.getStateAtHeight(height)
-	if state != blockStateNew {
-		return fmt.Errorf("block %d should be in blockStateNew but is %s", height, state)
-	}
-
-	peer, ok := sc.peers[peerID]
-	if !ok {
-		return fmt.Errorf("cannot find peer %s", peerID)
-	}
-
-	if peer.state != peerStateReady {
-		return fmt.Errorf("cannot schedule %d from %s in %s", height, peerID, peer.state)
-	}
-
-	if height > peer.height {
-		return fmt.Errorf("cannot request height %d from peer %s that is at height %d",
-			height, peerID, peer.height)
-	}
-
-	if height < peer.base {
-		return fmt.Errorf("cannot request height %d for peer %s with base %d",
-			height, peerID, peer.base)
-	}
-
-	sc.setStateAtHeight(height, blockStatePending)
-	sc.pendingBlocks[height] = peerID
-	sc.pendingTime[height] = time
-
-	return nil
-}
-
-func (sc *scheduler) markProcessed(height int64) error {
-	// It is possible that a peer error or timeout is handled after the processor
-	// has processed the block but before the scheduler received this event, so
-	// when pcBlockProcessed event is received, the block had been requested
-	// again => don't check the block state.
-	sc.lastAdvance = time.Now()
-	sc.height = height + 1
-	delete(sc.pendingBlocks, height)
-	delete(sc.pendingTime, height)
-	delete(sc.receivedBlocks, height)
-	delete(sc.blockStates, height)
-	sc.addNewBlocks()
-	return nil
-}
-
-func (sc *scheduler) allBlocksProcessed() bool {
-	if len(sc.peers) == 0 {
-		return false
-	}
-	return sc.height >= sc.maxHeight()
-}
-
-// returns max peer height or the last processed block, i.e. sc.height
-func (sc *scheduler) maxHeight() int64 {
-	max := sc.height - 1
-	for _, peer := range sc.peers {
-		if peer.state != peerStateReady {
-			continue
-		}
-		if max < peer.height {
-			max = peer.height
-		}
-	}
-	return max
-}
-
-// lowest block in sc.blockStates with state == blockStateNew or -1 if no new blocks
-func (sc *scheduler) nextHeightToSchedule() int64 {
-	var min int64 = math.MaxInt64
-	for height, state := range sc.blockStates {
-		if state == blockStateNew && height < min {
-			min = height
-		}
-	}
-	if min == math.MaxInt64 {
-		min = -1
-	}
-	return min
-}
-
-func (sc *scheduler) pendingFrom(peerID types.NodeID) []int64 {
-	var heights []int64
-	for height, pendingPeerID := range sc.pendingBlocks {
-		if pendingPeerID == peerID {
-			heights = append(heights, height)
-		}
-	}
-	return heights
-}
-
-func (sc *scheduler) selectPeer(height int64) (types.NodeID, error) {
-	peers := sc.getPeersWithHeight(height)
-	if len(peers) == 0 {
-		return "", fmt.Errorf("cannot find peer for height %d", height)
-	}
-
-	// create a map from number of pending requests to a list
-	// of peers having that number of pending requests.
-	pendingFrom := make(map[int][]types.NodeID)
-	for _, peerID := range peers {
-		numPending := len(sc.pendingFrom(peerID))
-		pendingFrom[numPending] = append(pendingFrom[numPending], peerID)
-	}
-
-	// find the set of peers with minimum number of pending requests.
-	var minPending int64 = math.MaxInt64
-	for mp := range pendingFrom {
-		if int64(mp) < minPending {
-			minPending = int64(mp)
-		}
-	}
-
-	sort.Sort(PeerByID(pendingFrom[int(minPending)]))
-	return pendingFrom[int(minPending)][0], nil
-}
-
-// PeerByID is a list of peers sorted by peerID.
-type PeerByID []types.NodeID
-
-func (peers PeerByID) Len() int {
-	return len(peers)
-}
-func (peers PeerByID) Less(i, j int) bool {
-	return bytes.Compare([]byte(peers[i]), []byte(peers[j])) == -1
-}
-
-func (peers PeerByID) Swap(i, j int) {
-	peers[i], peers[j] = peers[j], peers[i]
-}
-
-// Handlers
-
-// This handler gets the block, performs some validation and then passes it on to the processor.
-func (sc *scheduler) handleBlockResponse(event bcBlockResponse) (Event, error) {
-	err := sc.touchPeer(event.peerID, event.time)
-	if err != nil {
-		// peer does not exist OR not ready
-		return noOp, nil
-	}
-
-	err = sc.markReceived(event.peerID, event.block.Height, event.size, event.time)
-	if err != nil {
-		sc.removePeer(event.peerID)
-		return scPeerError{peerID: event.peerID, reason: err}, nil
-	}
-
-	return scBlockReceived{peerID: event.peerID, block: event.block}, nil
-}
-
-func (sc *scheduler) handleNoBlockResponse(event bcNoBlockResponse) (Event, error) {
-	// No such peer or peer was removed.
-	peer, ok := sc.peers[event.peerID]
-	if !ok || peer.state == peerStateRemoved {
-		return noOp, nil
-	}
-
-	// The peer may have been just removed due to errors, low speed or timeouts.
-	sc.removePeer(event.peerID)
-
-	return scPeerError{peerID: event.peerID,
-		reason: fmt.Errorf("peer %v with base %d height %d claims no block for %d",
-			event.peerID, peer.base, peer.height, event.height)}, nil
-}
-
-func (sc *scheduler) handleBlockProcessed(event pcBlockProcessed) (Event, error) {
-	if event.height != sc.height {
-		panic(fmt.Sprintf("processed height %d, but expected height %d", event.height, sc.height))
-	}
-
-	err := sc.markProcessed(event.height)
-	if err != nil {
-		return scSchedulerFail{reason: err}, nil
-	}
-
-	if sc.allBlocksProcessed() {
-		return scFinishedEv{reason: "processed all blocks"}, nil
-	}
-
-	return noOp, nil
-}
-
-// Handles an error from the processor. The processor had already cleaned the blocks from
-// the peers included in this event. Just attempt to remove the peers.
-func (sc *scheduler) handleBlockProcessError(event pcBlockVerificationFailure) (Event, error) {
-	// The peers may have been just removed due to errors, low speed or timeouts.
-	sc.removePeer(event.firstPeerID)
-	if event.firstPeerID != event.secondPeerID {
-		sc.removePeer(event.secondPeerID)
-	}
-
-	if sc.allBlocksProcessed() {
-		return scFinishedEv{reason: "error on last block"}, nil
-	}
-
-	return noOp, nil
-}
-
-func (sc *scheduler) handleAddNewPeer(event bcAddNewPeer) (Event, error) {
-	sc.ensurePeer(event.peerID)
-	return noOp, nil
-}
-
-func (sc *scheduler) handleRemovePeer(event bcRemovePeer) (Event, error) {
-	sc.removePeer(event.peerID)
-
-	if sc.allBlocksProcessed() {
-		return scFinishedEv{reason: "removed peer"}, nil
-	}
-
-	// Return scPeerError so the peer (and all associated blocks) is removed from
-	// the processor.
-	return scPeerError{peerID: event.peerID, reason: errors.New("peer was stopped")}, nil
-}
-
-func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
-	// Check behavior of peer responsible to deliver block at sc.height.
-	timeHeightAsked, ok := sc.pendingTime[sc.height]
-	if ok && time.Since(timeHeightAsked) > sc.peerTimeout {
-		// A request was sent to a peer for block at sc.height but a response was not received
-		// from that peer within sc.peerTimeout. Remove the peer. This is to ensure that a peer
-		// will be timed out even if it sends blocks at higher heights but prevents progress by
-		// not sending the block at current height.
-		sc.removePeer(sc.pendingBlocks[sc.height])
-	}
-
-	prunablePeers := sc.prunablePeers(sc.peerTimeout, sc.minRecvRate, event.time)
-	if len(prunablePeers) == 0 {
-		return noOp, nil
-	}
-	for _, peerID := range prunablePeers {
-		sc.removePeer(peerID)
-	}
-
-	// If all blocks are processed we should finish.
-	if sc.allBlocksProcessed() {
-		return scFinishedEv{reason: "after try prune"}, nil
-	}
-
-	return scPeersPruned{peers: prunablePeers}, nil
-}
-
-func (sc *scheduler) handleResetState(event bcResetState) (Event, error) {
-	initHeight := event.state.LastBlockHeight + 1
-	if initHeight == 1 {
-		initHeight = event.state.InitialHeight
-	}
-	sc.initHeight = initHeight
-	sc.height = initHeight
-	sc.lastAdvance = time.Now()
-	sc.addNewBlocks()
-	return noOp, nil
-}
-
-func (sc *scheduler) handleTrySchedule(event rTrySchedule) (Event, error) {
-	if time.Since(sc.lastAdvance) > sc.syncTimeout {
-		return scFinishedEv{reason: "timeout, no advance"}, nil
-	}
-
-	nextHeight := sc.nextHeightToSchedule()
-	if nextHeight == -1 {
-		return noOp, nil
-	}
-
-	bestPeerID, err := sc.selectPeer(nextHeight)
-	if err != nil {
-		return scSchedulerFail{reason: err}, nil
-	}
-	if err := sc.markPending(bestPeerID, nextHeight, event.time); err != nil {
-		return scSchedulerFail{reason: err}, nil // XXX: peerError might be more appropriate
-	}
-	return scBlockRequest{peerID: bestPeerID, height: nextHeight}, nil
-
-}
-
-func (sc *scheduler) handleStatusResponse(event bcStatusResponse) (Event, error) {
-	err := sc.setPeerRange(event.peerID, event.base, event.height)
-	if err != nil {
-		return scPeerError{peerID: event.peerID, reason: err}, nil
-	}
-	return noOp, nil
-}
-
-func (sc *scheduler) handle(event Event) (Event, error) {
-	switch event := event.(type) {
-	case bcResetState:
-		nextEvent, err := sc.handleResetState(event)
-		return nextEvent, err
-	case bcStatusResponse:
-		nextEvent, err := sc.handleStatusResponse(event)
-		return nextEvent, err
-	case bcBlockResponse:
-		nextEvent, err := sc.handleBlockResponse(event)
-		return nextEvent, err
-	case bcNoBlockResponse:
-		nextEvent, err := sc.handleNoBlockResponse(event)
-		return nextEvent, err
-	case rTrySchedule:
-		nextEvent, err := sc.handleTrySchedule(event)
-		return nextEvent, err
-	case bcAddNewPeer:
-		nextEvent, err := sc.handleAddNewPeer(event)
-		return nextEvent, err
-	case bcRemovePeer:
-		nextEvent, err := sc.handleRemovePeer(event)
-		return nextEvent, err
-	case rTryPrunePeer:
-		nextEvent, err := sc.handleTryPrunePeer(event)
-		return nextEvent, err
-	case pcBlockProcessed:
-		nextEvent, err := sc.handleBlockProcessed(event)
-		return nextEvent, err
-	case pcBlockVerificationFailure:
-		nextEvent, err := sc.handleBlockProcessError(event)
-		return nextEvent, err
-	default:
-		return scSchedulerFail{reason: fmt.Errorf("unknown event %v", event)}, nil
-	}
-}

internal/blocksync/v2/types.go

@@ -1,65 +0,0 @@
-package v2
-
-import (
-	"github.com/Workiva/go-datastructures/queue"
-)
-
-// Event is the type that can be added to the priority queue.
-type Event queue.Item
-
-type priority interface {
-	Compare(other queue.Item) int
-	Priority() int
-}
-
-type priorityLow struct{}
-type priorityNormal struct{}
-type priorityHigh struct{}
-
-func (p priorityLow) Priority() int {
-	return 1
-}
-
-func (p priorityNormal) Priority() int {
-	return 2
-}
-
-func (p priorityHigh) Priority() int {
-	return 3
-}
-
-func (p priorityLow) Compare(other queue.Item) int {
-	op := other.(priority)
-	if p.Priority() > op.Priority() {
-		return 1
-	} else if p.Priority() == op.Priority() {
-		return 0
-	}
-	return -1
-}
-
-func (p priorityNormal) Compare(other queue.Item) int {
-	op := other.(priority)
-	if p.Priority() > op.Priority() {
-		return 1
-	} else if p.Priority() == op.Priority() {
-		return 0
-	}
-	return -1
-}
-
-func (p priorityHigh) Compare(other queue.Item) int {
-	op := other.(priority)
-	if p.Priority() > op.Priority() {
-		return 1
-	} else if p.Priority() == op.Priority() {
-		return 0
-	}
-	return -1
-}
-
-type noOpEvent struct {
-	priorityLow
-}
-
-var noOp = noOpEvent{}

internal/consensus/byzantine_test.go

@@ -10,20 +10,21 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	abcicli "github.com/tendermint/tendermint/abci/client"
+	dbm "github.com/tendermint/tm-db"
+
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/internal/evidence"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
 	"github.com/tendermint/tendermint/internal/p2p"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
-	dbm "github.com/tendermint/tm-db"
 )
 
 // Byzantine node sends two different prevotes (nil and blockID) to the same
@@ -61,9 +62,9 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 			blockStore := store.NewBlockStore(blockDB)
 
 			// one for mempool, one for consensus
-			mtx := new(tmsync.RWMutex)
-			proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
-			proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+			mtx := new(tmsync.Mutex)
+			proxyAppConnMem := abciclient.NewLocalClient(mtx, app)
+			proxyAppConnCon := abciclient.NewLocalClient(mtx, app)
 
 			// Make Mempool
 			mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)

internal/consensus/common_test.go

@@ -7,24 +7,24 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/require"
-
-	"path"
-
 	dbm "github.com/tendermint/tm-db"
 
-	abcicli "github.com/tendermint/tendermint/abci/client"
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	tmbytes "github.com/tendermint/tendermint/libs/bytes"
 	"github.com/tendermint/tendermint/libs/log"
@@ -33,8 +33,6 @@ import (
 	tmtime "github.com/tendermint/tendermint/libs/time"
 	"github.com/tendermint/tendermint/privval"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -49,10 +47,10 @@ const (
 // test.
 type cleanupFunc func()
 
-func configSetup(t *testing.T) *cfg.Config {
+func configSetup(t *testing.T) *config.Config {
 	t.Helper()
 
-	config := ResetConfig("consensus_reactor_test")
+	cfg := ResetConfig("consensus_reactor_test")
 
 	consensusReplayConfig := ResetConfig("consensus_replay_test")
 	configStateTest := ResetConfig("consensus_state_test")
@@ -60,13 +58,13 @@ func configSetup(t *testing.T) *cfg.Config {
 	configByzantineTest := ResetConfig("consensus_byzantine_test")
 
 	t.Cleanup(func() {
-		os.RemoveAll(config.RootDir)
+		os.RemoveAll(cfg.RootDir)
 		os.RemoveAll(consensusReplayConfig.RootDir)
 		os.RemoveAll(configStateTest.RootDir)
 		os.RemoveAll(configMempoolTest.RootDir)
 		os.RemoveAll(configByzantineTest.RootDir)
 	})
-	return config
+	return cfg
 }
 
 func ensureDir(dir string, mode os.FileMode) {
@@ -75,8 +73,8 @@ func ensureDir(dir string, mode os.FileMode) {
 	}
 }
 
-func ResetConfig(name string) *cfg.Config {
-	return cfg.ResetTestRoot(name)
+func ResetConfig(name string) *config.Config {
+	return config.ResetTestRoot(name)
 }
 
 //-------------------------------------------------------------------------------
@@ -102,7 +100,7 @@ func newValidatorStub(privValidator types.PrivValidator, valIndex int32) *valida
 }
 
 func (vs *validatorStub) signVote(
-	config *cfg.Config,
+	cfg *config.Config,
 	voteType tmproto.SignedMsgType,
 	hash []byte,
 	header types.PartSetHeader) (*types.Vote, error) {
@@ -122,7 +120,7 @@ func (vs *validatorStub) signVote(
 		BlockID:          types.BlockID{Hash: hash, PartSetHeader: header},
 	}
 	v := vote.ToProto()
-	if err := vs.PrivValidator.SignVote(context.Background(), config.ChainID(), v); err != nil {
+	if err := vs.PrivValidator.SignVote(context.Background(), cfg.ChainID(), v); err != nil {
 		return nil, fmt.Errorf("sign vote failed: %w", err)
 	}
 
@@ -141,12 +139,12 @@ func (vs *validatorStub) signVote(
 // Sign vote for type/hash/header
 func signVote(
 	vs *validatorStub,
-	config *cfg.Config,
+	cfg *config.Config,
 	voteType tmproto.SignedMsgType,
 	hash []byte,
 	header types.PartSetHeader) *types.Vote {
 
-	v, err := vs.signVote(config, voteType, hash, header)
+	v, err := vs.signVote(cfg, voteType, hash, header)
 	if err != nil {
 		panic(fmt.Errorf("failed to sign vote: %v", err))
 	}
@@ -157,14 +155,14 @@ func signVote(
 }
 
 func signVotes(
-	config *cfg.Config,
+	cfg *config.Config,
 	voteType tmproto.SignedMsgType,
 	hash []byte,
 	header types.PartSetHeader,
 	vss ...*validatorStub) []*types.Vote {
 	votes := make([]*types.Vote, len(vss))
 	for i, vs := range vss {
-		votes[i] = signVote(vs, config, voteType, hash, header)
+		votes[i] = signVote(vs, cfg, voteType, hash, header)
 	}
 	return votes
 }
@@ -255,14 +253,14 @@ func addVotes(to *State, votes ...*types.Vote) {
 }
 
 func signAddVotes(
-	config *cfg.Config,
+	cfg *config.Config,
 	to *State,
 	voteType tmproto.SignedMsgType,
 	hash []byte,
 	header types.PartSetHeader,
 	vss ...*validatorStub,
 ) {
-	votes := signVotes(config, voteType, hash, header, vss...)
+	votes := signVotes(cfg, voteType, hash, header, vss...)
 	addVotes(to, votes...)
 }
 
@@ -387,12 +385,12 @@ func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message {
 // consensus states
 
 func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State {
-	config := cfg.ResetTestRoot("consensus_state_test")
-	return newStateWithConfig(config, state, pv, app)
+	cfg := config.ResetTestRoot("consensus_state_test")
+	return newStateWithConfig(cfg, state, pv, app)
 }
 
 func newStateWithConfig(
-	thisConfig *cfg.Config,
+	thisConfig *config.Config,
 	state sm.State,
 	pv types.PrivValidator,
 	app abci.Application,
@@ -402,16 +400,16 @@ func newStateWithConfig(
 }
 
 func newStateWithConfigAndBlockStore(
-	thisConfig *cfg.Config,
+	thisConfig *config.Config,
 	state sm.State,
 	pv types.PrivValidator,
 	app abci.Application,
 	blockStore *store.BlockStore,
 ) *State {
 	// one for mempool, one for consensus
-	mtx := new(tmsync.RWMutex)
-	proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
-	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+	mtx := new(tmsync.Mutex)
+	proxyAppConnMem := abciclient.NewLocalClient(mtx, app)
+	proxyAppConnCon := abciclient.NewLocalClient(mtx, app)
 
 	// Make Mempool
 	mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
@@ -444,10 +442,10 @@ func newStateWithConfigAndBlockStore(
 	return cs
 }
 
-func loadPrivValidator(config *cfg.Config) *privval.FilePV {
-	privValidatorKeyFile := config.PrivValidator.KeyFile()
+func loadPrivValidator(cfg *config.Config) *privval.FilePV {
+	privValidatorKeyFile := cfg.PrivValidator.KeyFile()
 	ensureDir(filepath.Dir(privValidatorKeyFile), 0700)
-	privValidatorStateFile := config.PrivValidator.StateFile()
+	privValidatorStateFile := cfg.PrivValidator.StateFile()
 	privValidator, err := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
 	if err != nil {
 		panic(err)
@@ -456,9 +454,9 @@ func loadPrivValidator(config *cfg.Config) *privval.FilePV {
 	return privValidator
 }
 
-func randState(config *cfg.Config, nValidators int) (*State, []*validatorStub) {
+func randState(cfg *config.Config, nValidators int) (*State, []*validatorStub) {
 	// Get State
-	state, privVals := randGenesisState(config, nValidators, false, 10)
+	state, privVals := randGenesisState(cfg, nValidators, false, 10)
 
 	vss := make([]*validatorStub, nValidators)
 
@@ -704,15 +702,15 @@ func consensusLogger() log.Logger {
 
 func randConsensusState(
 	t *testing.T,
-	config *cfg.Config,
+	cfg *config.Config,
 	nValidators int,
 	testName string,
 	tickerFunc func() TimeoutTicker,
 	appFunc func() abci.Application,
-	configOpts ...func(*cfg.Config),
+	configOpts ...func(*config.Config),
 ) ([]*State, cleanupFunc) {
 
-	genDoc, privVals := factory.RandGenesisDoc(config, nValidators, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, nValidators, false, 30)
 	css := make([]*State, nValidators)
 	logger := consensusLogger()
 
@@ -759,18 +757,18 @@ func randConsensusState(
 
 // nPeers = nValidators + nNotValidator
 func randConsensusNetWithPeers(
-	config *cfg.Config,
+	cfg *config.Config,
 	nValidators,
 	nPeers int,
 	testName string,
 	tickerFunc func() TimeoutTicker,
 	appFunc func(string) abci.Application,
-) ([]*State, *types.GenesisDoc, *cfg.Config, cleanupFunc) {
-	genDoc, privVals := factory.RandGenesisDoc(config, nValidators, false, testMinPower)
+) ([]*State, *types.GenesisDoc, *config.Config, cleanupFunc) {
+	genDoc, privVals := factory.RandGenesisDoc(cfg, nValidators, false, testMinPower)
 	css := make([]*State, nPeers)
 	logger := consensusLogger()
 
-	var peer0Config *cfg.Config
+	var peer0Config *config.Config
 	configRootDirs := make([]string, 0, nPeers)
 	for i := 0; i < nPeers; i++ {
 		state, _ := sm.MakeGenesisState(genDoc)
@@ -799,7 +797,7 @@ func randConsensusNetWithPeers(
 			}
 		}
 
-		app := appFunc(path.Join(config.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
+		app := appFunc(path.Join(cfg.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
 		vals := types.TM2PB.ValidatorUpdates(state.Validators)
 		if _, ok := app.(*kvstore.PersistentKVStoreApplication); ok {
 			// simulate handshake, receive app version. If don't do this, replay test will fail
@@ -820,12 +818,12 @@ func randConsensusNetWithPeers(
 }
 
 func randGenesisState(
-	config *cfg.Config,
+	cfg *config.Config,
 	numValidators int,
 	randPower bool,
 	minPower int64) (sm.State, []types.PrivValidator) {
 
-	genDoc, privValidators := factory.RandGenesisDoc(config, numValidators, randPower, minPower)
+	genDoc, privValidators := factory.RandGenesisDoc(cfg, numValidators, randPower, minPower)
 	s0, _ := sm.MakeGenesisState(genDoc)
 	return s0, privValidators
 }

internal/consensus/mempool_test.go

@@ -10,20 +10,19 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/tendermint/tendermint/abci/example/code"
 	abci "github.com/tendermint/tendermint/abci/types"
-	mempl "github.com/tendermint/tendermint/internal/mempool"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/internal/mempool"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/types"
 )
 
 // for testing
-func assertMempool(txn txNotifier) mempl.Mempool {
-	return txn.(mempl.Mempool)
+func assertMempool(txn txNotifier) mempool.Mempool {
+	return txn.(mempool.Mempool)
 }
 
 func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
@@ -113,7 +112,7 @@ func deliverTxsRange(cs *State, start, end int) {
 	for i := start; i < end; i++ {
 		txBytes := make([]byte, 8)
 		binary.BigEndian.PutUint64(txBytes, uint64(i))
-		err := assertMempool(cs.txNotifier).CheckTx(context.Background(), txBytes, nil, mempl.TxInfo{})
+		err := assertMempool(cs.txNotifier).CheckTx(context.Background(), txBytes, nil, mempool.TxInfo{})
 		if err != nil {
 			panic(fmt.Sprintf("Error after CheckTx: %v", err))
 		}
@@ -179,7 +178,7 @@ func TestMempoolRmBadTx(t *testing.T) {
 				return
 			}
 			checkTxRespCh <- struct{}{}
-		}, mempl.TxInfo{})
+		}, mempool.TxInfo{})
 		if err != nil {
 			t.Errorf("error after CheckTx: %v", err)
 			return

internal/consensus/metrics.go

@@ -61,6 +61,9 @@ type Metrics struct {
 
 	// Number of blockparts transmitted by peer.
 	BlockParts metrics.Counter
+
+	// Histogram of time taken per step annotated with reason that the step proceeded.
+	StepTime metrics.Histogram
 }
 
 // PrometheusMetrics returns Metrics build using Prometheus client library.
@@ -187,6 +190,12 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
 			Name:      "block_parts",
 			Help:      "Number of blockparts transmitted by peer.",
 		}, append(labels, "peer_id")).With(labelsAndValues...),
+		StepTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "step_time",
+			Help:      "Time spent per step.",
+		}, append(labels, "step", "reason")).With(labelsAndValues...),
 	}
 }
 

internal/consensus/mocks/cons_sync_reactor.go

@@ -4,7 +4,7 @@ package mocks
 
 import (
 	mock "github.com/stretchr/testify/mock"
-	state "github.com/tendermint/tendermint/state"
+	state "github.com/tendermint/tendermint/internal/state"
 )
 
 // ConsSyncReactor is an autogenerated mock type for the ConsSyncReactor type

internal/consensus/mocks/fast_sync_reactor.go

@@ -4,7 +4,7 @@ package mocks
 
 import (
 	mock "github.com/stretchr/testify/mock"
-	state "github.com/tendermint/tendermint/state"
+	state "github.com/tendermint/tendermint/internal/state"
 
 	time "time"
 )

internal/consensus/msgs.go

@@ -77,7 +77,7 @@ func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error {
 			m.LastCommitRound, initialHeight)
 	}
 	if m.Height > initialHeight && m.LastCommitRound < 0 {
-		return fmt.Errorf("LastCommitRound can only be negative for initial height %v", // nolint
+		return fmt.Errorf("LastCommitRound can only be negative for initial height %v",
 			initialHeight)
 	}
 	return nil

internal/consensus/reactor.go

@@ -8,77 +8,62 @@ import (
 	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/p2p"
+	sm "github.com/tendermint/tendermint/internal/state"
 	"github.com/tendermint/tendermint/libs/bits"
 	tmevents "github.com/tendermint/tendermint/libs/events"
 	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/libs/service"
 	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
 var (
 	_ service.Service = (*Reactor)(nil)
 	_ p2p.Wrapper     = (*tmcons.Message)(nil)
+)
 
-	// ChannelShims contains a map of ChannelDescriptorShim objects, where each
-	// object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
-	// p2p proto.Message the new p2p Channel is responsible for handling.
-	//
-	//
-	// TODO: Remove once p2p refactor is complete.
-	// ref: https://github.com/tendermint/tendermint/issues/5670
-	ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
-		StateChannel: {
-			MsgType: new(tmcons.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(StateChannel),
-				Priority:            8,
-				SendQueueCapacity:   64,
-				RecvMessageCapacity: maxMsgSize,
-				RecvBufferCapacity:  128,
-				MaxSendBytes:        12000,
-			},
+// GetChannelDescriptor produces an instance of a descriptor for this
+// package's required channels.
+func GetChannelDescriptors() []*p2p.ChannelDescriptor {
+	return []*p2p.ChannelDescriptor{
+		{
+			ID:                  StateChannel,
+			MessageType:         new(tmcons.Message),
+			Priority:            8,
+			SendQueueCapacity:   64,
+			RecvMessageCapacity: maxMsgSize,
+			RecvBufferCapacity:  128,
 		},
-		DataChannel: {
-			MsgType: new(tmcons.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				// TODO: Consider a split between gossiping current block and catchup
-				// stuff. Once we gossip the whole block there is nothing left to send
-				// until next height or round.
-				ID:                  byte(DataChannel),
-				Priority:            12,
-				SendQueueCapacity:   64,
-				RecvBufferCapacity:  512,
-				RecvMessageCapacity: maxMsgSize,
-				MaxSendBytes:        40000,
-			},
+		{
+			// TODO: Consider a split between gossiping current block and catchup
+			// stuff. Once we gossip the whole block there is nothing left to send
+			// until next height or round.
+			ID:                  DataChannel,
+			MessageType:         new(tmcons.Message),
+			Priority:            12,
+			SendQueueCapacity:   64,
+			RecvBufferCapacity:  512,
+			RecvMessageCapacity: maxMsgSize,
 		},
-		VoteChannel: {
-			MsgType: new(tmcons.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(VoteChannel),
-				Priority:            10,
-				SendQueueCapacity:   64,
-				RecvBufferCapacity:  128,
-				RecvMessageCapacity: maxMsgSize,
-				MaxSendBytes:        150,
-			},
+		{
+			ID:                  VoteChannel,
+			MessageType:         new(tmcons.Message),
+			Priority:            10,
+			SendQueueCapacity:   64,
+			RecvBufferCapacity:  128,
+			RecvMessageCapacity: maxMsgSize,
 		},
-		VoteSetBitsChannel: {
-			MsgType: new(tmcons.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(VoteSetBitsChannel),
-				Priority:            5,
-				SendQueueCapacity:   8,
-				RecvBufferCapacity:  128,
-				RecvMessageCapacity: maxMsgSize,
-				MaxSendBytes:        50,
-			},
+		{
+			ID:                  VoteSetBitsChannel,
+			MessageType:         new(tmcons.Message),
+			Priority:            5,
+			SendQueueCapacity:   8,
+			RecvBufferCapacity:  128,
+			RecvMessageCapacity: maxMsgSize,
 		},
 	}
-)
+}
 
 const (
 	StateChannel       = p2p.ChannelID(0x20)
@@ -230,17 +215,15 @@ func (r *Reactor) OnStop() {
 	}
 
 	r.mtx.Lock()
-	peers := r.peers
+	// Close and wait for each of the peers to shutdown.
+	// This is safe to perform with the lock since none of the peers require the
+	// lock to complete any of the methods that the waitgroup is waiting on.
+	for _, state := range r.peers {
+		state.closer.Close()
+		state.broadcastWG.Wait()
+	}
 	r.mtx.Unlock()
 
-	// wait for all spawned peer goroutines to gracefully exit
-	for _, ps := range peers {
-		ps.closer.Close()
-	}
-	for _, ps := range peers {
-		ps.broadcastWG.Wait()
-	}
-
 	// Close the StateChannel goroutine separately since it uses its own channel
 	// to signal closure.
 	close(r.stateCloseCh)
@@ -1096,7 +1079,7 @@ func (r *Reactor) handleDataMessage(envelope p2p.Envelope, msgI Message) error {
 	}
 
 	if r.WaitSync() {
-		logger.Info("ignoring message received during sync", "msg", msgI)
+		logger.Info("ignoring message received during sync", "msg", fmt.Sprintf("%T", msgI))
 		return nil
 	}
 

internal/consensus/reactor_test.go

@@ -12,26 +12,26 @@ import (
 	"github.com/fortytw2/leaktest"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
 
-	abcicli "github.com/tendermint/tendermint/abci/client"
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
-	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/crypto/encoding"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/mempool"
 	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
 	"github.com/tendermint/tendermint/internal/p2p"
 	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
+	sm "github.com/tendermint/tendermint/internal/state"
+	statemocks "github.com/tendermint/tendermint/internal/state/mocks"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
-	sm "github.com/tendermint/tendermint/state"
-	statemocks "github.com/tendermint/tendermint/state/mocks"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
-	dbm "github.com/tendermint/tm-db"
 )
 
 var (
@@ -50,9 +50,11 @@ type reactorTestSuite struct {
 	voteSetBitsChannels map[types.NodeID]*p2p.Channel
 }
 
-func chDesc(chID p2p.ChannelID) p2p.ChannelDescriptor {
-	return p2p.ChannelDescriptor{
-		ID: byte(chID),
+func chDesc(chID p2p.ChannelID, size int) *p2p.ChannelDescriptor {
+	return &p2p.ChannelDescriptor{
+		ID:                 chID,
+		MessageType:        new(tmcons.Message),
+		RecvBufferCapacity: size,
 	}
 }
 
@@ -67,10 +69,10 @@ func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSu
 		blocksyncSubs: make(map[types.NodeID]types.Subscription, numNodes),
 	}
 
-	rts.stateChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(StateChannel), new(tmcons.Message), size)
-	rts.dataChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(DataChannel), new(tmcons.Message), size)
-	rts.voteChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteChannel), new(tmcons.Message), size)
-	rts.voteSetBitsChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteSetBitsChannel), new(tmcons.Message), size)
+	rts.stateChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(StateChannel, size))
+	rts.dataChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(DataChannel, size))
+	rts.voteChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteChannel, size))
+	rts.voteSetBitsChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteSetBitsChannel, size))
 
 	_, cancel := context.WithCancel(context.Background())
 
@@ -273,11 +275,11 @@ func ensureBlockSyncStatus(t *testing.T, msg tmpubsub.Message, complete bool, he
 }
 
 func TestReactorBasic(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	n := 4
 	states, cleanup := randConsensusState(t,
-		config, n, "consensus_reactor_test",
+		cfg, n, "consensus_reactor_test",
 		newMockTickerFunc(true), newKVStore)
 	t.Cleanup(cleanup)
 
@@ -316,14 +318,14 @@ func TestReactorBasic(t *testing.T) {
 }
 
 func TestReactorWithEvidence(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	n := 4
 	testName := "consensus_reactor_test"
 	tickerFunc := newMockTickerFunc(true)
 	appFunc := newKVStore
 
-	genDoc, privVals := factory.RandGenesisDoc(config, n, false, 30)
+	genDoc, privVals := factory.RandGenesisDoc(cfg, n, false, 30)
 	states := make([]*State, n)
 	logger := consensusLogger()
 
@@ -346,9 +348,9 @@ func TestReactorWithEvidence(t *testing.T) {
 		blockStore := store.NewBlockStore(blockDB)
 
 		// one for mempool, one for consensus
-		mtx := new(tmsync.RWMutex)
-		proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
-		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+		mtx := new(tmsync.Mutex)
+		proxyAppConnMem := abciclient.NewLocalClient(mtx, app)
+		proxyAppConnCon := abciclient.NewLocalClient(mtx, app)
 
 		mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
 		mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
@@ -360,7 +362,7 @@ func TestReactorWithEvidence(t *testing.T) {
 		// everyone includes evidence of another double signing
 		vIdx := (i + 1) % n
 
-		ev := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID())
+		ev := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], cfg.ChainID())
 		evpool := &statemocks.EvidencePool{}
 		evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil)
 		evpool.On("PendingEvidence", mock.AnythingOfType("int64")).Return([]types.Evidence{
@@ -412,17 +414,17 @@ func TestReactorWithEvidence(t *testing.T) {
 }
 
 func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	n := 4
 	states, cleanup := randConsensusState(
 		t,
-		config,
+		cfg,
 		n,
 		"consensus_reactor_test",
 		newMockTickerFunc(true),
 		newKVStore,
-		func(c *cfg.Config) {
+		func(c *config.Config) {
 			c.Consensus.CreateEmptyBlocks = false
 		},
 	)
@@ -462,11 +464,11 @@ func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
 }
 
 func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	n := 4
 	states, cleanup := randConsensusState(t,
-		config, n, "consensus_reactor_test",
+		cfg, n, "consensus_reactor_test",
 		newMockTickerFunc(true), newKVStore)
 	t.Cleanup(cleanup)
 
@@ -521,12 +523,12 @@ func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
 }
 
 func TestReactorVotingPowerChange(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	n := 4
 	states, cleanup := randConsensusState(
 		t,
-		config,
+		cfg,
 		n,
 		"consensus_voting_power_changes_test",
 		newMockTickerFunc(true),
@@ -573,7 +575,7 @@ func TestReactorVotingPowerChange(t *testing.T) {
 	val1PubKey, err := states[0].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
 
-	val1PubKeyABCI, err := cryptoenc.PubKeyToProto(val1PubKey)
+	val1PubKeyABCI, err := encoding.PubKeyToProto(val1PubKey)
 	require.NoError(t, err)
 
 	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
@@ -622,12 +624,12 @@ func TestReactorVotingPowerChange(t *testing.T) {
 }
 
 func TestReactorValidatorSetChanges(t *testing.T) {
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	nPeers := 7
 	nVals := 4
 	states, _, _, cleanup := randConsensusNetWithPeers(
-		config,
+		cfg,
 		nVals,
 		nPeers,
 		"consensus_val_set_changes_test",
@@ -668,7 +670,7 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	newValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
 
-	valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1)
+	valPubKey1ABCI, err := encoding.PubKeyToProto(newValidatorPubKey1)
 	require.NoError(t, err)
 
 	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
@@ -701,7 +703,7 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	updateValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
 
-	updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1)
+	updatePubKey1ABCI, err := encoding.PubKeyToProto(updateValidatorPubKey1)
 	require.NoError(t, err)
 
 	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
@@ -721,7 +723,7 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	newValidatorPubKey2, err := states[nVals+1].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
 
-	newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2)
+	newVal2ABCI, err := encoding.PubKeyToProto(newValidatorPubKey2)
 	require.NoError(t, err)
 
 	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
@@ -729,7 +731,7 @@ func TestReactorValidatorSetChanges(t *testing.T) {
 	newValidatorPubKey3, err := states[nVals+2].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
 
-	newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3)
+	newVal3ABCI, err := encoding.PubKeyToProto(newValidatorPubKey3)
 	require.NoError(t, err)
 
 	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)

internal/consensus/replay_file.go

@@ -12,13 +12,13 @@ import (
 
 	dbm "github.com/tendermint/tm-db"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/proxy"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/libs/log"
 	tmos "github.com/tendermint/tendermint/libs/os"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -31,8 +31,8 @@ const (
 // replay messages interactively or all at once
 
 // replay the wal file
-func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) {
-	consensusState := newConsensusStateForReplay(config, csConfig)
+func RunReplayFile(cfg config.BaseConfig, csConfig *config.ConsensusConfig, console bool) {
+	consensusState := newConsensusStateForReplay(cfg, csConfig)
 
 	if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil {
 		tmos.Exit(fmt.Sprintf("Error during consensus replay: %v", err))
@@ -286,22 +286,22 @@ func (pb *playback) replayConsoleLoop() int {
 //--------------------------------------------------------------------------------
 
 // convenience for replay mode
-func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *State {
-	dbType := dbm.BackendType(config.DBBackend)
+func newConsensusStateForReplay(cfg config.BaseConfig, csConfig *config.ConsensusConfig) *State {
+	dbType := dbm.BackendType(cfg.DBBackend)
 	// Get BlockStore
-	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, cfg.DBDir())
 	if err != nil {
 		tmos.Exit(err.Error())
 	}
 	blockStore := store.NewBlockStore(blockStoreDB)
 
 	// Get State
-	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	stateDB, err := dbm.NewDB("state", dbType, cfg.DBDir())
 	if err != nil {
 		tmos.Exit(err.Error())
 	}
 	stateStore := sm.NewStore(stateDB)
-	gdoc, err := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	gdoc, err := sm.MakeGenesisDocFromFile(cfg.GenesisFile())
 	if err != nil {
 		tmos.Exit(err.Error())
 	}
@@ -311,8 +311,8 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 	}
 
 	// Create proxyAppConn connection (consensus, mempool, query)
-	clientCreator, _ := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
-	proxyApp := proxy.NewAppConns(clientCreator)
+	clientCreator, _ := proxy.DefaultClientCreator(cfg.ProxyApp, cfg.ABCI, cfg.DBDir())
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	err = proxyApp.Start()
 	if err != nil {
 		tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))

internal/consensus/replay_stubs.go

@@ -5,7 +5,7 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/internal/libs/clist"
-	mempl "github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -13,22 +13,23 @@ import (
 
 type emptyMempool struct{}
 
-var _ mempl.Mempool = emptyMempool{}
+var _ mempool.Mempool = emptyMempool{}
 
 func (emptyMempool) Lock()     {}
 func (emptyMempool) Unlock()   {}
 func (emptyMempool) Size() int { return 0 }
-func (emptyMempool) CheckTx(_ context.Context, _ types.Tx, _ func(*abci.Response), _ mempl.TxInfo) error {
+func (emptyMempool) CheckTx(_ context.Context, _ types.Tx, _ func(*abci.Response), _ mempool.TxInfo) error {
 	return nil
 }
+func (emptyMempool) RemoveTxByKey(txKey types.TxKey) error   { return nil }
 func (emptyMempool) ReapMaxBytesMaxGas(_, _ int64) types.Txs { return types.Txs{} }
 func (emptyMempool) ReapMaxTxs(n int) types.Txs              { return types.Txs{} }
 func (emptyMempool) Update(
 	_ int64,
 	_ types.Txs,
 	_ []*abci.ResponseDeliverTx,
-	_ mempl.PreCheckFunc,
-	_ mempl.PostCheckFunc,
+	_ mempool.PreCheckFunc,
+	_ mempool.PostCheckFunc,
 ) error {
 	return nil
 }

internal/consensus/replay_test.go

@@ -17,12 +17,13 @@ import (
 	"github.com/stretchr/testify/require"
 	dbm "github.com/tendermint/tm-db"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/crypto"
-	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
-	mempl "github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -47,7 +48,7 @@ import (
 // and which ones we need the wal for - then we'd also be able to only flush the
 // wal writer when we need to, instead of with every message.
 
-func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *cfg.Config,
+func startNewStateAndWaitForBlock(t *testing.T, consensusReplayConfig *config.Config,
 	lastBlockHeight int64, blockDB dbm.DB, stateStore sm.Store) {
 	logger := log.TestingLogger()
 	state, err := sm.MakeGenesisStateFromFile(consensusReplayConfig.GenesisFile())
@@ -96,7 +97,7 @@ func sendTxs(ctx context.Context, cs *State) {
 			return
 		default:
 			tx := []byte{byte(i)}
-			if err := assertMempool(cs.txNotifier).CheckTx(context.Background(), tx, nil, mempl.TxInfo{}); err != nil {
+			if err := assertMempool(cs.txNotifier).CheckTx(context.Background(), tx, nil, mempool.TxInfo{}); err != nil {
 				panic(err)
 			}
 			i++
@@ -130,7 +131,7 @@ func TestWALCrash(t *testing.T) {
 	}
 }
 
-func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *cfg.Config,
+func crashWALandCheckLiveness(t *testing.T, consensusReplayConfig *config.Config,
 	initFn func(dbm.DB, *State, context.Context), heightToStop int64) {
 	walPanicked := make(chan error)
 	crashingWal := &crashingWAL{panicCh: walPanicked, heightToStop: heightToStop}
@@ -279,12 +280,12 @@ func (w *crashingWAL) Wait()        { w.next.Wait() }
 //------------------------------------------------------------------------------------------
 type simulatorTestSuite struct {
 	GenesisState sm.State
-	Config       *cfg.Config
+	Config       *config.Config
 	Chain        []*types.Block
 	Commits      []*types.Commit
 	CleanupFunc  cleanupFunc
 
-	Mempool mempl.Mempool
+	Mempool mempool.Mempool
 	Evpool  sm.EvidencePool
 }
 
@@ -304,7 +305,7 @@ var modes = []uint{0, 1, 2, 3}
 // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay
 func setupSimulator(t *testing.T) *simulatorTestSuite {
 	t.Helper()
-	config := configSetup(t)
+	cfg := configSetup(t)
 
 	sim := &simulatorTestSuite{
 		Mempool: emptyMempool{},
@@ -314,14 +315,14 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	nPeers := 7
 	nVals := 4
 
-	css, genDoc, config, cleanup := randConsensusNetWithPeers(
-		config,
+	css, genDoc, cfg, cleanup := randConsensusNetWithPeers(
+		cfg,
 		nVals,
 		nPeers,
 		"replay_test",
 		newMockTickerFunc(true),
 		newPersistentKVStoreWithPath)
-	sim.Config = config
+	sim.Config = cfg
 	sim.GenesisState, _ = sm.MakeGenesisState(genDoc)
 	sim.CleanupFunc = cleanup
 
@@ -354,10 +355,10 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	incrementHeight(vss...)
 	newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
-	valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1)
+	valPubKey1ABCI, err := encoding.PubKeyToProto(newValidatorPubKey1)
 	require.NoError(t, err)
 	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx1, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx1, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 	propBlock, _ := css[0].createProposalBlock() // changeProposer(t, cs1, vs2)
 	propBlockParts := propBlock.MakePartSet(partSize)
@@ -365,7 +366,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 
 	proposal := types.NewProposal(vss[1].Height, round, -1, blockID)
 	p := proposal.ToProto()
-	if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil {
+	if err := vss[1].SignProposal(context.Background(), cfg.ChainID(), p); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 	proposal.Signature = p.Signature
@@ -386,10 +387,10 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	incrementHeight(vss...)
 	updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
-	updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1)
+	updatePubKey1ABCI, err := encoding.PubKeyToProto(updateValidatorPubKey1)
 	require.NoError(t, err)
 	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), updateValidatorTx1, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), updateValidatorTx1, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 	propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2)
 	propBlockParts = propBlock.MakePartSet(partSize)
@@ -397,7 +398,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 
 	proposal = types.NewProposal(vss[2].Height, round, -1, blockID)
 	p = proposal.ToProto()
-	if err := vss[2].SignProposal(context.Background(), config.ChainID(), p); err != nil {
+	if err := vss[2].SignProposal(context.Background(), cfg.ChainID(), p); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 	proposal.Signature = p.Signature
@@ -418,17 +419,17 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	incrementHeight(vss...)
 	newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
-	newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2)
+	newVal2ABCI, err := encoding.PubKeyToProto(newValidatorPubKey2)
 	require.NoError(t, err)
 	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx2, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx2, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 	newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey(context.Background())
 	require.NoError(t, err)
-	newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3)
+	newVal3ABCI, err := encoding.PubKeyToProto(newValidatorPubKey3)
 	require.NoError(t, err)
 	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx3, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), newValidatorTx3, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 	propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2)
 	propBlockParts = propBlock.MakePartSet(partSize)
@@ -456,7 +457,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 
 	proposal = types.NewProposal(vss[3].Height, round, -1, blockID)
 	p = proposal.ToProto()
-	if err := vss[3].SignProposal(context.Background(), config.ChainID(), p); err != nil {
+	if err := vss[3].SignProposal(context.Background(), cfg.ChainID(), p); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 	proposal.Signature = p.Signature
@@ -468,7 +469,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	ensureNewProposal(proposalCh, height, round)
 
 	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx2, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx2, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 
 	rs = css[0].GetRoundState()
@@ -507,7 +508,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	height++
 	incrementHeight(vss...)
 	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
-	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx3, nil, mempl.TxInfo{})
+	err = assertMempool(css[0].txNotifier).CheckTx(context.Background(), removeValidatorTx3, nil, mempool.TxInfo{})
 	assert.Nil(t, err)
 	propBlock, _ = css[0].createProposalBlock() // changeProposer(t, cs1, vs2)
 	propBlockParts = propBlock.MakePartSet(partSize)
@@ -519,7 +520,7 @@ func setupSimulator(t *testing.T) *simulatorTestSuite {
 	selfIndex = valIndexFn(0)
 	proposal = types.NewProposal(vss[1].Height, round, -1, blockID)
 	p = proposal.ToProto()
-	if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil {
+	if err := vss[1].SignProposal(context.Background(), cfg.ChainID(), p); err != nil {
 		t.Fatal("failed to sign bad proposal", err)
 	}
 	proposal.Signature = p.Signature
@@ -569,7 +570,7 @@ func tempWALWithData(data []byte) string {
 }
 
 func applyBlock(stateStore sm.Store,
-	mempool mempl.Mempool,
+	mempool mempool.Mempool,
 	evpool sm.EvidencePool,
 	st sm.State,
 	blk *types.Block,
@@ -589,7 +590,7 @@ func applyBlock(stateStore sm.Store,
 func buildAppStateFromChain(
 	proxyApp proxy.AppConns,
 	stateStore sm.Store,
-	mempool mempl.Mempool,
+	mempool mempool.Mempool,
 	evpool sm.EvidencePool,
 	state sm.State,
 	chain []*types.Block,
@@ -636,8 +637,8 @@ func buildAppStateFromChain(
 }
 
 func buildTMStateFromChain(
-	config *cfg.Config,
-	mempool mempl.Mempool,
+	cfg *config.Config,
+	mempool mempool.Mempool,
 	evpool sm.EvidencePool,
 	stateStore sm.Store,
 	state sm.State,
@@ -647,11 +648,11 @@ func buildTMStateFromChain(
 	blockStore *mockBlockStore) sm.State {
 	// run the whole chain against this client to build up the tendermint state
 	kvstoreApp := kvstore.NewPersistentKVStoreApplication(
-		filepath.Join(config.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode)))
+		filepath.Join(cfg.DBDir(), fmt.Sprintf("replay_test_%d_%d_t", nBlocks, mode)))
 	defer kvstoreApp.Close()
-	clientCreator := proxy.NewLocalClientCreator(kvstoreApp)
+	clientCreator := abciclient.NewLocalCreator(kvstoreApp)
 
-	proxyApp := proxy.NewAppConns(clientCreator)
+	proxyApp := proxy.NewAppConns(clientCreator, proxy.NopMetrics())
 	if err := proxyApp.Start(); err != nil {
 		panic(err)
 	}
@@ -842,14 +843,14 @@ func readPieceFromWAL(msg *TimedWALMessage) interface{} {
 
 // fresh state and mock store
 func stateAndStore(
-	config *cfg.Config,
+	cfg *config.Config,
 	pubKey crypto.PubKey,
 	appVersion uint64) (dbm.DB, sm.State, *mockBlockStore) {
 	stateDB := dbm.NewMemDB()
 	stateStore := sm.NewStore(stateDB)
-	state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile())
+	state, _ := sm.MakeGenesisStateFromFile(cfg.GenesisFile())
 	state.Version.Consensus.App = appVersion
-	store := newMockBlockStore(config, state.ConsensusParams)
+	store := newMockBlockStore(cfg, state.ConsensusParams)
 	if err := stateStore.Save(state); err != nil {
 		panic(err)
 	}
@@ -860,7 +861,7 @@ func stateAndStore(
 // mock block store
 
 type mockBlockStore struct {
-	config  *cfg.Config
+	cfg     *config.Config
 	params  types.ConsensusParams
 	chain   []*types.Block
 	commits []*types.Commit
@@ -868,8 +869,8 @@ type mockBlockStore struct {
 }
 
 // TODO: NewBlockStore(db.NewMemDB) ...
-func newMockBlockStore(config *cfg.Config, params types.ConsensusParams) *mockBlockStore {
-	return &mockBlockStore{config, params, nil, nil, 0}
+func newMockBlockStore(cfg *config.Config, params types.ConsensusParams) *mockBlockStore {
+	return &mockBlockStore{cfg, params, nil, nil, 0}
 }
 
 func (bs *mockBlockStore) Height() int64                       { return int64(len(bs.chain)) }

internal/consensus/state.go

@@ -12,11 +12,12 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/crypto"
 	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
 	"github.com/tendermint/tendermint/internal/libs/fail"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	sm "github.com/tendermint/tendermint/internal/state"
 	tmevents "github.com/tendermint/tendermint/libs/events"
 	tmjson "github.com/tendermint/tendermint/libs/json"
 	"github.com/tendermint/tendermint/libs/log"
@@ -27,7 +28,6 @@ import (
 	"github.com/tendermint/tendermint/privval"
 	tmgrpc "github.com/tendermint/tendermint/privval/grpc"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -80,7 +80,7 @@ type State struct {
 	service.BaseService
 
 	// config details
-	config            *cfg.ConsensusConfig
+	config            *config.ConsensusConfig
 	privValidator     types.PrivValidator // for signing votes
 	privValidatorType types.PrivValidatorType
 
@@ -137,7 +137,7 @@ type State struct {
 	done chan struct{}
 
 	// synchronous pubsub between consensus state and reactor.
-	// state only emits EventNewRoundStep and EventVote
+	// state only emits EventNewRoundStep, EventValidBlock, and EventVote
 	evsw tmevents.EventSwitch
 
 	// for reporting metrics
@@ -152,7 +152,7 @@ type StateOption func(*State)
 
 // NewState returns a new State.
 func NewState(
-	config *cfg.ConsensusConfig,
+	cfg *config.ConsensusConfig,
 	state sm.State,
 	blockExec *sm.BlockExecutor,
 	blockStore sm.BlockStore,
@@ -161,7 +161,7 @@ func NewState(
 	options ...StateOption,
 ) *State {
 	cs := &State{
-		config:           config,
+		config:           cfg,
 		blockExec:        blockExec,
 		blockStore:       blockStore,
 		txNotifier:       txNotifier,
@@ -241,8 +241,12 @@ func (cs *State) GetLastHeight() int64 {
 // GetRoundState returns a shallow copy of the internal consensus state.
 func (cs *State) GetRoundState() *cstypes.RoundState {
 	cs.mtx.RLock()
+	defer cs.mtx.RUnlock()
+
+	// NOTE: this might be dodgy, as RoundState itself isn't thread
+	// safe as it contains a number of pointers and is explicitly
+	// not thread safe.
 	rs := cs.RoundState // copy
-	cs.mtx.RUnlock()
 	return &rs
 }
 
@@ -916,8 +920,8 @@ func (cs *State) handleMsg(mi msgInfo) {
 			"height", cs.Height,
 			"round", cs.Round,
 			"peer", peerID,
+			"msg_type", fmt.Sprintf("%T", msg),
 			"err", err,
-			"msg", msg,
 		)
 	}
 }

internal/consensus/state_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	"github.com/tendermint/tendermint/crypto/tmhash"
 	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
-	p2pmock "github.com/tendermint/tendermint/internal/p2p/mock"
 	"github.com/tendermint/tendermint/libs/log"
 	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -331,7 +330,7 @@ func TestStateFullRound1(t *testing.T) {
 		t.Error(err)
 	}
 
-	voteCh := subscribeUnBuffered(cs.eventBus, types.EventQueryVote)
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
 	propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal)
 	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
 
@@ -361,7 +360,7 @@ func TestStateFullRoundNil(t *testing.T) {
 	cs, vss := randState(config, 1)
 	height, round := cs.Height, cs.Round
 
-	voteCh := subscribeUnBuffered(cs.eventBus, types.EventQueryVote)
+	voteCh := subscribe(cs.eventBus, types.EventQueryVote)
 
 	cs.enterPrevote(height, round)
 	cs.startRoutines(4)
@@ -382,7 +381,7 @@ func TestStateFullRound2(t *testing.T) {
 	vs2 := vss[1]
 	height, round := cs1.Height, cs1.Round
 
-	voteCh := subscribeUnBuffered(cs1.eventBus, types.EventQueryVote)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
 	newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock)
 
 	// start round and wait for propose and prevote
@@ -428,7 +427,7 @@ func TestStateLockNoPOL(t *testing.T) {
 
 	timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose)
 	timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait)
-	voteCh := subscribeUnBuffered(cs1.eventBus, types.EventQueryVote)
+	voteCh := subscribe(cs1.eventBus, types.EventQueryVote)
 	proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal)
 	newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound)
 
@@ -1864,7 +1863,8 @@ func TestStateOutputsBlockPartsStats(t *testing.T) {
 
 	// create dummy peer
 	cs, _ := randState(config, 1)
-	peer := p2pmock.NewPeer(nil)
+	peerID, err := types.NewNodeID("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+	require.NoError(t, err)
 
 	// 1) new block part
 	parts := types.NewPartSetFromData(tmrand.Bytes(100), 10)
@@ -1875,26 +1875,26 @@ func TestStateOutputsBlockPartsStats(t *testing.T) {
 	}
 
 	cs.ProposalBlockParts = types.NewPartSetFromHeader(parts.Header())
-	cs.handleMsg(msgInfo{msg, peer.ID()})
+	cs.handleMsg(msgInfo{msg, peerID})
 
 	statsMessage := <-cs.statsMsgQueue
 	require.Equal(t, msg, statsMessage.Msg, "")
-	require.Equal(t, peer.ID(), statsMessage.PeerID, "")
+	require.Equal(t, peerID, statsMessage.PeerID, "")
 
 	// sending the same part from different peer
 	cs.handleMsg(msgInfo{msg, "peer2"})
 
 	// sending the part with the same height, but different round
 	msg.Round = 1
-	cs.handleMsg(msgInfo{msg, peer.ID()})
+	cs.handleMsg(msgInfo{msg, peerID})
 
 	// sending the part from the smaller height
 	msg.Height = 0
-	cs.handleMsg(msgInfo{msg, peer.ID()})
+	cs.handleMsg(msgInfo{msg, peerID})
 
 	// sending the part from the bigger height
 	msg.Height = 3
-	cs.handleMsg(msgInfo{msg, peer.ID()})
+	cs.handleMsg(msgInfo{msg, peerID})
 
 	select {
 	case <-cs.statsMsgQueue:
@@ -1909,18 +1909,19 @@ func TestStateOutputVoteStats(t *testing.T) {
 
 	cs, vss := randState(config, 2)
 	// create dummy peer
-	peer := p2pmock.NewPeer(nil)
+	peerID, err := types.NewNodeID("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
+	require.NoError(t, err)
 
 	randBytes := tmrand.Bytes(tmhash.Size)
 
 	vote := signVote(vss[1], config, tmproto.PrecommitType, randBytes, types.PartSetHeader{})
 
 	voteMessage := &VoteMessage{vote}
-	cs.handleMsg(msgInfo{voteMessage, peer.ID()})
+	cs.handleMsg(msgInfo{voteMessage, peerID})
 
 	statsMessage := <-cs.statsMsgQueue
 	require.Equal(t, voteMessage, statsMessage.Msg, "")
-	require.Equal(t, peer.ID(), statsMessage.PeerID, "")
+	require.Equal(t, peerID, statsMessage.PeerID, "")
 
 	// sending the same part from different peer
 	cs.handleMsg(msgInfo{&VoteMessage{vote}, "peer2"})
@@ -1929,7 +1930,7 @@ func TestStateOutputVoteStats(t *testing.T) {
 	incrementHeight(vss[1])
 	vote = signVote(vss[1], config, tmproto.PrecommitType, randBytes, types.PartSetHeader{})
 
-	cs.handleMsg(msgInfo{&VoteMessage{vote}, peer.ID()})
+	cs.handleMsg(msgInfo{&VoteMessage{vote}, peerID})
 
 	select {
 	case <-cs.statsMsgQueue:
@@ -1971,12 +1972,3 @@ func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan tmpubsub.Messa
 	}
 	return sub.Out()
 }
-
-// subscribe subscribes test client to the given query and returns a channel with cap = 0.
-func subscribeUnBuffered(eventBus *types.EventBus, q tmpubsub.Query) <-chan tmpubsub.Message {
-	sub, err := eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, q)
-	if err != nil {
-		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, q))
-	}
-	return sub.Out()
-}

internal/consensus/types/height_vote_set_test.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"testing"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/crypto/tmhash"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -15,19 +15,19 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-var config *cfg.Config // NOTE: must be reset for each _test.go file
+var cfg *config.Config // NOTE: must be reset for each _test.go file
 
 func TestMain(m *testing.M) {
-	config = cfg.ResetTestRoot("consensus_height_vote_set_test")
+	cfg = config.ResetTestRoot("consensus_height_vote_set_test")
 	code := m.Run()
-	os.RemoveAll(config.RootDir)
+	os.RemoveAll(cfg.RootDir)
 	os.Exit(code)
 }
 
 func TestPeerCatchupRounds(t *testing.T) {
 	valSet, privVals := factory.RandValidatorSet(10, 1)
 
-	hvs := NewHeightVoteSet(config.ChainID(), 1, valSet)
+	hvs := NewHeightVoteSet(cfg.ChainID(), 1, valSet)
 
 	vote999_0 := makeVoteHR(t, 1, 0, 999, privVals)
 	added, err := hvs.AddVote(vote999_0, "peer1")
@@ -75,7 +75,7 @@ func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []ty
 		Type:             tmproto.PrecommitType,
 		BlockID:          types.BlockID{Hash: randBytes, PartSetHeader: types.PartSetHeader{}},
 	}
-	chainID := config.ChainID()
+	chainID := cfg.ChainID()
 
 	v := vote.ToProto()
 	err = privVal.SignVote(context.Background(), chainID, v)

internal/consensus/wal_fuzz.go

@@ -1,3 +1,4 @@
+//go:build gofuzz
 // +build gofuzz
 
 package consensus

internal/consensus/wal_generator.go

@@ -11,15 +11,16 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	db "github.com/tendermint/tm-db"
+	dbm "github.com/tendermint/tm-db"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/proxy"
+	sm "github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/libs/log"
 	"github.com/tendermint/tendermint/privval"
-	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -29,9 +30,9 @@ import (
 // (byteBufferWAL) and waits until numBlocks are created.
 // If the node fails to produce given numBlocks, it returns an error.
 func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
-	config := getConfig(t)
+	cfg := getConfig(t)
 
-	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), "wal_generator"))
+	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(cfg.DBDir(), "wal_generator"))
 	t.Cleanup(func() { require.NoError(t, app.Close()) })
 
 	logger := log.TestingLogger().With("wal_generator", "wal_generator")
@@ -40,17 +41,17 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
 	// NOTE: we can't import node package because of circular dependency.
 	// NOTE: we don't do handshake so need to set state.Version.Consensus.App directly.
-	privValidatorKeyFile := config.PrivValidator.KeyFile()
-	privValidatorStateFile := config.PrivValidator.StateFile()
+	privValidatorKeyFile := cfg.PrivValidator.KeyFile()
+	privValidatorStateFile := cfg.PrivValidator.StateFile()
 	privValidator, err := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
 	if err != nil {
 		return err
 	}
-	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
+	genDoc, err := types.GenesisDocFromFile(cfg.GenesisFile())
 	if err != nil {
 		return fmt.Errorf("failed to read genesis file: %w", err)
 	}
-	blockStoreDB := db.NewMemDB()
+	blockStoreDB := dbm.NewMemDB()
 	stateDB := blockStoreDB
 	stateStore := sm.NewStore(stateDB)
 	state, err := sm.MakeGenesisState(genDoc)
@@ -64,7 +65,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 
 	blockStore := store.NewBlockStore(blockStoreDB)
 
-	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app))
+	proxyApp := proxy.NewAppConns(abciclient.NewLocalCreator(app), proxy.NopMetrics())
 	proxyApp.SetLogger(logger.With("module", "proxy"))
 	if err := proxyApp.Start(); err != nil {
 		return fmt.Errorf("failed to start proxy app connections: %w", err)
@@ -88,7 +89,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 	mempool := emptyMempool{}
 	evpool := sm.EmptyEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore)
-	consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
+	consensusState := NewState(cfg.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
 	consensusState.SetLogger(logger)
 	consensusState.SetEventBus(eventBus)
 	if privValidator != nil && privValidator != (*privval.FilePV)(nil) {
@@ -144,22 +145,22 @@ func randPort() int {
 	return base + mrand.Intn(spread)
 }
 
-func makeAddrs() (string, string, string) {
+// makeAddrs constructs local TCP addresses for node services.
+// It uses consecutive ports from a random starting point, so that concurrent
+// instances are less likely to collide.
+func makeAddrs() (p2pAddr, rpcAddr string) {
+	const addrTemplate = "tcp://127.0.0.1:%d"
 	start := randPort()
-	return fmt.Sprintf("tcp://127.0.0.1:%d", start),
-		fmt.Sprintf("tcp://127.0.0.1:%d", start+1),
-		fmt.Sprintf("tcp://127.0.0.1:%d", start+2)
+	return fmt.Sprintf(addrTemplate, start), fmt.Sprintf(addrTemplate, start+1)
 }
 
 // getConfig returns a config for test cases
-func getConfig(t *testing.T) *cfg.Config {
-	c := cfg.ResetTestRoot(t.Name())
+func getConfig(t *testing.T) *config.Config {
+	c := config.ResetTestRoot(t.Name())
 
-	// and we use random ports to run in parallel
-	tm, rpc, grpc := makeAddrs()
-	c.P2P.ListenAddress = tm
-	c.RPC.ListenAddress = rpc
-	c.RPC.GRPCListenAddress = grpc
+	p2pAddr, rpcAddr := makeAddrs()
+	c.P2P.ListenAddress = p2pAddr
+	c.RPC.ListenAddress = rpcAddr
 	return c
 }
 

internal/evidence/pool.go

@@ -14,9 +14,9 @@ import (
 	dbm "github.com/tendermint/tm-db"
 
 	clist "github.com/tendermint/tendermint/internal/libs/clist"
+	sm "github.com/tendermint/tendermint/internal/state"
 	"github.com/tendermint/tendermint/libs/log"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -516,10 +516,13 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) {
 
 		// Check the height of the conflicting votes and fetch the corresponding time and validator set
 		// to produce the valid evidence
-		var dve *types.DuplicateVoteEvidence
+		var (
+			dve *types.DuplicateVoteEvidence
+			err error
+		)
 		switch {
 		case voteSet.VoteA.Height == state.LastBlockHeight:
-			dve = types.NewDuplicateVoteEvidence(
+			dve, err = types.NewDuplicateVoteEvidence(
 				voteSet.VoteA,
 				voteSet.VoteB,
 				state.LastBlockTime,
@@ -527,8 +530,8 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) {
 			)
 
 		case voteSet.VoteA.Height < state.LastBlockHeight:
-			valSet, err := evpool.stateDB.LoadValidators(voteSet.VoteA.Height)
-			if err != nil {
+			valSet, dbErr := evpool.stateDB.LoadValidators(voteSet.VoteA.Height)
+			if dbErr != nil {
 				evpool.logger.Error("failed to load validator set for conflicting votes",
 					"height", voteSet.VoteA.Height, "err", err)
 				continue
@@ -538,7 +541,7 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) {
 				evpool.logger.Error("failed to load block time for conflicting votes", "height", voteSet.VoteA.Height)
 				continue
 			}
-			dve = types.NewDuplicateVoteEvidence(
+			dve, err = types.NewDuplicateVoteEvidence(
 				voteSet.VoteA,
 				voteSet.VoteB,
 				blockMeta.Header.Time,
@@ -554,6 +557,10 @@ func (evpool *Pool) processConsensusBuffer(state sm.State) {
 				"state.LastBlockHeight", state.LastBlockHeight)
 			continue
 		}
+		if err != nil {
+			evpool.logger.Error("error in generating evidence from votes", "err", err)
+			continue
+		}
 
 		// check if we already have this evidence
 		if evpool.isPending(dve) {
@@ -608,7 +615,7 @@ func prefixToBytes(prefix int64) []byte {
 }
 
 func keyCommitted(evidence types.Evidence) []byte {
-	var height int64 = evidence.Height()
+	height := evidence.Height()
 	key, err := orderedcode.Append(nil, prefixCommitted, height, string(evidence.Hash()))
 	if err != nil {
 		panic(err)
@@ -617,7 +624,7 @@ func keyCommitted(evidence types.Evidence) []byte {
 }
 
 func keyPending(evidence types.Evidence) []byte {
-	var height int64 = evidence.Height()
+	height := evidence.Height()
 	key, err := orderedcode.Append(nil, prefixPending, height, string(evidence.Hash()))
 	if err != nil {
 		panic(err)

internal/evidence/pool_test.go

@@ -13,12 +13,12 @@ import (
 
 	"github.com/tendermint/tendermint/internal/evidence"
 	"github.com/tendermint/tendermint/internal/evidence/mocks"
+	sm "github.com/tendermint/tendermint/internal/state"
+	smmocks "github.com/tendermint/tendermint/internal/state/mocks"
+	sf "github.com/tendermint/tendermint/internal/state/test/factory"
+	"github.com/tendermint/tendermint/internal/store"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
-	sm "github.com/tendermint/tendermint/state"
-	smmocks "github.com/tendermint/tendermint/state/mocks"
-	sf "github.com/tendermint/tendermint/state/test/factory"
-	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 	"github.com/tendermint/tendermint/version"
 )

internal/evidence/reactor.go

@@ -15,29 +15,7 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-var (
-	_ service.Service = (*Reactor)(nil)
-
-	// ChannelShims contains a map of ChannelDescriptorShim objects, where each
-	// object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
-	// p2p proto.Message the new p2p Channel is responsible for handling.
-	//
-	//
-	// TODO: Remove once p2p refactor is complete.
-	// ref: https://github.com/tendermint/tendermint/issues/5670
-	ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
-		EvidenceChannel: {
-			MsgType: new(tmproto.EvidenceList),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(EvidenceChannel),
-				Priority:            6,
-				RecvMessageCapacity: maxMsgSize,
-				RecvBufferCapacity:  32,
-				MaxSendBytes:        400,
-			},
-		},
-	}
-)
+var _ service.Service = (*Reactor)(nil)
 
 const (
 	EvidenceChannel = p2p.ChannelID(0x38)
@@ -51,6 +29,18 @@ const (
 	broadcastEvidenceIntervalS = 10
 )
 
+// GetChannelDescriptor produces an instance of a descriptor for this
+// package's required channels.
+func GetChannelDescriptor() *p2p.ChannelDescriptor {
+	return &p2p.ChannelDescriptor{
+		ID:                  EvidenceChannel,
+		MessageType:         new(tmproto.EvidenceList),
+		Priority:            6,
+		RecvMessageCapacity: maxMsgSize,
+		RecvBufferCapacity:  32,
+	}
+}
+
 // Reactor handles evpool evidence broadcasting amongst peers.
 type Reactor struct {
 	service.BaseService

internal/evidence/reactor_test.go

@@ -20,9 +20,9 @@ import (
 	"github.com/tendermint/tendermint/internal/evidence/mocks"
 	"github.com/tendermint/tendermint/internal/p2p"
 	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
+	sm "github.com/tendermint/tendermint/internal/state"
 	"github.com/tendermint/tendermint/libs/log"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -62,11 +62,8 @@ func setup(t *testing.T, stateStores []sm.Store, chBuf uint) *reactorTestSuite {
 		peerChans:      make(map[types.NodeID]chan p2p.PeerUpdate, numStateStores),
 	}
 
-	chDesc := p2p.ChannelDescriptor{ID: byte(evidence.EvidenceChannel)}
-	rts.evidenceChannels = rts.network.MakeChannelsNoCleanup(t,
-		chDesc,
-		new(tmproto.EvidenceList),
-		int(chBuf))
+	chDesc := &p2p.ChannelDescriptor{ID: evidence.EvidenceChannel, MessageType: new(tmproto.EvidenceList)}
+	rts.evidenceChannels = rts.network.MakeChannelsNoCleanup(t, chDesc)
 	require.Len(t, rts.network.RandomNode().PeerManager.Peers(), 0)
 
 	idx := 0
@@ -534,12 +531,13 @@ func TestEvidenceListSerialization(t *testing.T) {
 
 	valSet := types.NewValidatorSet([]*types.Validator{val})
 
-	dupl := types.NewDuplicateVoteEvidence(
+	dupl, err := types.NewDuplicateVoteEvidence(
 		exampleVote(1),
 		exampleVote(2),
 		defaultEvidenceTime,
 		valSet,
 	)
+	require.NoError(t, err)
 
 	testCases := map[string]struct {
 		evidenceList []types.Evidence

internal/evidence/verify_test.go

@@ -8,18 +8,17 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/tmhash"
 	"github.com/tendermint/tendermint/internal/evidence"
 	"github.com/tendermint/tendermint/internal/evidence/mocks"
+	sm "github.com/tendermint/tendermint/internal/state"
+	smmocks "github.com/tendermint/tendermint/internal/state/mocks"
 	"github.com/tendermint/tendermint/internal/test/factory"
 	"github.com/tendermint/tendermint/libs/log"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
-	sm "github.com/tendermint/tendermint/state"
-	smmocks "github.com/tendermint/tendermint/state/mocks"
 	"github.com/tendermint/tendermint/types"
 )
 

internal/inspect/doc.go

@@ -0,0 +1,36 @@
+/*
+Package inspect provides a tool for investigating the state of a
+failed Tendermint node.
+
+This package provides the Inspector type. The Inspector type runs a subset of the Tendermint
+RPC endpoints that are useful for debugging issues with Tendermint consensus.
+
+When a node running the Tendermint consensus engine detects an inconsistent consensus state,
+the entire node will crash. The Tendermint consensus engine cannot run in this
+inconsistent state so the node will not be able to start up again.
+
+The RPC endpoints provided by the Inspector type allow for a node operator to inspect
+the block store and state store to better understand what may have caused the inconsistent state.
+
+
+The Inspector type's lifecycle is controlled by a context.Context
+  ins := inspect.NewFromConfig(rpcConfig)
+  ctx, cancelFunc:= context.WithCancel(context.Background())
+
+  // Run blocks until the Inspector server is shut down.
+  go ins.Run(ctx)
+  ...
+
+  // calling the cancel function will stop the running inspect server
+  cancelFunc()
+
+Inspector serves its RPC endpoints on the address configured in the RPC configuration
+
+  rpcConfig.ListenAddress = "tcp://127.0.0.1:26657"
+  ins := inspect.NewFromConfig(rpcConfig)
+  go ins.Run(ctx)
+
+The list of available RPC endpoints can then be viewed by navigating to
+http://127.0.0.1:26657/ in the web browser.
+*/
+package inspect

internal/inspect/inspect.go

@@ -0,0 +1,148 @@
+package inspect
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/inspect/rpc"
+	rpccore "github.com/tendermint/tendermint/internal/rpc/core"
+	"github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/state/indexer"
+	"github.com/tendermint/tendermint/internal/state/indexer/sink"
+	"github.com/tendermint/tendermint/internal/store"
+	"github.com/tendermint/tendermint/libs/log"
+	tmstrings "github.com/tendermint/tendermint/libs/strings"
+	"github.com/tendermint/tendermint/types"
+
+	"golang.org/x/sync/errgroup"
+)
+
+// Inspector manages an RPC service that exports methods to debug a failed node.
+// After a node shuts down due to a consensus failure, it will no longer start
+// up its state cannot easily be inspected. An Inspector value provides a similar interface
+// to the node, using the underlying Tendermint data stores, without bringing up
+// any other components. A caller can query the Inspector service to inspect the
+// persisted state and debug the failure.
+type Inspector struct {
+	routes rpccore.RoutesMap
+
+	config *config.RPCConfig
+
+	indexerService *indexer.Service
+	eventBus       *types.EventBus
+	logger         log.Logger
+}
+
+// New returns an Inspector that serves RPC on the specified BlockStore and StateStore.
+// The Inspector type does not modify the state or block stores.
+// The sinks are used to enable block and transaction querying via the RPC server.
+// The caller is responsible for starting and stopping the Inspector service.
+///
+//nolint:lll
+func New(cfg *config.RPCConfig, bs state.BlockStore, ss state.Store, es []indexer.EventSink, logger log.Logger) *Inspector {
+	routes := rpc.Routes(*cfg, ss, bs, es, logger)
+	eb := types.NewEventBus()
+	eb.SetLogger(logger.With("module", "events"))
+	is := indexer.NewIndexerService(es, eb)
+	is.SetLogger(logger.With("module", "txindex"))
+	return &Inspector{
+		routes:         routes,
+		config:         cfg,
+		logger:         logger,
+		eventBus:       eb,
+		indexerService: is,
+	}
+}
+
+// NewFromConfig constructs an Inspector using the values defined in the passed in config.
+func NewFromConfig(logger log.Logger, cfg *config.Config) (*Inspector, error) {
+	bsDB, err := config.DefaultDBProvider(&config.DBContext{ID: "blockstore", Config: cfg})
+	if err != nil {
+		return nil, err
+	}
+	bs := store.NewBlockStore(bsDB)
+	sDB, err := config.DefaultDBProvider(&config.DBContext{ID: "state", Config: cfg})
+	if err != nil {
+		return nil, err
+	}
+	genDoc, err := types.GenesisDocFromFile(cfg.GenesisFile())
+	if err != nil {
+		return nil, err
+	}
+	sinks, err := sink.EventSinksFromConfig(cfg, config.DefaultDBProvider, genDoc.ChainID)
+	if err != nil {
+		return nil, err
+	}
+	ss := state.NewStore(sDB)
+	return New(cfg.RPC, bs, ss, sinks, logger), nil
+}
+
+// Run starts the Inspector servers and blocks until the servers shut down. The passed
+// in context is used to control the lifecycle of the servers.
+func (ins *Inspector) Run(ctx context.Context) error {
+	err := ins.eventBus.Start()
+	if err != nil {
+		return fmt.Errorf("error starting event bus: %s", err)
+	}
+	defer func() {
+		err := ins.eventBus.Stop()
+		if err != nil {
+			ins.logger.Error("event bus stopped with error", "err", err)
+		}
+	}()
+	err = ins.indexerService.Start()
+	if err != nil {
+		return fmt.Errorf("error starting indexer service: %s", err)
+	}
+	defer func() {
+		err := ins.indexerService.Stop()
+		if err != nil {
+			ins.logger.Error("indexer service stopped with error", "err", err)
+		}
+	}()
+	return startRPCServers(ctx, ins.config, ins.logger, ins.routes)
+}
+
+func startRPCServers(ctx context.Context, cfg *config.RPCConfig, logger log.Logger, routes rpccore.RoutesMap) error {
+	g, tctx := errgroup.WithContext(ctx)
+	listenAddrs := tmstrings.SplitAndTrimEmpty(cfg.ListenAddress, ",", " ")
+	rh := rpc.Handler(cfg, routes, logger)
+	for _, listenerAddr := range listenAddrs {
+		server := rpc.Server{
+			Logger:  logger,
+			Config:  cfg,
+			Handler: rh,
+			Addr:    listenerAddr,
+		}
+		if cfg.IsTLSEnabled() {
+			keyFile := cfg.KeyFile()
+			certFile := cfg.CertFile()
+			listenerAddr := listenerAddr
+			g.Go(func() error {
+				logger.Info("RPC HTTPS server starting", "address", listenerAddr,
+					"certfile", certFile, "keyfile", keyFile)
+				err := server.ListenAndServeTLS(tctx, certFile, keyFile)
+				if !errors.Is(err, net.ErrClosed) {
+					return err
+				}
+				logger.Info("RPC HTTPS server stopped", "address", listenerAddr)
+				return nil
+			})
+		} else {
+			listenerAddr := listenerAddr
+			g.Go(func() error {
+				logger.Info("RPC HTTP server starting", "address", listenerAddr)
+				err := server.ListenAndServe(tctx)
+				if !errors.Is(err, net.ErrClosed) {
+					return err
+				}
+				logger.Info("RPC HTTP server stopped", "address", listenerAddr)
+				return nil
+			})
+		}
+	}
+	return g.Wait()
+}

internal/inspect/inspect_test.go

@@ -0,0 +1,587 @@
+package inspect_test
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/fortytw2/leaktest"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+	abcitypes "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/inspect"
+	"github.com/tendermint/tendermint/internal/state/indexer"
+	indexermocks "github.com/tendermint/tendermint/internal/state/indexer/mocks"
+	statemocks "github.com/tendermint/tendermint/internal/state/mocks"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/pubsub/query"
+	"github.com/tendermint/tendermint/proto/tendermint/state"
+	httpclient "github.com/tendermint/tendermint/rpc/client/http"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestInspectConstructor(t *testing.T) {
+	cfg := config.ResetTestRoot("test")
+	testLogger := log.TestingLogger()
+	t.Cleanup(leaktest.Check(t))
+	defer func() { _ = os.RemoveAll(cfg.RootDir) }()
+	t.Run("from config", func(t *testing.T) {
+		logger := testLogger.With(t.Name())
+		d, err := inspect.NewFromConfig(logger, cfg)
+		require.NoError(t, err)
+		require.NotNil(t, d)
+	})
+
+}
+
+func TestInspectRun(t *testing.T) {
+	cfg := config.ResetTestRoot("test")
+	testLogger := log.TestingLogger()
+	t.Cleanup(leaktest.Check(t))
+	defer func() { _ = os.RemoveAll(cfg.RootDir) }()
+	t.Run("from config", func(t *testing.T) {
+		logger := testLogger.With(t.Name())
+		d, err := inspect.NewFromConfig(logger, cfg)
+		require.NoError(t, err)
+		ctx, cancel := context.WithCancel(context.Background())
+		stoppedWG := &sync.WaitGroup{}
+		stoppedWG.Add(1)
+		go func() {
+			require.NoError(t, d.Run(ctx))
+			stoppedWG.Done()
+		}()
+		cancel()
+		stoppedWG.Wait()
+	})
+
+}
+
+func TestBlock(t *testing.T) {
+	testHeight := int64(1)
+	testBlock := new(types.Block)
+	testBlock.Header.Height = testHeight
+	testBlock.Header.LastCommitHash = []byte("test hash")
+	stateStoreMock := &statemocks.Store{}
+
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Height").Return(testHeight)
+	blockStoreMock.On("Base").Return(int64(0))
+	blockStoreMock.On("LoadBlockMeta", testHeight).Return(&types.BlockMeta{})
+	blockStoreMock.On("LoadBlock", testHeight).Return(testBlock)
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	resultBlock, err := cli.Block(context.Background(), &testHeight)
+	require.NoError(t, err)
+	require.Equal(t, testBlock.Height, resultBlock.Block.Height)
+	require.Equal(t, testBlock.LastCommitHash, resultBlock.Block.LastCommitHash)
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestTxSearch(t *testing.T) {
+	testHash := []byte("test")
+	testTx := []byte("tx")
+	testQuery := fmt.Sprintf("tx.hash='%s'", string(testHash))
+	testTxResult := &abcitypes.TxResult{
+		Height: 1,
+		Index:  100,
+		Tx:     testTx,
+	}
+
+	stateStoreMock := &statemocks.Store{}
+	blockStoreMock := &statemocks.BlockStore{}
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	eventSinkMock.On("Type").Return(indexer.KV)
+	eventSinkMock.On("SearchTxEvents", mock.Anything,
+		mock.MatchedBy(func(q *query.Query) bool { return testQuery == q.String() })).
+		Return([]*abcitypes.TxResult{testTxResult}, nil)
+
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+
+	var page = 1
+	resultTxSearch, err := cli.TxSearch(context.Background(), testQuery, false, &page, &page, "")
+	require.NoError(t, err)
+	require.Len(t, resultTxSearch.Txs, 1)
+	require.Equal(t, types.Tx(testTx), resultTxSearch.Txs[0].Tx)
+
+	cancel()
+	wg.Wait()
+
+	eventSinkMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+	blockStoreMock.AssertExpectations(t)
+}
+func TestTx(t *testing.T) {
+	testHash := []byte("test")
+	testTx := []byte("tx")
+
+	stateStoreMock := &statemocks.Store{}
+	blockStoreMock := &statemocks.BlockStore{}
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	eventSinkMock.On("Type").Return(indexer.KV)
+	eventSinkMock.On("GetTxByHash", testHash).Return(&abcitypes.TxResult{
+		Tx: testTx,
+	}, nil)
+
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+
+	res, err := cli.Tx(context.Background(), testHash, false)
+	require.NoError(t, err)
+	require.Equal(t, types.Tx(testTx), res.Tx)
+
+	cancel()
+	wg.Wait()
+
+	eventSinkMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+	blockStoreMock.AssertExpectations(t)
+}
+func TestConsensusParams(t *testing.T) {
+	testHeight := int64(1)
+	testMaxGas := int64(55)
+	stateStoreMock := &statemocks.Store{}
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Height").Return(testHeight)
+	blockStoreMock.On("Base").Return(int64(0))
+	stateStoreMock.On("LoadConsensusParams", testHeight).Return(types.ConsensusParams{
+		Block: types.BlockParams{
+			MaxGas: testMaxGas,
+		},
+	}, nil)
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	params, err := cli.ConsensusParams(context.Background(), &testHeight)
+	require.NoError(t, err)
+	require.Equal(t, params.ConsensusParams.Block.MaxGas, testMaxGas)
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestBlockResults(t *testing.T) {
+	testHeight := int64(1)
+	testGasUsed := int64(100)
+	stateStoreMock := &statemocks.Store{}
+	//	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	stateStoreMock.On("LoadABCIResponses", testHeight).Return(&state.ABCIResponses{
+		DeliverTxs: []*abcitypes.ResponseDeliverTx{
+			{
+				GasUsed: testGasUsed,
+			},
+		},
+		EndBlock:   &abcitypes.ResponseEndBlock{},
+		BeginBlock: &abcitypes.ResponseBeginBlock{},
+	}, nil)
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Base").Return(int64(0))
+	blockStoreMock.On("Height").Return(testHeight)
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	res, err := cli.BlockResults(context.Background(), &testHeight)
+	require.NoError(t, err)
+	require.Equal(t, res.TotalGasUsed, testGasUsed)
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestCommit(t *testing.T) {
+	testHeight := int64(1)
+	testRound := int32(101)
+	stateStoreMock := &statemocks.Store{}
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Base").Return(int64(0))
+	blockStoreMock.On("Height").Return(testHeight)
+	blockStoreMock.On("LoadBlockMeta", testHeight).Return(&types.BlockMeta{}, nil)
+	blockStoreMock.On("LoadSeenCommit").Return(&types.Commit{
+		Height: testHeight,
+		Round:  testRound,
+	}, nil)
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	res, err := cli.Commit(context.Background(), &testHeight)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, res.SignedHeader.Commit.Round, testRound)
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestBlockByHash(t *testing.T) {
+	testHeight := int64(1)
+	testHash := []byte("test hash")
+	testBlock := new(types.Block)
+	testBlock.Header.Height = testHeight
+	testBlock.Header.LastCommitHash = testHash
+	stateStoreMock := &statemocks.Store{}
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("LoadBlockMeta", testHeight).Return(&types.BlockMeta{
+		BlockID: types.BlockID{
+			Hash: testHash,
+		},
+		Header: types.Header{
+			Height: testHeight,
+		},
+	}, nil)
+	blockStoreMock.On("LoadBlockByHash", testHash).Return(testBlock, nil)
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	res, err := cli.BlockByHash(context.Background(), testHash)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, []byte(res.BlockID.Hash), testHash)
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestBlockchain(t *testing.T) {
+	testHeight := int64(1)
+	testBlock := new(types.Block)
+	testBlockHash := []byte("test hash")
+	testBlock.Header.Height = testHeight
+	testBlock.Header.LastCommitHash = testBlockHash
+	stateStoreMock := &statemocks.Store{}
+
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Height").Return(testHeight)
+	blockStoreMock.On("Base").Return(int64(0))
+	blockStoreMock.On("LoadBlockMeta", testHeight).Return(&types.BlockMeta{
+		BlockID: types.BlockID{
+			Hash: testBlockHash,
+		},
+	})
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+	res, err := cli.BlockchainInfo(context.Background(), 0, 100)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, testBlockHash, []byte(res.BlockMetas[0].BlockID.Hash))
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestValidators(t *testing.T) {
+	testHeight := int64(1)
+	testVotingPower := int64(100)
+	testValidators := types.ValidatorSet{
+		Validators: []*types.Validator{
+			{
+				VotingPower: testVotingPower,
+			},
+		},
+	}
+	stateStoreMock := &statemocks.Store{}
+	stateStoreMock.On("LoadValidators", testHeight).Return(&testValidators, nil)
+
+	blockStoreMock := &statemocks.BlockStore{}
+	blockStoreMock.On("Height").Return(testHeight)
+	blockStoreMock.On("Base").Return(int64(0))
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+
+	testPage := 1
+	testPerPage := 100
+	res, err := cli.Validators(context.Background(), &testHeight, &testPage, &testPerPage)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, testVotingPower, res.Validators[0].VotingPower)
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func TestBlockSearch(t *testing.T) {
+	testHeight := int64(1)
+	testBlockHash := []byte("test hash")
+	testQuery := "block.height = 1"
+	stateStoreMock := &statemocks.Store{}
+
+	blockStoreMock := &statemocks.BlockStore{}
+	eventSinkMock := &indexermocks.EventSink{}
+	eventSinkMock.On("Stop").Return(nil)
+	eventSinkMock.On("Type").Return(indexer.KV)
+	blockStoreMock.On("LoadBlock", testHeight).Return(&types.Block{
+		Header: types.Header{
+			Height: testHeight,
+		},
+	}, nil)
+	blockStoreMock.On("LoadBlockMeta", testHeight).Return(&types.BlockMeta{
+		BlockID: types.BlockID{
+			Hash: testBlockHash,
+		},
+	})
+	eventSinkMock.On("SearchBlockEvents", mock.Anything,
+		mock.MatchedBy(func(q *query.Query) bool { return testQuery == q.String() })).
+		Return([]int64{testHeight}, nil)
+	rpcConfig := config.TestRPCConfig()
+	l := log.TestingLogger()
+	d := inspect.New(rpcConfig, blockStoreMock, stateStoreMock, []indexer.EventSink{eventSinkMock}, l)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
+
+	startedWG := &sync.WaitGroup{}
+	startedWG.Add(1)
+	go func() {
+		startedWG.Done()
+		defer wg.Done()
+		require.NoError(t, d.Run(ctx))
+	}()
+	// FIXME: used to induce context switch.
+	// Determine more deterministic method for prompting a context switch
+	startedWG.Wait()
+	requireConnect(t, rpcConfig.ListenAddress, 20)
+	cli, err := httpclient.New(rpcConfig.ListenAddress)
+	require.NoError(t, err)
+
+	testPage := 1
+	testPerPage := 100
+	testOrderBy := "desc"
+	res, err := cli.BlockSearch(context.Background(), testQuery, &testPage, &testPerPage, testOrderBy)
+	require.NoError(t, err)
+	require.NotNil(t, res)
+	require.Equal(t, testBlockHash, []byte(res.Blocks[0].BlockID.Hash))
+
+	cancel()
+	wg.Wait()
+
+	blockStoreMock.AssertExpectations(t)
+	stateStoreMock.AssertExpectations(t)
+}
+
+func requireConnect(t testing.TB, addr string, retries int) {
+	parts := strings.SplitN(addr, "://", 2)
+	if len(parts) != 2 {
+		t.Fatalf("malformed address to dial: %s", addr)
+	}
+	var err error
+	for i := 0; i < retries; i++ {
+		var conn net.Conn
+		conn, err = net.Dial(parts[0], parts[1])
+		if err == nil {
+			conn.Close()
+			return
+		}
+		// FIXME attempt to yield and let the other goroutine continue execution.
+		time.Sleep(time.Microsecond * 100)
+	}
+	t.Fatalf("unable to connect to server %s after %d tries: %s", addr, retries, err)
+}

internal/inspect/rpc/rpc.go

@@ -0,0 +1,143 @@
+package rpc
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/rs/cors"
+
+	"github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/internal/consensus"
+	"github.com/tendermint/tendermint/internal/rpc/core"
+	"github.com/tendermint/tendermint/internal/state"
+	"github.com/tendermint/tendermint/internal/state/indexer"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/rpc/jsonrpc/server"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Server defines parameters for running an Inspector rpc server.
+type Server struct {
+	Addr    string // TCP address to listen on, ":http" if empty
+	Handler http.Handler
+	Logger  log.Logger
+	Config  *config.RPCConfig
+}
+
+// Routes returns the set of routes used by the Inspector server.
+//
+//nolint: lll
+func Routes(cfg config.RPCConfig, s state.Store, bs state.BlockStore, es []indexer.EventSink, logger log.Logger) core.RoutesMap {
+	env := &core.Environment{
+		Config:           cfg,
+		EventSinks:       es,
+		StateStore:       s,
+		BlockStore:       bs,
+		ConsensusReactor: waitSyncCheckerImpl{},
+		Logger:           logger,
+	}
+	return core.RoutesMap{
+		"blockchain":       server.NewRPCFunc(env.BlockchainInfo, "minHeight,maxHeight", true),
+		"consensus_params": server.NewRPCFunc(env.ConsensusParams, "height", true),
+		"block":            server.NewRPCFunc(env.Block, "height", true),
+		"block_by_hash":    server.NewRPCFunc(env.BlockByHash, "hash", true),
+		"block_results":    server.NewRPCFunc(env.BlockResults, "height", true),
+		"commit":           server.NewRPCFunc(env.Commit, "height", true),
+		"validators":       server.NewRPCFunc(env.Validators, "height,page,per_page", true),
+		"tx":               server.NewRPCFunc(env.Tx, "hash,prove", true),
+		"tx_search":        server.NewRPCFunc(env.TxSearch, "query,prove,page,per_page,order_by", false),
+		"block_search":     server.NewRPCFunc(env.BlockSearch, "query,page,per_page,order_by", false),
+	}
+}
+
+// Handler returns the http.Handler configured for use with an Inspector server. Handler
+// registers the routes on the http.Handler and also registers the websocket handler
+// and the CORS handler if specified by the configuration options.
+func Handler(rpcConfig *config.RPCConfig, routes core.RoutesMap, logger log.Logger) http.Handler {
+	mux := http.NewServeMux()
+	wmLogger := logger.With("protocol", "websocket")
+
+	var eventBus types.EventBusSubscriber
+
+	websocketDisconnectFn := func(remoteAddr string) {
+		err := eventBus.UnsubscribeAll(context.Background(), remoteAddr)
+		if err != nil && err != pubsub.ErrSubscriptionNotFound {
+			wmLogger.Error("Failed to unsubscribe addr from events", "addr", remoteAddr, "err", err)
+		}
+	}
+	wm := server.NewWebsocketManager(routes,
+		server.OnDisconnect(websocketDisconnectFn),
+		server.ReadLimit(rpcConfig.MaxBodyBytes))
+	wm.SetLogger(wmLogger)
+	mux.HandleFunc("/websocket", wm.WebsocketHandler)
+
+	server.RegisterRPCFuncs(mux, routes, logger)
+	var rootHandler http.Handler = mux
+	if rpcConfig.IsCorsEnabled() {
+		rootHandler = addCORSHandler(rpcConfig, mux)
+	}
+	return rootHandler
+}
+
+func addCORSHandler(rpcConfig *config.RPCConfig, h http.Handler) http.Handler {
+	corsMiddleware := cors.New(cors.Options{
+		AllowedOrigins: rpcConfig.CORSAllowedOrigins,
+		AllowedMethods: rpcConfig.CORSAllowedMethods,
+		AllowedHeaders: rpcConfig.CORSAllowedHeaders,
+	})
+	h = corsMiddleware.Handler(h)
+	return h
+}
+
+type waitSyncCheckerImpl struct{}
+
+func (waitSyncCheckerImpl) WaitSync() bool {
+	return false
+}
+
+func (waitSyncCheckerImpl) GetPeerState(peerID types.NodeID) (*consensus.PeerState, bool) {
+	return nil, false
+}
+
+// ListenAndServe listens on the address specified in srv.Addr and handles any
+// incoming requests over HTTP using the Inspector rpc handler specified on the server.
+func (srv *Server) ListenAndServe(ctx context.Context) error {
+	listener, err := server.Listen(srv.Addr, srv.Config.MaxOpenConnections)
+	if err != nil {
+		return err
+	}
+	go func() {
+		<-ctx.Done()
+		listener.Close()
+	}()
+	return server.Serve(listener, srv.Handler, srv.Logger, serverRPCConfig(srv.Config))
+}
+
+// ListenAndServeTLS listens on the address specified in srv.Addr. ListenAndServeTLS handles
+// incoming requests over HTTPS using the Inspector rpc handler specified on the server.
+func (srv *Server) ListenAndServeTLS(ctx context.Context, certFile, keyFile string) error {
+	listener, err := server.Listen(srv.Addr, srv.Config.MaxOpenConnections)
+	if err != nil {
+		return err
+	}
+	go func() {
+		<-ctx.Done()
+		listener.Close()
+	}()
+	return server.ServeTLS(listener, srv.Handler, certFile, keyFile, srv.Logger, serverRPCConfig(srv.Config))
+}
+
+func serverRPCConfig(r *config.RPCConfig) *server.Config {
+	cfg := server.DefaultConfig()
+	cfg.MaxBodyBytes = r.MaxBodyBytes
+	cfg.MaxHeaderBytes = r.MaxHeaderBytes
+	// If necessary adjust global WriteTimeout to ensure it's greater than
+	// TimeoutBroadcastTxCommit.
+	// See https://github.com/tendermint/tendermint/issues/3435
+	if cfg.WriteTimeout <= r.TimeoutBroadcastTxCommit {
+		cfg.WriteTimeout = r.TimeoutBroadcastTxCommit + 1*time.Second
+	}
+	return cfg
+}

internal/libs/clist/clist_property_test.go

@@ -0,0 +1,72 @@
+package clist_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"pgregory.net/rapid"
+
+	"github.com/tendermint/tendermint/internal/libs/clist"
+)
+
+func TestCListProperties(t *testing.T) {
+	rapid.Check(t, rapid.Run(&clistModel{}))
+}
+
+// clistModel is used by the rapid state machine testing framework.
+// clistModel contains both the clist that is being tested and a slice of *clist.CElements
+// that will be used to model the expected clist behavior.
+type clistModel struct {
+	clist *clist.CList
+
+	model []*clist.CElement
+}
+
+// Init is a method used by the rapid state machine testing library.
+// Init is called when the test starts to initialize the data that will be used
+// in the state machine test.
+func (m *clistModel) Init(t *rapid.T) {
+	m.clist = clist.New()
+	m.model = []*clist.CElement{}
+}
+
+// PushBack defines an action that will be randomly selected across by the rapid state
+// machines testing library. Every call to PushBack calls PushBack on the clist and
+// performs a similar action on the model data.
+func (m *clistModel) PushBack(t *rapid.T) {
+	value := rapid.String().Draw(t, "value").(string)
+	el := m.clist.PushBack(value)
+	m.model = append(m.model, el)
+}
+
+// Remove defines an action that will be randomly selected across by the rapid state
+// machine testing library. Every call to Remove selects an element from the model
+// and calls Remove on the CList with that element. The same element is removed from
+// the model to keep the objects in sync.
+func (m *clistModel) Remove(t *rapid.T) {
+	if len(m.model) == 0 {
+		return
+	}
+	ix := rapid.IntRange(0, len(m.model)-1).Draw(t, "index").(int)
+	value := m.model[ix]
+	m.model = append(m.model[:ix], m.model[ix+1:]...)
+	m.clist.Remove(value)
+}
+
+// Check is a method required by the rapid state machine testing library.
+// Check is run after each action and is used to verify that the state of the object,
+// in this case a clist.CList matches the state of the objec.
+func (m *clistModel) Check(t *rapid.T) {
+	require.Equal(t, len(m.model), m.clist.Len())
+	if len(m.model) == 0 {
+		return
+	}
+	require.Equal(t, m.model[0], m.clist.Front())
+	require.Equal(t, m.model[len(m.model)-1], m.clist.Back())
+
+	iter := m.clist.Front()
+	for _, val := range m.model {
+		require.Equal(t, val, iter)
+		iter = iter.Next()
+	}
+}

internal/libs/protoio/io_test.go

@@ -71,7 +71,7 @@ func iotest(writer protoio.WriteCloser, reader protoio.ReadCloser) error {
 			return err
 		}
 		if n != len(bz)+visize {
-			return fmt.Errorf("WriteMsg() wrote %v bytes, expected %v", n, len(bz)+visize) // nolint
+			return fmt.Errorf("WriteMsg() wrote %v bytes, expected %v", n, len(bz)+visize)
 		}
 		lens[i] = n
 	}

internal/libs/sync/deadlock.go

@@ -1,3 +1,4 @@
+//go:build deadlock
 // +build deadlock
 
 package sync

internal/libs/sync/sync.go

@@ -1,3 +1,4 @@
+//go:build !deadlock
 // +build !deadlock
 
 package sync

internal/mempool/cache.go

@@ -31,14 +31,14 @@ var _ TxCache = (*LRUTxCache)(nil)
 type LRUTxCache struct {
 	mtx      tmsync.Mutex
 	size     int
-	cacheMap map[[TxKeySize]byte]*list.Element
+	cacheMap map[types.TxKey]*list.Element
 	list     *list.List
 }
 
 func NewLRUTxCache(cacheSize int) *LRUTxCache {
 	return &LRUTxCache{
 		size:     cacheSize,
-		cacheMap: make(map[[TxKeySize]byte]*list.Element, cacheSize),
+		cacheMap: make(map[types.TxKey]*list.Element, cacheSize),
 		list:     list.New(),
 	}
 }
@@ -53,7 +53,7 @@ func (c *LRUTxCache) Reset() {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 
-	c.cacheMap = make(map[[TxKeySize]byte]*list.Element, c.size)
+	c.cacheMap = make(map[types.TxKey]*list.Element, c.size)
 	c.list.Init()
 }
 
@@ -61,7 +61,7 @@ func (c *LRUTxCache) Push(tx types.Tx) bool {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 
-	key := TxKey(tx)
+	key := tx.Key()
 
 	moved, ok := c.cacheMap[key]
 	if ok {
@@ -72,7 +72,7 @@ func (c *LRUTxCache) Push(tx types.Tx) bool {
 	if c.list.Len() >= c.size {
 		front := c.list.Front()
 		if front != nil {
-			frontKey := front.Value.([TxKeySize]byte)
+			frontKey := front.Value.(types.TxKey)
 			delete(c.cacheMap, frontKey)
 			c.list.Remove(front)
 		}
@@ -88,7 +88,7 @@ func (c *LRUTxCache) Remove(tx types.Tx) {
 	c.mtx.Lock()
 	defer c.mtx.Unlock()
 
-	key := TxKey(tx)
+	key := tx.Key()
 	e := c.cacheMap[key]
 	delete(c.cacheMap, key)
 

internal/mempool/ids.go

@@ -7,17 +7,15 @@ import (
 	"github.com/tendermint/tendermint/types"
 )
 
-// nolint: golint
-// TODO: Rename type.
-type MempoolIDs struct {
+type IDs struct {
 	mtx       tmsync.RWMutex
 	peerMap   map[types.NodeID]uint16
 	nextID    uint16              // assumes that a node will never have over 65536 active peers
 	activeIDs map[uint16]struct{} // used to check if a given peerID key is used
 }
 
-func NewMempoolIDs() *MempoolIDs {
-	return &MempoolIDs{
+func NewMempoolIDs() *IDs {
+	return &IDs{
 		peerMap: make(map[types.NodeID]uint16),
 
 		// reserve UnknownPeerID for mempoolReactor.BroadcastTx
@@ -28,7 +26,7 @@ func NewMempoolIDs() *MempoolIDs {
 
 // ReserveForPeer searches for the next unused ID and assigns it to the provided
 // peer.
-func (ids *MempoolIDs) ReserveForPeer(peerID types.NodeID) {
+func (ids *IDs) ReserveForPeer(peerID types.NodeID) {
 	ids.mtx.Lock()
 	defer ids.mtx.Unlock()
 
@@ -38,7 +36,7 @@ func (ids *MempoolIDs) ReserveForPeer(peerID types.NodeID) {
 }
 
 // Reclaim returns the ID reserved for the peer back to unused pool.
-func (ids *MempoolIDs) Reclaim(peerID types.NodeID) {
+func (ids *IDs) Reclaim(peerID types.NodeID) {
 	ids.mtx.Lock()
 	defer ids.mtx.Unlock()
 
@@ -50,7 +48,7 @@ func (ids *MempoolIDs) Reclaim(peerID types.NodeID) {
 }
 
 // GetForPeer returns an ID reserved for the peer.
-func (ids *MempoolIDs) GetForPeer(peerID types.NodeID) uint16 {
+func (ids *IDs) GetForPeer(peerID types.NodeID) uint16 {
 	ids.mtx.RLock()
 	defer ids.mtx.RUnlock()
 
@@ -59,7 +57,7 @@ func (ids *MempoolIDs) GetForPeer(peerID types.NodeID) uint16 {
 
 // nextPeerID returns the next unused peer ID to use. We assume that the mutex
 // is already held.
-func (ids *MempoolIDs) nextPeerID() uint16 {
+func (ids *IDs) nextPeerID() uint16 {
 	if len(ids.activeIDs) == MaxActiveIDs {
 		panic(fmt.Sprintf("node has maximum %d active IDs and wanted to get one more", MaxActiveIDs))
 	}

internal/mempool/mempool.go

@@ -32,6 +32,10 @@ type Mempool interface {
 	// its validity and whether it should be added to the mempool.
 	CheckTx(ctx context.Context, tx types.Tx, callback func(*abci.Response), txInfo TxInfo) error
 
+	// RemoveTxByKey removes a transaction, identified by its key,
+	// from the mempool.
+	RemoveTxByKey(txKey types.TxKey) error
+
 	// ReapMaxBytesMaxGas reaps transactions from the mempool up to maxBytes
 	// bytes total with the condition that the total gasWanted must be less than
 	// maxGas.

internal/mempool/mock/mempool.go

@@ -5,29 +5,30 @@ import (
 
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/internal/libs/clist"
-	mempl "github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/types"
 )
 
 // Mempool is an empty implementation of a Mempool, useful for testing.
 type Mempool struct{}
 
-var _ mempl.Mempool = Mempool{}
+var _ mempool.Mempool = Mempool{}
 
 func (Mempool) Lock()     {}
 func (Mempool) Unlock()   {}
 func (Mempool) Size() int { return 0 }
-func (Mempool) CheckTx(_ context.Context, _ types.Tx, _ func(*abci.Response), _ mempl.TxInfo) error {
+func (Mempool) CheckTx(_ context.Context, _ types.Tx, _ func(*abci.Response), _ mempool.TxInfo) error {
 	return nil
 }
+func (Mempool) RemoveTxByKey(txKey types.TxKey) error   { return nil }
 func (Mempool) ReapMaxBytesMaxGas(_, _ int64) types.Txs { return types.Txs{} }
 func (Mempool) ReapMaxTxs(n int) types.Txs              { return types.Txs{} }
 func (Mempool) Update(
 	_ int64,
 	_ types.Txs,
 	_ []*abci.ResponseDeliverTx,
-	_ mempl.PreCheckFunc,
-	_ mempl.PostCheckFunc,
+	_ mempool.PreCheckFunc,
+	_ mempool.PostCheckFunc,
 ) error {
 	return nil
 }

internal/mempool/tx.go

@@ -1,24 +1,9 @@
 package mempool
 
 import (
-	"crypto/sha256"
-
 	"github.com/tendermint/tendermint/types"
 )
 
-// TxKeySize defines the size of the transaction's key used for indexing.
-const TxKeySize = sha256.Size
-
-// TxKey is the fixed length array key used as an index.
-func TxKey(tx types.Tx) [TxKeySize]byte {
-	return sha256.Sum256(tx)
-}
-
-// TxHashFromBytes returns the hash of a transaction from raw bytes.
-func TxHashFromBytes(tx []byte) []byte {
-	return types.Tx(tx).Hash()
-}
-
 // TxInfo are parameters that get passed when attempting to add a tx to the
 // mempool.
 type TxInfo struct {

internal/mempool/v0/bench_test.go

@@ -6,14 +6,14 @@ import (
 	"sync/atomic"
 	"testing"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	"github.com/tendermint/tendermint/internal/mempool"
-	"github.com/tendermint/tendermint/proxy"
 )
 
 func BenchmarkReap(b *testing.B) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 	mp.config.Size = 100000
@@ -34,7 +34,7 @@ func BenchmarkReap(b *testing.B) {
 
 func BenchmarkCheckTx(b *testing.B) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -56,7 +56,7 @@ func BenchmarkCheckTx(b *testing.B) {
 
 func BenchmarkParallelCheckTx(b *testing.B) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -81,7 +81,7 @@ func BenchmarkParallelCheckTx(b *testing.B) {
 
 func BenchmarkCheckDuplicateTx(b *testing.B) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 

internal/mempool/v0/cache_test.go

@@ -7,16 +7,16 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/internal/mempool"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
 func TestCacheAfterUpdate(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -61,7 +61,7 @@ func TestCacheAfterUpdate(t *testing.T) {
 			require.NotEqual(t, len(tc.txsInCache), counter,
 				"cache larger than expected on testcase %d", tcIndex)
 
-			nodeVal := node.Value.([sha256.Size]byte)
+			nodeVal := node.Value.(types.TxKey)
 			expectedBz := sha256.Sum256([]byte{byte(tc.txsInCache[len(tc.txsInCache)-counter-1])})
 			// Reference for reading the errors:
 			// >>> sha256('\x00').hexdigest()
@@ -71,7 +71,7 @@ func TestCacheAfterUpdate(t *testing.T) {
 			// >>> sha256('\x02').hexdigest()
 			// 'dbc1b4c900ffe48d575b5da5c638040125f65db0fe3e24494b76ea986457d986'
 
-			require.Equal(t, expectedBz, nodeVal, "Equality failed on index %d, tc %d", counter, tcIndex)
+			require.EqualValues(t, expectedBz, nodeVal, "Equality failed on index %d, tc %d", counter, tcIndex)
 			counter++
 			node = node.Next()
 		}

internal/mempool/v0/clist_mempool.go

@@ -3,19 +3,19 @@ package v0
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 	"sync/atomic"
 
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/internal/proxy"
 	"github.com/tendermint/tendermint/libs/log"
 	tmmath "github.com/tendermint/tendermint/libs/math"
-	pubmempool "github.com/tendermint/tendermint/pkg/mempool"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -33,7 +33,7 @@ type CListMempool struct {
 	notifiedTxsAvailable bool
 	txsAvailable         chan struct{} // fires once for each height, when the mempool is not empty
 
-	config *cfg.MempoolConfig
+	config *config.MempoolConfig
 
 	// Exclusive mutex for Update method to prevent concurrent execution of
 	// CheckTx or ReapMaxBytesMaxGas(ReapMaxTxs) methods.
@@ -70,14 +70,14 @@ type CListMempoolOption func(*CListMempool)
 // NewCListMempool returns a new mempool with the given configuration and
 // connection to an application.
 func NewCListMempool(
-	config *cfg.MempoolConfig,
+	cfg *config.MempoolConfig,
 	proxyAppConn proxy.AppConnMempool,
 	height int64,
 	options ...CListMempoolOption,
 ) *CListMempool {
 
 	mp := &CListMempool{
-		config:        config,
+		config:        cfg,
 		proxyAppConn:  proxyAppConn,
 		txs:           clist.New(),
 		height:        height,
@@ -87,8 +87,8 @@ func NewCListMempool(
 		metrics:       mempool.NopMetrics(),
 	}
 
-	if config.CacheSize > 0 {
-		mp.cache = mempool.NewLRUTxCache(config.CacheSize)
+	if cfg.CacheSize > 0 {
+		mp.cache = mempool.NewLRUTxCache(cfg.CacheSize)
 	} else {
 		mp.cache = mempool.NopTxCache{}
 	}
@@ -217,7 +217,7 @@ func (mem *CListMempool) CheckTx(
 	}
 
 	if txSize > mem.config.MaxTxBytes {
-		return pubmempool.ErrTxTooLarge{
+		return types.ErrTxTooLarge{
 			Max:    mem.config.MaxTxBytes,
 			Actual: txSize,
 		}
@@ -225,7 +225,7 @@ func (mem *CListMempool) CheckTx(
 
 	if mem.preCheck != nil {
 		if err := mem.preCheck(tx); err != nil {
-			return pubmempool.ErrPreCheck{
+			return types.ErrPreCheck{
 				Reason: err,
 			}
 		}
@@ -241,14 +241,14 @@ func (mem *CListMempool) CheckTx(
 		// Note it's possible a tx is still in the cache but no longer in the mempool
 		// (eg. after committing a block, txs are removed from mempool but not cache),
 		// so we only record the sender for txs still in the mempool.
-		if e, ok := mem.txsMap.Load(mempool.TxKey(tx)); ok {
+		if e, ok := mem.txsMap.Load(tx.Key()); ok {
 			memTx := e.(*clist.CElement).Value.(*mempoolTx)
 			_, loaded := memTx.senders.LoadOrStore(txInfo.SenderID, true)
 			// TODO: consider punishing peer for dups,
 			// its non-trivial since invalid txs can become valid,
 			// but they can spam the same tx with little cost to them atm.
 			if loaded {
-				return pubmempool.ErrTxInCache
+				return types.ErrTxInCache
 			}
 		}
 
@@ -328,7 +328,7 @@ func (mem *CListMempool) reqResCb(
 //  - resCbFirstTime (lock not held) if tx is valid
 func (mem *CListMempool) addTx(memTx *mempoolTx) {
 	e := mem.txs.PushBack(memTx)
-	mem.txsMap.Store(mempool.TxKey(memTx.tx), e)
+	mem.txsMap.Store(memTx.tx.Key(), e)
 	atomic.AddInt64(&mem.txsBytes, int64(len(memTx.tx)))
 	mem.metrics.TxSizeBytes.Observe(float64(len(memTx.tx)))
 }
@@ -339,7 +339,7 @@ func (mem *CListMempool) addTx(memTx *mempoolTx) {
 func (mem *CListMempool) removeTx(tx types.Tx, elem *clist.CElement, removeFromCache bool) {
 	mem.txs.Remove(elem)
 	elem.DetachPrev()
-	mem.txsMap.Delete(mempool.TxKey(tx))
+	mem.txsMap.Delete(tx.Key())
 	atomic.AddInt64(&mem.txsBytes, int64(-len(tx)))
 
 	if removeFromCache {
@@ -348,13 +348,16 @@ func (mem *CListMempool) removeTx(tx types.Tx, elem *clist.CElement, removeFromC
 }
 
 // RemoveTxByKey removes a transaction from the mempool by its TxKey index.
-func (mem *CListMempool) RemoveTxByKey(txKey [mempool.TxKeySize]byte, removeFromCache bool) {
+func (mem *CListMempool) RemoveTxByKey(txKey types.TxKey) error {
 	if e, ok := mem.txsMap.Load(txKey); ok {
 		memTx := e.(*clist.CElement).Value.(*mempoolTx)
 		if memTx != nil {
-			mem.removeTx(memTx.tx, e.(*clist.CElement), removeFromCache)
+			mem.removeTx(memTx.tx, e.(*clist.CElement), false)
+			return nil
 		}
+		return errors.New("transaction not found")
 	}
+	return errors.New("invalid transaction found")
 }
 
 func (mem *CListMempool) isFull(txSize int) error {
@@ -364,7 +367,7 @@ func (mem *CListMempool) isFull(txSize int) error {
 	)
 
 	if memSize >= mem.config.Size || int64(txSize)+txsBytes > mem.config.MaxTxsBytes {
-		return pubmempool.ErrMempoolIsFull{
+		return types.ErrMempoolIsFull{
 			NumTxs:      memSize,
 			MaxTxs:      mem.config.Size,
 			TxsBytes:    txsBytes,
@@ -410,7 +413,7 @@ func (mem *CListMempool) resCbFirstTime(
 			mem.addTx(memTx)
 			mem.logger.Debug(
 				"added good transaction",
-				"tx", mempool.TxHashFromBytes(tx),
+				"tx", types.Tx(tx).Hash(),
 				"res", r,
 				"height", memTx.height,
 				"total", mem.Size(),
@@ -420,7 +423,7 @@ func (mem *CListMempool) resCbFirstTime(
 			// ignore bad transaction
 			mem.logger.Debug(
 				"rejected bad transaction",
-				"tx", mempool.TxHashFromBytes(tx),
+				"tx", types.Tx(tx).Hash(),
 				"peerID", peerP2PID,
 				"res", r,
 				"err", postCheckErr,
@@ -461,7 +464,7 @@ func (mem *CListMempool) resCbRecheck(req *abci.Request, res *abci.Response) {
 			// Good, nothing to do.
 		} else {
 			// Tx became invalidated due to newly committed block.
-			mem.logger.Debug("tx is no longer valid", "tx", mempool.TxHashFromBytes(tx), "res", r, "err", postCheckErr)
+			mem.logger.Debug("tx is no longer valid", "tx", types.Tx(tx).Hash(), "res", r, "err", postCheckErr)
 			// NOTE: we remove tx from the cache because it might be good later
 			mem.removeTx(tx, mem.recheckCursor, !mem.config.KeepInvalidTxsInCache)
 		}
@@ -599,7 +602,7 @@ func (mem *CListMempool) Update(
 		// Mempool after:
 		//   100
 		// https://github.com/tendermint/tendermint/issues/3322.
-		if e, ok := mem.txsMap.Load(mempool.TxKey(tx)); ok {
+		if e, ok := mem.txsMap.Load(tx.Key()); ok {
 			mem.removeTx(tx, e.(*clist.CElement), false)
 		}
 	}

internal/mempool/v0/clist_mempool_test.go

@@ -15,16 +15,15 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abciserver "github.com/tendermint/tendermint/abci/server"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 	"github.com/tendermint/tendermint/libs/service"
-	pubmempool "github.com/tendermint/tendermint/pkg/mempool"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -32,22 +31,22 @@ import (
 // test.
 type cleanupFunc func()
 
-func newMempoolWithApp(cc proxy.ClientCreator) (*CListMempool, cleanupFunc) {
-	return newMempoolWithAppAndConfig(cc, cfg.ResetTestRoot("mempool_test"))
+func newMempoolWithApp(cc abciclient.Creator) (*CListMempool, cleanupFunc) {
+	return newMempoolWithAppAndConfig(cc, config.ResetTestRoot("mempool_test"))
 }
 
-func newMempoolWithAppAndConfig(cc proxy.ClientCreator, config *cfg.Config) (*CListMempool, cleanupFunc) {
-	appConnMem, _ := cc.NewABCIClient()
+func newMempoolWithAppAndConfig(cc abciclient.Creator, cfg *config.Config) (*CListMempool, cleanupFunc) {
+	appConnMem, _ := cc()
 	appConnMem.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "mempool"))
 	err := appConnMem.Start()
 	if err != nil {
 		panic(err)
 	}
 
-	mp := NewCListMempool(config.Mempool, appConnMem, 0)
+	mp := NewCListMempool(cfg.Mempool, appConnMem, 0)
 	mp.SetLogger(log.TestingLogger())
 
-	return mp, func() { os.RemoveAll(config.RootDir) }
+	return mp, func() { os.RemoveAll(cfg.RootDir) }
 }
 
 func ensureNoFire(t *testing.T, ch <-chan struct{}, timeoutMS int) {
@@ -82,7 +81,7 @@ func checkTxs(t *testing.T, mp mempool.Mempool, count int, peerID uint16) types.
 			// Skip invalid txs.
 			// TestMempoolFilters will fail otherwise. It asserts a number of txs
 			// returned.
-			if pubmempool.IsPreCheckError(err) {
+			if types.IsPreCheckError(err) {
 				continue
 			}
 			t.Fatalf("CheckTx failed: %v while checking #%d tx", err, i)
@@ -93,7 +92,7 @@ func checkTxs(t *testing.T, mp mempool.Mempool, count int, peerID uint16) types.
 
 func TestReapMaxBytesMaxGas(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -142,7 +141,7 @@ func TestReapMaxBytesMaxGas(t *testing.T) {
 
 func TestMempoolFilters(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 	emptyTxArr := []types.Tx{[]byte{}}
@@ -181,7 +180,7 @@ func TestMempoolFilters(t *testing.T) {
 
 func TestMempoolUpdate(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -217,8 +216,8 @@ func TestMempoolUpdate(t *testing.T) {
 
 func TestMempool_KeepInvalidTxsInCache(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
-	wcfg := cfg.DefaultConfig()
+	cc := abciclient.NewLocalCreator(app)
+	wcfg := config.DefaultConfig()
 	wcfg.Mempool.KeepInvalidTxsInCache = true
 	mp, cleanup := newMempoolWithAppAndConfig(cc, wcfg)
 	defer cleanup()
@@ -265,7 +264,7 @@ func TestMempool_KeepInvalidTxsInCache(t *testing.T) {
 
 func TestTxsAvailable(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 	mp.EnableTxsAvailable()
@@ -309,12 +308,12 @@ func TestTxsAvailable(t *testing.T) {
 
 func TestSerialReap(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 
 	mp, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
-	appConnCon, _ := cc.NewABCIClient()
+	appConnCon, _ := cc()
 	appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus"))
 	err := appConnCon.Start()
 	require.Nil(t, err)
@@ -420,7 +419,7 @@ func TestSerialReap(t *testing.T) {
 
 func TestMempool_CheckTxChecksTxSize(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 	mempl, cleanup := newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -455,7 +454,7 @@ func TestMempool_CheckTxChecksTxSize(t *testing.T) {
 		if !testCase.err {
 			require.NoError(t, err, caseString)
 		} else {
-			require.Equal(t, err, pubmempool.ErrTxTooLarge{
+			require.Equal(t, err, types.ErrTxTooLarge{
 				Max:    maxTxSize,
 				Actual: testCase.len,
 			}, caseString)
@@ -465,10 +464,10 @@ func TestMempool_CheckTxChecksTxSize(t *testing.T) {
 
 func TestMempoolTxsBytes(t *testing.T) {
 	app := kvstore.NewApplication()
-	cc := proxy.NewLocalClientCreator(app)
-	config := cfg.ResetTestRoot("mempool_test")
-	config.Mempool.MaxTxsBytes = 10
-	mp, cleanup := newMempoolWithAppAndConfig(cc, config)
+	cc := abciclient.NewLocalCreator(app)
+	cfg := config.ResetTestRoot("mempool_test")
+	cfg.Mempool.MaxTxsBytes = 10
+	mp, cleanup := newMempoolWithAppAndConfig(cc, cfg)
 	defer cleanup()
 
 	// 1. zero by default
@@ -503,12 +502,12 @@ func TestMempoolTxsBytes(t *testing.T) {
 
 	err = mp.CheckTx(context.Background(), []byte{0x05}, nil, mempool.TxInfo{})
 	if assert.Error(t, err) {
-		assert.IsType(t, pubmempool.ErrMempoolIsFull{}, err)
+		assert.IsType(t, types.ErrMempoolIsFull{}, err)
 	}
 
 	// 6. zero after tx is rechecked and removed due to not being valid anymore
 	app2 := kvstore.NewApplication()
-	cc = proxy.NewLocalClientCreator(app2)
+	cc = abciclient.NewLocalCreator(app2)
 	mp, cleanup = newMempoolWithApp(cc)
 	defer cleanup()
 
@@ -519,7 +518,7 @@ func TestMempoolTxsBytes(t *testing.T) {
 	require.NoError(t, err)
 	assert.EqualValues(t, 8, mp.SizeBytes())
 
-	appConnCon, _ := cc.NewABCIClient()
+	appConnCon, _ := cc()
 	appConnCon.SetLogger(log.TestingLogger().With("module", "abci-client", "connection", "consensus"))
 	err = appConnCon.Start()
 	require.Nil(t, err)
@@ -545,9 +544,9 @@ func TestMempoolTxsBytes(t *testing.T) {
 	err = mp.CheckTx(context.Background(), []byte{0x06}, nil, mempool.TxInfo{})
 	require.NoError(t, err)
 	assert.EqualValues(t, 9, mp.SizeBytes())
-	mp.RemoveTxByKey(mempool.TxKey([]byte{0x07}), true)
+	assert.Error(t, mp.RemoveTxByKey(types.Tx([]byte{0x07}).Key()))
 	assert.EqualValues(t, 9, mp.SizeBytes())
-	mp.RemoveTxByKey(mempool.TxKey([]byte{0x06}), true)
+	assert.NoError(t, mp.RemoveTxByKey(types.Tx([]byte{0x06}).Key()))
 	assert.EqualValues(t, 8, mp.SizeBytes())
 
 }
@@ -565,8 +564,8 @@ func TestMempoolRemoteAppConcurrency(t *testing.T) {
 			t.Error(err)
 		}
 	})
-	config := cfg.ResetTestRoot("mempool_test")
-	mp, cleanup := newMempoolWithAppAndConfig(cc, config)
+	cfg := config.ResetTestRoot("mempool_test")
+	mp, cleanup := newMempoolWithAppAndConfig(cc, cfg)
 	defer cleanup()
 
 	// generate small number of txs
@@ -578,7 +577,7 @@ func TestMempoolRemoteAppConcurrency(t *testing.T) {
 	}
 
 	// simulate a group of peers sending them over and over
-	N := config.Mempool.Size
+	N := cfg.Mempool.Size
 	maxPeers := 5
 	for i := 0; i < N; i++ {
 		peerID := mrand.Intn(maxPeers)
@@ -598,10 +597,10 @@ func newRemoteApp(
 	addr string,
 	app abci.Application,
 ) (
-	clientCreator proxy.ClientCreator,
+	clientCreator abciclient.Creator,
 	server service.Service,
 ) {
-	clientCreator = proxy.NewRemoteClientCreator(addr, "socket", true)
+	clientCreator = abciclient.NewRemoteCreator(addr, "socket", true)
 
 	// Start server
 	server = abciserver.NewSocketServer(addr, app)

internal/mempool/v0/reactor.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 	"time"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/mempool"
@@ -37,9 +37,9 @@ type PeerManager interface {
 type Reactor struct {
 	service.BaseService
 
-	config  *cfg.MempoolConfig
+	cfg     *config.MempoolConfig
 	mempool *CListMempool
-	ids     *mempool.MempoolIDs
+	ids     *mempool.IDs
 
 	// XXX: Currently, this is the only way to get information about a peer. Ideally,
 	// we rely on message-oriented communication to get necessary peer data.
@@ -61,7 +61,7 @@ type Reactor struct {
 // NewReactor returns a reference to a new reactor.
 func NewReactor(
 	logger log.Logger,
-	config *cfg.MempoolConfig,
+	cfg *config.MempoolConfig,
 	peerMgr PeerManager,
 	mp *CListMempool,
 	mempoolCh *p2p.Channel,
@@ -69,7 +69,7 @@ func NewReactor(
 ) *Reactor {
 
 	r := &Reactor{
-		config:       config,
+		cfg:          cfg,
 		peerMgr:      peerMgr,
 		mempool:      mp,
 		ids:          mempool.NewMempoolIDs(),
@@ -83,32 +83,22 @@ func NewReactor(
 	return r
 }
 
-// GetChannelShims returns a map of ChannelDescriptorShim objects, where each
-// object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
-// p2p proto.Message the new p2p Channel is responsible for handling.
-//
-//
-// TODO: Remove once p2p refactor is complete.
-// ref: https://github.com/tendermint/tendermint/issues/5670
-func GetChannelShims(config *cfg.MempoolConfig) map[p2p.ChannelID]*p2p.ChannelDescriptorShim {
-	largestTx := make([]byte, config.MaxTxBytes)
+// GetChannelDescriptor produces an instance of a descriptor for this
+// package's required channels.
+func GetChannelDescriptor(cfg *config.MempoolConfig) *p2p.ChannelDescriptor {
+	largestTx := make([]byte, cfg.MaxTxBytes)
 	batchMsg := protomem.Message{
 		Sum: &protomem.Message_Txs{
 			Txs: &protomem.Txs{Txs: [][]byte{largestTx}},
 		},
 	}
 
-	return map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
-		mempool.MempoolChannel: {
-			MsgType: new(protomem.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(mempool.MempoolChannel),
-				Priority:            5,
-				RecvMessageCapacity: batchMsg.Size(),
-				RecvBufferCapacity:  128,
-				MaxSendBytes:        5000,
-			},
-		},
+	return &p2p.ChannelDescriptor{
+		ID:                  mempool.MempoolChannel,
+		MessageType:         new(protomem.Message),
+		Priority:            5,
+		RecvMessageCapacity: batchMsg.Size(),
+		RecvBufferCapacity:  128,
 	}
 }
 
@@ -117,7 +107,7 @@ func GetChannelShims(config *cfg.MempoolConfig) map[p2p.ChannelID]*p2p.ChannelDe
 // messages on that p2p channel accordingly. The caller must be sure to execute
 // OnStop to ensure the outbound p2p Channels are closed.
 func (r *Reactor) OnStart() error {
-	if !r.config.Broadcast {
+	if !r.cfg.Broadcast {
 		r.Logger.Info("tx broadcasting is disabled")
 	}
 
@@ -171,7 +161,7 @@ func (r *Reactor) handleMempoolMessage(envelope p2p.Envelope) error {
 
 		for _, tx := range protoTxs {
 			if err := r.mempool.CheckTx(context.Background(), types.Tx(tx), nil, txInfo); err != nil {
-				logger.Error("checktx failed for tx", "tx", fmt.Sprintf("%X", mempool.TxHashFromBytes(tx)), "err", err)
+				logger.Error("checktx failed for tx", "tx", fmt.Sprintf("%X", types.Tx(tx).Hash()), "err", err)
 			}
 		}
 
@@ -254,7 +244,7 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
 			return
 		}
 
-		if r.config.Broadcast {
+		if r.cfg.Broadcast {
 			// Check if we've already started a goroutine for this peer, if not we create
 			// a new done channel so we can explicitly close the goroutine if the peer
 			// is later removed, we increment the waitgroup so the reactor can stop
@@ -378,7 +368,7 @@ func (r *Reactor) broadcastTxRoutine(peerID types.NodeID, closer *tmsync.Closer)
 			}
 			r.Logger.Debug(
 				"gossiped tx to peer",
-				"tx", fmt.Sprintf("%X", mempool.TxHashFromBytes(memTx.tx)),
+				"tx", fmt.Sprintf("%X", memTx.tx.Hash()),
 				"peer", peerID,
 			)
 		}

internal/mempool/v0/reactor_test.go

@@ -8,16 +8,16 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/internal/p2p"
 	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 	protomem "github.com/tendermint/tendermint/proto/tendermint/mempool"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -36,7 +36,7 @@ type reactorTestSuite struct {
 	nodes []types.NodeID
 }
 
-func setup(t *testing.T, cfg *cfg.MempoolConfig, numNodes int, chBuf uint) *reactorTestSuite {
+func setup(t *testing.T, config *config.MempoolConfig, numNodes int, chBuf uint) *reactorTestSuite {
 	t.Helper()
 
 	rts := &reactorTestSuite{
@@ -50,12 +50,13 @@ func setup(t *testing.T, cfg *cfg.MempoolConfig, numNodes int, chBuf uint) *reac
 		peerUpdates:    make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
 	}
 
-	chDesc := p2p.ChannelDescriptor{ID: byte(mempool.MempoolChannel)}
-	rts.mempoolChnnels = rts.network.MakeChannelsNoCleanup(t, chDesc, new(protomem.Message), int(chBuf))
+	chDesc := GetChannelDescriptor(config)
+	chDesc.RecvBufferCapacity = int(chBuf)
+	rts.mempoolChnnels = rts.network.MakeChannelsNoCleanup(t, chDesc)
 
 	for nodeID := range rts.network.Nodes {
 		rts.kvstores[nodeID] = kvstore.NewApplication()
-		cc := proxy.NewLocalClientCreator(rts.kvstores[nodeID])
+		cc := abciclient.NewLocalCreator(rts.kvstores[nodeID])
 
 		mempool, memCleanup := newMempoolWithApp(cc)
 		t.Cleanup(memCleanup)
@@ -68,7 +69,7 @@ func setup(t *testing.T, cfg *cfg.MempoolConfig, numNodes int, chBuf uint) *reac
 
 		rts.reactors[nodeID] = NewReactor(
 			rts.logger.With("nodeID", nodeID),
-			cfg,
+			config,
 			rts.network.Nodes[nodeID].PeerManager,
 			mempool,
 			rts.mempoolChnnels[nodeID],
@@ -158,9 +159,9 @@ func (rts *reactorTestSuite) waitForTxns(t *testing.T, txs types.Txs, ids ...typ
 func TestReactorBroadcastTxs(t *testing.T) {
 	numTxs := 1000
 	numNodes := 10
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
-	rts := setup(t, config.Mempool, numNodes, 0)
+	rts := setup(t, cfg.Mempool, numNodes, 0)
 
 	primary := rts.nodes[0]
 	secondaries := rts.nodes[1:]
@@ -185,9 +186,9 @@ func TestReactorBroadcastTxs(t *testing.T) {
 func TestReactorConcurrency(t *testing.T) {
 	numTxs := 5
 	numNodes := 2
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
-	rts := setup(t, config.Mempool, numNodes, 0)
+	rts := setup(t, cfg.Mempool, numNodes, 0)
 
 	primary := rts.nodes[0]
 	secondary := rts.nodes[1]
@@ -244,9 +245,9 @@ func TestReactorConcurrency(t *testing.T) {
 func TestReactorNoBroadcastToSender(t *testing.T) {
 	numTxs := 1000
 	numNodes := 2
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
-	rts := setup(t, config.Mempool, numNodes, uint(numTxs))
+	rts := setup(t, cfg.Mempool, numNodes, uint(numTxs))
 
 	primary := rts.nodes[0]
 	secondary := rts.nodes[1]
@@ -267,16 +268,16 @@ func TestReactorNoBroadcastToSender(t *testing.T) {
 
 func TestReactor_MaxTxBytes(t *testing.T) {
 	numNodes := 2
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
-	rts := setup(t, config.Mempool, numNodes, 0)
+	rts := setup(t, cfg.Mempool, numNodes, 0)
 
 	primary := rts.nodes[0]
 	secondary := rts.nodes[1]
 
 	// Broadcast a tx, which has the max size and ensure it's received by the
 	// second reactor.
-	tx1 := tmrand.Bytes(config.Mempool.MaxTxBytes)
+	tx1 := tmrand.Bytes(cfg.Mempool.MaxTxBytes)
 	err := rts.reactors[primary].mempool.CheckTx(
 		context.Background(),
 		tx1,
@@ -297,7 +298,7 @@ func TestReactor_MaxTxBytes(t *testing.T) {
 	rts.reactors[secondary].mempool.Flush()
 
 	// broadcast a tx, which is beyond the max size and ensure it's not sent
-	tx2 := tmrand.Bytes(config.Mempool.MaxTxBytes + 1)
+	tx2 := tmrand.Bytes(cfg.Mempool.MaxTxBytes + 1)
 	err = rts.mempools[primary].CheckTx(context.Background(), tx2, nil, mempool.TxInfo{SenderID: mempool.UnknownPeerID})
 	require.Error(t, err)
 
@@ -305,11 +306,11 @@ func TestReactor_MaxTxBytes(t *testing.T) {
 }
 
 func TestDontExhaustMaxActiveIDs(t *testing.T) {
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
 	// we're creating a single node network, but not starting the
 	// network.
-	rts := setup(t, config.Mempool, 1, mempool.MaxActiveIDs+1)
+	rts := setup(t, cfg.Mempool, 1, mempool.MaxActiveIDs+1)
 
 	nodeID := rts.nodes[0]
 
@@ -374,9 +375,9 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) {
 		t.Skip("skipping test in short mode")
 	}
 
-	config := cfg.TestConfig()
+	cfg := config.TestConfig()
 
-	rts := setup(t, config.Mempool, 2, 0)
+	rts := setup(t, cfg.Mempool, 2, 0)
 
 	primary := rts.nodes[0]
 	secondary := rts.nodes[1]

internal/mempool/v1/mempool.go

@@ -3,6 +3,7 @@ package v1
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"sync/atomic"
 	"time"
@@ -12,10 +13,9 @@ import (
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/internal/proxy"
 	"github.com/tendermint/tendermint/libs/log"
 	tmmath "github.com/tendermint/tendermint/libs/math"
-	pubmempool "github.com/tendermint/tendermint/pkg/mempool"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -239,7 +239,7 @@ func (txmp *TxMempool) CheckTx(
 
 	txSize := len(tx)
 	if txSize > txmp.config.MaxTxBytes {
-		return pubmempool.ErrTxTooLarge{
+		return types.ErrTxTooLarge{
 			Max:    txmp.config.MaxTxBytes,
 			Actual: txSize,
 		}
@@ -247,7 +247,7 @@ func (txmp *TxMempool) CheckTx(
 
 	if txmp.preCheck != nil {
 		if err := txmp.preCheck(tx); err != nil {
-			return pubmempool.ErrPreCheck{
+			return types.ErrPreCheck{
 				Reason: err,
 			}
 		}
@@ -257,7 +257,7 @@ func (txmp *TxMempool) CheckTx(
 		return err
 	}
 
-	txHash := mempool.TxKey(tx)
+	txHash := tx.Key()
 
 	// We add the transaction to the mempool's cache and if the transaction already
 	// exists, i.e. false is returned, then we check if we've seen this transaction
@@ -267,7 +267,7 @@ func (txmp *TxMempool) CheckTx(
 		if wtx != nil && ok {
 			// We already have the transaction stored and the we've already seen this
 			// transaction from txInfo.SenderID.
-			return pubmempool.ErrTxInCache
+			return types.ErrTxInCache
 		}
 
 		txmp.logger.Debug("tx exists already in cache", "tx_hash", tx.Hash())
@@ -305,6 +305,19 @@ func (txmp *TxMempool) CheckTx(
 	return nil
 }
 
+func (txmp *TxMempool) RemoveTxByKey(txKey types.TxKey) error {
+	txmp.Lock()
+	defer txmp.Unlock()
+
+	// remove the committed transaction from the transaction store and indexes
+	if wtx := txmp.txStore.GetTxByHash(txKey); wtx != nil {
+		txmp.removeTx(wtx, false)
+		return nil
+	}
+
+	return errors.New("transaction not found")
+}
+
 // Flush flushes out the mempool. It acquires a read-lock, fetches all the
 // transactions currently in the transaction store and removes each transaction
 // from the store and all indexes and finally resets the cache.
@@ -452,7 +465,7 @@ func (txmp *TxMempool) Update(
 		}
 
 		// remove the committed transaction from the transaction store and indexes
-		if wtx := txmp.txStore.GetTxByHash(mempool.TxKey(tx)); wtx != nil {
+		if wtx := txmp.txStore.GetTxByHash(tx.Key()); wtx != nil {
 			txmp.removeTx(wtx, false)
 		}
 	}
@@ -630,7 +643,7 @@ func (txmp *TxMempool) defaultTxCallback(req *abci.Request, res *abci.Response)
 		tx := req.GetCheckTx().Tx
 		wtx := txmp.recheckCursor.Value.(*WrappedTx)
 		if !bytes.Equal(tx, wtx.tx) {
-			panic(fmt.Sprintf("re-CheckTx transaction mismatch; got: %X, expected: %X", wtx.tx.Hash(), mempool.TxKey(tx)))
+			panic(fmt.Sprintf("re-CheckTx transaction mismatch; got: %X, expected: %X", wtx.tx.Hash(), types.Tx(tx).Key()))
 		}
 
 		// Only evaluate transactions that have not been removed. This can happen
@@ -648,7 +661,7 @@ func (txmp *TxMempool) defaultTxCallback(req *abci.Request, res *abci.Response)
 				txmp.logger.Debug(
 					"existing transaction no longer valid; failed re-CheckTx callback",
 					"priority", wtx.priority,
-					"tx", fmt.Sprintf("%X", mempool.TxHashFromBytes(wtx.tx)),
+					"tx", fmt.Sprintf("%X", wtx.tx.Hash()),
 					"err", err,
 					"code", checkTxRes.CheckTx.Code,
 				)
@@ -728,7 +741,7 @@ func (txmp *TxMempool) canAddTx(wtx *WrappedTx) error {
 	)
 
 	if numTxs >= txmp.config.Size || int64(wtx.Size())+sizeBytes > txmp.config.MaxTxsBytes {
-		return pubmempool.ErrMempoolIsFull{
+		return types.ErrMempoolIsFull{
 			NumTxs:      numTxs,
 			MaxTxs:      txmp.config.Size,
 			TxsBytes:    sizeBytes,
@@ -785,13 +798,13 @@ func (txmp *TxMempool) removeTx(wtx *WrappedTx, removeFromCache bool) {
 // the height and time based indexes.
 func (txmp *TxMempool) purgeExpiredTxs(blockHeight int64) {
 	now := time.Now()
-	expiredTxs := make(map[[mempool.TxKeySize]byte]*WrappedTx)
+	expiredTxs := make(map[types.TxKey]*WrappedTx)
 
 	if txmp.config.TTLNumBlocks > 0 {
 		purgeIdx := -1
 		for i, wtx := range txmp.heightIndex.txs {
 			if (blockHeight - wtx.height) > txmp.config.TTLNumBlocks {
-				expiredTxs[mempool.TxKey(wtx.tx)] = wtx
+				expiredTxs[wtx.tx.Key()] = wtx
 				purgeIdx = i
 			} else {
 				// since the index is sorted, we know no other txs can be be purged
@@ -808,7 +821,7 @@ func (txmp *TxMempool) purgeExpiredTxs(blockHeight int64) {
 		purgeIdx := -1
 		for i, wtx := range txmp.timestampIndex.txs {
 			if now.Sub(wtx.timestamp) > txmp.config.TTLDuration {
-				expiredTxs[mempool.TxKey(wtx.tx)] = wtx
+				expiredTxs[wtx.tx.Key()] = wtx
 				purgeIdx = i
 			} else {
 				// since the index is sorted, we know no other txs can be be purged

internal/mempool/v1/mempool_test.go

@@ -15,13 +15,14 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+
+	abciclient "github.com/tendermint/tendermint/abci/client"
 	"github.com/tendermint/tendermint/abci/example/code"
 	"github.com/tendermint/tendermint/abci/example/kvstore"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/proxy"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -76,12 +77,12 @@ func setup(t testing.TB, cacheSize int, options ...TxMempoolOption) *TxMempool {
 	t.Helper()
 
 	app := &application{kvstore.NewApplication()}
-	cc := proxy.NewLocalClientCreator(app)
+	cc := abciclient.NewLocalCreator(app)
 
 	cfg := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|"))
 	cfg.Mempool.CacheSize = cacheSize
 
-	appConnMem, err := cc.NewABCIClient()
+	appConnMem, err := cc()
 	require.NoError(t, err)
 	require.NoError(t, appConnMem.Start())
 
@@ -225,10 +226,10 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) {
 	require.Equal(t, len(tTxs), txmp.Size())
 	require.Equal(t, int64(5690), txmp.SizeBytes())
 
-	txMap := make(map[[mempool.TxKeySize]byte]testTx)
+	txMap := make(map[types.TxKey]testTx)
 	priorities := make([]int64, len(tTxs))
 	for i, tTx := range tTxs {
-		txMap[mempool.TxKey(tTx.tx)] = tTx
+		txMap[tTx.tx.Key()] = tTx
 		priorities[i] = tTx.priority
 	}
 
@@ -240,7 +241,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) {
 	ensurePrioritized := func(reapedTxs types.Txs) {
 		reapedPriorities := make([]int64, len(reapedTxs))
 		for i, rTx := range reapedTxs {
-			reapedPriorities[i] = txMap[mempool.TxKey(rTx)].priority
+			reapedPriorities[i] = txMap[rTx.Key()].priority
 		}
 
 		require.Equal(t, priorities[:len(reapedPriorities)], reapedPriorities)
@@ -275,10 +276,10 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) {
 	require.Equal(t, len(tTxs), txmp.Size())
 	require.Equal(t, int64(5690), txmp.SizeBytes())
 
-	txMap := make(map[[mempool.TxKeySize]byte]testTx)
+	txMap := make(map[types.TxKey]testTx)
 	priorities := make([]int64, len(tTxs))
 	for i, tTx := range tTxs {
-		txMap[mempool.TxKey(tTx.tx)] = tTx
+		txMap[tTx.tx.Key()] = tTx
 		priorities[i] = tTx.priority
 	}
 
@@ -290,7 +291,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) {
 	ensurePrioritized := func(reapedTxs types.Txs) {
 		reapedPriorities := make([]int64, len(reapedTxs))
 		for i, rTx := range reapedTxs {
-			reapedPriorities[i] = txMap[mempool.TxKey(rTx)].priority
+			reapedPriorities[i] = txMap[rTx.Key()].priority
 		}
 
 		require.Equal(t, priorities[:len(reapedPriorities)], reapedPriorities)

internal/mempool/v1/reactor.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 	"time"
 
-	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/config"
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/internal/mempool"
@@ -37,9 +37,9 @@ type PeerManager interface {
 type Reactor struct {
 	service.BaseService
 
-	config  *cfg.MempoolConfig
+	cfg     *config.MempoolConfig
 	mempool *TxMempool
-	ids     *mempool.MempoolIDs
+	ids     *mempool.IDs
 
 	// XXX: Currently, this is the only way to get information about a peer. Ideally,
 	// we rely on message-oriented communication to get necessary peer data.
@@ -54,6 +54,10 @@ type Reactor struct {
 	// goroutines.
 	peerWG sync.WaitGroup
 
+	// observePanic is a function for observing panics that were recovered in methods on
+	// Reactor. observePanic is called with the recovered value.
+	observePanic func(interface{})
+
 	mtx          tmsync.Mutex
 	peerRoutines map[types.NodeID]*tmsync.Closer
 }
@@ -61,7 +65,7 @@ type Reactor struct {
 // NewReactor returns a reference to a new reactor.
 func NewReactor(
 	logger log.Logger,
-	config *cfg.MempoolConfig,
+	cfg *config.MempoolConfig,
 	peerMgr PeerManager,
 	txmp *TxMempool,
 	mempoolCh *p2p.Channel,
@@ -69,7 +73,7 @@ func NewReactor(
 ) *Reactor {
 
 	r := &Reactor{
-		config:       config,
+		cfg:          cfg,
 		peerMgr:      peerMgr,
 		mempool:      txmp,
 		ids:          mempool.NewMempoolIDs(),
@@ -77,38 +81,31 @@ func NewReactor(
 		peerUpdates:  peerUpdates,
 		closeCh:      make(chan struct{}),
 		peerRoutines: make(map[types.NodeID]*tmsync.Closer),
+		observePanic: defaultObservePanic,
 	}
 
 	r.BaseService = *service.NewBaseService(logger, "Mempool", r)
 	return r
 }
 
-// GetChannelShims returns a map of ChannelDescriptorShim objects, where each
-// object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding
-// p2p proto.Message the new p2p Channel is responsible for handling.
-//
-//
-// TODO: Remove once p2p refactor is complete.
-// ref: https://github.com/tendermint/tendermint/issues/5670
-func GetChannelShims(config *cfg.MempoolConfig) map[p2p.ChannelID]*p2p.ChannelDescriptorShim {
-	largestTx := make([]byte, config.MaxTxBytes)
+func defaultObservePanic(r interface{}) {}
+
+// GetChannelDescriptor produces an instance of a descriptor for this
+// package's required channels.
+func GetChannelDescriptor(cfg *config.MempoolConfig) *p2p.ChannelDescriptor {
+	largestTx := make([]byte, cfg.MaxTxBytes)
 	batchMsg := protomem.Message{
 		Sum: &protomem.Message_Txs{
 			Txs: &protomem.Txs{Txs: [][]byte{largestTx}},
 		},
 	}
 
-	return map[p2p.ChannelID]*p2p.ChannelDescriptorShim{
-		mempool.MempoolChannel: {
-			MsgType: new(protomem.Message),
-			Descriptor: &p2p.ChannelDescriptor{
-				ID:                  byte(mempool.MempoolChannel),
-				Priority:            5,
-				RecvMessageCapacity: batchMsg.Size(),
-				RecvBufferCapacity:  128,
-				MaxSendBytes:        5000,
-			},
-		},
+	return &p2p.ChannelDescriptor{
+		ID:                  mempool.MempoolChannel,
+		MessageType:         new(protomem.Message),
+		Priority:            5,
+		RecvMessageCapacity: batchMsg.Size(),
+		RecvBufferCapacity:  128,
 	}
 }
 
@@ -117,7 +114,7 @@ func GetChannelShims(config *cfg.MempoolConfig) map[p2p.ChannelID]*p2p.ChannelDe
 // messages on that p2p channel accordingly. The caller must be sure to execute
 // OnStop to ensure the outbound p2p Channels are closed.
 func (r *Reactor) OnStart() error {
-	if !r.config.Broadcast {
+	if !r.cfg.Broadcast {
 		r.Logger.Info("tx broadcasting is disabled")
 	}
 
@@ -171,7 +168,7 @@ func (r *Reactor) handleMempoolMessage(envelope p2p.Envelope) error {
 
 		for _, tx := range protoTxs {
 			if err := r.mempool.CheckTx(context.Background(), types.Tx(tx), nil, txInfo); err != nil {
-				logger.Error("checktx failed for tx", "tx", fmt.Sprintf("%X", mempool.TxHashFromBytes(tx)), "err", err)
+				logger.Error("checktx failed for tx", "tx", fmt.Sprintf("%X", types.Tx(tx).Hash()), "err", err)
 			}
 		}
 
@@ -188,6 +185,7 @@ func (r *Reactor) handleMempoolMessage(envelope p2p.Envelope) error {
 func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) {
 	defer func() {
 		if e := recover(); e != nil {
+			r.observePanic(e)
 			err = fmt.Errorf("panic in processing message: %v", e)
 			r.Logger.Error(
 				"recovering from processing message panic",
@@ -254,7 +252,7 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
 			return
 		}
 
-		if r.config.Broadcast {
+		if r.cfg.Broadcast {
 			// Check if we've already started a goroutine for this peer, if not we create
 			// a new done channel so we can explicitly close the goroutine if the peer
 			// is later removed, we increment the waitgroup so the reactor can stop
@@ -318,6 +316,7 @@ func (r *Reactor) broadcastTxRoutine(peerID types.NodeID, closer *tmsync.Closer)
 		r.peerWG.Done()
 
 		if e := recover(); e != nil {
+			r.observePanic(e)
 			r.Logger.Error(
 				"recovering from broadcasting mempool loop",
 				"err", e,
@@ -377,7 +376,7 @@ func (r *Reactor) broadcastTxRoutine(peerID types.NodeID, closer *tmsync.Closer)
 			}
 			r.Logger.Debug(
 				"gossiped tx to peer",
-				"tx", fmt.Sprintf("%X", mempool.TxHashFromBytes(memTx.tx)),
+				"tx", fmt.Sprintf("%X", memTx.tx.Hash()),
 				"peer", peerID,
 			)
 		}

internal/mempool/v1/reactor_test.go

@@ -0,0 +1,145 @@
+package v1
+
+import (
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	"github.com/tendermint/tendermint/config"
+	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/types"
+)
+
+type reactorTestSuite struct {
+	network *p2ptest.Network
+	logger  log.Logger
+
+	reactors        map[types.NodeID]*Reactor
+	mempoolChannels map[types.NodeID]*p2p.Channel
+	mempools        map[types.NodeID]*TxMempool
+	kvstores        map[types.NodeID]*kvstore.Application
+
+	peerChans   map[types.NodeID]chan p2p.PeerUpdate
+	peerUpdates map[types.NodeID]*p2p.PeerUpdates
+
+	nodes []types.NodeID
+}
+
+func setupReactors(t *testing.T, numNodes int, chBuf uint) *reactorTestSuite {
+	t.Helper()
+
+	cfg := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|"))
+	t.Cleanup(func() {
+		os.RemoveAll(cfg.RootDir)
+	})
+
+	rts := &reactorTestSuite{
+		logger:          log.TestingLogger().With("testCase", t.Name()),
+		network:         p2ptest.MakeNetwork(t, p2ptest.NetworkOptions{NumNodes: numNodes}),
+		reactors:        make(map[types.NodeID]*Reactor, numNodes),
+		mempoolChannels: make(map[types.NodeID]*p2p.Channel, numNodes),
+		mempools:        make(map[types.NodeID]*TxMempool, numNodes),
+		kvstores:        make(map[types.NodeID]*kvstore.Application, numNodes),
+		peerChans:       make(map[types.NodeID]chan p2p.PeerUpdate, numNodes),
+		peerUpdates:     make(map[types.NodeID]*p2p.PeerUpdates, numNodes),
+	}
+
+	chDesc := GetChannelDescriptor(cfg.Mempool)
+	rts.mempoolChannels = rts.network.MakeChannelsNoCleanup(t, chDesc)
+
+	for nodeID := range rts.network.Nodes {
+		rts.kvstores[nodeID] = kvstore.NewApplication()
+
+		mempool := setup(t, 0)
+		rts.mempools[nodeID] = mempool
+
+		rts.peerChans[nodeID] = make(chan p2p.PeerUpdate)
+		rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
+		rts.network.Nodes[nodeID].PeerManager.Register(rts.peerUpdates[nodeID])
+
+		rts.reactors[nodeID] = NewReactor(
+			rts.logger.With("nodeID", nodeID),
+			cfg.Mempool,
+			rts.network.Nodes[nodeID].PeerManager,
+			mempool,
+			rts.mempoolChannels[nodeID],
+			rts.peerUpdates[nodeID],
+		)
+
+		rts.nodes = append(rts.nodes, nodeID)
+
+		require.NoError(t, rts.reactors[nodeID].Start())
+		require.True(t, rts.reactors[nodeID].IsRunning())
+	}
+
+	require.Len(t, rts.reactors, numNodes)
+
+	t.Cleanup(func() {
+		for nodeID := range rts.reactors {
+			if rts.reactors[nodeID].IsRunning() {
+				require.NoError(t, rts.reactors[nodeID].Stop())
+				require.False(t, rts.reactors[nodeID].IsRunning())
+			}
+		}
+	})
+
+	return rts
+}
+
+func (rts *reactorTestSuite) start(t *testing.T) {
+	t.Helper()
+	rts.network.Start(t)
+	require.Len(t,
+		rts.network.RandomNode().PeerManager.Peers(),
+		len(rts.nodes)-1,
+		"network does not have expected number of nodes")
+}
+
+func TestReactorBroadcastDoesNotPanic(t *testing.T) {
+	numNodes := 2
+	rts := setupReactors(t, numNodes, 0)
+
+	observePanic := func(r interface{}) {
+		t.Fatal("panic detected in reactor")
+	}
+
+	primary := rts.nodes[0]
+	secondary := rts.nodes[1]
+	primaryReactor := rts.reactors[primary]
+	primaryMempool := primaryReactor.mempool
+	secondaryReactor := rts.reactors[secondary]
+
+	primaryReactor.observePanic = observePanic
+	secondaryReactor.observePanic = observePanic
+
+	firstTx := &WrappedTx{}
+	primaryMempool.insertTx(firstTx)
+
+	// run the router
+	rts.start(t)
+
+	closer := tmsync.NewCloser()
+	primaryReactor.peerWG.Add(1)
+	go primaryReactor.broadcastTxRoutine(secondary, closer)
+
+	wg := &sync.WaitGroup{}
+	for i := 0; i < 50; i++ {
+		next := &WrappedTx{}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			primaryMempool.insertTx(next)
+		}()
+	}
+
+	err := primaryReactor.Stop()
+	require.NoError(t, err)
+	primaryReactor.peerWG.Wait()
+	wg.Wait()
+}

internal/mempool/v1/tx.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/tendermint/tendermint/internal/libs/clist"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-	"github.com/tendermint/tendermint/internal/mempool"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -17,7 +16,7 @@ type WrappedTx struct {
 	tx types.Tx
 
 	// hash defines the transaction hash and the primary key used in the mempool
-	hash [mempool.TxKeySize]byte
+	hash types.TxKey
 
 	// height defines the height at which the transaction was validated at
 	height int64
@@ -66,14 +65,14 @@ func (wtx *WrappedTx) Size() int {
 //   need mutative access.
 type TxStore struct {
 	mtx       tmsync.RWMutex
-	hashTxs   map[[mempool.TxKeySize]byte]*WrappedTx // primary index
-	senderTxs map[string]*WrappedTx                  // sender is defined by the ABCI application
+	hashTxs   map[types.TxKey]*WrappedTx // primary index
+	senderTxs map[string]*WrappedTx      // sender is defined by the ABCI application
 }
 
 func NewTxStore() *TxStore {
 	return &TxStore{
 		senderTxs: make(map[string]*WrappedTx),
-		hashTxs:   make(map[[mempool.TxKeySize]byte]*WrappedTx),
+		hashTxs:   make(map[types.TxKey]*WrappedTx),
 	}
 }
 
@@ -110,7 +109,7 @@ func (txs *TxStore) GetTxBySender(sender string) *WrappedTx {
 }
 
 // GetTxByHash returns a *WrappedTx by the transaction's hash.
-func (txs *TxStore) GetTxByHash(hash [mempool.TxKeySize]byte) *WrappedTx {
+func (txs *TxStore) GetTxByHash(hash types.TxKey) *WrappedTx {
 	txs.mtx.RLock()
 	defer txs.mtx.RUnlock()
 
@@ -119,7 +118,7 @@ func (txs *TxStore) GetTxByHash(hash [mempool.TxKeySize]byte) *WrappedTx {
 
 // IsTxRemoved returns true if a transaction by hash is marked as removed and
 // false otherwise.
-func (txs *TxStore) IsTxRemoved(hash [mempool.TxKeySize]byte) bool {
+func (txs *TxStore) IsTxRemoved(hash types.TxKey) bool {
 	txs.mtx.RLock()
 	defer txs.mtx.RUnlock()
 
@@ -142,7 +141,7 @@ func (txs *TxStore) SetTx(wtx *WrappedTx) {
 		txs.senderTxs[wtx.sender] = wtx
 	}
 
-	txs.hashTxs[mempool.TxKey(wtx.tx)] = wtx
+	txs.hashTxs[wtx.tx.Key()] = wtx
 }
 
 // RemoveTx removes a *WrappedTx from the transaction store. It deletes all
@@ -155,13 +154,13 @@ func (txs *TxStore) RemoveTx(wtx *WrappedTx) {
 		delete(txs.senderTxs, wtx.sender)
 	}
 
-	delete(txs.hashTxs, mempool.TxKey(wtx.tx))
+	delete(txs.hashTxs, wtx.tx.Key())
 	wtx.removed = true
 }
 
 // TxHasPeer returns true if a transaction by hash has a given peer ID and false
 // otherwise. If the transaction does not exist, false is returned.
-func (txs *TxStore) TxHasPeer(hash [mempool.TxKeySize]byte, peerID uint16) bool {
+func (txs *TxStore) TxHasPeer(hash types.TxKey, peerID uint16) bool {
 	txs.mtx.RLock()
 	defer txs.mtx.RUnlock()
 
@@ -179,7 +178,7 @@ func (txs *TxStore) TxHasPeer(hash [mempool.TxKeySize]byte, peerID uint16) bool
 // We return true if we've already recorded the given peer for this transaction
 // and false otherwise. If the transaction does not exist by hash, we return
 // (nil, false).
-func (txs *TxStore) GetOrSetPeerByTxHash(hash [mempool.TxKeySize]byte, peerID uint16) (*WrappedTx, bool) {
+func (txs *TxStore) GetOrSetPeerByTxHash(hash types.TxKey, peerID uint16) (*WrappedTx, bool) {
 	txs.mtx.Lock()
 	defer txs.mtx.Unlock()
 

internal/mempool/v1/tx_test.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
-	"github.com/tendermint/tendermint/internal/mempool"
+	"github.com/tendermint/tendermint/types"
 )
 
 func TestTxStore_GetTxBySender(t *testing.T) {
@@ -39,7 +39,7 @@ func TestTxStore_GetTxByHash(t *testing.T) {
 		timestamp: time.Now(),
 	}
 
-	key := mempool.TxKey(wtx.tx)
+	key := wtx.tx.Key()
 	res := txs.GetTxByHash(key)
 	require.Nil(t, res)
 
@@ -58,7 +58,7 @@ func TestTxStore_SetTx(t *testing.T) {
 		timestamp: time.Now(),
 	}
 
-	key := mempool.TxKey(wtx.tx)
+	key := wtx.tx.Key()
 	txs.SetTx(wtx)
 
 	res := txs.GetTxByHash(key)
@@ -81,10 +81,10 @@ func TestTxStore_GetOrSetPeerByTxHash(t *testing.T) {
 		timestamp: time.Now(),
 	}
 
-	key := mempool.TxKey(wtx.tx)
+	key := wtx.tx.Key()
 	txs.SetTx(wtx)
 
-	res, ok := txs.GetOrSetPeerByTxHash(mempool.TxKey([]byte("test_tx_2")), 15)
+	res, ok := txs.GetOrSetPeerByTxHash(types.Tx([]byte("test_tx_2")).Key(), 15)
 	require.Nil(t, res)
 	require.False(t, ok)
 
@@ -110,7 +110,7 @@ func TestTxStore_RemoveTx(t *testing.T) {
 
 	txs.SetTx(wtx)
 
-	key := mempool.TxKey(wtx.tx)
+	key := wtx.tx.Key()
 	res := txs.GetTxByHash(key)
 	require.NotNil(t, res)
 

internal/p2p/base_reactor.go

@@ -1,74 +0,0 @@
-package p2p
-
-import (
-	"github.com/tendermint/tendermint/internal/p2p/conn"
-	"github.com/tendermint/tendermint/libs/service"
-)
-
-// Reactor is responsible for handling incoming messages on one or more
-// Channel. Switch calls GetChannels when reactor is added to it. When a new
-// peer joins our node, InitPeer and AddPeer are called. RemovePeer is called
-// when the peer is stopped. Receive is called when a message is received on a
-// channel associated with this reactor.
-//
-// Peer#Send or Peer#TrySend should be used to send the message to a peer.
-type Reactor interface {
-	service.Service // Start, Stop
-
-	// SetSwitch allows setting a switch.
-	SetSwitch(*Switch)
-
-	// GetChannels returns the list of MConnection.ChannelDescriptor. Make sure
-	// that each ID is unique across all the reactors added to the switch.
-	GetChannels() []*conn.ChannelDescriptor
-
-	// InitPeer is called by the switch before the peer is started. Use it to
-	// initialize data for the peer (e.g. peer state).
-	//
-	// NOTE: The switch won't call AddPeer nor RemovePeer if it fails to start
-	// the peer. Do not store any data associated with the peer in the reactor
-	// itself unless you don't want to have a state, which is never cleaned up.
-	InitPeer(peer Peer) Peer
-
-	// AddPeer is called by the switch after the peer is added and successfully
-	// started. Use it to start goroutines communicating with the peer.
-	AddPeer(peer Peer)
-
-	// RemovePeer is called by the switch when the peer is stopped (due to error
-	// or other reason).
-	RemovePeer(peer Peer, reason interface{})
-
-	// Receive is called by the switch when msgBytes is received from the peer.
-	//
-	// NOTE reactor can not keep msgBytes around after Receive completes without
-	// copying.
-	//
-	// CONTRACT: msgBytes are not nil.
-	//
-	// XXX: do not call any methods that can block or incur heavy processing.
-	// https://github.com/tendermint/tendermint/issues/2888
-	Receive(chID byte, peer Peer, msgBytes []byte)
-}
-
-//--------------------------------------
-
-type BaseReactor struct {
-	service.BaseService // Provides Start, Stop, .Quit
-	Switch              *Switch
-}
-
-func NewBaseReactor(name string, impl Reactor) *BaseReactor {
-	return &BaseReactor{
-		BaseService: *service.NewBaseService(nil, name, impl),
-		Switch:      nil,
-	}
-}
-
-func (br *BaseReactor) SetSwitch(sw *Switch) {
-	br.Switch = sw
-}
-func (*BaseReactor) GetChannels() []*conn.ChannelDescriptor        { return nil }
-func (*BaseReactor) AddPeer(peer Peer)                             {}
-func (*BaseReactor) RemovePeer(peer Peer, reason interface{})      {}
-func (*BaseReactor) Receive(chID byte, peer Peer, msgBytes []byte) {}
-func (*BaseReactor) InitPeer(peer Peer) Peer                       { return peer }

internal/p2p/conn/conn_go110.go

@@ -1,3 +1,4 @@
+//go:build go1.10
 // +build go1.10
 
 package conn

internal/p2p/conn/conn_notgo110.go

@@ -1,3 +1,4 @@
+//go:build !go1.10
 // +build !go1.10
 
 package conn

internal/p2p/conn/connection.go

@@ -48,7 +48,7 @@ const (
 	defaultPongTimeout         = 45 * time.Second
 )
 
-type receiveCbFunc func(chID byte, msgBytes []byte)
+type receiveCbFunc func(chID ChannelID, msgBytes []byte)
 type errorCbFunc func(interface{})
 
 /*
@@ -64,15 +64,11 @@ initialization of the connection.
 
 There are two methods for sending messages:
 	func (m MConnection) Send(chID byte, msgBytes []byte) bool {}
-	func (m MConnection) TrySend(chID byte, msgBytes []byte}) bool {}
 
 `Send(chID, msgBytes)` is a blocking call that waits until `msg` is
 successfully queued for the channel with the given id byte `chID`, or until the
 request times out.  The message `msg` is serialized using Protobuf.
 
-`TrySend(chID, msgBytes)` is a nonblocking call that returns false if the
-channel's queue is full.
-
 Inbound message bytes are handled with an onReceive callback function.
 */
 type MConnection struct {
@@ -85,8 +81,8 @@ type MConnection struct {
 	recvMonitor   *flow.Monitor
 	send          chan struct{}
 	pong          chan struct{}
-	channels      []*Channel
-	channelsIdx   map[byte]*Channel
+	channels      []*channel
+	channelsIdx   map[ChannelID]*channel
 	onReceive     receiveCbFunc
 	onError       errorCbFunc
 	errored       uint32
@@ -190,8 +186,8 @@ func NewMConnectionWithConfig(
 	}
 
 	// Create channels
-	var channelsIdx = map[byte]*Channel{}
-	var channels = []*Channel{}
+	var channelsIdx = map[ChannelID]*channel{}
+	var channels = []*channel{}
 
 	for _, desc := range chDescs {
 		channel := newChannel(mconn, *desc)
@@ -265,43 +261,6 @@ func (c *MConnection) stopServices() (alreadyStopped bool) {
 	return false
 }
 
-// FlushStop replicates the logic of OnStop.
-// It additionally ensures that all successful
-// .Send() calls will get flushed before closing
-// the connection.
-func (c *MConnection) FlushStop() {
-	if c.stopServices() {
-		return
-	}
-
-	// this block is unique to FlushStop
-	{
-		// wait until the sendRoutine exits
-		// so we dont race on calling sendSomePacketMsgs
-		<-c.doneSendRoutine
-
-		// Send and flush all pending msgs.
-		// Since sendRoutine has exited, we can call this
-		// safely
-		eof := c.sendSomePacketMsgs()
-		for !eof {
-			eof = c.sendSomePacketMsgs()
-		}
-		c.flush()
-
-		// Now we can close the connection
-	}
-
-	c.conn.Close()
-
-	// We can't close pong safely here because
-	// recvRoutine may write to it after we've stopped.
-	// Though it doesn't need to get closed at all,
-	// we close it @ recvRoutine.
-
-	// c.Stop()
-}
-
 // OnStop implements BaseService
 func (c *MConnection) OnStop() {
 	if c.stopServices() {
@@ -348,7 +307,7 @@ func (c *MConnection) stopForError(r interface{}) {
 }
 
 // Queues a message to be sent to channel.
-func (c *MConnection) Send(chID byte, msgBytes []byte) bool {
+func (c *MConnection) Send(chID ChannelID, msgBytes []byte) bool {
 	if !c.IsRunning() {
 		return false
 	}
@@ -375,49 +334,6 @@ func (c *MConnection) Send(chID byte, msgBytes []byte) bool {
 	return success
 }
 
-// Queues a message to be sent to channel.
-// Nonblocking, returns true if successful.
-func (c *MConnection) TrySend(chID byte, msgBytes []byte) bool {
-	if !c.IsRunning() {
-		return false
-	}
-
-	c.Logger.Debug("TrySend", "channel", chID, "conn", c, "msgBytes", msgBytes)
-
-	// Send message to channel.
-	channel, ok := c.channelsIdx[chID]
-	if !ok {
-		c.Logger.Error(fmt.Sprintf("Cannot send bytes, unknown channel %X", chID))
-		return false
-	}
-
-	ok = channel.trySendBytes(msgBytes)
-	if ok {
-		// Wake up sendRoutine if necessary
-		select {
-		case c.send <- struct{}{}:
-		default:
-		}
-	}
-
-	return ok
-}
-
-// CanSend returns true if you can send more data onto the chID, false
-// otherwise. Use only as a heuristic.
-func (c *MConnection) CanSend(chID byte) bool {
-	if !c.IsRunning() {
-		return false
-	}
-
-	channel, ok := c.channelsIdx[chID]
-	if !ok {
-		c.Logger.Error(fmt.Sprintf("Unknown channel %X", chID))
-		return false
-	}
-	return channel.canSend()
-}
-
 // sendRoutine polls for packets to send from channels.
 func (c *MConnection) sendRoutine() {
 	defer c._recover()
@@ -520,7 +436,7 @@ func (c *MConnection) sendPacketMsg() bool {
 	// Choose a channel to create a PacketMsg from.
 	// The chosen channel will be the one whose recentlySent/priority is the least.
 	var leastRatio float32 = math.MaxFloat32
-	var leastChannel *Channel
+	var leastChannel *channel
 	for _, channel := range c.channels {
 		// If nothing to send, skip this channel
 		if !channel.isSendPending() {
@@ -624,7 +540,7 @@ FOR_LOOP:
 				// never block
 			}
 		case *tmp2p.Packet_PacketMsg:
-			channelID := byte(pkt.PacketMsg.ChannelID)
+			channelID := ChannelID(pkt.PacketMsg.ChannelID)
 			channel, ok := c.channelsIdx[channelID]
 			if pkt.PacketMsg.ChannelID < 0 || pkt.PacketMsg.ChannelID > math.MaxUint8 || !ok || channel == nil {
 				err := fmt.Errorf("unknown channel %X", pkt.PacketMsg.ChannelID)
@@ -682,13 +598,6 @@ func (c *MConnection) maxPacketMsgSize() int {
 	return len(bz)
 }
 
-type ConnectionStatus struct {
-	Duration    time.Duration
-	SendMonitor flow.Status
-	RecvMonitor flow.Status
-	Channels    []ChannelStatus
-}
-
 type ChannelStatus struct {
 	ID                byte
 	SendQueueCapacity int
@@ -697,30 +606,16 @@ type ChannelStatus struct {
 	RecentlySent      int64
 }
 
-func (c *MConnection) Status() ConnectionStatus {
-	var status ConnectionStatus
-	status.Duration = time.Since(c.created)
-	status.SendMonitor = c.sendMonitor.Status()
-	status.RecvMonitor = c.recvMonitor.Status()
-	status.Channels = make([]ChannelStatus, len(c.channels))
-	for i, channel := range c.channels {
-		status.Channels[i] = ChannelStatus{
-			ID:                channel.desc.ID,
-			SendQueueCapacity: cap(channel.sendQueue),
-			SendQueueSize:     int(atomic.LoadInt32(&channel.sendQueueSize)),
-			Priority:          channel.desc.Priority,
-			RecentlySent:      atomic.LoadInt64(&channel.recentlySent),
-		}
-	}
-	return status
-}
-
 //-----------------------------------------------------------------------------
+// ChannelID is an arbitrary channel ID.
+type ChannelID uint16
 
 type ChannelDescriptor struct {
-	ID       byte
+	ID       ChannelID
 	Priority int
 
+	MessageType proto.Message
+
 	// TODO: Remove once p2p refactor is complete.
 	SendQueueCapacity   int
 	RecvMessageCapacity int
@@ -728,10 +623,6 @@ type ChannelDescriptor struct {
 	// RecvBufferCapacity defines the max buffer size of inbound messages for a
 	// given p2p Channel queue.
 	RecvBufferCapacity int
-
-	// MaxSendBytes defines the maximum number of bytes that can be sent at any
-	// given moment from a Channel to a peer.
-	MaxSendBytes uint
 }
 
 func (chDesc ChannelDescriptor) FillDefaults() (filled ChannelDescriptor) {
@@ -748,28 +639,32 @@ func (chDesc ChannelDescriptor) FillDefaults() (filled ChannelDescriptor) {
 	return
 }
 
-// TODO: lowercase.
 // NOTE: not goroutine-safe.
-type Channel struct {
+type channel struct {
+	// Exponential moving average.
+	// This field must be accessed atomically.
+	// It is first in the struct to ensure correct alignment.
+	// See https://github.com/tendermint/tendermint/issues/7000.
+	recentlySent int64
+
 	conn          *MConnection
 	desc          ChannelDescriptor
 	sendQueue     chan []byte
 	sendQueueSize int32 // atomic.
 	recving       []byte
 	sending       []byte
-	recentlySent  int64 // exponential moving average
 
 	maxPacketMsgPayloadSize int
 
 	Logger log.Logger
 }
 
-func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel {
+func newChannel(conn *MConnection, desc ChannelDescriptor) *channel {
 	desc = desc.FillDefaults()
 	if desc.Priority <= 0 {
 		panic("Channel default priority must be a positive integer")
 	}
-	return &Channel{
+	return &channel{
 		conn:                    conn,
 		desc:                    desc,
 		sendQueue:               make(chan []byte, desc.SendQueueCapacity),
@@ -778,14 +673,14 @@ func newChannel(conn *MConnection, desc ChannelDescriptor) *Channel {
 	}
 }
 
-func (ch *Channel) SetLogger(l log.Logger) {
+func (ch *channel) SetLogger(l log.Logger) {
 	ch.Logger = l
 }
 
 // Queues message to send to this channel.
 // Goroutine-safe
 // Times out (and returns false) after defaultSendTimeout
-func (ch *Channel) sendBytes(bytes []byte) bool {
+func (ch *channel) sendBytes(bytes []byte) bool {
 	select {
 	case ch.sendQueue <- bytes:
 		atomic.AddInt32(&ch.sendQueueSize, 1)
@@ -795,34 +690,10 @@ func (ch *Channel) sendBytes(bytes []byte) bool {
 	}
 }
 
-// Queues message to send to this channel.
-// Nonblocking, returns true if successful.
-// Goroutine-safe
-func (ch *Channel) trySendBytes(bytes []byte) bool {
-	select {
-	case ch.sendQueue <- bytes:
-		atomic.AddInt32(&ch.sendQueueSize, 1)
-		return true
-	default:
-		return false
-	}
-}
-
-// Goroutine-safe
-func (ch *Channel) loadSendQueueSize() (size int) {
-	return int(atomic.LoadInt32(&ch.sendQueueSize))
-}
-
-// Goroutine-safe
-// Use only as a heuristic.
-func (ch *Channel) canSend() bool {
-	return ch.loadSendQueueSize() < defaultSendQueueCapacity
-}
-
 // Returns true if any PacketMsgs are pending to be sent.
 // Call before calling nextPacketMsg()
 // Goroutine-safe
-func (ch *Channel) isSendPending() bool {
+func (ch *channel) isSendPending() bool {
 	if len(ch.sending) == 0 {
 		if len(ch.sendQueue) == 0 {
 			return false
@@ -834,7 +705,7 @@ func (ch *Channel) isSendPending() bool {
 
 // Creates a new PacketMsg to send.
 // Not goroutine-safe
-func (ch *Channel) nextPacketMsg() tmp2p.PacketMsg {
+func (ch *channel) nextPacketMsg() tmp2p.PacketMsg {
 	packet := tmp2p.PacketMsg{ChannelID: int32(ch.desc.ID)}
 	maxSize := ch.maxPacketMsgPayloadSize
 	packet.Data = ch.sending[:tmmath.MinInt(maxSize, len(ch.sending))]
@@ -851,7 +722,7 @@ func (ch *Channel) nextPacketMsg() tmp2p.PacketMsg {
 
 // Writes next PacketMsg to w and updates c.recentlySent.
 // Not goroutine-safe
-func (ch *Channel) writePacketMsgTo(w io.Writer) (n int, err error) {
+func (ch *channel) writePacketMsgTo(w io.Writer) (n int, err error) {
 	packet := ch.nextPacketMsg()
 	n, err = protoio.NewDelimitedWriter(w).WriteMsg(mustWrapPacket(&packet))
 	atomic.AddInt64(&ch.recentlySent, int64(n))
@@ -861,7 +732,7 @@ func (ch *Channel) writePacketMsgTo(w io.Writer) (n int, err error) {
 // Handles incoming PacketMsgs. It returns a message bytes if message is
 // complete, which is owned by the caller and will not be modified.
 // Not goroutine-safe
-func (ch *Channel) recvPacketMsg(packet tmp2p.PacketMsg) ([]byte, error) {
+func (ch *channel) recvPacketMsg(packet tmp2p.PacketMsg) ([]byte, error) {
 	ch.Logger.Debug("Read PacketMsg", "conn", ch.conn, "packet", packet)
 	var recvCap, recvReceived = ch.desc.RecvMessageCapacity, len(ch.recving) + len(packet.Data)
 	if recvCap < recvReceived {
@@ -878,7 +749,7 @@ func (ch *Channel) recvPacketMsg(packet tmp2p.PacketMsg) ([]byte, error) {
 
 // Call this periodically to update stats for throttling purposes.
 // Not goroutine-safe
-func (ch *Channel) updateStats() {
+func (ch *channel) updateStats() {
 	// Exponential decay of stats.
 	// TODO: optimize.
 	atomic.StoreInt64(&ch.recentlySent, int64(float64(atomic.LoadInt64(&ch.recentlySent))*0.8))

internal/p2p/conn/connection_test.go

@@ -20,7 +20,7 @@ import (
 const maxPingPongPacketSize = 1024 // bytes
 
 func createTestMConnection(conn net.Conn) *MConnection {
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 	}
 	onError := func(r interface{}) {
 	}
@@ -31,7 +31,7 @@ func createTestMConnection(conn net.Conn) *MConnection {
 
 func createMConnectionWithCallbacks(
 	conn net.Conn,
-	onReceive func(chID byte, msgBytes []byte),
+	onReceive func(chID ChannelID, msgBytes []byte),
 	onError func(r interface{}),
 ) *MConnection {
 	cfg := DefaultMConnConfig()
@@ -69,9 +69,6 @@ func TestMConnectionSendFlushStop(t *testing.T) {
 		errCh <- err
 	}()
 
-	// stop the conn - it should flush all conns
-	clientConn.FlushStop()
-
 	timer := time.NewTimer(3 * time.Second)
 	select {
 	case <-errCh:
@@ -97,16 +94,14 @@ func TestMConnectionSend(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	assert.True(t, mconn.CanSend(0x01))
 
 	msg = []byte("Spider-Man")
-	assert.True(t, mconn.TrySend(0x01, msg))
+	assert.True(t, mconn.Send(0x01, msg))
 	_, err = server.Read(make([]byte, len(msg)))
 	if err != nil {
 		t.Error(err)
 	}
 
-	assert.False(t, mconn.CanSend(0x05), "CanSend should return false because channel is unknown")
 	assert.False(t, mconn.Send(0x05, []byte("Absorbing Man")), "Send should return false because channel is unknown")
 }
 
@@ -116,7 +111,7 @@ func TestMConnectionReceive(t *testing.T) {
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -145,27 +140,13 @@ func TestMConnectionReceive(t *testing.T) {
 	}
 }
 
-func TestMConnectionStatus(t *testing.T) {
-	server, client := NetPipe()
-	t.Cleanup(closeAll(t, client, server))
-
-	mconn := createTestMConnection(client)
-	err := mconn.Start()
-	require.Nil(t, err)
-	t.Cleanup(stopAll(t, mconn))
-
-	status := mconn.Status()
-	assert.NotNil(t, status)
-	assert.Zero(t, status.Channels[0].SendQueueSize)
-}
-
 func TestMConnectionPongTimeoutResultsInError(t *testing.T) {
 	server, client := net.Pipe()
 	t.Cleanup(closeAll(t, client, server))
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -203,7 +184,7 @@ func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) {
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -257,7 +238,7 @@ func TestMConnectionMultiplePings(t *testing.T) {
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -304,7 +285,7 @@ func TestMConnectionPingPongs(t *testing.T) {
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -361,7 +342,7 @@ func TestMConnectionStopsAndReturnsError(t *testing.T) {
 
 	receivedCh := make(chan []byte)
 	errorsCh := make(chan interface{})
-	onReceive := func(chID byte, msgBytes []byte) {
+	onReceive := func(chID ChannelID, msgBytes []byte) {
 		receivedCh <- msgBytes
 	}
 	onError := func(r interface{}) {
@@ -390,7 +371,7 @@ func TestMConnectionStopsAndReturnsError(t *testing.T) {
 func newClientAndServerConnsForReadErrors(t *testing.T, chOnErr chan struct{}) (*MConnection, *MConnection) {
 	server, client := NetPipe()
 
-	onReceive := func(chID byte, msgBytes []byte) {}
+	onReceive := func(chID ChannelID, msgBytes []byte) {}
 	onError := func(r interface{}) {}
 
 	// create client conn with two channels
@@ -462,7 +443,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) {
 	mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr)
 	t.Cleanup(stopAll(t, mconnClient, mconnServer))
 
-	mconnServer.onReceive = func(chID byte, msgBytes []byte) {
+	mconnServer.onReceive = func(chID ChannelID, msgBytes []byte) {
 		chOnRcv <- struct{}{}
 	}
 
@@ -514,18 +495,15 @@ func TestMConnectionTrySend(t *testing.T) {
 
 	msg := []byte("Semicolon-Woman")
 	resultCh := make(chan string, 2)
-	assert.True(t, mconn.TrySend(0x01, msg))
+	assert.True(t, mconn.Send(0x01, msg))
 	_, err = server.Read(make([]byte, len(msg)))
 	require.NoError(t, err)
-	assert.True(t, mconn.CanSend(0x01))
-	assert.True(t, mconn.TrySend(0x01, msg))
-	assert.False(t, mconn.CanSend(0x01))
+	assert.True(t, mconn.Send(0x01, msg))
 	go func() {
-		mconn.TrySend(0x01, msg)
+		mconn.Send(0x01, msg)
 		resultCh <- "TrySend"
 	}()
-	assert.False(t, mconn.CanSend(0x01))
-	assert.False(t, mconn.TrySend(0x01, msg))
+	assert.False(t, mconn.Send(0x01, msg))
 	assert.Equal(t, "TrySend", <-resultCh)
 }
 
@@ -560,7 +538,7 @@ func TestMConnectionChannelOverflow(t *testing.T) {
 	mconnClient, mconnServer := newClientAndServerConnsForReadErrors(t, chOnErr)
 	t.Cleanup(stopAll(t, mconnClient, mconnServer))
 
-	mconnServer.onReceive = func(chID byte, msgBytes []byte) {
+	mconnServer.onReceive = func(chID ChannelID, msgBytes []byte) {
 		chOnRcv <- struct{}{}
 	}
 

internal/p2p/conn/evil_secret_connection_test.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
-	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/crypto/encoding"
 	"github.com/tendermint/tendermint/internal/libs/protoio"
 	tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p"
 )
@@ -113,7 +113,7 @@ func (c *evilConn) Read(data []byte) (n int, err error) {
 	case 1:
 		signature := c.signChallenge()
 		if !c.badAuthSignature {
-			pkpb, err := cryptoenc.PubKeyToProto(c.privKey.PubKey())
+			pkpb, err := encoding.PubKeyToProto(c.privKey.PubKey())
 			if err != nil {
 				panic(err)
 			}

internal/p2p/conn/secret_connection.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
-	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/crypto/encoding"
 	"github.com/tendermint/tendermint/internal/libs/protoio"
 	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
 	"github.com/tendermint/tendermint/libs/async"
@@ -406,7 +406,7 @@ func shareAuthSignature(sc io.ReadWriter, pubKey crypto.PubKey, signature []byte
 	// Send our info and receive theirs in tandem.
 	var trs, _ = async.Parallel(
 		func(_ int) (val interface{}, abort bool, err error) {
-			pbpk, err := cryptoenc.PubKeyToProto(pubKey)
+			pbpk, err := encoding.PubKeyToProto(pubKey)
 			if err != nil {
 				return nil, true, err
 			}
@@ -423,7 +423,7 @@ func shareAuthSignature(sc io.ReadWriter, pubKey crypto.PubKey, signature []byte
 				return nil, true, err // abort
 			}
 
-			pk, err := cryptoenc.PubKeyFromProto(pba.PubKey)
+			pk, err := encoding.PubKeyFromProto(pba.PubKey)
 			if err != nil {
 				return nil, true, err // abort
 			}

internal/p2p/conn/secret_connection_test.go

@@ -195,7 +195,7 @@ func TestSecretConnectionReadWrite(t *testing.T) {
 	compareWritesReads := func(writes []string, reads []string) {
 		for {
 			// Pop next write & corresponding reads
-			var read, write string = "", writes[0]
+			var read, write = "", writes[0]
 			var readCount = 0
 			for _, readChunk := range reads {
 				read += readChunk

internal/p2p/conn_set.go

@@ -1,82 +0,0 @@
-package p2p
-
-import (
-	"net"
-
-	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-)
-
-// ConnSet is a lookup table for connections and all their ips.
-type ConnSet interface {
-	Has(net.Conn) bool
-	HasIP(net.IP) bool
-	Set(net.Conn, []net.IP)
-	Remove(net.Conn)
-	RemoveAddr(net.Addr)
-}
-
-type connSetItem struct {
-	conn net.Conn
-	ips  []net.IP
-}
-
-type connSet struct {
-	tmsync.RWMutex
-
-	conns map[string]connSetItem
-}
-
-// NewConnSet returns a ConnSet implementation.
-func NewConnSet() ConnSet {
-	return &connSet{
-		conns: map[string]connSetItem{},
-	}
-}
-
-func (cs *connSet) Has(c net.Conn) bool {
-	cs.RLock()
-	defer cs.RUnlock()
-
-	_, ok := cs.conns[c.RemoteAddr().String()]
-
-	return ok
-}
-
-func (cs *connSet) HasIP(ip net.IP) bool {
-	cs.RLock()
-	defer cs.RUnlock()
-
-	for _, c := range cs.conns {
-		for _, known := range c.ips {
-			if known.Equal(ip) {
-				return true
-			}
-		}
-	}
-
-	return false
-}
-
-func (cs *connSet) Remove(c net.Conn) {
-	cs.Lock()
-	defer cs.Unlock()
-
-	delete(cs.conns, c.RemoteAddr().String())
-}
-
-func (cs *connSet) RemoveAddr(addr net.Addr) {
-	cs.Lock()
-	defer cs.Unlock()
-
-	delete(cs.conns, addr.String())
-}
-
-func (cs *connSet) Set(c net.Conn, ips []net.IP) {
-	cs.Lock()
-	defer cs.Unlock()
-
-	cs.conns[c.RemoteAddr().String()] = connSetItem{
-		conn: c,
-		ips:  ips,
-	}
-}

internal/p2p/errors.go

@@ -17,7 +17,7 @@ func (e ErrFilterTimeout) Error() string {
 // ErrRejected indicates that a Peer was rejected carrying additional
 // information as to the reason.
 type ErrRejected struct {
-	addr              NetAddress
+	addr              NodeAddress
 	conn              net.Conn
 	err               error
 	id                types.NodeID
@@ -30,7 +30,7 @@ type ErrRejected struct {
 }
 
 // Addr returns the NetAddress for the rejected Peer.
-func (e ErrRejected) Addr() NetAddress {
+func (e ErrRejected) Addr() NodeAddress {
 	return e.addr
 }
 
@@ -120,15 +120,15 @@ func (e ErrSwitchDuplicatePeerIP) Error() string {
 
 // ErrSwitchConnectToSelf to be raised when trying to connect to itself.
 type ErrSwitchConnectToSelf struct {
-	Addr *NetAddress
+	Addr *NodeAddress
 }
 
 func (e ErrSwitchConnectToSelf) Error() string {
-	return fmt.Sprintf("connect to self: %v", e.Addr)
+	return fmt.Sprintf("connect to self: %s", e.Addr)
 }
 
 type ErrSwitchAuthenticationFailure struct {
-	Dialed *NetAddress
+	Dialed *NodeAddress
 	Got    types.NodeID
 }
 

internal/p2p/mock/peer.go

@@ -1,70 +0,0 @@
-package mock
-
-import (
-	"net"
-
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/internal/p2p/conn"
-	"github.com/tendermint/tendermint/libs/service"
-	"github.com/tendermint/tendermint/types"
-)
-
-type Peer struct {
-	*service.BaseService
-	ip                   net.IP
-	id                   types.NodeID
-	addr                 *p2p.NetAddress
-	kv                   map[string]interface{}
-	Outbound, Persistent bool
-}
-
-// NewPeer creates and starts a new mock peer. If the ip
-// is nil, random routable address is used.
-func NewPeer(ip net.IP) *Peer {
-	var netAddr *p2p.NetAddress
-	if ip == nil {
-		_, netAddr = p2p.CreateRoutableAddr()
-	} else {
-		netAddr = types.NewNetAddressIPPort(ip, 26656)
-	}
-	nodeKey := types.GenNodeKey()
-	netAddr.ID = nodeKey.ID
-	mp := &Peer{
-		ip:   ip,
-		id:   nodeKey.ID,
-		addr: netAddr,
-		kv:   make(map[string]interface{}),
-	}
-	mp.BaseService = service.NewBaseService(nil, "MockPeer", mp)
-	if err := mp.Start(); err != nil {
-		panic(err)
-	}
-	return mp
-}
-
-func (mp *Peer) FlushStop()                              { mp.Stop() } //nolint:errcheck //ignore error
-func (mp *Peer) TrySend(chID byte, msgBytes []byte) bool { return true }
-func (mp *Peer) Send(chID byte, msgBytes []byte) bool    { return true }
-func (mp *Peer) NodeInfo() types.NodeInfo {
-	return types.NodeInfo{
-		NodeID:     mp.addr.ID,
-		ListenAddr: mp.addr.DialString(),
-	}
-}
-func (mp *Peer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} }
-func (mp *Peer) ID() types.NodeID              { return mp.id }
-func (mp *Peer) IsOutbound() bool              { return mp.Outbound }
-func (mp *Peer) IsPersistent() bool            { return mp.Persistent }
-func (mp *Peer) Get(key string) interface{} {
-	if value, ok := mp.kv[key]; ok {
-		return value
-	}
-	return nil
-}
-func (mp *Peer) Set(key string, value interface{}) {
-	mp.kv[key] = value
-}
-func (mp *Peer) RemoteIP() net.IP            { return mp.ip }
-func (mp *Peer) SocketAddr() *p2p.NetAddress { return mp.addr }
-func (mp *Peer) RemoteAddr() net.Addr        { return &net.TCPAddr{IP: mp.ip, Port: 8800} }
-func (mp *Peer) CloseConn() error            { return nil }

internal/p2p/mock/reactor.go

@@ -1,23 +0,0 @@
-package mock
-
-import (
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/internal/p2p/conn"
-	"github.com/tendermint/tendermint/libs/log"
-)
-
-type Reactor struct {
-	p2p.BaseReactor
-}
-
-func NewReactor() *Reactor {
-	r := &Reactor{}
-	r.BaseReactor = *p2p.NewBaseReactor("Mock-PEX", r)
-	r.SetLogger(log.TestingLogger())
-	return r
-}
-
-func (r *Reactor) GetChannels() []*conn.ChannelDescriptor            { return []*conn.ChannelDescriptor{} }
-func (r *Reactor) AddPeer(peer p2p.Peer)                             {}
-func (r *Reactor) RemovePeer(peer p2p.Peer, reason interface{})      {}
-func (r *Reactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) {}

internal/p2p/mocks/connection.go

@@ -35,20 +35,6 @@ func (_m *Connection) Close() error {
 	return r0
 }
 
-// FlushClose provides a mock function with given fields:
-func (_m *Connection) FlushClose() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
 // Handshake provides a mock function with given fields: _a0, _a1, _a2
 func (_m *Connection) Handshake(_a0 context.Context, _a1 types.NodeInfo, _a2 crypto.PrivKey) (types.NodeInfo, crypto.PubKey, error) {
 	ret := _m.Called(_a0, _a1, _a2)
@@ -94,14 +80,14 @@ func (_m *Connection) LocalEndpoint() p2p.Endpoint {
 }
 
 // ReceiveMessage provides a mock function with given fields:
-func (_m *Connection) ReceiveMessage() (p2p.ChannelID, []byte, error) {
+func (_m *Connection) ReceiveMessage() (conn.ChannelID, []byte, error) {
 	ret := _m.Called()
 
-	var r0 p2p.ChannelID
-	if rf, ok := ret.Get(0).(func() p2p.ChannelID); ok {
+	var r0 conn.ChannelID
+	if rf, ok := ret.Get(0).(func() conn.ChannelID); ok {
 		r0 = rf()
 	} else {
-		r0 = ret.Get(0).(p2p.ChannelID)
+		r0 = ret.Get(0).(conn.ChannelID)
 	}
 
 	var r1 []byte
@@ -138,35 +124,14 @@ func (_m *Connection) RemoteEndpoint() p2p.Endpoint {
 }
 
 // SendMessage provides a mock function with given fields: _a0, _a1
-func (_m *Connection) SendMessage(_a0 p2p.ChannelID, _a1 []byte) (bool, error) {
+func (_m *Connection) SendMessage(_a0 conn.ChannelID, _a1 []byte) error {
 	ret := _m.Called(_a0, _a1)
 
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(p2p.ChannelID, []byte) bool); ok {
+	var r0 error
+	if rf, ok := ret.Get(0).(func(conn.ChannelID, []byte) error); ok {
 		r0 = rf(_a0, _a1)
 	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(p2p.ChannelID, []byte) error); ok {
-		r1 = rf(_a0, _a1)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
-// Status provides a mock function with given fields:
-func (_m *Connection) Status() conn.ConnectionStatus {
-	ret := _m.Called()
-
-	var r0 conn.ConnectionStatus
-	if rf, ok := ret.Get(0).(func() conn.ConnectionStatus); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(conn.ConnectionStatus)
+		r0 = ret.Error(0)
 	}
 
 	return r0
@@ -185,24 +150,3 @@ func (_m *Connection) String() string {
 
 	return r0
 }
-
-// TrySendMessage provides a mock function with given fields: _a0, _a1
-func (_m *Connection) TrySendMessage(_a0 p2p.ChannelID, _a1 []byte) (bool, error) {
-	ret := _m.Called(_a0, _a1)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(p2p.ChannelID, []byte) bool); ok {
-		r0 = rf(_a0, _a1)
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	var r1 error
-	if rf, ok := ret.Get(1).(func(p2p.ChannelID, []byte) error); ok {
-		r1 = rf(_a0, _a1)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}

internal/p2p/mocks/peer.go

@@ -1,334 +0,0 @@
-// Code generated by mockery. DO NOT EDIT.
-
-package mocks
-
-import (
-	conn "github.com/tendermint/tendermint/internal/p2p/conn"
-	log "github.com/tendermint/tendermint/libs/log"
-
-	mock "github.com/stretchr/testify/mock"
-
-	net "net"
-
-	types "github.com/tendermint/tendermint/types"
-)
-
-// Peer is an autogenerated mock type for the Peer type
-type Peer struct {
-	mock.Mock
-}
-
-// CloseConn provides a mock function with given fields:
-func (_m *Peer) CloseConn() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// FlushStop provides a mock function with given fields:
-func (_m *Peer) FlushStop() {
-	_m.Called()
-}
-
-// Get provides a mock function with given fields: _a0
-func (_m *Peer) Get(_a0 string) interface{} {
-	ret := _m.Called(_a0)
-
-	var r0 interface{}
-	if rf, ok := ret.Get(0).(func(string) interface{}); ok {
-		r0 = rf(_a0)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(interface{})
-		}
-	}
-
-	return r0
-}
-
-// ID provides a mock function with given fields:
-func (_m *Peer) ID() types.NodeID {
-	ret := _m.Called()
-
-	var r0 types.NodeID
-	if rf, ok := ret.Get(0).(func() types.NodeID); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(types.NodeID)
-	}
-
-	return r0
-}
-
-// IsOutbound provides a mock function with given fields:
-func (_m *Peer) IsOutbound() bool {
-	ret := _m.Called()
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func() bool); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}
-
-// IsPersistent provides a mock function with given fields:
-func (_m *Peer) IsPersistent() bool {
-	ret := _m.Called()
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func() bool); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}
-
-// IsRunning provides a mock function with given fields:
-func (_m *Peer) IsRunning() bool {
-	ret := _m.Called()
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func() bool); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}
-
-// NodeInfo provides a mock function with given fields:
-func (_m *Peer) NodeInfo() types.NodeInfo {
-	ret := _m.Called()
-
-	var r0 types.NodeInfo
-	if rf, ok := ret.Get(0).(func() types.NodeInfo); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(types.NodeInfo)
-	}
-
-	return r0
-}
-
-// OnReset provides a mock function with given fields:
-func (_m *Peer) OnReset() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// OnStart provides a mock function with given fields:
-func (_m *Peer) OnStart() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// OnStop provides a mock function with given fields:
-func (_m *Peer) OnStop() {
-	_m.Called()
-}
-
-// Quit provides a mock function with given fields:
-func (_m *Peer) Quit() <-chan struct{} {
-	ret := _m.Called()
-
-	var r0 <-chan struct{}
-	if rf, ok := ret.Get(0).(func() <-chan struct{}); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(<-chan struct{})
-		}
-	}
-
-	return r0
-}
-
-// RemoteAddr provides a mock function with given fields:
-func (_m *Peer) RemoteAddr() net.Addr {
-	ret := _m.Called()
-
-	var r0 net.Addr
-	if rf, ok := ret.Get(0).(func() net.Addr); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(net.Addr)
-		}
-	}
-
-	return r0
-}
-
-// RemoteIP provides a mock function with given fields:
-func (_m *Peer) RemoteIP() net.IP {
-	ret := _m.Called()
-
-	var r0 net.IP
-	if rf, ok := ret.Get(0).(func() net.IP); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(net.IP)
-		}
-	}
-
-	return r0
-}
-
-// Reset provides a mock function with given fields:
-func (_m *Peer) Reset() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Send provides a mock function with given fields: _a0, _a1
-func (_m *Peer) Send(_a0 byte, _a1 []byte) bool {
-	ret := _m.Called(_a0, _a1)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(byte, []byte) bool); ok {
-		r0 = rf(_a0, _a1)
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}
-
-// Set provides a mock function with given fields: _a0, _a1
-func (_m *Peer) Set(_a0 string, _a1 interface{}) {
-	_m.Called(_a0, _a1)
-}
-
-// SetLogger provides a mock function with given fields: _a0
-func (_m *Peer) SetLogger(_a0 log.Logger) {
-	_m.Called(_a0)
-}
-
-// SocketAddr provides a mock function with given fields:
-func (_m *Peer) SocketAddr() *types.NetAddress {
-	ret := _m.Called()
-
-	var r0 *types.NetAddress
-	if rf, ok := ret.Get(0).(func() *types.NetAddress); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).(*types.NetAddress)
-		}
-	}
-
-	return r0
-}
-
-// Start provides a mock function with given fields:
-func (_m *Peer) Start() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// Status provides a mock function with given fields:
-func (_m *Peer) Status() conn.ConnectionStatus {
-	ret := _m.Called()
-
-	var r0 conn.ConnectionStatus
-	if rf, ok := ret.Get(0).(func() conn.ConnectionStatus); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(conn.ConnectionStatus)
-	}
-
-	return r0
-}
-
-// Stop provides a mock function with given fields:
-func (_m *Peer) Stop() error {
-	ret := _m.Called()
-
-	var r0 error
-	if rf, ok := ret.Get(0).(func() error); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Error(0)
-	}
-
-	return r0
-}
-
-// String provides a mock function with given fields:
-func (_m *Peer) String() string {
-	ret := _m.Called()
-
-	var r0 string
-	if rf, ok := ret.Get(0).(func() string); ok {
-		r0 = rf()
-	} else {
-		r0 = ret.Get(0).(string)
-	}
-
-	return r0
-}
-
-// TrySend provides a mock function with given fields: _a0, _a1
-func (_m *Peer) TrySend(_a0 byte, _a1 []byte) bool {
-	ret := _m.Called(_a0, _a1)
-
-	var r0 bool
-	if rf, ok := ret.Get(0).(func(byte, []byte) bool); ok {
-		r0 = rf(_a0, _a1)
-	} else {
-		r0 = ret.Get(0).(bool)
-	}
-
-	return r0
-}
-
-// Wait provides a mock function with given fields:
-func (_m *Peer) Wait() {
-	_m.Called()
-}

internal/p2p/mocks/transport.go

@@ -5,7 +5,10 @@ package mocks
 import (
 	context "context"
 
+	conn "github.com/tendermint/tendermint/internal/p2p/conn"
+
 	mock "github.com/stretchr/testify/mock"
+
 	p2p "github.com/tendermint/tendermint/internal/p2p"
 )
 
@@ -37,6 +40,11 @@ func (_m *Transport) Accept() (p2p.Connection, error) {
 	return r0, r1
 }
 
+// AddChannelDescriptors provides a mock function with given fields: _a0
+func (_m *Transport) AddChannelDescriptors(_a0 []*conn.ChannelDescriptor) {
+	_m.Called(_a0)
+}
+
 // Close provides a mock function with given fields:
 func (_m *Transport) Close() error {
 	ret := _m.Called()

internal/p2p/netaddress.go

@@ -1,11 +0,0 @@
-// Modified for Tendermint
-// Originally Copyright (c) 2013-2014 Conformal Systems LLC.
-// https://github.com/conformal/btcd/blob/master/LICENSE
-
-package p2p
-
-import (
-	"github.com/tendermint/tendermint/types"
-)
-
-type NetAddress = types.NetAddress

internal/p2p/p2p_test.go

@@ -6,6 +6,7 @@ import (
 	"github.com/tendermint/tendermint/crypto"
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -14,12 +15,12 @@ import (
 var (
 	ctx    = context.Background()
 	chID   = p2p.ChannelID(1)
-	chDesc = p2p.ChannelDescriptor{
-		ID:                  byte(chID),
+	chDesc = &p2p.ChannelDescriptor{
+		ID:                  chID,
+		MessageType:         &p2ptest.Message{},
 		Priority:            5,
 		SendQueueCapacity:   10,
 		RecvMessageCapacity: 10,
-		MaxSendBytes:        1000,
 	}
 
 	selfKey  crypto.PrivKey = ed25519.GenPrivKeyFromSecret([]byte{0xf9, 0x1b, 0x08, 0xaa, 0x38, 0xee, 0x34, 0xdd})

internal/p2p/p2ptest/network.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gogo/protobuf/proto"
 	"github.com/stretchr/testify/require"
 	dbm "github.com/tendermint/tm-db"
 
@@ -137,13 +136,11 @@ func (n *Network) NodeIDs() []types.NodeID {
 // doing error checks and cleanups.
 func (n *Network) MakeChannels(
 	t *testing.T,
-	chDesc p2p.ChannelDescriptor,
-	messageType proto.Message,
-	size int,
+	chDesc *p2p.ChannelDescriptor,
 ) map[types.NodeID]*p2p.Channel {
 	channels := map[types.NodeID]*p2p.Channel{}
 	for _, node := range n.Nodes {
-		channels[node.NodeID] = node.MakeChannel(t, chDesc, messageType, size)
+		channels[node.NodeID] = node.MakeChannel(t, chDesc)
 	}
 	return channels
 }
@@ -153,13 +150,11 @@ func (n *Network) MakeChannels(
 // all the channels.
 func (n *Network) MakeChannelsNoCleanup(
 	t *testing.T,
-	chDesc p2p.ChannelDescriptor,
-	messageType proto.Message,
-	size int,
+	chDesc *p2p.ChannelDescriptor,
 ) map[types.NodeID]*p2p.Channel {
 	channels := map[types.NodeID]*p2p.Channel{}
 	for _, node := range n.Nodes {
-		channels[node.NodeID] = node.MakeChannelNoCleanup(t, chDesc, messageType, size)
+		channels[node.NodeID] = node.MakeChannelNoCleanup(t, chDesc)
 	}
 	return channels
 }
@@ -281,11 +276,13 @@ func (n *Network) MakeNode(t *testing.T, opts NodeOptions) *Node {
 // MakeChannel opens a channel, with automatic error handling and cleanup. On
 // test cleanup, it also checks that the channel is empty, to make sure
 // all expected messages have been asserted.
-func (n *Node) MakeChannel(t *testing.T, chDesc p2p.ChannelDescriptor,
-	messageType proto.Message, size int) *p2p.Channel {
-	channel, err := n.Router.OpenChannel(chDesc, messageType, size)
+func (n *Node) MakeChannel(
+	t *testing.T,
+	chDesc *p2p.ChannelDescriptor,
+) *p2p.Channel {
+	channel, err := n.Router.OpenChannel(chDesc)
 	require.NoError(t, err)
-	require.Contains(t, n.Router.NodeInfo().Channels, chDesc.ID)
+	require.Contains(t, n.Router.NodeInfo().Channels, byte(chDesc.ID))
 	t.Cleanup(func() {
 		RequireEmpty(t, channel)
 		channel.Close()
@@ -297,12 +294,9 @@ func (n *Node) MakeChannel(t *testing.T, chDesc p2p.ChannelDescriptor,
 // caller must ensure proper cleanup of the channel.
 func (n *Node) MakeChannelNoCleanup(
 	t *testing.T,
-	chDesc p2p.ChannelDescriptor,
-	messageType proto.Message,
-	size int,
+	chDesc *p2p.ChannelDescriptor,
 ) *p2p.Channel {
-
-	channel, err := n.Router.OpenChannel(chDesc, messageType, size)
+	channel, err := n.Router.OpenChannel(chDesc)
 	require.NoError(t, err)
 	return channel
 }
@@ -333,12 +327,12 @@ func (n *Node) MakePeerUpdatesNoRequireEmpty(t *testing.T) *p2p.PeerUpdates {
 	return sub
 }
 
-func MakeChannelDesc(chID p2p.ChannelID) p2p.ChannelDescriptor {
-	return p2p.ChannelDescriptor{
-		ID:                  byte(chID),
+func MakeChannelDesc(chID p2p.ChannelID) *p2p.ChannelDescriptor {
+	return &p2p.ChannelDescriptor{
+		ID:                  chID,
+		MessageType:         &Message{},
 		Priority:            5,
 		SendQueueCapacity:   10,
 		RecvMessageCapacity: 10,
-		MaxSendBytes:        1000,
 	}
 }

internal/p2p/p2ptest/require.go

@@ -24,6 +24,8 @@ func RequireEmpty(t *testing.T, channels ...*p2p.Channel) {
 
 // RequireReceive requires that the given envelope is received on the channel.
 func RequireReceive(t *testing.T, channel *p2p.Channel, expect p2p.Envelope) {
+	t.Helper()
+
 	timer := time.NewTimer(time.Second) // not time.After due to goroutine leaks
 	defer timer.Stop()
 

internal/p2p/peer.go

@@ -1,371 +0,0 @@
-package p2p
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"runtime/debug"
-	"time"
-
-	tmconn "github.com/tendermint/tendermint/internal/p2p/conn"
-	"github.com/tendermint/tendermint/libs/cmap"
-	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/libs/service"
-	"github.com/tendermint/tendermint/types"
-)
-
-//go:generate ../../scripts/mockery_generate.sh Peer
-
-const metricsTickerDuration = 10 * time.Second
-
-// Peer is an interface representing a peer connected on a reactor.
-type Peer interface {
-	service.Service
-	FlushStop()
-
-	ID() types.NodeID     // peer's cryptographic ID
-	RemoteIP() net.IP     // remote IP of the connection
-	RemoteAddr() net.Addr // remote address of the connection
-
-	IsOutbound() bool   // did we dial the peer
-	IsPersistent() bool // do we redial this peer when we disconnect
-
-	CloseConn() error // close original connection
-
-	NodeInfo() types.NodeInfo // peer's info
-	Status() tmconn.ConnectionStatus
-	SocketAddr() *NetAddress // actual address of the socket
-
-	Send(byte, []byte) bool
-	TrySend(byte, []byte) bool
-
-	Set(string, interface{})
-	Get(string) interface{}
-}
-
-//----------------------------------------------------------
-
-// peerConn contains the raw connection and its config.
-type peerConn struct {
-	outbound   bool
-	persistent bool
-	conn       Connection
-	ip         net.IP // cached RemoteIP()
-}
-
-func newPeerConn(outbound, persistent bool, conn Connection) peerConn {
-	return peerConn{
-		outbound:   outbound,
-		persistent: persistent,
-		conn:       conn,
-	}
-}
-
-// Return the IP from the connection RemoteAddr
-func (pc peerConn) RemoteIP() net.IP {
-	if pc.ip == nil {
-		pc.ip = pc.conn.RemoteEndpoint().IP
-	}
-	return pc.ip
-}
-
-// peer implements Peer.
-//
-// Before using a peer, you will need to perform a handshake on connection.
-type peer struct {
-	service.BaseService
-
-	// raw peerConn and the multiplex connection
-	peerConn
-
-	// peer's node info and the channel it knows about
-	// channels = nodeInfo.Channels
-	// cached to avoid copying nodeInfo in hasChannel
-	nodeInfo    types.NodeInfo
-	channels    []byte
-	reactors    map[byte]Reactor
-	onPeerError func(Peer, interface{})
-
-	// User data
-	Data *cmap.CMap
-
-	metrics       *Metrics
-	metricsTicker *time.Ticker
-}
-
-type PeerOption func(*peer)
-
-func newPeer(
-	nodeInfo types.NodeInfo,
-	pc peerConn,
-	reactorsByCh map[byte]Reactor,
-	onPeerError func(Peer, interface{}),
-	options ...PeerOption,
-) *peer {
-	p := &peer{
-		peerConn:      pc,
-		nodeInfo:      nodeInfo,
-		channels:      nodeInfo.Channels,
-		reactors:      reactorsByCh,
-		onPeerError:   onPeerError,
-		Data:          cmap.NewCMap(),
-		metricsTicker: time.NewTicker(metricsTickerDuration),
-		metrics:       NopMetrics(),
-	}
-
-	p.BaseService = *service.NewBaseService(nil, "Peer", p)
-	for _, option := range options {
-		option(p)
-	}
-
-	return p
-}
-
-// onError calls the peer error callback.
-func (p *peer) onError(err interface{}) {
-	p.onPeerError(p, err)
-}
-
-// String representation.
-func (p *peer) String() string {
-	if p.outbound {
-		return fmt.Sprintf("Peer{%v %v out}", p.conn, p.ID())
-	}
-
-	return fmt.Sprintf("Peer{%v %v in}", p.conn, p.ID())
-}
-
-//---------------------------------------------------
-// Implements service.Service
-
-// SetLogger implements BaseService.
-func (p *peer) SetLogger(l log.Logger) {
-	p.Logger = l
-}
-
-// OnStart implements BaseService.
-func (p *peer) OnStart() error {
-	if err := p.BaseService.OnStart(); err != nil {
-		return err
-	}
-
-	go p.processMessages()
-	go p.metricsReporter()
-
-	return nil
-}
-
-// processMessages processes messages received from the connection.
-func (p *peer) processMessages() {
-	defer func() {
-		if r := recover(); r != nil {
-			p.Logger.Error("peer message processing panic", "err", r, "stack", string(debug.Stack()))
-			p.onError(fmt.Errorf("panic during peer message processing: %v", r))
-		}
-	}()
-
-	for {
-		chID, msg, err := p.conn.ReceiveMessage()
-		if err != nil {
-			p.onError(err)
-			return
-		}
-		reactor, ok := p.reactors[byte(chID)]
-		if !ok {
-			p.onError(fmt.Errorf("unknown channel %v", chID))
-			return
-		}
-		reactor.Receive(byte(chID), p, msg)
-	}
-}
-
-// FlushStop mimics OnStop but additionally ensures that all successful
-// .Send() calls will get flushed before closing the connection.
-// NOTE: it is not safe to call this method more than once.
-func (p *peer) FlushStop() {
-	p.metricsTicker.Stop()
-	p.BaseService.OnStop()
-	if err := p.conn.FlushClose(); err != nil {
-		p.Logger.Debug("error while stopping peer", "err", err)
-	}
-}
-
-// OnStop implements BaseService.
-func (p *peer) OnStop() {
-	p.metricsTicker.Stop()
-	p.BaseService.OnStop()
-	if err := p.conn.Close(); err != nil {
-		p.Logger.Debug("error while stopping peer", "err", err)
-	}
-}
-
-//---------------------------------------------------
-// Implements Peer
-
-// ID returns the peer's ID - the hex encoded hash of its pubkey.
-func (p *peer) ID() types.NodeID {
-	return p.nodeInfo.ID()
-}
-
-// IsOutbound returns true if the connection is outbound, false otherwise.
-func (p *peer) IsOutbound() bool {
-	return p.peerConn.outbound
-}
-
-// IsPersistent returns true if the peer is persitent, false otherwise.
-func (p *peer) IsPersistent() bool {
-	return p.peerConn.persistent
-}
-
-// NodeInfo returns a copy of the peer's NodeInfo.
-func (p *peer) NodeInfo() types.NodeInfo {
-	return p.nodeInfo
-}
-
-// SocketAddr returns the address of the socket.
-// For outbound peers, it's the address dialed (after DNS resolution).
-// For inbound peers, it's the address returned by the underlying connection
-// (not what's reported in the peer's NodeInfo).
-func (p *peer) SocketAddr() *NetAddress {
-	endpoint := p.peerConn.conn.RemoteEndpoint()
-	return &NetAddress{
-		ID:   p.ID(),
-		IP:   endpoint.IP,
-		Port: endpoint.Port,
-	}
-}
-
-// Status returns the peer's ConnectionStatus.
-func (p *peer) Status() tmconn.ConnectionStatus {
-	return p.conn.Status()
-}
-
-// Send msg bytes to the channel identified by chID byte. Returns false if the
-// send queue is full after timeout, specified by MConnection.
-func (p *peer) Send(chID byte, msgBytes []byte) bool {
-	if !p.IsRunning() {
-		// see Switch#Broadcast, where we fetch the list of peers and loop over
-		// them - while we're looping, one peer may be removed and stopped.
-		return false
-	} else if !p.hasChannel(chID) {
-		return false
-	}
-	res, err := p.conn.SendMessage(ChannelID(chID), msgBytes)
-	if err == io.EOF {
-		return false
-	} else if err != nil {
-		p.onError(err)
-		return false
-	}
-	if res {
-		labels := []string{
-			"peer_id", string(p.ID()),
-			"chID", fmt.Sprintf("%#x", chID),
-		}
-		p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes)))
-	}
-	return res
-}
-
-// TrySend msg bytes to the channel identified by chID byte. Immediately returns
-// false if the send queue is full.
-func (p *peer) TrySend(chID byte, msgBytes []byte) bool {
-	if !p.IsRunning() {
-		return false
-	} else if !p.hasChannel(chID) {
-		return false
-	}
-	res, err := p.conn.TrySendMessage(ChannelID(chID), msgBytes)
-	if err == io.EOF {
-		return false
-	} else if err != nil {
-		p.onError(err)
-		return false
-	}
-	if res {
-		labels := []string{
-			"peer_id", string(p.ID()),
-			"chID", fmt.Sprintf("%#x", chID),
-		}
-		p.metrics.PeerSendBytesTotal.With(labels...).Add(float64(len(msgBytes)))
-	}
-	return res
-}
-
-// Get the data for a given key.
-func (p *peer) Get(key string) interface{} {
-	return p.Data.Get(key)
-}
-
-// Set sets the data for the given key.
-func (p *peer) Set(key string, data interface{}) {
-	p.Data.Set(key, data)
-}
-
-// hasChannel returns true if the peer reported
-// knowing about the given chID.
-func (p *peer) hasChannel(chID byte) bool {
-	for _, ch := range p.channels {
-		if ch == chID {
-			return true
-		}
-	}
-	// NOTE: probably will want to remove this
-	// but could be helpful while the feature is new
-	p.Logger.Debug(
-		"Unknown channel for peer",
-		"channel",
-		chID,
-		"channels",
-		p.channels,
-	)
-	return false
-}
-
-// CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all.
-func (p *peer) CloseConn() error {
-	return p.peerConn.conn.Close()
-}
-
-//---------------------------------------------------
-// methods only used for testing
-// TODO: can we remove these?
-
-// CloseConn closes the underlying connection
-func (pc *peerConn) CloseConn() {
-	pc.conn.Close()
-}
-
-// RemoteAddr returns peer's remote network address.
-func (p *peer) RemoteAddr() net.Addr {
-	endpoint := p.conn.RemoteEndpoint()
-	return &net.TCPAddr{
-		IP:   endpoint.IP,
-		Port: int(endpoint.Port),
-	}
-}
-
-//---------------------------------------------------
-
-func PeerMetrics(metrics *Metrics) PeerOption {
-	return func(p *peer) {
-		p.metrics = metrics
-	}
-}
-
-func (p *peer) metricsReporter() {
-	for {
-		select {
-		case <-p.metricsTicker.C:
-			status := p.conn.Status()
-			var sendQueueSize float64
-			for _, chStatus := range status.Channels {
-				sendQueueSize += float64(chStatus.SendQueueSize)
-			}
-
-			p.metrics.PeerPendingSendBytes.With("peer_id", string(p.ID())).Set(sendQueueSize)
-		case <-p.Quit():
-			return
-		}
-	}
-}

internal/p2p/peer_set.go

@@ -1,149 +0,0 @@
-package p2p
-
-import (
-	"net"
-
-	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-	"github.com/tendermint/tendermint/types"
-)
-
-// IPeerSet has a (immutable) subset of the methods of PeerSet.
-type IPeerSet interface {
-	Has(key types.NodeID) bool
-	HasIP(ip net.IP) bool
-	Get(key types.NodeID) Peer
-	List() []Peer
-	Size() int
-}
-
-//-----------------------------------------------------------------------------
-
-// PeerSet is a special structure for keeping a table of peers.
-// Iteration over the peers is super fast and thread-safe.
-type PeerSet struct {
-	mtx    tmsync.Mutex
-	lookup map[types.NodeID]*peerSetItem
-	list   []Peer
-}
-
-type peerSetItem struct {
-	peer  Peer
-	index int
-}
-
-// NewPeerSet creates a new peerSet with a list of initial capacity of 256 items.
-func NewPeerSet() *PeerSet {
-	return &PeerSet{
-		lookup: make(map[types.NodeID]*peerSetItem),
-		list:   make([]Peer, 0, 256),
-	}
-}
-
-// Add adds the peer to the PeerSet.
-// It returns an error carrying the reason, if the peer is already present.
-func (ps *PeerSet) Add(peer Peer) error {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-
-	if ps.lookup[peer.ID()] != nil {
-		return ErrSwitchDuplicatePeerID{peer.ID()}
-	}
-
-	index := len(ps.list)
-	// Appending is safe even with other goroutines
-	// iterating over the ps.list slice.
-	ps.list = append(ps.list, peer)
-	ps.lookup[peer.ID()] = &peerSetItem{peer, index}
-	return nil
-}
-
-// Has returns true if the set contains the peer referred to by this
-// peerKey, otherwise false.
-func (ps *PeerSet) Has(peerKey types.NodeID) bool {
-	ps.mtx.Lock()
-	_, ok := ps.lookup[peerKey]
-	ps.mtx.Unlock()
-	return ok
-}
-
-// HasIP returns true if the set contains the peer referred to by this IP
-// address, otherwise false.
-func (ps *PeerSet) HasIP(peerIP net.IP) bool {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-
-	return ps.hasIP(peerIP)
-}
-
-// hasIP does not acquire a lock so it can be used in public methods which
-// already lock.
-func (ps *PeerSet) hasIP(peerIP net.IP) bool {
-	for _, item := range ps.lookup {
-		if item.peer.RemoteIP().Equal(peerIP) {
-			return true
-		}
-	}
-
-	return false
-}
-
-// Get looks up a peer by the provided peerKey. Returns nil if peer is not
-// found.
-func (ps *PeerSet) Get(peerKey types.NodeID) Peer {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-	item, ok := ps.lookup[peerKey]
-	if ok {
-		return item.peer
-	}
-	return nil
-}
-
-// Remove discards peer by its Key, if the peer was previously memoized.
-// Returns true if the peer was removed, and false if it was not found.
-// in the set.
-func (ps *PeerSet) Remove(peer Peer) bool {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-
-	item := ps.lookup[peer.ID()]
-	if item == nil {
-		return false
-	}
-
-	index := item.index
-	// Create a new copy of the list but with one less item.
-	// (we must copy because we'll be mutating the list).
-	newList := make([]Peer, len(ps.list)-1)
-	copy(newList, ps.list)
-	// If it's the last peer, that's an easy special case.
-	if index == len(ps.list)-1 {
-		ps.list = newList
-		delete(ps.lookup, peer.ID())
-		return true
-	}
-
-	// Replace the popped item with the last item in the old list.
-	lastPeer := ps.list[len(ps.list)-1]
-	lastPeerKey := lastPeer.ID()
-	lastPeerItem := ps.lookup[lastPeerKey]
-	newList[index] = lastPeer
-	lastPeerItem.index = index
-	ps.list = newList
-	delete(ps.lookup, peer.ID())
-	return true
-}
-
-// Size returns the number of unique items in the peerSet.
-func (ps *PeerSet) Size() int {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-	return len(ps.list)
-}
-
-// List returns the threadsafe list of peers.
-func (ps *PeerSet) List() []Peer {
-	ps.mtx.Lock()
-	defer ps.mtx.Unlock()
-	return ps.list
-}

internal/p2p/peer_set_test.go

@@ -1,190 +0,0 @@
-package p2p
-
-import (
-	"net"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/tendermint/tendermint/libs/service"
-	"github.com/tendermint/tendermint/types"
-)
-
-// mockPeer for testing the PeerSet
-type mockPeer struct {
-	service.BaseService
-	ip net.IP
-	id types.NodeID
-}
-
-func (mp *mockPeer) FlushStop()                              { mp.Stop() } //nolint:errcheck // ignore error
-func (mp *mockPeer) TrySend(chID byte, msgBytes []byte) bool { return true }
-func (mp *mockPeer) Send(chID byte, msgBytes []byte) bool    { return true }
-func (mp *mockPeer) NodeInfo() types.NodeInfo                { return types.NodeInfo{} }
-func (mp *mockPeer) Status() ConnectionStatus                { return ConnectionStatus{} }
-func (mp *mockPeer) ID() types.NodeID                        { return mp.id }
-func (mp *mockPeer) IsOutbound() bool                        { return false }
-func (mp *mockPeer) IsPersistent() bool                      { return true }
-func (mp *mockPeer) Get(s string) interface{}                { return s }
-func (mp *mockPeer) Set(string, interface{})                 {}
-func (mp *mockPeer) RemoteIP() net.IP                        { return mp.ip }
-func (mp *mockPeer) SocketAddr() *NetAddress                 { return nil }
-func (mp *mockPeer) RemoteAddr() net.Addr                    { return &net.TCPAddr{IP: mp.ip, Port: 8800} }
-func (mp *mockPeer) CloseConn() error                        { return nil }
-
-// Returns a mock peer
-func newMockPeer(ip net.IP) *mockPeer {
-	if ip == nil {
-		ip = net.IP{127, 0, 0, 1}
-	}
-	nodeKey := types.GenNodeKey()
-	return &mockPeer{
-		ip: ip,
-		id: nodeKey.ID,
-	}
-}
-
-func TestPeerSetAddRemoveOne(t *testing.T) {
-	t.Parallel()
-
-	peerSet := NewPeerSet()
-
-	var peerList []Peer
-	for i := 0; i < 5; i++ {
-		p := newMockPeer(net.IP{127, 0, 0, byte(i)})
-		if err := peerSet.Add(p); err != nil {
-			t.Error(err)
-		}
-		peerList = append(peerList, p)
-	}
-
-	n := len(peerList)
-	// 1. Test removing from the front
-	for i, peerAtFront := range peerList {
-		removed := peerSet.Remove(peerAtFront)
-		assert.True(t, removed)
-		wantSize := n - i - 1
-		for j := 0; j < 2; j++ {
-			assert.Equal(t, false, peerSet.Has(peerAtFront.ID()), "#%d Run #%d: failed to remove peer", i, j)
-			assert.Equal(t, wantSize, peerSet.Size(), "#%d Run #%d: failed to remove peer and decrement size", i, j)
-			// Test the route of removing the now non-existent element
-			removed := peerSet.Remove(peerAtFront)
-			assert.False(t, removed)
-		}
-	}
-
-	// 2. Next we are testing removing the peer at the end
-	// a) Replenish the peerSet
-	for _, peer := range peerList {
-		if err := peerSet.Add(peer); err != nil {
-			t.Error(err)
-		}
-	}
-
-	// b) In reverse, remove each element
-	for i := n - 1; i >= 0; i-- {
-		peerAtEnd := peerList[i]
-		removed := peerSet.Remove(peerAtEnd)
-		assert.True(t, removed)
-		assert.Equal(t, false, peerSet.Has(peerAtEnd.ID()), "#%d: failed to remove item at end", i)
-		assert.Equal(t, i, peerSet.Size(), "#%d: differing sizes after peerSet.Remove(atEndPeer)", i)
-	}
-}
-
-func TestPeerSetAddRemoveMany(t *testing.T) {
-	t.Parallel()
-	peerSet := NewPeerSet()
-
-	peers := []Peer{}
-	N := 100
-	for i := 0; i < N; i++ {
-		peer := newMockPeer(net.IP{127, 0, 0, byte(i)})
-		if err := peerSet.Add(peer); err != nil {
-			t.Errorf("failed to add new peer")
-		}
-		if peerSet.Size() != i+1 {
-			t.Errorf("failed to add new peer and increment size")
-		}
-		peers = append(peers, peer)
-	}
-
-	for i, peer := range peers {
-		removed := peerSet.Remove(peer)
-		assert.True(t, removed)
-		if peerSet.Has(peer.ID()) {
-			t.Errorf("failed to remove peer")
-		}
-		if peerSet.Size() != len(peers)-i-1 {
-			t.Errorf("failed to remove peer and decrement size")
-		}
-	}
-}
-
-func TestPeerSetAddDuplicate(t *testing.T) {
-	t.Parallel()
-	peerSet := NewPeerSet()
-	peer := newMockPeer(nil)
-
-	n := 20
-	errsChan := make(chan error)
-	// Add the same asynchronously to test the
-	// concurrent guarantees of our APIs, and
-	// our expectation in the end is that only
-	// one addition succeeded, but the rest are
-	// instances of ErrSwitchDuplicatePeer.
-	for i := 0; i < n; i++ {
-		go func() {
-			errsChan <- peerSet.Add(peer)
-		}()
-	}
-
-	// Now collect and tally the results
-	errsTally := make(map[string]int)
-	for i := 0; i < n; i++ {
-		err := <-errsChan
-
-		switch err.(type) {
-		case ErrSwitchDuplicatePeerID:
-			errsTally["duplicateID"]++
-		default:
-			errsTally["other"]++
-		}
-	}
-
-	// Our next procedure is to ensure that only one addition
-	// succeeded and that the rest are each ErrSwitchDuplicatePeer.
-	wantErrCount, gotErrCount := n-1, errsTally["duplicateID"]
-	assert.Equal(t, wantErrCount, gotErrCount, "invalid ErrSwitchDuplicatePeer count")
-
-	wantNilErrCount, gotNilErrCount := 1, errsTally["other"]
-	assert.Equal(t, wantNilErrCount, gotNilErrCount, "invalid nil errCount")
-}
-
-func TestPeerSetGet(t *testing.T) {
-	t.Parallel()
-
-	var (
-		peerSet = NewPeerSet()
-		peer    = newMockPeer(nil)
-	)
-
-	assert.Nil(t, peerSet.Get(peer.ID()), "expecting a nil lookup, before .Add")
-
-	if err := peerSet.Add(peer); err != nil {
-		t.Fatalf("Failed to add new peer: %v", err)
-	}
-
-	var wg sync.WaitGroup
-	for i := 0; i < 10; i++ {
-		// Add them asynchronously to test the
-		// concurrent guarantees of our APIs.
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			have, want := peerSet.Get(peer.ID()), peer
-			assert.Equal(t, have, want, "%d: have %v, want %v", i, have, want)
-		}(i)
-	}
-	wg.Wait()
-}

internal/p2p/peer_test.go

@@ -1,239 +0,0 @@
-package p2p
-
-import (
-	"context"
-	"fmt"
-	golog "log"
-	"net"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/tendermint/tendermint/crypto"
-	"github.com/tendermint/tendermint/crypto/ed25519"
-	"github.com/tendermint/tendermint/libs/bytes"
-	"github.com/tendermint/tendermint/libs/log"
-	"github.com/tendermint/tendermint/types"
-
-	"github.com/tendermint/tendermint/config"
-	tmconn "github.com/tendermint/tendermint/internal/p2p/conn"
-)
-
-func TestPeerBasic(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	// simulate remote peer
-	rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg}
-	rp.Start()
-	t.Cleanup(rp.Stop)
-
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), cfg, tmconn.DefaultMConnConfig())
-	require.Nil(err)
-
-	err = p.Start()
-	require.Nil(err)
-	t.Cleanup(func() {
-		if err := p.Stop(); err != nil {
-			t.Error(err)
-		}
-	})
-
-	assert.True(p.IsRunning())
-	assert.True(p.IsOutbound())
-	assert.False(p.IsPersistent())
-	p.persistent = true
-	assert.True(p.IsPersistent())
-	assert.Equal(rp.Addr().DialString(), p.RemoteAddr().String())
-	assert.Equal(rp.ID(), p.ID())
-}
-
-func TestPeerSend(t *testing.T) {
-	assert, require := assert.New(t), require.New(t)
-
-	config := cfg
-
-	// simulate remote peer
-	rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: config}
-	rp.Start()
-	t.Cleanup(rp.Stop)
-
-	p, err := createOutboundPeerAndPerformHandshake(rp.Addr(), config, tmconn.DefaultMConnConfig())
-	require.Nil(err)
-
-	err = p.Start()
-	require.Nil(err)
-
-	t.Cleanup(func() {
-		if err := p.Stop(); err != nil {
-			t.Error(err)
-		}
-	})
-
-	assert.True(p.Send(testCh, []byte("Asylum")))
-}
-
-func createOutboundPeerAndPerformHandshake(
-	addr *NetAddress,
-	config *config.P2PConfig,
-	mConfig tmconn.MConnConfig,
-) (*peer, error) {
-	chDescs := []*tmconn.ChannelDescriptor{
-		{ID: testCh, Priority: 1},
-	}
-	pk := ed25519.GenPrivKey()
-	ourNodeInfo := testNodeInfo(types.NodeIDFromPubKey(pk.PubKey()), "host_peer")
-	transport := NewMConnTransport(log.TestingLogger(), mConfig, chDescs, MConnTransportOptions{})
-	reactorsByCh := map[byte]Reactor{testCh: NewTestReactor(chDescs, true)}
-	pc, err := testOutboundPeerConn(transport, addr, config, false, pk)
-	if err != nil {
-		return nil, err
-	}
-	peerInfo, _, err := pc.conn.Handshake(context.Background(), ourNodeInfo, pk)
-	if err != nil {
-		return nil, err
-	}
-
-	p := newPeer(peerInfo, pc, reactorsByCh, func(p Peer, r interface{}) {})
-	p.SetLogger(log.TestingLogger().With("peer", addr))
-	return p, nil
-}
-
-func testDial(addr *NetAddress, cfg *config.P2PConfig) (net.Conn, error) {
-	if cfg.TestDialFail {
-		return nil, fmt.Errorf("dial err (peerConfig.DialFail == true)")
-	}
-
-	conn, err := addr.DialTimeout(cfg.DialTimeout)
-	if err != nil {
-		return nil, err
-	}
-	return conn, nil
-}
-
-func testOutboundPeerConn(
-	transport *MConnTransport,
-	addr *NetAddress,
-	config *config.P2PConfig,
-	persistent bool,
-	ourNodePrivKey crypto.PrivKey,
-) (peerConn, error) {
-
-	var pc peerConn
-	conn, err := testDial(addr, config)
-	if err != nil {
-		return pc, fmt.Errorf("error creating peer: %w", err)
-	}
-
-	pc, err = testPeerConn(transport, conn, true, persistent)
-	if err != nil {
-		if cerr := conn.Close(); cerr != nil {
-			return pc, fmt.Errorf("%v: %w", cerr.Error(), err)
-		}
-		return pc, err
-	}
-
-	return pc, nil
-}
-
-type remotePeer struct {
-	PrivKey    crypto.PrivKey
-	Config     *config.P2PConfig
-	Network    string
-	addr       *NetAddress
-	channels   bytes.HexBytes
-	listenAddr string
-	listener   net.Listener
-}
-
-func (rp *remotePeer) Addr() *NetAddress {
-	return rp.addr
-}
-
-func (rp *remotePeer) ID() types.NodeID {
-	return types.NodeIDFromPubKey(rp.PrivKey.PubKey())
-}
-
-func (rp *remotePeer) Start() {
-	if rp.listenAddr == "" {
-		rp.listenAddr = "127.0.0.1:0"
-	}
-
-	l, e := net.Listen("tcp", rp.listenAddr) // any available address
-	if e != nil {
-		golog.Fatalf("net.Listen tcp :0: %+v", e)
-	}
-	rp.listener = l
-	rp.addr = types.NewNetAddress(types.NodeIDFromPubKey(rp.PrivKey.PubKey()), l.Addr())
-	if rp.channels == nil {
-		rp.channels = []byte{testCh}
-	}
-	go rp.accept()
-}
-
-func (rp *remotePeer) Stop() {
-	rp.listener.Close()
-}
-
-func (rp *remotePeer) Dial(addr *NetAddress) (net.Conn, error) {
-	transport := NewMConnTransport(log.TestingLogger(), MConnConfig(rp.Config),
-		[]*ChannelDescriptor{}, MConnTransportOptions{})
-	conn, err := addr.DialTimeout(1 * time.Second)
-	if err != nil {
-		return nil, err
-	}
-	pc, err := testInboundPeerConn(transport, conn)
-	if err != nil {
-		return nil, err
-	}
-	_, _, err = pc.conn.Handshake(context.Background(), rp.nodeInfo(), rp.PrivKey)
-	if err != nil {
-		return nil, err
-	}
-	return conn, err
-}
-
-func (rp *remotePeer) accept() {
-	transport := NewMConnTransport(log.TestingLogger(), MConnConfig(rp.Config),
-		[]*ChannelDescriptor{}, MConnTransportOptions{})
-	conns := []net.Conn{}
-
-	for {
-		conn, err := rp.listener.Accept()
-		if err != nil {
-			golog.Printf("Failed to accept conn: %+v", err)
-			for _, conn := range conns {
-				_ = conn.Close()
-			}
-			return
-		}
-
-		pc, err := testInboundPeerConn(transport, conn)
-		if err != nil {
-			golog.Printf("Failed to create a peer: %+v", err)
-		}
-		_, _, err = pc.conn.Handshake(context.Background(), rp.nodeInfo(), rp.PrivKey)
-		if err != nil {
-			golog.Printf("Failed to handshake a peer: %+v", err)
-		}
-
-		conns = append(conns, conn)
-	}
-}
-
-func (rp *remotePeer) nodeInfo() types.NodeInfo {
-	ni := types.NodeInfo{
-		ProtocolVersion: defaultProtocolVersion,
-		NodeID:          rp.Addr().ID,
-		ListenAddr:      rp.listener.Addr().String(),
-		Network:         "testing",
-		Version:         "1.2.3-rc0-deadbeef",
-		Channels:        rp.channels,
-		Moniker:         "remote_peer",
-	}
-	if rp.Network != "" {
-		ni.Network = rp.Network
-	}
-	return ni
-}

internal/p2p/peermanager.go

@@ -180,7 +180,7 @@ func (o *PeerManagerOptions) Validate() error {
 
 	if o.MaxPeers > 0 {
 		if o.MaxConnected == 0 || o.MaxConnected+o.MaxConnectedUpgrade > o.MaxPeers {
-			return fmt.Errorf("MaxConnected %v and MaxConnectedUpgrade %v can't exceed MaxPeers %v", // nolint
+			return fmt.Errorf("MaxConnected %v and MaxConnectedUpgrade %v can't exceed MaxPeers %v",
 				o.MaxConnected, o.MaxConnectedUpgrade, o.MaxPeers)
 		}
 	}
@@ -190,7 +190,7 @@ func (o *PeerManagerOptions) Validate() error {
 			return errors.New("can't set MaxRetryTime without MinRetryTime")
 		}
 		if o.MinRetryTime > o.MaxRetryTime {
-			return fmt.Errorf("MinRetryTime %v is greater than MaxRetryTime %v", // nolint
+			return fmt.Errorf("MinRetryTime %v is greater than MaxRetryTime %v",
 				o.MinRetryTime, o.MaxRetryTime)
 		}
 	}
@@ -200,7 +200,7 @@ func (o *PeerManagerOptions) Validate() error {
 			return errors.New("can't set MaxRetryTimePersistent without MinRetryTime")
 		}
 		if o.MinRetryTime > o.MaxRetryTimePersistent {
-			return fmt.Errorf("MinRetryTime %v is greater than MaxRetryTimePersistent %v", // nolint
+			return fmt.Errorf("MinRetryTime %v is greater than MaxRetryTimePersistent %v",
 				o.MinRetryTime, o.MaxRetryTimePersistent)
 		}
 	}

internal/p2p/peermanager_scoring_test.go

@@ -6,9 +6,10 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+
 	"github.com/tendermint/tendermint/crypto/ed25519"
 	"github.com/tendermint/tendermint/types"
-	dbm "github.com/tendermint/tm-db"
 )
 
 func TestPeerScoring(t *testing.T) {

internal/p2p/pex/addrbook.go

@@ -1,948 +0,0 @@
-// Modified for Tendermint
-// Originally Copyright (c) 2013-2014 Conformal Systems LLC.
-// https://github.com/conformal/btcd/blob/master/LICENSE
-
-package pex
-
-import (
-	"encoding/binary"
-	"fmt"
-	"hash"
-	"math"
-	mrand "math/rand"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/minio/highwayhash"
-	"github.com/tendermint/tendermint/crypto"
-	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
-	"github.com/tendermint/tendermint/internal/p2p"
-	tmmath "github.com/tendermint/tendermint/libs/math"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
-	"github.com/tendermint/tendermint/libs/service"
-	"github.com/tendermint/tendermint/types"
-)
-
-const (
-	bucketTypeNew = 0x01
-	bucketTypeOld = 0x02
-)
-
-// AddrBook is an address book used for tracking peers
-// so we can gossip about them to others and select
-// peers to dial.
-// TODO: break this up?
-type AddrBook interface {
-	service.Service
-
-	// Add our own addresses so we don't later add ourselves
-	AddOurAddress(*p2p.NetAddress)
-	// Check if it is our address
-	OurAddress(*p2p.NetAddress) bool
-
-	AddPrivateIDs([]string)
-
-	// Add and remove an address
-	AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error
-	RemoveAddress(*p2p.NetAddress)
-
-	// Check if the address is in the book
-	HasAddress(*p2p.NetAddress) bool
-
-	// Do we need more peers?
-	NeedMoreAddrs() bool
-	// Is Address Book Empty? Answer should not depend on being in your own
-	// address book, or private peers
-	Empty() bool
-
-	// Pick an address to dial
-	PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress
-
-	// Mark address
-	MarkGood(types.NodeID)
-	MarkAttempt(*p2p.NetAddress)
-	MarkBad(*p2p.NetAddress, time.Duration) // Move peer to bad peers list
-	// Add bad peers back to addrBook
-	ReinstateBadPeers()
-
-	IsGood(*p2p.NetAddress) bool
-	IsBanned(*p2p.NetAddress) bool
-
-	// Send a selection of addresses to peers
-	GetSelection() []*p2p.NetAddress
-	// Send a selection of addresses with bias
-	GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress
-
-	Size() int
-
-	// Persist to disk
-	Save()
-}
-
-var _ AddrBook = (*addrBook)(nil)
-
-// addrBook - concurrency safe peer address manager.
-// Implements AddrBook.
-type addrBook struct {
-	service.BaseService
-
-	// accessed concurrently
-	mtx        tmsync.Mutex
-	ourAddrs   map[string]struct{}
-	privateIDs map[types.NodeID]struct{}
-	addrLookup map[types.NodeID]*knownAddress // new & old
-	badPeers   map[types.NodeID]*knownAddress // blacklisted peers
-	bucketsOld []map[string]*knownAddress
-	bucketsNew []map[string]*knownAddress
-	nOld       int
-	nNew       int
-
-	// immutable after creation
-	filePath          string
-	key               string // random prefix for bucket placement
-	routabilityStrict bool
-	hasher            hash.Hash64
-
-	wg sync.WaitGroup
-}
-
-func mustNewHasher() hash.Hash64 {
-	key := crypto.CRandBytes(highwayhash.Size)
-	hasher, err := highwayhash.New64(key)
-	if err != nil {
-		panic(err)
-	}
-	return hasher
-}
-
-// NewAddrBook creates a new address book.
-// Use Start to begin processing asynchronous address updates.
-func NewAddrBook(filePath string, routabilityStrict bool) AddrBook {
-	am := &addrBook{
-		ourAddrs:          make(map[string]struct{}),
-		privateIDs:        make(map[types.NodeID]struct{}),
-		addrLookup:        make(map[types.NodeID]*knownAddress),
-		badPeers:          make(map[types.NodeID]*knownAddress),
-		filePath:          filePath,
-		routabilityStrict: routabilityStrict,
-	}
-	am.init()
-	am.BaseService = *service.NewBaseService(nil, "AddrBook", am)
-	return am
-}
-
-// Initialize the buckets.
-// When modifying this, don't forget to update loadFromFile()
-func (a *addrBook) init() {
-	a.key = crypto.CRandHex(24) // 24/2 * 8 = 96 bits
-	// New addr buckets
-	a.bucketsNew = make([]map[string]*knownAddress, newBucketCount)
-	for i := range a.bucketsNew {
-		a.bucketsNew[i] = make(map[string]*knownAddress)
-	}
-	// Old addr buckets
-	a.bucketsOld = make([]map[string]*knownAddress, oldBucketCount)
-	for i := range a.bucketsOld {
-		a.bucketsOld[i] = make(map[string]*knownAddress)
-	}
-	a.hasher = mustNewHasher()
-}
-
-// OnStart implements Service.
-func (a *addrBook) OnStart() error {
-	if err := a.BaseService.OnStart(); err != nil {
-		return err
-	}
-	a.loadFromFile(a.filePath)
-
-	// wg.Add to ensure that any invocation of .Wait()
-	// later on will wait for saveRoutine to terminate.
-	a.wg.Add(1)
-	go a.saveRoutine()
-
-	return nil
-}
-
-// OnStop implements Service.
-func (a *addrBook) OnStop() {
-	a.BaseService.OnStop()
-}
-
-func (a *addrBook) Wait() {
-	a.wg.Wait()
-}
-
-func (a *addrBook) FilePath() string {
-	return a.filePath
-}
-
-//-------------------------------------------------------
-
-// AddOurAddress one of our addresses.
-func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	a.Logger.Info("Add our address to book", "addr", addr)
-	a.ourAddrs[addr.String()] = struct{}{}
-}
-
-// OurAddress returns true if it is our address.
-func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	_, ok := a.ourAddrs[addr.String()]
-	return ok
-}
-
-func (a *addrBook) AddPrivateIDs(ids []string) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	for _, id := range ids {
-		a.privateIDs[types.NodeID(id)] = struct{}{}
-	}
-}
-
-// AddAddress implements AddrBook
-// Add address to a "new" bucket. If it's already in one, only add it probabilistically.
-// Returns error if the addr is non-routable. Does not add self.
-// NOTE: addr must not be nil
-func (a *addrBook) AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	return a.addAddress(addr, src)
-}
-
-// RemoveAddress implements AddrBook - removes the address from the book.
-func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	a.removeAddress(addr)
-}
-
-// IsGood returns true if peer was ever marked as good and haven't
-// done anything wrong since then.
-func (a *addrBook) IsGood(addr *p2p.NetAddress) bool {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	return a.addrLookup[addr.ID].isOld()
-}
-
-// IsBanned returns true if the peer is currently banned
-func (a *addrBook) IsBanned(addr *p2p.NetAddress) bool {
-	a.mtx.Lock()
-	_, ok := a.badPeers[addr.ID]
-	a.mtx.Unlock()
-
-	return ok
-}
-
-// HasAddress returns true if the address is in the book.
-func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	ka := a.addrLookup[addr.ID]
-	return ka != nil
-}
-
-// NeedMoreAddrs implements AddrBook - returns true if there are not have enough addresses in the book.
-func (a *addrBook) NeedMoreAddrs() bool {
-	return a.Size() < needAddressThreshold
-}
-
-// Empty implements AddrBook - returns true if there are no addresses in the address book.
-// Does not count the peer appearing in its own address book, or private peers.
-func (a *addrBook) Empty() bool {
-	return a.Size() == 0
-}
-
-// PickAddress implements AddrBook. It picks an address to connect to.
-// The address is picked randomly from an old or new bucket according
-// to the biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to that range)
-// and determines how biased we are to pick an address from a new bucket.
-// PickAddress returns nil if the AddrBook is empty or if we try to pick
-// from an empty bucket.
-// nolint:gosec // G404: Use of weak random number generator
-func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	bookSize := a.size()
-	if bookSize <= 0 {
-		if bookSize < 0 {
-			panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld))
-		}
-		return nil
-	}
-	if biasTowardsNewAddrs > 100 {
-		biasTowardsNewAddrs = 100
-	}
-	if biasTowardsNewAddrs < 0 {
-		biasTowardsNewAddrs = 0
-	}
-
-	// Bias between new and old addresses.
-	oldCorrelation := math.Sqrt(float64(a.nOld)) * (100.0 - float64(biasTowardsNewAddrs))
-	newCorrelation := math.Sqrt(float64(a.nNew)) * float64(biasTowardsNewAddrs)
-
-	// pick a random peer from a random bucket
-	var bucket map[string]*knownAddress
-	pickFromOldBucket := (newCorrelation+oldCorrelation)*mrand.Float64() < oldCorrelation
-	if (pickFromOldBucket && a.nOld == 0) ||
-		(!pickFromOldBucket && a.nNew == 0) {
-		return nil
-	}
-	// loop until we pick a random non-empty bucket
-	for len(bucket) == 0 {
-		if pickFromOldBucket {
-			bucket = a.bucketsOld[mrand.Intn(len(a.bucketsOld))]
-		} else {
-			bucket = a.bucketsNew[mrand.Intn(len(a.bucketsNew))]
-		}
-	}
-	// pick a random index and loop over the map to return that index
-	randIndex := mrand.Intn(len(bucket))
-	for _, ka := range bucket {
-		if randIndex == 0 {
-			return ka.Addr
-		}
-		randIndex--
-	}
-	return nil
-}
-
-// MarkGood implements AddrBook - it marks the peer as good and
-// moves it into an "old" bucket.
-func (a *addrBook) MarkGood(id types.NodeID) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	ka := a.addrLookup[id]
-	if ka == nil {
-		return
-	}
-	ka.markGood()
-	if ka.isNew() {
-		if err := a.moveToOld(ka); err != nil {
-			a.Logger.Error("Error moving address to old", "err", err)
-		}
-	}
-}
-
-// MarkAttempt implements AddrBook - it marks that an attempt was made to connect to the address.
-func (a *addrBook) MarkAttempt(addr *p2p.NetAddress) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	ka := a.addrLookup[addr.ID]
-	if ka == nil {
-		return
-	}
-	ka.markAttempt()
-}
-
-// MarkBad implements AddrBook. Kicks address out from book, places
-// the address in the badPeers pool.
-func (a *addrBook) MarkBad(addr *p2p.NetAddress, banTime time.Duration) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	if a.addBadPeer(addr, banTime) {
-		a.removeAddress(addr)
-	}
-}
-
-// ReinstateBadPeers removes bad peers from ban list and places them into a new
-// bucket.
-func (a *addrBook) ReinstateBadPeers() {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	for _, ka := range a.badPeers {
-		if ka.isBanned() {
-			continue
-		}
-
-		bucket, err := a.calcNewBucket(ka.Addr, ka.Src)
-		if err != nil {
-			a.Logger.Error("Failed to calculate new bucket (bad peer won't be reinstantiated)",
-				"addr", ka.Addr, "err", err)
-			continue
-		}
-
-		if err := a.addToNewBucket(ka, bucket); err != nil {
-			a.Logger.Error("Error adding peer to new bucket", "err", err)
-		}
-		delete(a.badPeers, ka.ID())
-
-		a.Logger.Info("Reinstated address", "addr", ka.Addr)
-	}
-}
-
-// GetSelection implements AddrBook.
-// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols.
-// Must never return a nil address.
-func (a *addrBook) GetSelection() []*p2p.NetAddress {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	bookSize := a.size()
-	if bookSize <= 0 {
-		if bookSize < 0 {
-			panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld))
-		}
-		return nil
-	}
-
-	numAddresses := tmmath.MaxInt(
-		tmmath.MinInt(minGetSelection, bookSize),
-		bookSize*getSelectionPercent/100)
-	numAddresses = tmmath.MinInt(maxGetSelection, numAddresses)
-
-	// XXX: instead of making a list of all addresses, shuffling, and slicing a random chunk,
-	// could we just select a random numAddresses of indexes?
-	allAddr := make([]*p2p.NetAddress, bookSize)
-	i := 0
-	for _, ka := range a.addrLookup {
-		allAddr[i] = ka.Addr
-		i++
-	}
-
-	// Fisher-Yates shuffle the array. We only need to do the first
-	// `numAddresses' since we are throwing the rest.
-	for i := 0; i < numAddresses; i++ {
-		// pick a number between current index and the end
-		// nolint:gosec // G404: Use of weak random number generator
-		j := mrand.Intn(len(allAddr)-i) + i
-		allAddr[i], allAddr[j] = allAddr[j], allAddr[i]
-	}
-
-	// slice off the limit we are willing to share.
-	return allAddr[:numAddresses]
-}
-
-func percentageOfNum(p, n int) int {
-	return int(math.Round((float64(p) / float64(100)) * float64(n)))
-}
-
-// GetSelectionWithBias implements AddrBook.
-// It randomly selects some addresses (old & new). Suitable for peer-exchange protocols.
-// Must never return a nil address.
-//
-// Each address is picked randomly from an old or new bucket according to the
-// biasTowardsNewAddrs argument, which must be between [0, 100] (or else is truncated to
-// that range) and determines how biased we are to pick an address from a new
-// bucket.
-func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddress {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	bookSize := a.size()
-	if bookSize <= 0 {
-		if bookSize < 0 {
-			panic(fmt.Sprintf("Addrbook size %d (new: %d + old: %d) is less than 0", a.nNew+a.nOld, a.nNew, a.nOld))
-		}
-		return nil
-	}
-
-	if biasTowardsNewAddrs > 100 {
-		biasTowardsNewAddrs = 100
-	}
-	if biasTowardsNewAddrs < 0 {
-		biasTowardsNewAddrs = 0
-	}
-
-	numAddresses := tmmath.MaxInt(
-		tmmath.MinInt(minGetSelection, bookSize),
-		bookSize*getSelectionPercent/100)
-	numAddresses = tmmath.MinInt(maxGetSelection, numAddresses)
-
-	// number of new addresses that, if possible, should be in the beginning of the selection
-	// if there are no enough old addrs, will choose new addr instead.
-	numRequiredNewAdd := tmmath.MaxInt(percentageOfNum(biasTowardsNewAddrs, numAddresses), numAddresses-a.nOld)
-	selection := a.randomPickAddresses(bucketTypeNew, numRequiredNewAdd)
-	selection = append(selection, a.randomPickAddresses(bucketTypeOld, numAddresses-len(selection))...)
-	return selection
-}
-
-//------------------------------------------------
-
-// Size returns the number of addresses in the book.
-func (a *addrBook) Size() int {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	return a.size()
-}
-
-func (a *addrBook) size() int {
-	return a.nNew + a.nOld
-}
-
-//----------------------------------------------------------
-
-// Save persists the address book to disk.
-func (a *addrBook) Save() {
-	a.saveToFile(a.filePath) // thread safe
-}
-
-func (a *addrBook) saveRoutine() {
-	defer a.wg.Done()
-
-	saveFileTicker := time.NewTicker(dumpAddressInterval)
-out:
-	for {
-		select {
-		case <-saveFileTicker.C:
-			a.saveToFile(a.filePath)
-		case <-a.Quit():
-			break out
-		}
-	}
-	saveFileTicker.Stop()
-	a.saveToFile(a.filePath)
-}
-
-//----------------------------------------------------------
-
-func (a *addrBook) getBucket(bucketType byte, bucketIdx int) map[string]*knownAddress {
-	switch bucketType {
-	case bucketTypeNew:
-		return a.bucketsNew[bucketIdx]
-	case bucketTypeOld:
-		return a.bucketsOld[bucketIdx]
-	default:
-		panic("Invalid bucket type")
-	}
-}
-
-// Adds ka to new bucket. Returns false if it couldn't do it cuz buckets full.
-// NOTE: currently it always returns true.
-func (a *addrBook) addToNewBucket(ka *knownAddress, bucketIdx int) error {
-	// Consistency check to ensure we don't add an already known address
-	if ka.isOld() {
-		return errAddrBookOldAddressNewBucket{ka.Addr, bucketIdx}
-	}
-
-	addrStr := ka.Addr.String()
-	bucket := a.getBucket(bucketTypeNew, bucketIdx)
-
-	// Already exists?
-	if _, ok := bucket[addrStr]; ok {
-		return nil
-	}
-
-	// Enforce max addresses.
-	if len(bucket) > newBucketSize {
-		a.Logger.Info("new bucket is full, expiring new")
-		a.expireNew(bucketIdx)
-	}
-
-	// Add to bucket.
-	bucket[addrStr] = ka
-	// increment nNew if the peer doesnt already exist in a bucket
-	if ka.addBucketRef(bucketIdx) == 1 {
-		a.nNew++
-	}
-
-	// Add it to addrLookup
-	a.addrLookup[ka.ID()] = ka
-	return nil
-}
-
-// Adds ka to old bucket. Returns false if it couldn't do it cuz buckets full.
-func (a *addrBook) addToOldBucket(ka *knownAddress, bucketIdx int) bool {
-	// Sanity check
-	if ka.isNew() {
-		a.Logger.Error(fmt.Sprintf("Cannot add new address to old bucket: %v", ka))
-		return false
-	}
-	if len(ka.Buckets) != 0 {
-		a.Logger.Error(fmt.Sprintf("Cannot add already old address to another old bucket: %v", ka))
-		return false
-	}
-
-	addrStr := ka.Addr.String()
-	bucket := a.getBucket(bucketTypeOld, bucketIdx)
-
-	// Already exists?
-	if _, ok := bucket[addrStr]; ok {
-		return true
-	}
-
-	// Enforce max addresses.
-	if len(bucket) > oldBucketSize {
-		return false
-	}
-
-	// Add to bucket.
-	bucket[addrStr] = ka
-	if ka.addBucketRef(bucketIdx) == 1 {
-		a.nOld++
-	}
-
-	// Ensure in addrLookup
-	a.addrLookup[ka.ID()] = ka
-
-	return true
-}
-
-func (a *addrBook) removeFromBucket(ka *knownAddress, bucketType byte, bucketIdx int) {
-	if ka.BucketType != bucketType {
-		a.Logger.Error(fmt.Sprintf("Bucket type mismatch: %v", ka))
-		return
-	}
-	bucket := a.getBucket(bucketType, bucketIdx)
-	delete(bucket, ka.Addr.String())
-	if ka.removeBucketRef(bucketIdx) == 0 {
-		if bucketType == bucketTypeNew {
-			a.nNew--
-		} else {
-			a.nOld--
-		}
-		delete(a.addrLookup, ka.ID())
-	}
-}
-
-func (a *addrBook) removeFromAllBuckets(ka *knownAddress) {
-	for _, bucketIdx := range ka.Buckets {
-		bucket := a.getBucket(ka.BucketType, bucketIdx)
-		delete(bucket, ka.Addr.String())
-	}
-	ka.Buckets = nil
-	if ka.BucketType == bucketTypeNew {
-		a.nNew--
-	} else {
-		a.nOld--
-	}
-	delete(a.addrLookup, ka.ID())
-}
-
-//----------------------------------------------------------
-
-func (a *addrBook) pickOldest(bucketType byte, bucketIdx int) *knownAddress {
-	bucket := a.getBucket(bucketType, bucketIdx)
-	var oldest *knownAddress
-	for _, ka := range bucket {
-		if oldest == nil || ka.LastAttempt.Before(oldest.LastAttempt) {
-			oldest = ka
-		}
-	}
-	return oldest
-}
-
-// adds the address to a "new" bucket. if its already in one,
-// it only adds it probabilistically
-func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error {
-	if addr == nil || src == nil {
-		return ErrAddrBookNilAddr{addr, src}
-	}
-
-	if err := addr.Valid(); err != nil {
-		return ErrAddrBookInvalidAddr{Addr: addr, AddrErr: err}
-	}
-
-	if _, ok := a.badPeers[addr.ID]; ok {
-		return ErrAddressBanned{addr}
-	}
-
-	if _, ok := a.privateIDs[addr.ID]; ok {
-		return ErrAddrBookPrivate{addr}
-	}
-
-	if _, ok := a.privateIDs[src.ID]; ok {
-		return ErrAddrBookPrivateSrc{src}
-	}
-
-	// TODO: we should track ourAddrs by ID and by IP:PORT and refuse both.
-	if _, ok := a.ourAddrs[addr.String()]; ok {
-		return ErrAddrBookSelf{addr}
-	}
-
-	if a.routabilityStrict && !addr.Routable() {
-		return ErrAddrBookNonRoutable{addr}
-	}
-
-	ka := a.addrLookup[addr.ID]
-	if ka != nil {
-		// If its already old and the address ID's are the same, ignore it.
-		// Thereby avoiding issues with a node on the network attempting to change
-		// the IP of a known node ID. (Which could yield an eclipse attack on the node)
-		if ka.isOld() && ka.Addr.ID == addr.ID {
-			return nil
-		}
-		// Already in max new buckets.
-		if len(ka.Buckets) == maxNewBucketsPerAddress {
-			return nil
-		}
-		// The more entries we have, the less likely we are to add more.
-		factor := int32(2 * len(ka.Buckets))
-		// nolint:gosec // G404: Use of weak random number generator
-		if mrand.Int31n(factor) != 0 {
-			return nil
-		}
-	} else {
-		ka = newKnownAddress(addr, src)
-	}
-
-	bucket, err := a.calcNewBucket(addr, src)
-	if err != nil {
-		return err
-	}
-	return a.addToNewBucket(ka, bucket)
-}
-
-func (a *addrBook) randomPickAddresses(bucketType byte, num int) []*p2p.NetAddress {
-	var buckets []map[string]*knownAddress
-	switch bucketType {
-	case bucketTypeNew:
-		buckets = a.bucketsNew
-	case bucketTypeOld:
-		buckets = a.bucketsOld
-	default:
-		panic("unexpected bucketType")
-	}
-	total := 0
-	for _, bucket := range buckets {
-		total += len(bucket)
-	}
-	addresses := make([]*knownAddress, 0, total)
-	for _, bucket := range buckets {
-		for _, ka := range bucket {
-			addresses = append(addresses, ka)
-		}
-	}
-	selection := make([]*p2p.NetAddress, 0, num)
-	chosenSet := make(map[string]bool, num)
-	rand := tmrand.NewRand()
-	rand.Shuffle(total, func(i, j int) {
-		addresses[i], addresses[j] = addresses[j], addresses[i]
-	})
-	for _, addr := range addresses {
-		if chosenSet[addr.Addr.String()] {
-			continue
-		}
-		chosenSet[addr.Addr.String()] = true
-		selection = append(selection, addr.Addr)
-		if len(selection) >= num {
-			return selection
-		}
-	}
-	return selection
-}
-
-// Make space in the new buckets by expiring the really bad entries.
-// If no bad entries are available we remove the oldest.
-func (a *addrBook) expireNew(bucketIdx int) {
-	for addrStr, ka := range a.bucketsNew[bucketIdx] {
-		// If an entry is bad, throw it away
-		if ka.isBad() {
-			a.Logger.Info(fmt.Sprintf("expiring bad address %v", addrStr))
-			a.removeFromBucket(ka, bucketTypeNew, bucketIdx)
-			return
-		}
-	}
-
-	// If we haven't thrown out a bad entry, throw out the oldest entry
-	oldest := a.pickOldest(bucketTypeNew, bucketIdx)
-	a.removeFromBucket(oldest, bucketTypeNew, bucketIdx)
-}
-
-// Promotes an address from new to old. If the destination bucket is full,
-// demote the oldest one to a "new" bucket.
-// TODO: Demote more probabilistically?
-func (a *addrBook) moveToOld(ka *knownAddress) error {
-	// Sanity check
-	if ka.isOld() {
-		a.Logger.Error(fmt.Sprintf("Cannot promote address that is already old %v", ka))
-		return nil
-	}
-	if len(ka.Buckets) == 0 {
-		a.Logger.Error(fmt.Sprintf("Cannot promote address that isn't in any new buckets %v", ka))
-		return nil
-	}
-
-	// Remove from all (new) buckets.
-	a.removeFromAllBuckets(ka)
-	// It's officially old now.
-	ka.BucketType = bucketTypeOld
-
-	// Try to add it to its oldBucket destination.
-	oldBucketIdx, err := a.calcOldBucket(ka.Addr)
-	if err != nil {
-		return err
-	}
-	added := a.addToOldBucket(ka, oldBucketIdx)
-	if !added {
-		// No room; move the oldest to a new bucket
-		oldest := a.pickOldest(bucketTypeOld, oldBucketIdx)
-		a.removeFromBucket(oldest, bucketTypeOld, oldBucketIdx)
-		newBucketIdx, err := a.calcNewBucket(oldest.Addr, oldest.Src)
-		if err != nil {
-			return err
-		}
-		if err := a.addToNewBucket(oldest, newBucketIdx); err != nil {
-			a.Logger.Error("Error adding peer to old bucket", "err", err)
-		}
-
-		// Finally, add our ka to old bucket again.
-		added = a.addToOldBucket(ka, oldBucketIdx)
-		if !added {
-			a.Logger.Error(fmt.Sprintf("Could not re-add ka %v to oldBucketIdx %v", ka, oldBucketIdx))
-		}
-	}
-	return nil
-}
-
-func (a *addrBook) removeAddress(addr *p2p.NetAddress) {
-	ka := a.addrLookup[addr.ID]
-	if ka == nil {
-		return
-	}
-	a.Logger.Info("Remove address from book", "addr", addr)
-	a.removeFromAllBuckets(ka)
-}
-
-func (a *addrBook) addBadPeer(addr *p2p.NetAddress, banTime time.Duration) bool {
-	// check it exists in addrbook
-	ka := a.addrLookup[addr.ID]
-	// check address is not already there
-	if ka == nil {
-		return false
-	}
-
-	if _, alreadyBadPeer := a.badPeers[addr.ID]; !alreadyBadPeer {
-		// add to bad peer list
-		ka.ban(banTime)
-		a.badPeers[addr.ID] = ka
-		a.Logger.Info("Add address to blacklist", "addr", addr)
-	}
-	return true
-}
-
-//---------------------------------------------------------------------
-// calculate bucket placements
-
-// hash(key + sourcegroup + int64(hash(key + group + sourcegroup)) % bucket_per_group) % num_new_buckets
-func (a *addrBook) calcNewBucket(addr, src *p2p.NetAddress) (int, error) {
-	data1 := []byte{}
-	data1 = append(data1, []byte(a.key)...)
-	data1 = append(data1, []byte(a.groupKey(addr))...)
-	data1 = append(data1, []byte(a.groupKey(src))...)
-	hash1, err := a.hash(data1)
-	if err != nil {
-		return 0, err
-	}
-	hash64 := binary.BigEndian.Uint64(hash1)
-	hash64 %= newBucketsPerGroup
-	var hashbuf [8]byte
-	binary.BigEndian.PutUint64(hashbuf[:], hash64)
-	data2 := []byte{}
-	data2 = append(data2, []byte(a.key)...)
-	data2 = append(data2, a.groupKey(src)...)
-	data2 = append(data2, hashbuf[:]...)
-
-	hash2, err := a.hash(data2)
-	if err != nil {
-		return 0, err
-	}
-	result := int(binary.BigEndian.Uint64(hash2) % newBucketCount)
-	return result, nil
-}
-
-// hash(key + group + int64(hash(key + addr)) % buckets_per_group) % num_old_buckets
-func (a *addrBook) calcOldBucket(addr *p2p.NetAddress) (int, error) {
-	data1 := []byte{}
-	data1 = append(data1, []byte(a.key)...)
-	data1 = append(data1, []byte(addr.String())...)
-	hash1, err := a.hash(data1)
-	if err != nil {
-		return 0, err
-	}
-	hash64 := binary.BigEndian.Uint64(hash1)
-	hash64 %= oldBucketsPerGroup
-	var hashbuf [8]byte
-	binary.BigEndian.PutUint64(hashbuf[:], hash64)
-	data2 := []byte{}
-	data2 = append(data2, []byte(a.key)...)
-	data2 = append(data2, a.groupKey(addr)...)
-	data2 = append(data2, hashbuf[:]...)
-
-	hash2, err := a.hash(data2)
-	if err != nil {
-		return 0, err
-	}
-	result := int(binary.BigEndian.Uint64(hash2) % oldBucketCount)
-	return result, nil
-}
-
-// Return a string representing the network group of this address.
-// This is the /16 for IPv4 (e.g. 1.2.0.0), the /32 (/36 for he.net) for IPv6, the string
-// "local" for a local address and the string "unroutable" for an unroutable
-// address.
-func (a *addrBook) groupKey(na *p2p.NetAddress) string {
-	return groupKeyFor(na, a.routabilityStrict)
-}
-
-func groupKeyFor(na *p2p.NetAddress, routabilityStrict bool) string {
-	if routabilityStrict && na.Local() {
-		return "local"
-	}
-	if routabilityStrict && !na.Routable() {
-		return "unroutable"
-	}
-
-	if ipv4 := na.IP.To4(); ipv4 != nil {
-		return na.IP.Mask(net.CIDRMask(16, 32)).String()
-	}
-
-	if na.RFC6145() || na.RFC6052() {
-		// last four bytes are the ip address
-		ip := na.IP[12:16]
-		return ip.Mask(net.CIDRMask(16, 32)).String()
-	}
-
-	if na.RFC3964() {
-		ip := na.IP[2:6]
-		return ip.Mask(net.CIDRMask(16, 32)).String()
-	}
-
-	if na.RFC4380() {
-		// teredo tunnels have the last 4 bytes as the v4 address XOR
-		// 0xff.
-		ip := net.IP(make([]byte, 4))
-		for i, byte := range na.IP[12:16] {
-			ip[i] = byte ^ 0xff
-		}
-		return ip.Mask(net.CIDRMask(16, 32)).String()
-	}
-
-	if na.OnionCatTor() {
-		// group is keyed off the first 4 bits of the actual onion key.
-		return fmt.Sprintf("tor:%d", na.IP[6]&((1<<4)-1))
-	}
-
-	// OK, so now we know ourselves to be a IPv6 address.
-	// bitcoind uses /32 for everything, except for Hurricane Electric's
-	// (he.net) IP range, which it uses /36 for.
-	bits := 32
-	heNet := &net.IPNet{IP: net.ParseIP("2001:470::"), Mask: net.CIDRMask(32, 128)}
-	if heNet.Contains(na.IP) {
-		bits = 36
-	}
-	ipv6Mask := net.CIDRMask(bits, 128)
-	return na.IP.Mask(ipv6Mask).String()
-}
-
-func (a *addrBook) hash(b []byte) ([]byte, error) {
-	a.hasher.Reset()
-	a.hasher.Write(b)
-	return a.hasher.Sum(nil), nil
-}

internal/p2p/pex/addrbook_test.go

@@ -1,777 +0,0 @@
-package pex
-
-import (
-	"encoding/hex"
-	"fmt"
-	"io/ioutil"
-	"math"
-	mrand "math/rand"
-	"net"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/libs/log"
-	tmmath "github.com/tendermint/tendermint/libs/math"
-	tmrand "github.com/tendermint/tendermint/libs/rand"
-	"github.com/tendermint/tendermint/types"
-)
-
-// FIXME These tests should not rely on .(*addrBook) assertions
-
-func TestAddrBookPickAddress(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-
-	// 0 addresses
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	assert.Zero(t, book.Size())
-
-	addr := book.PickAddress(50)
-	assert.Nil(t, addr, "expected no address")
-
-	randAddrs := randNetAddressPairs(t, 1)
-	addrSrc := randAddrs[0]
-	err := book.AddAddress(addrSrc.addr, addrSrc.src)
-	require.NoError(t, err)
-
-	// pick an address when we only have new address
-	addr = book.PickAddress(0)
-	assert.NotNil(t, addr, "expected an address")
-	addr = book.PickAddress(50)
-	assert.NotNil(t, addr, "expected an address")
-	addr = book.PickAddress(100)
-	assert.NotNil(t, addr, "expected an address")
-
-	// pick an address when we only have old address
-	book.MarkGood(addrSrc.addr.ID)
-	addr = book.PickAddress(0)
-	assert.NotNil(t, addr, "expected an address")
-	addr = book.PickAddress(50)
-	assert.NotNil(t, addr, "expected an address")
-
-	// in this case, nNew==0 but we biased 100% to new, so we return nil
-	addr = book.PickAddress(100)
-	assert.Nil(t, addr, "did not expected an address")
-}
-
-func TestAddrBookSaveLoad(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-
-	// 0 addresses
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	book.Save()
-
-	book = NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	err := book.Start()
-	require.NoError(t, err)
-
-	assert.True(t, book.Empty())
-
-	// 100 addresses
-	randAddrs := randNetAddressPairs(t, 100)
-
-	for _, addrSrc := range randAddrs {
-		err := book.AddAddress(addrSrc.addr, addrSrc.src)
-		require.NoError(t, err)
-	}
-
-	assert.Equal(t, 100, book.Size())
-	book.Save()
-
-	book = NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	err = book.Start()
-	require.NoError(t, err)
-
-	assert.Equal(t, 100, book.Size())
-}
-
-func TestAddrBookLookup(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	randAddrs := randNetAddressPairs(t, 100)
-
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	for _, addrSrc := range randAddrs {
-		addr := addrSrc.addr
-		src := addrSrc.src
-		err := book.AddAddress(addr, src)
-		require.NoError(t, err)
-
-		ka := book.HasAddress(addr)
-		assert.True(t, ka, "Expected to find KnownAddress %v but wasn't there.", addr)
-	}
-}
-
-func TestAddrBookPromoteToOld(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	randAddrs := randNetAddressPairs(t, 100)
-
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	for _, addrSrc := range randAddrs {
-		err := book.AddAddress(addrSrc.addr, addrSrc.src)
-		require.NoError(t, err)
-	}
-
-	// Attempt all addresses.
-	for _, addrSrc := range randAddrs {
-		book.MarkAttempt(addrSrc.addr)
-	}
-
-	// Promote half of them
-	for i, addrSrc := range randAddrs {
-		if i%2 == 0 {
-			book.MarkGood(addrSrc.addr.ID)
-		}
-	}
-
-	// TODO: do more testing :)
-
-	selection := book.GetSelection()
-	t.Logf("selection: %v", selection)
-
-	if len(selection) > book.Size() {
-		t.Errorf("selection could not be bigger than the book")
-	}
-
-	selection = book.GetSelectionWithBias(30)
-	t.Logf("selection: %v", selection)
-
-	if len(selection) > book.Size() {
-		t.Errorf("selection with bias could not be bigger than the book")
-	}
-
-	assert.Equal(t, book.Size(), 100, "expecting book size to be 100")
-}
-
-func TestAddrBookHandlesDuplicates(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-
-	book.SetLogger(log.TestingLogger())
-
-	randAddrs := randNetAddressPairs(t, 100)
-
-	differentSrc := randIPv4Address(t)
-	for _, addrSrc := range randAddrs {
-		err := book.AddAddress(addrSrc.addr, addrSrc.src)
-		require.NoError(t, err)
-		err = book.AddAddress(addrSrc.addr, addrSrc.src) // duplicate
-		require.NoError(t, err)
-		err = book.AddAddress(addrSrc.addr, differentSrc) // different src
-		require.NoError(t, err)
-	}
-
-	assert.Equal(t, 100, book.Size())
-}
-
-type netAddressPair struct {
-	addr *p2p.NetAddress
-	src  *p2p.NetAddress
-}
-
-func randNetAddressPairs(t *testing.T, n int) []netAddressPair {
-	randAddrs := make([]netAddressPair, n)
-	for i := 0; i < n; i++ {
-		randAddrs[i] = netAddressPair{addr: randIPv4Address(t), src: randIPv4Address(t)}
-	}
-	return randAddrs
-}
-
-func randIPv4Address(t *testing.T) *p2p.NetAddress {
-	for {
-		ip := fmt.Sprintf("%v.%v.%v.%v",
-			mrand.Intn(254)+1,
-			mrand.Intn(255),
-			mrand.Intn(255),
-			mrand.Intn(255),
-		)
-		port := mrand.Intn(65535-1) + 1
-		id := types.NodeID(hex.EncodeToString(tmrand.Bytes(types.NodeIDByteLength)))
-		idAddr := id.AddressString(fmt.Sprintf("%v:%v", ip, port))
-		addr, err := types.NewNetAddressString(idAddr)
-		assert.Nil(t, err, "error generating rand network address")
-		if addr.Routable() {
-			return addr
-		}
-	}
-}
-
-func TestAddrBookRemoveAddress(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-
-	addr := randIPv4Address(t)
-	err := book.AddAddress(addr, addr)
-	require.NoError(t, err)
-	assert.Equal(t, 1, book.Size())
-
-	book.RemoveAddress(addr)
-	assert.Equal(t, 0, book.Size())
-
-	nonExistingAddr := randIPv4Address(t)
-	book.RemoveAddress(nonExistingAddr)
-	assert.Equal(t, 0, book.Size())
-}
-
-func TestAddrBookGetSelectionWithOneMarkedGood(t *testing.T) {
-	// create a book with 10 addresses, 1 good/old and 9 new
-	book, _ := createAddrBookWithMOldAndNNewAddrs(t, 1, 9)
-	addrs := book.GetSelectionWithBias(biasToSelectNewPeers)
-	assert.NotNil(t, addrs)
-	assertMOldAndNNewAddrsInSelection(t, 1, 9, addrs, book)
-}
-
-func TestAddrBookGetSelectionWithOneNotMarkedGood(t *testing.T) {
-	// create a book with 10 addresses, 9 good/old and 1 new
-	book, _ := createAddrBookWithMOldAndNNewAddrs(t, 9, 1)
-	addrs := book.GetSelectionWithBias(biasToSelectNewPeers)
-	assert.NotNil(t, addrs)
-	assertMOldAndNNewAddrsInSelection(t, 9, 1, addrs, book)
-}
-
-func TestAddrBookGetSelectionReturnsNilWhenAddrBookIsEmpty(t *testing.T) {
-	book, _ := createAddrBookWithMOldAndNNewAddrs(t, 0, 0)
-	addrs := book.GetSelectionWithBias(biasToSelectNewPeers)
-	assert.Nil(t, addrs)
-}
-
-func TestAddrBookGetSelection(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-
-	// 1) empty book
-	assert.Empty(t, book.GetSelection())
-
-	// 2) add one address
-	addr := randIPv4Address(t)
-	err := book.AddAddress(addr, addr)
-	require.NoError(t, err)
-
-	assert.Equal(t, 1, len(book.GetSelection()))
-	assert.Equal(t, addr, book.GetSelection()[0])
-
-	// 3) add a bunch of addresses
-	randAddrs := randNetAddressPairs(t, 100)
-	for _, addrSrc := range randAddrs {
-		err := book.AddAddress(addrSrc.addr, addrSrc.src)
-		require.NoError(t, err)
-	}
-
-	// check there is no duplicates
-	addrs := make(map[string]*p2p.NetAddress)
-	selection := book.GetSelection()
-	for _, addr := range selection {
-		if dup, ok := addrs[addr.String()]; ok {
-			t.Fatalf("selection %v contains duplicates %v", selection, dup)
-		}
-		addrs[addr.String()] = addr
-	}
-
-	if len(selection) > book.Size() {
-		t.Errorf("selection %v could not be bigger than the book", selection)
-	}
-}
-
-func TestAddrBookGetSelectionWithBias(t *testing.T) {
-	const biasTowardsNewAddrs = 30
-
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-
-	// 1) empty book
-	selection := book.GetSelectionWithBias(biasTowardsNewAddrs)
-	assert.Empty(t, selection)
-
-	// 2) add one address
-	addr := randIPv4Address(t)
-	err := book.AddAddress(addr, addr)
-	require.NoError(t, err)
-
-	selection = book.GetSelectionWithBias(biasTowardsNewAddrs)
-	assert.Equal(t, 1, len(selection))
-	assert.Equal(t, addr, selection[0])
-
-	// 3) add a bunch of addresses
-	randAddrs := randNetAddressPairs(t, 100)
-	for _, addrSrc := range randAddrs {
-		err := book.AddAddress(addrSrc.addr, addrSrc.src)
-		require.NoError(t, err)
-	}
-
-	// check there is no duplicates
-	addrs := make(map[string]*p2p.NetAddress)
-	selection = book.GetSelectionWithBias(biasTowardsNewAddrs)
-	for _, addr := range selection {
-		if dup, ok := addrs[addr.String()]; ok {
-			t.Fatalf("selection %v contains duplicates %v", selection, dup)
-		}
-		addrs[addr.String()] = addr
-	}
-
-	if len(selection) > book.Size() {
-		t.Fatalf("selection %v could not be bigger than the book", selection)
-	}
-
-	// 4) mark 80% of the addresses as good
-	randAddrsLen := len(randAddrs)
-	for i, addrSrc := range randAddrs {
-		if int((float64(i)/float64(randAddrsLen))*100) >= 20 {
-			book.MarkGood(addrSrc.addr.ID)
-		}
-	}
-
-	selection = book.GetSelectionWithBias(biasTowardsNewAddrs)
-
-	// check that ~70% of addresses returned are good
-	good := 0
-	for _, addr := range selection {
-		if book.IsGood(addr) {
-			good++
-		}
-	}
-
-	got, expected := int((float64(good)/float64(len(selection)))*100), 100-biasTowardsNewAddrs
-
-	// compute some slack to protect against small differences due to rounding:
-	slack := int(math.Round(float64(100) / float64(len(selection))))
-	if got > expected+slack {
-		t.Fatalf(
-			"got more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)",
-			got,
-			expected,
-			good,
-			len(selection),
-		)
-	}
-	if got < expected-slack {
-		t.Fatalf(
-			"got fewer good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)",
-			got,
-			expected,
-			good,
-			len(selection),
-		)
-	}
-}
-
-func TestAddrBookHasAddress(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	addr := randIPv4Address(t)
-	err := book.AddAddress(addr, addr)
-	require.NoError(t, err)
-
-	assert.True(t, book.HasAddress(addr))
-
-	book.RemoveAddress(addr)
-
-	assert.False(t, book.HasAddress(addr))
-}
-
-func testCreatePrivateAddrs(t *testing.T, numAddrs int) ([]*p2p.NetAddress, []string) {
-	t.Helper()
-	addrs := make([]*p2p.NetAddress, numAddrs)
-	for i := 0; i < numAddrs; i++ {
-		addrs[i] = randIPv4Address(t)
-	}
-
-	private := make([]string, numAddrs)
-	for i, addr := range addrs {
-		private[i] = string(addr.ID)
-	}
-	return addrs, private
-}
-
-func TestBanBadPeers(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-
-	addr := randIPv4Address(t)
-	_ = book.AddAddress(addr, addr)
-
-	book.MarkBad(addr, 1*time.Second)
-	// addr should not reachable
-	assert.False(t, book.HasAddress(addr))
-	assert.True(t, book.IsBanned(addr))
-
-	err := book.AddAddress(addr, addr)
-	// book should not add address from the blacklist
-	assert.Error(t, err)
-
-	time.Sleep(1 * time.Second)
-	book.ReinstateBadPeers()
-	// address should be reinstated in the new bucket
-	assert.EqualValues(t, 1, book.Size())
-	assert.True(t, book.HasAddress(addr))
-	assert.False(t, book.IsGood(addr))
-}
-
-func TestAddrBookEmpty(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	// Check that empty book is empty
-	require.True(t, book.Empty())
-	// Check that book with our address is empty
-	book.AddOurAddress(randIPv4Address(t))
-	require.True(t, book.Empty())
-	// Check that book with private addrs is empty
-	_, privateIds := testCreatePrivateAddrs(t, 5)
-	book.AddPrivateIDs(privateIds)
-	require.True(t, book.Empty())
-
-	// Check that book with address is not empty
-	err := book.AddAddress(randIPv4Address(t), randIPv4Address(t))
-	require.NoError(t, err)
-	require.False(t, book.Empty())
-}
-
-func TestPrivatePeers(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-
-	addrs, private := testCreatePrivateAddrs(t, 10)
-	book.AddPrivateIDs(private)
-
-	// private addrs must not be added
-	for _, addr := range addrs {
-		err := book.AddAddress(addr, addr)
-		if assert.Error(t, err) {
-			_, ok := err.(ErrAddrBookPrivate)
-			assert.True(t, ok)
-		}
-	}
-
-	// addrs coming from private peers must not be added
-	err := book.AddAddress(randIPv4Address(t), addrs[0])
-	if assert.Error(t, err) {
-		_, ok := err.(ErrAddrBookPrivateSrc)
-		assert.True(t, ok)
-	}
-}
-
-func testAddrBookAddressSelection(t *testing.T, bookSize int) {
-	// generate all combinations of old (m) and new addresses
-	for nBookOld := 0; nBookOld <= bookSize; nBookOld++ {
-		nBookNew := bookSize - nBookOld
-		dbgStr := fmt.Sprintf("book of size %d (new %d, old %d)", bookSize, nBookNew, nBookOld)
-
-		// create book and get selection
-		book, _ := createAddrBookWithMOldAndNNewAddrs(t, nBookOld, nBookNew)
-		addrs := book.GetSelectionWithBias(biasToSelectNewPeers)
-		assert.NotNil(t, addrs, "%s - expected a non-nil selection", dbgStr)
-		nAddrs := len(addrs)
-		assert.NotZero(t, nAddrs, "%s - expected at least one address in selection", dbgStr)
-
-		// check there's no nil addresses
-		for _, addr := range addrs {
-			if addr == nil {
-				t.Fatalf("%s - got nil address in selection %v", dbgStr, addrs)
-			}
-		}
-
-		// XXX: shadowing
-		nOld, nNew := countOldAndNewAddrsInSelection(addrs, book)
-
-		// Given:
-		// n - num new addrs, m - num old addrs
-		// k - num new addrs expected in the beginning (based on bias %)
-		// i=min(n, max(k,r-m)), aka expNew
-		// j=min(m, r-i), aka expOld
-		//
-		// We expect this layout:
-		// indices:      0...i-1   i...i+j-1
-		// addresses:    N0..Ni-1  O0..Oj-1
-		//
-		// There is at least one partition and at most three.
-		var (
-			k      = percentageOfNum(biasToSelectNewPeers, nAddrs)
-			expNew = tmmath.MinInt(nNew, tmmath.MaxInt(k, nAddrs-nBookOld))
-			expOld = tmmath.MinInt(nOld, nAddrs-expNew)
-		)
-
-		// Verify that the number of old and new addresses are as expected
-		if nNew != expNew {
-			t.Fatalf("%s - expected new addrs %d, got %d", dbgStr, expNew, nNew)
-		}
-		if nOld != expOld {
-			t.Fatalf("%s - expected old addrs %d, got %d", dbgStr, expOld, nOld)
-		}
-
-		// Verify that the order of addresses is as expected
-		// Get the sequence types and lengths of the selection
-		seqLens, seqTypes, err := analyseSelectionLayout(book, addrs)
-		assert.NoError(t, err, "%s", dbgStr)
-
-		// Build a list with the expected lengths of partitions and another with the expected types, e.g.:
-		// expSeqLens = [10, 22], expSeqTypes = [1, 2]
-		// means we expect 10 new (type 1) addresses followed by 22 old (type 2) addresses.
-		var expSeqLens []int
-		var expSeqTypes []int
-
-		switch {
-		case expOld == 0: // all new addresses
-			expSeqLens = []int{nAddrs}
-			expSeqTypes = []int{1}
-		case expNew == 0: // all old addresses
-			expSeqLens = []int{nAddrs}
-			expSeqTypes = []int{2}
-		case nAddrs-expNew-expOld == 0: // new addresses, old addresses
-			expSeqLens = []int{expNew, expOld}
-			expSeqTypes = []int{1, 2}
-		}
-
-		assert.Equal(t, expSeqLens, seqLens,
-			"%s - expected sequence lengths of old/new %v, got %v",
-			dbgStr, expSeqLens, seqLens)
-		assert.Equal(t, expSeqTypes, seqTypes,
-			"%s - expected sequence types (1-new, 2-old) was %v, got %v",
-			dbgStr, expSeqTypes, seqTypes)
-	}
-}
-
-func TestMultipleAddrBookAddressSelection(t *testing.T) {
-	// test books with smaller size, < N
-	const N = 32
-	for bookSize := 1; bookSize < N; bookSize++ {
-		testAddrBookAddressSelection(t, bookSize)
-	}
-
-	// Test for two books with sizes from following ranges
-	ranges := [...][]int{{33, 100}, {100, 175}}
-	bookSizes := make([]int, 0, len(ranges))
-	for _, r := range ranges {
-		bookSizes = append(bookSizes, mrand.Intn(r[1]-r[0])+r[0])
-	}
-	t.Logf("Testing address selection for the following book sizes %v\n", bookSizes)
-	for _, bookSize := range bookSizes {
-		testAddrBookAddressSelection(t, bookSize)
-	}
-}
-
-func TestAddrBookAddDoesNotOverwriteOldIP(t *testing.T) {
-	fname := createTempFileName(t, "addrbook_test")
-
-	// This test creates adds a peer to the address book and marks it good
-	// It then attempts to override the peer's IP, by adding a peer with the same ID
-	// but different IP. We distinguish the IP's by "RealIP" and "OverrideAttemptIP"
-	peerID := "678503e6c8f50db7279c7da3cb9b072aac4bc0d5"
-	peerRealIP := "1.1.1.1:26656"
-	peerOverrideAttemptIP := "2.2.2.2:26656"
-	SrcAddr := "b0dd378c3fbc4c156cd6d302a799f0d2e4227201@159.89.121.174:26656"
-
-	// There is a chance that AddAddress will ignore the new peer its given.
-	// So we repeat trying to override the peer several times,
-	// to ensure we aren't in a case that got probabilistically ignored
-	numOverrideAttempts := 10
-
-	peerRealAddr, err := types.NewNetAddressString(peerID + "@" + peerRealIP)
-	require.Nil(t, err)
-
-	peerOverrideAttemptAddr, err := types.NewNetAddressString(peerID + "@" + peerOverrideAttemptIP)
-	require.Nil(t, err)
-
-	src, err := types.NewNetAddressString(SrcAddr)
-	require.Nil(t, err)
-
-	book := NewAddrBook(fname, true)
-	book.SetLogger(log.TestingLogger())
-	err = book.AddAddress(peerRealAddr, src)
-	require.Nil(t, err)
-	book.MarkAttempt(peerRealAddr)
-	book.MarkGood(peerRealAddr.ID)
-
-	// Double check that adding a peer again doesn't error
-	err = book.AddAddress(peerRealAddr, src)
-	require.Nil(t, err)
-
-	// Try changing ip but keeping the same node id. (change 1.1.1.1 to 2.2.2.2)
-	// This should just be ignored, and not error.
-	for i := 0; i < numOverrideAttempts; i++ {
-		err = book.AddAddress(peerOverrideAttemptAddr, src)
-		require.Nil(t, err)
-	}
-	// Now check that the IP was not overridden.
-	// This is done by sampling several peers from addr book
-	// and ensuring they all have the correct IP.
-	// In the expected functionality, this test should only have 1 Peer, hence will pass.
-	for i := 0; i < numOverrideAttempts; i++ {
-		selection := book.GetSelection()
-		for _, addr := range selection {
-			require.Equal(t, addr.IP, peerRealAddr.IP)
-		}
-	}
-}
-
-func TestAddrBookGroupKey(t *testing.T) {
-	// non-strict routability
-	testCases := []struct {
-		name   string
-		ip     string
-		expKey string
-	}{
-		// IPv4 normal.
-		{"ipv4 normal class a", "12.1.2.3", "12.1.0.0"},
-		{"ipv4 normal class b", "173.1.2.3", "173.1.0.0"},
-		{"ipv4 normal class c", "196.1.2.3", "196.1.0.0"},
-
-		// IPv6/IPv4 translations.
-		{"ipv6 rfc3964 with ipv4 encap", "2002:0c01:0203::", "12.1.0.0"},
-		{"ipv6 rfc4380 toredo ipv4", "2001:0:1234::f3fe:fdfc", "12.1.0.0"},
-		{"ipv6 rfc6052 well-known prefix with ipv4", "64:ff9b::0c01:0203", "12.1.0.0"},
-		{"ipv6 rfc6145 translated ipv4", "::ffff:0:0c01:0203", "12.1.0.0"},
-
-		// Tor.
-		{"ipv6 tor onioncat", "fd87:d87e:eb43:1234::5678", "tor:2"},
-		{"ipv6 tor onioncat 2", "fd87:d87e:eb43:1245::6789", "tor:2"},
-		{"ipv6 tor onioncat 3", "fd87:d87e:eb43:1345::6789", "tor:3"},
-
-		// IPv6 normal.
-		{"ipv6 normal", "2602:100::1", "2602:100::"},
-		{"ipv6 normal 2", "2602:0100::1234", "2602:100::"},
-		{"ipv6 hurricane electric", "2001:470:1f10:a1::2", "2001:470:1000::"},
-		{"ipv6 hurricane electric 2", "2001:0470:1f10:a1::2", "2001:470:1000::"},
-	}
-
-	for i, tc := range testCases {
-		nip := net.ParseIP(tc.ip)
-		key := groupKeyFor(types.NewNetAddressIPPort(nip, 26656), false)
-		assert.Equal(t, tc.expKey, key, "#%d", i)
-	}
-
-	// strict routability
-	testCases = []struct {
-		name   string
-		ip     string
-		expKey string
-	}{
-		// Local addresses.
-		{"ipv4 localhost", "127.0.0.1", "local"},
-		{"ipv6 localhost", "::1", "local"},
-		{"ipv4 zero", "0.0.0.0", "local"},
-		{"ipv4 first octet zero", "0.1.2.3", "local"},
-
-		// Unroutable addresses.
-		{"ipv4 invalid bcast", "255.255.255.255", "unroutable"},
-		{"ipv4 rfc1918 10/8", "10.1.2.3", "unroutable"},
-		{"ipv4 rfc1918 172.16/12", "172.16.1.2", "unroutable"},
-		{"ipv4 rfc1918 192.168/16", "192.168.1.2", "unroutable"},
-		{"ipv6 rfc3849 2001:db8::/32", "2001:db8::1234", "unroutable"},
-		{"ipv4 rfc3927 169.254/16", "169.254.1.2", "unroutable"},
-		{"ipv6 rfc4193 fc00::/7", "fc00::1234", "unroutable"},
-		{"ipv6 rfc4843 2001:10::/28", "2001:10::1234", "unroutable"},
-		{"ipv6 rfc4862 fe80::/64", "fe80::1234", "unroutable"},
-	}
-
-	for i, tc := range testCases {
-		nip := net.ParseIP(tc.ip)
-		key := groupKeyFor(types.NewNetAddressIPPort(nip, 26656), true)
-		assert.Equal(t, tc.expKey, key, "#%d", i)
-	}
-}
-
-func assertMOldAndNNewAddrsInSelection(t *testing.T, m, n int, addrs []*p2p.NetAddress, book *addrBook) {
-	nOld, nNew := countOldAndNewAddrsInSelection(addrs, book)
-	assert.Equal(t, m, nOld, "old addresses")
-	assert.Equal(t, n, nNew, "new addresses")
-}
-
-func createTempFileName(t *testing.T, prefix string) string {
-	t.Helper()
-	f, err := ioutil.TempFile("", prefix)
-	if err != nil {
-		panic(err)
-	}
-
-	fname := f.Name()
-	if err := f.Close(); err != nil {
-		t.Fatal(err)
-	}
-
-	t.Cleanup(func() { _ = os.Remove(fname) })
-
-	return fname
-}
-
-func createAddrBookWithMOldAndNNewAddrs(t *testing.T, nOld, nNew int) (book *addrBook, fname string) {
-	t.Helper()
-	fname = createTempFileName(t, "addrbook_test")
-
-	book = NewAddrBook(fname, true).(*addrBook)
-	book.SetLogger(log.TestingLogger())
-	assert.Zero(t, book.Size())
-
-	randAddrs := randNetAddressPairs(t, nOld)
-	for _, addr := range randAddrs {
-		err := book.AddAddress(addr.addr, addr.src)
-		require.NoError(t, err)
-		book.MarkGood(addr.addr.ID)
-	}
-
-	randAddrs = randNetAddressPairs(t, nNew)
-	for _, addr := range randAddrs {
-		err := book.AddAddress(addr.addr, addr.src)
-		require.NoError(t, err)
-	}
-
-	return
-}
-
-func countOldAndNewAddrsInSelection(addrs []*p2p.NetAddress, book *addrBook) (nOld, nNew int) {
-	for _, addr := range addrs {
-		if book.IsGood(addr) {
-			nOld++
-		} else {
-			nNew++
-		}
-	}
-	return
-}
-
-// Analyze the layout of the selection specified by 'addrs'
-// Returns:
-// - seqLens - the lengths of the sequences of addresses of same type
-// - seqTypes - the types of sequences in selection
-func analyseSelectionLayout(book *addrBook, addrs []*p2p.NetAddress) (seqLens, seqTypes []int, err error) {
-	// address types are: 0 - nil, 1 - new, 2 - old
-	var (
-		prevType      = 0
-		currentSeqLen = 0
-	)
-
-	for _, addr := range addrs {
-		addrType := 0
-		if book.IsGood(addr) {
-			addrType = 2
-		} else {
-			addrType = 1
-		}
-		if addrType != prevType && prevType != 0 {
-			seqLens = append(seqLens, currentSeqLen)
-			seqTypes = append(seqTypes, prevType)
-			currentSeqLen = 0
-		}
-		currentSeqLen++
-		prevType = addrType
-	}
-
-	seqLens = append(seqLens, currentSeqLen)
-	seqTypes = append(seqTypes, prevType)
-
-	return
-}

internal/p2p/pex/bench_test.go

@@ -1,24 +0,0 @@
-package pex
-
-import (
-	"testing"
-
-	"github.com/tendermint/tendermint/types"
-)
-
-func BenchmarkAddrBook_hash(b *testing.B) {
-	book := &addrBook{
-		ourAddrs:          make(map[string]struct{}),
-		privateIDs:        make(map[types.NodeID]struct{}),
-		addrLookup:        make(map[types.NodeID]*knownAddress),
-		badPeers:          make(map[types.NodeID]*knownAddress),
-		filePath:          "",
-		routabilityStrict: true,
-	}
-	book.init()
-	msg := []byte(`foobar`)
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, _ = book.hash(msg)
-	}
-}

internal/p2p/pex/doc.go

@@ -7,19 +7,14 @@ The PEX reactor is a continuous service which periodically requests addresses
 and serves addresses to other peers. There are two versions of this service
 aligning with the two p2p frameworks that Tendermint currently supports.
 
-V1 is coupled with the Switch (which handles peer connections and routing of
-messages) and, alongside exchanging peer information in the form of port/IP
-pairs, also has the responsibility of dialing peers and ensuring that a
-node has a sufficient amount of peers connected.
-
-V2 is embedded with the new p2p stack and uses the peer manager to advertise
+The reactor is embedded with the new p2p stack and uses the peer manager to advertise
 peers as well as add new peers to the peer store. The V2 reactor passes a
 different set of proto messages which include a list of
 [urls](https://golang.org/pkg/net/url/#URL).These can be used to save a set of
 endpoints that each peer uses. The V2 reactor has backwards compatibility with
 V1. It can also handle V1 messages.
 
-The V2 reactor is able to tweak the intensity of it's search by decreasing or
+The reactor is able to tweak the intensity of it's search by decreasing or
 increasing the interval between each request. It tracks connected peers via a
 linked list, sending a request to the node at the front of the list and adding
 it to the back of the list once a response is received. Using this method, a

internal/p2p/pex/errors.go

@@ -1,89 +0,0 @@
-package pex
-
-import (
-	"errors"
-	"fmt"
-
-	"github.com/tendermint/tendermint/internal/p2p"
-)
-
-type ErrAddrBookNonRoutable struct {
-	Addr *p2p.NetAddress
-}
-
-func (err ErrAddrBookNonRoutable) Error() string {
-	return fmt.Sprintf("Cannot add non-routable address %v", err.Addr)
-}
-
-type errAddrBookOldAddressNewBucket struct {
-	Addr     *p2p.NetAddress
-	BucketID int
-}
-
-func (err errAddrBookOldAddressNewBucket) Error() string {
-	return fmt.Sprintf("failed consistency check!"+
-		" Cannot add pre-existing address %v into new bucket %v",
-		err.Addr, err.BucketID)
-}
-
-type ErrAddrBookSelf struct {
-	Addr *p2p.NetAddress
-}
-
-func (err ErrAddrBookSelf) Error() string {
-	return fmt.Sprintf("Cannot add ourselves with address %v", err.Addr)
-}
-
-type ErrAddrBookPrivate struct {
-	Addr *p2p.NetAddress
-}
-
-func (err ErrAddrBookPrivate) Error() string {
-	return fmt.Sprintf("Cannot add private peer with address %v", err.Addr)
-}
-
-func (err ErrAddrBookPrivate) PrivateAddr() bool {
-	return true
-}
-
-type ErrAddrBookPrivateSrc struct {
-	Src *p2p.NetAddress
-}
-
-func (err ErrAddrBookPrivateSrc) Error() string {
-	return fmt.Sprintf("Cannot add peer coming from private peer with address %v", err.Src)
-}
-
-func (err ErrAddrBookPrivateSrc) PrivateAddr() bool {
-	return true
-}
-
-type ErrAddrBookNilAddr struct {
-	Addr *p2p.NetAddress
-	Src  *p2p.NetAddress
-}
-
-func (err ErrAddrBookNilAddr) Error() string {
-	return fmt.Sprintf("Cannot add a nil address. Got (addr, src) = (%v, %v)", err.Addr, err.Src)
-}
-
-type ErrAddrBookInvalidAddr struct {
-	Addr    *p2p.NetAddress
-	AddrErr error
-}
-
-func (err ErrAddrBookInvalidAddr) Error() string {
-	return fmt.Sprintf("Cannot add invalid address %v: %v", err.Addr, err.AddrErr)
-}
-
-// ErrAddressBanned is thrown when the address has been banned and therefore cannot be used
-type ErrAddressBanned struct {
-	Addr *p2p.NetAddress
-}
-
-func (err ErrAddressBanned) Error() string {
-	return fmt.Sprintf("Address: %v is currently banned", err.Addr)
-}
-
-// ErrUnsolicitedList is thrown when a peer provides a list of addresses that have not been asked for.
-var ErrUnsolicitedList = errors.New("unsolicited pexAddrsMessage")

internal/p2p/pex/file.go

@@ -1,83 +0,0 @@
-package pex
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/tendermint/tendermint/internal/libs/tempfile"
-)
-
-/* Loading & Saving */
-
-type addrBookJSON struct {
-	Key   string          `json:"key"`
-	Addrs []*knownAddress `json:"addrs"`
-}
-
-func (a *addrBook) saveToFile(filePath string) {
-	a.mtx.Lock()
-	defer a.mtx.Unlock()
-
-	a.Logger.Info("Saving AddrBook to file", "size", a.size())
-
-	addrs := make([]*knownAddress, 0, len(a.addrLookup))
-	for _, ka := range a.addrLookup {
-		addrs = append(addrs, ka)
-	}
-	aJSON := &addrBookJSON{
-		Key:   a.key,
-		Addrs: addrs,
-	}
-
-	jsonBytes, err := json.MarshalIndent(aJSON, "", "\t")
-	if err != nil {
-		a.Logger.Error("Failed to save AddrBook to file", "err", err)
-		return
-	}
-	err = tempfile.WriteFileAtomic(filePath, jsonBytes, 0644)
-	if err != nil {
-		a.Logger.Error("Failed to save AddrBook to file", "file", filePath, "err", err)
-	}
-}
-
-// Returns false if file does not exist.
-// cmn.Panics if file is corrupt.
-func (a *addrBook) loadFromFile(filePath string) bool {
-	// If doesn't exist, do nothing.
-	_, err := os.Stat(filePath)
-	if os.IsNotExist(err) {
-		return false
-	}
-
-	// Load addrBookJSON{}
-	r, err := os.Open(filePath)
-	if err != nil {
-		panic(fmt.Sprintf("Error opening file %s: %v", filePath, err))
-	}
-	defer r.Close()
-	aJSON := &addrBookJSON{}
-	dec := json.NewDecoder(r)
-	err = dec.Decode(aJSON)
-	if err != nil {
-		panic(fmt.Sprintf("Error reading file %s: %v", filePath, err))
-	}
-
-	// Restore all the fields...
-	// Restore the key
-	a.key = aJSON.Key
-	// Restore .bucketsNew & .bucketsOld
-	for _, ka := range aJSON.Addrs {
-		for _, bucketIndex := range ka.Buckets {
-			bucket := a.getBucket(ka.BucketType, bucketIndex)
-			bucket[ka.Addr.String()] = ka
-		}
-		a.addrLookup[ka.ID()] = ka
-		if ka.BucketType == bucketTypeNew {
-			a.nNew++
-		} else {
-			a.nOld++
-		}
-	}
-	return true
-}

internal/p2p/pex/known_address.go

@@ -1,141 +0,0 @@
-package pex
-
-import (
-	"time"
-
-	"github.com/tendermint/tendermint/internal/p2p"
-	"github.com/tendermint/tendermint/types"
-)
-
-// knownAddress tracks information about a known network address
-// that is used to determine how viable an address is.
-type knownAddress struct {
-	Addr        *p2p.NetAddress `json:"addr"`
-	Src         *p2p.NetAddress `json:"src"`
-	Buckets     []int           `json:"buckets"`
-	Attempts    int32           `json:"attempts"`
-	BucketType  byte            `json:"bucket_type"`
-	LastAttempt time.Time       `json:"last_attempt"`
-	LastSuccess time.Time       `json:"last_success"`
-	LastBanTime time.Time       `json:"last_ban_time"`
-}
-
-func newKnownAddress(addr *p2p.NetAddress, src *p2p.NetAddress) *knownAddress {
-	return &knownAddress{
-		Addr:        addr,
-		Src:         src,
-		Attempts:    0,
-		LastAttempt: time.Now(),
-		BucketType:  bucketTypeNew,
-		Buckets:     nil,
-	}
-}
-
-func (ka *knownAddress) ID() types.NodeID {
-	return ka.Addr.ID
-}
-
-func (ka *knownAddress) isOld() bool {
-	return ka.BucketType == bucketTypeOld
-}
-
-func (ka *knownAddress) isNew() bool {
-	return ka.BucketType == bucketTypeNew
-}
-
-func (ka *knownAddress) markAttempt() {
-	now := time.Now()
-	ka.LastAttempt = now
-	ka.Attempts++
-}
-
-func (ka *knownAddress) markGood() {
-	now := time.Now()
-	ka.LastAttempt = now
-	ka.Attempts = 0
-	ka.LastSuccess = now
-}
-
-func (ka *knownAddress) ban(banTime time.Duration) {
-	if ka.LastBanTime.Before(time.Now().Add(banTime)) {
-		ka.LastBanTime = time.Now().Add(banTime)
-	}
-}
-
-func (ka *knownAddress) isBanned() bool {
-	return ka.LastBanTime.After(time.Now())
-}
-
-func (ka *knownAddress) addBucketRef(bucketIdx int) int {
-	for _, bucket := range ka.Buckets {
-		if bucket == bucketIdx {
-			// TODO refactor to return error?
-			// log.Warn(Fmt("Bucket already exists in ka.Buckets: %v", ka))
-			return -1
-		}
-	}
-	ka.Buckets = append(ka.Buckets, bucketIdx)
-	return len(ka.Buckets)
-}
-
-func (ka *knownAddress) removeBucketRef(bucketIdx int) int {
-	buckets := []int{}
-	for _, bucket := range ka.Buckets {
-		if bucket != bucketIdx {
-			buckets = append(buckets, bucket)
-		}
-	}
-	if len(buckets) != len(ka.Buckets)-1 {
-		// TODO refactor to return error?
-		// log.Warn(Fmt("bucketIdx not found in ka.Buckets: %v", ka))
-		return -1
-	}
-	ka.Buckets = buckets
-	return len(ka.Buckets)
-}
-
-/*
-   An address is bad if the address in question is a New address, has not been tried in the last
-   minute, and meets one of the following criteria:
-
-   1) It claims to be from the future
-   2) It hasn't been seen in over a week
-   3) It has failed at least three times and never succeeded
-   4) It has failed ten times in the last week
-
-   All addresses that meet these criteria are assumed to be worthless and not
-   worth keeping hold of.
-
-*/
-func (ka *knownAddress) isBad() bool {
-	// Is Old --> good
-	if ka.BucketType == bucketTypeOld {
-		return false
-	}
-
-	// Has been attempted in the last minute --> good
-	if ka.LastAttempt.After(time.Now().Add(-1 * time.Minute)) {
-		return false
-	}
-
-	// TODO: From the future?
-
-	// Too old?
-	// TODO: should be a timestamp of last seen, not just last attempt
-	if ka.LastAttempt.Before(time.Now().Add(-1 * numMissingDays * time.Hour * 24)) {
-		return true
-	}
-
-	// Never succeeded?
-	if ka.LastSuccess.IsZero() && ka.Attempts >= numRetries {
-		return true
-	}
-
-	// Hasn't succeeded in too long?
-	if ka.LastSuccess.Before(time.Now().Add(-1*minBadDays*time.Hour*24)) &&
-		ka.Attempts >= maxFailures {
-		return true
-	}
-
-	return false
-}