initial movement

internal/blocksync/msgs.go

@@ -0,0 +1,110 @@
+package blockchain
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// NOTE: keep up to date with bcproto.BlockResponse
+	BlockResponseMessagePrefixSize   = 4
+	BlockResponseMessageFieldKeySize = 1
+	MaxMsgSize                       = types.MaxBlockSizeBytes +
+		BlockResponseMessagePrefixSize +
+		BlockResponseMessageFieldKeySize
+)
+
+// EncodeMsg encodes a Protobuf message
+func EncodeMsg(pb proto.Message) ([]byte, error) {
+	msg := bcproto.Message{}
+
+	switch pb := pb.(type) {
+	case *bcproto.BlockRequest:
+		msg.Sum = &bcproto.Message_BlockRequest{BlockRequest: pb}
+	case *bcproto.BlockResponse:
+		msg.Sum = &bcproto.Message_BlockResponse{BlockResponse: pb}
+	case *bcproto.NoBlockResponse:
+		msg.Sum = &bcproto.Message_NoBlockResponse{NoBlockResponse: pb}
+	case *bcproto.StatusRequest:
+		msg.Sum = &bcproto.Message_StatusRequest{StatusRequest: pb}
+	case *bcproto.StatusResponse:
+		msg.Sum = &bcproto.Message_StatusResponse{StatusResponse: pb}
+	default:
+		return nil, fmt.Errorf("unknown message type %T", pb)
+	}
+
+	bz, err := proto.Marshal(&msg)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal %T: %w", pb, err)
+	}
+
+	return bz, nil
+}
+
+// DecodeMsg decodes a Protobuf message.
+func DecodeMsg(bz []byte) (proto.Message, error) {
+	pb := &bcproto.Message{}
+
+	err := proto.Unmarshal(bz, pb)
+	if err != nil {
+		return nil, err
+	}
+
+	switch msg := pb.Sum.(type) {
+	case *bcproto.Message_BlockRequest:
+		return msg.BlockRequest, nil
+	case *bcproto.Message_BlockResponse:
+		return msg.BlockResponse, nil
+	case *bcproto.Message_NoBlockResponse:
+		return msg.NoBlockResponse, nil
+	case *bcproto.Message_StatusRequest:
+		return msg.StatusRequest, nil
+	case *bcproto.Message_StatusResponse:
+		return msg.StatusResponse, nil
+	default:
+		return nil, fmt.Errorf("unknown message type %T", msg)
+	}
+}
+
+// ValidateMsg validates a message.
+func ValidateMsg(pb proto.Message) error {
+	if pb == nil {
+		return errors.New("message cannot be nil")
+	}
+
+	switch msg := pb.(type) {
+	case *bcproto.BlockRequest:
+		if msg.Height < 0 {
+			return errors.New("negative Height")
+		}
+	case *bcproto.BlockResponse:
+		_, err := types.BlockFromProto(msg.Block)
+		if err != nil {
+			return err
+		}
+	case *bcproto.NoBlockResponse:
+		if msg.Height < 0 {
+			return errors.New("negative Height")
+		}
+	case *bcproto.StatusResponse:
+		if msg.Base < 0 {
+			return errors.New("negative Base")
+		}
+		if msg.Height < 0 {
+			return errors.New("negative Height")
+		}
+		if msg.Base > msg.Height {
+			return fmt.Errorf("base %v cannot be greater than height %v", msg.Base, msg.Height)
+		}
+	case *bcproto.StatusRequest:
+		return nil
+	default:
+		return fmt.Errorf("unknown message type %T", msg)
+	}
+	return nil
+}

internal/blocksync/msgs_test.go

@@ -0,0 +1,125 @@
+package blockchain
+
+import (
+	"encoding/hex"
+	"math"
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestBcBlockRequestMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		testName      string
+		requestHeight int64
+		expectErr     bool
+	}{
+		{"Valid Request Message", 0, false},
+		{"Valid Request Message", 1, false},
+		{"Invalid Request Message", -1, true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			request := bcproto.BlockRequest{Height: tc.requestHeight}
+			assert.Equal(t, tc.expectErr, ValidateMsg(&request) != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+func TestBcNoBlockResponseMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		testName          string
+		nonResponseHeight int64
+		expectErr         bool
+	}{
+		{"Valid Non-Response Message", 0, false},
+		{"Valid Non-Response Message", 1, false},
+		{"Invalid Non-Response Message", -1, true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			nonResponse := bcproto.NoBlockResponse{Height: tc.nonResponseHeight}
+			assert.Equal(t, tc.expectErr, ValidateMsg(&nonResponse) != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+func TestBcStatusRequestMessageValidateBasic(t *testing.T) {
+	request := bcproto.StatusRequest{}
+	assert.NoError(t, ValidateMsg(&request))
+}
+
+func TestBcStatusResponseMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		testName       string
+		responseHeight int64
+		expectErr      bool
+	}{
+		{"Valid Response Message", 0, false},
+		{"Valid Response Message", 1, false},
+		{"Invalid Response Message", -1, true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			response := bcproto.StatusResponse{Height: tc.responseHeight}
+			assert.Equal(t, tc.expectErr, ValidateMsg(&response) != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+// nolint:lll // ignore line length in tests
+func TestBlockchainMessageVectors(t *testing.T) {
+	block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil)
+	block.Version.Block = 11 // overwrite updated protocol version
+
+	bpb, err := block.ToProto()
+	require.NoError(t, err)
+
+	testCases := []struct {
+		testName string
+		bmsg     proto.Message
+		expBytes string
+	}{
+		{"BlockRequestMessage", &bcproto.Message{Sum: &bcproto.Message_BlockRequest{
+			BlockRequest: &bcproto.BlockRequest{Height: 1}}}, "0a020801"},
+		{"BlockRequestMessage", &bcproto.Message{Sum: &bcproto.Message_BlockRequest{
+			BlockRequest: &bcproto.BlockRequest{Height: math.MaxInt64}}},
+			"0a0a08ffffffffffffffff7f"},
+		{"BlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_BlockResponse{
+			BlockResponse: &bcproto.BlockResponse{Block: bpb}}}, "1a700a6e0a5b0a02080b1803220b088092b8c398feffffff012a0212003a20c4da88e876062aa1543400d50d0eaa0dac88096057949cfb7bca7f3a48c04bf96a20e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855120d0a0b48656c6c6f20576f726c641a00"},
+		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
+			NoBlockResponse: &bcproto.NoBlockResponse{Height: 1}}}, "12020801"},
+		{"NoBlockResponseMessage", &bcproto.Message{Sum: &bcproto.Message_NoBlockResponse{
+			NoBlockResponse: &bcproto.NoBlockResponse{Height: math.MaxInt64}}},
+			"120a08ffffffffffffffff7f"},
+		{"StatusRequestMessage", &bcproto.Message{Sum: &bcproto.Message_StatusRequest{
+			StatusRequest: &bcproto.StatusRequest{}}},
+			"2200"},
+		{"StatusResponseMessage", &bcproto.Message{Sum: &bcproto.Message_StatusResponse{
+			StatusResponse: &bcproto.StatusResponse{Height: 1, Base: 2}}},
+			"2a0408011002"},
+		{"StatusResponseMessage", &bcproto.Message{Sum: &bcproto.Message_StatusResponse{
+			StatusResponse: &bcproto.StatusResponse{Height: math.MaxInt64, Base: math.MaxInt64}}},
+			"2a1408ffffffffffffffff7f10ffffffffffffffff7f"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			bz, _ := proto.Marshal(tc.bmsg)
+
+			require.Equal(t, tc.expBytes, hex.EncodeToString(bz))
+		})
+	}
+}

internal/blocksync/v0/pool.go

@@ -0,0 +1,647 @@
+package v0
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"sync/atomic"
+	"time"
+
+	flow "github.com/tendermint/tendermint/libs/flowrate"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/service"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+/*
+eg, L = latency = 0.1s
+	P = num peers = 10
+	FN = num full nodes
+	BS = 1kB block size
+	CB = 1 Mbit/s = 128 kB/s
+	CB/P = 12.8 kB
+	B/S = CB/P/BS = 12.8 blocks/s
+
+	12.8 * 0.1 = 1.28 blocks on conn
+*/
+
+const (
+	requestIntervalMS         = 2
+	maxTotalRequesters        = 600
+	maxPendingRequests        = maxTotalRequesters
+	maxPendingRequestsPerPeer = 20
+
+	// Minimum recv rate to ensure we're receiving blocks from a peer fast
+	// enough. If a peer is not sending us data at at least that rate, we
+	// consider them to have timedout and we disconnect.
+	//
+	// Assuming a DSL connection (not a good choice) 128 Kbps (upload) ~ 15 KB/s,
+	// sending data across atlantic ~ 7.5 KB/s.
+	minRecvRate = 7680
+
+	// Maximum difference between current and new block's height.
+	maxDiffBetweenCurrentAndReceivedBlockHeight = 100
+)
+
+var peerTimeout = 15 * time.Second // not const so we can override with tests
+
+/*
+	Peers self report their heights when we join the block pool.
+	Starting from our latest pool.height, we request blocks
+	in sequence from peers that reported higher heights than ours.
+	Every so often we ask peers what height they're on so we can keep going.
+
+	Requests are continuously made for blocks of higher heights until
+	the limit is reached. If most of the requests have no available peers, and we
+	are not at peer limits, we can probably switch to consensus reactor
+*/
+
+// BlockPool keeps track of the fast sync peers, block requests and block responses.
+type BlockPool struct {
+	service.BaseService
+	startTime time.Time
+
+	mtx tmsync.Mutex
+	// block requests
+	requesters map[int64]*bpRequester
+	height     int64 // the lowest key in requesters.
+	// peers
+	peers         map[p2p.ID]*bpPeer
+	maxPeerHeight int64 // the biggest reported height
+
+	// atomic
+	numPending int32 // number of requests pending assignment or block response
+
+	requestsCh chan<- BlockRequest
+	errorsCh   chan<- peerError
+}
+
+// NewBlockPool returns a new BlockPool with the height equal to start. Block
+// requests and errors will be sent to requestsCh and errorsCh accordingly.
+func NewBlockPool(start int64, requestsCh chan<- BlockRequest, errorsCh chan<- peerError) *BlockPool {
+	bp := &BlockPool{
+		peers: make(map[p2p.ID]*bpPeer),
+
+		requesters: make(map[int64]*bpRequester),
+		height:     start,
+		numPending: 0,
+
+		requestsCh: requestsCh,
+		errorsCh:   errorsCh,
+	}
+	bp.BaseService = *service.NewBaseService(nil, "BlockPool", bp)
+	return bp
+}
+
+// OnStart implements service.Service by spawning requesters routine and recording
+// pool's start time.
+func (pool *BlockPool) OnStart() error {
+	go pool.makeRequestersRoutine()
+	pool.startTime = time.Now()
+	return nil
+}
+
+// spawns requesters as needed
+func (pool *BlockPool) makeRequestersRoutine() {
+	for {
+		if !pool.IsRunning() {
+			break
+		}
+
+		_, numPending, lenRequesters := pool.GetStatus()
+		switch {
+		case numPending >= maxPendingRequests:
+			// sleep for a bit.
+			time.Sleep(requestIntervalMS * time.Millisecond)
+			// check for timed out peers
+			pool.removeTimedoutPeers()
+		case lenRequesters >= maxTotalRequesters:
+			// sleep for a bit.
+			time.Sleep(requestIntervalMS * time.Millisecond)
+			// check for timed out peers
+			pool.removeTimedoutPeers()
+		default:
+			// request for more blocks.
+			pool.makeNextRequester()
+		}
+	}
+}
+
+func (pool *BlockPool) removeTimedoutPeers() {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	for _, peer := range pool.peers {
+		if !peer.didTimeout && peer.numPending > 0 {
+			curRate := peer.recvMonitor.Status().CurRate
+			// curRate can be 0 on start
+			if curRate != 0 && curRate < minRecvRate {
+				err := errors.New("peer is not sending us data fast enough")
+				pool.sendError(err, peer.id)
+				pool.Logger.Error("SendTimeout", "peer", peer.id,
+					"reason", err,
+					"curRate", fmt.Sprintf("%d KB/s", curRate/1024),
+					"minRate", fmt.Sprintf("%d KB/s", minRecvRate/1024))
+				peer.didTimeout = true
+			}
+		}
+		if peer.didTimeout {
+			pool.removePeer(peer.id)
+		}
+	}
+}
+
+// GetStatus returns pool's height, numPending requests and the number of
+// requesters.
+func (pool *BlockPool) GetStatus() (height int64, numPending int32, lenRequesters int) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	return pool.height, atomic.LoadInt32(&pool.numPending), len(pool.requesters)
+}
+
+// IsCaughtUp returns true if this node is caught up, false - otherwise.
+// TODO: relax conditions, prevent abuse.
+func (pool *BlockPool) IsCaughtUp() bool {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	// Need at least 1 peer to be considered caught up.
+	if len(pool.peers) == 0 {
+		pool.Logger.Debug("Blockpool has no peers")
+		return false
+	}
+
+	// Some conditions to determine if we're caught up.
+	// Ensures we've either received a block or waited some amount of time,
+	// and that we're synced to the highest known height.
+	// Note we use maxPeerHeight - 1 because to sync block H requires block H+1
+	// to verify the LastCommit.
+	receivedBlockOrTimedOut := pool.height > 0 || time.Since(pool.startTime) > 5*time.Second
+	ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= (pool.maxPeerHeight-1)
+	isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers
+	return isCaughtUp
+}
+
+// PeekTwoBlocks returns blocks at pool.height and pool.height+1.
+// We need to see the second block's Commit to validate the first block.
+// So we peek two blocks at a time.
+// The caller will verify the commit.
+func (pool *BlockPool) PeekTwoBlocks() (first *types.Block, second *types.Block) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	if r := pool.requesters[pool.height]; r != nil {
+		first = r.getBlock()
+	}
+	if r := pool.requesters[pool.height+1]; r != nil {
+		second = r.getBlock()
+	}
+	return
+}
+
+// PopRequest pops the first block at pool.height.
+// It must have been validated by 'second'.Commit from PeekTwoBlocks().
+func (pool *BlockPool) PopRequest() {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	if r := pool.requesters[pool.height]; r != nil {
+		/*  The block can disappear at any time, due to removePeer().
+		if r := pool.requesters[pool.height]; r == nil || r.block == nil {
+			PanicSanity("PopRequest() requires a valid block")
+		}
+		*/
+		if err := r.Stop(); err != nil {
+			pool.Logger.Error("Error stopping requester", "err", err)
+		}
+		delete(pool.requesters, pool.height)
+		pool.height++
+	} else {
+		panic(fmt.Sprintf("Expected requester to pop, got nothing at height %v", pool.height))
+	}
+}
+
+// RedoRequest invalidates the block at pool.height,
+// Remove the peer and redo request from others.
+// Returns the ID of the removed peer.
+func (pool *BlockPool) RedoRequest(height int64) p2p.ID {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	request := pool.requesters[height]
+	peerID := request.getPeerID()
+	if peerID != p2p.ID("") {
+		// RemovePeer will redo all requesters associated with this peer.
+		pool.removePeer(peerID)
+	}
+	return peerID
+}
+
+// AddBlock validates that the block comes from the peer it was expected from and calls the requester to store it.
+// TODO: ensure that blocks come in order for each peer.
+func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	requester := pool.requesters[block.Height]
+	if requester == nil {
+		pool.Logger.Info(
+			"peer sent us a block we didn't expect",
+			"peer",
+			peerID,
+			"curHeight",
+			pool.height,
+			"blockHeight",
+			block.Height)
+		diff := pool.height - block.Height
+		if diff < 0 {
+			diff *= -1
+		}
+		if diff > maxDiffBetweenCurrentAndReceivedBlockHeight {
+			pool.sendError(errors.New("peer sent us a block we didn't expect with a height too far ahead/behind"), peerID)
+		}
+		return
+	}
+
+	if requester.setBlock(block, peerID) {
+		atomic.AddInt32(&pool.numPending, -1)
+		peer := pool.peers[peerID]
+		if peer != nil {
+			peer.decrPending(blockSize)
+		}
+	} else {
+		pool.Logger.Info("invalid peer", "peer", peerID, "blockHeight", block.Height)
+		pool.sendError(errors.New("invalid peer"), peerID)
+	}
+}
+
+// MaxPeerHeight returns the highest reported height.
+func (pool *BlockPool) MaxPeerHeight() int64 {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+	return pool.maxPeerHeight
+}
+
+// SetPeerRange sets the peer's alleged blockchain base and height.
+func (pool *BlockPool) SetPeerRange(peerID p2p.ID, base int64, height int64) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	peer := pool.peers[peerID]
+	if peer != nil {
+		peer.base = base
+		peer.height = height
+	} else {
+		peer = newBPPeer(pool, peerID, base, height)
+		peer.setLogger(pool.Logger.With("peer", peerID))
+		pool.peers[peerID] = peer
+	}
+
+	if height > pool.maxPeerHeight {
+		pool.maxPeerHeight = height
+	}
+}
+
+// RemovePeer removes the peer with peerID from the pool. If there's no peer
+// with peerID, function is a no-op.
+func (pool *BlockPool) RemovePeer(peerID p2p.ID) {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	pool.removePeer(peerID)
+}
+
+func (pool *BlockPool) removePeer(peerID p2p.ID) {
+	for _, requester := range pool.requesters {
+		if requester.getPeerID() == peerID {
+			requester.redo(peerID)
+		}
+	}
+
+	peer, ok := pool.peers[peerID]
+	if ok {
+		if peer.timeout != nil {
+			peer.timeout.Stop()
+		}
+
+		delete(pool.peers, peerID)
+
+		// Find a new peer with the biggest height and update maxPeerHeight if the
+		// peer's height was the biggest.
+		if peer.height == pool.maxPeerHeight {
+			pool.updateMaxPeerHeight()
+		}
+	}
+}
+
+// If no peers are left, maxPeerHeight is set to 0.
+func (pool *BlockPool) updateMaxPeerHeight() {
+	var max int64
+	for _, peer := range pool.peers {
+		if peer.height > max {
+			max = peer.height
+		}
+	}
+	pool.maxPeerHeight = max
+}
+
+// Pick an available peer with the given height available.
+// If no peers are available, returns nil.
+func (pool *BlockPool) pickIncrAvailablePeer(height int64) *bpPeer {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	for _, peer := range pool.peers {
+		if peer.didTimeout {
+			pool.removePeer(peer.id)
+			continue
+		}
+		if peer.numPending >= maxPendingRequestsPerPeer {
+			continue
+		}
+		if height < peer.base || height > peer.height {
+			continue
+		}
+		peer.incrPending()
+		return peer
+	}
+	return nil
+}
+
+func (pool *BlockPool) makeNextRequester() {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	nextHeight := pool.height + pool.requestersLen()
+	if nextHeight > pool.maxPeerHeight {
+		return
+	}
+
+	request := newBPRequester(pool, nextHeight)
+
+	pool.requesters[nextHeight] = request
+	atomic.AddInt32(&pool.numPending, 1)
+
+	err := request.Start()
+	if err != nil {
+		request.Logger.Error("Error starting request", "err", err)
+	}
+}
+
+func (pool *BlockPool) requestersLen() int64 {
+	return int64(len(pool.requesters))
+}
+
+func (pool *BlockPool) sendRequest(height int64, peerID p2p.ID) {
+	if !pool.IsRunning() {
+		return
+	}
+	pool.requestsCh <- BlockRequest{height, peerID}
+}
+
+func (pool *BlockPool) sendError(err error, peerID p2p.ID) {
+	if !pool.IsRunning() {
+		return
+	}
+	pool.errorsCh <- peerError{err, peerID}
+}
+
+// for debugging purposes
+//nolint:unused
+func (pool *BlockPool) debug() string {
+	pool.mtx.Lock()
+	defer pool.mtx.Unlock()
+
+	str := ""
+	nextHeight := pool.height + pool.requestersLen()
+	for h := pool.height; h < nextHeight; h++ {
+		if pool.requesters[h] == nil {
+			str += fmt.Sprintf("H(%v):X ", h)
+		} else {
+			str += fmt.Sprintf("H(%v):", h)
+			str += fmt.Sprintf("B?(%v) ", pool.requesters[h].block != nil)
+		}
+	}
+	return str
+}
+
+//-------------------------------------
+
+type bpPeer struct {
+	didTimeout  bool
+	numPending  int32
+	height      int64
+	base        int64
+	pool        *BlockPool
+	id          p2p.ID
+	recvMonitor *flow.Monitor
+
+	timeout *time.Timer
+
+	logger log.Logger
+}
+
+func newBPPeer(pool *BlockPool, peerID p2p.ID, base int64, height int64) *bpPeer {
+	peer := &bpPeer{
+		pool:       pool,
+		id:         peerID,
+		base:       base,
+		height:     height,
+		numPending: 0,
+		logger:     log.NewNopLogger(),
+	}
+	return peer
+}
+
+func (peer *bpPeer) setLogger(l log.Logger) {
+	peer.logger = l
+}
+
+func (peer *bpPeer) resetMonitor() {
+	peer.recvMonitor = flow.New(time.Second, time.Second*40)
+	initialValue := float64(minRecvRate) * math.E
+	peer.recvMonitor.SetREMA(initialValue)
+}
+
+func (peer *bpPeer) resetTimeout() {
+	if peer.timeout == nil {
+		peer.timeout = time.AfterFunc(peerTimeout, peer.onTimeout)
+	} else {
+		peer.timeout.Reset(peerTimeout)
+	}
+}
+
+func (peer *bpPeer) incrPending() {
+	if peer.numPending == 0 {
+		peer.resetMonitor()
+		peer.resetTimeout()
+	}
+	peer.numPending++
+}
+
+func (peer *bpPeer) decrPending(recvSize int) {
+	peer.numPending--
+	if peer.numPending == 0 {
+		peer.timeout.Stop()
+	} else {
+		peer.recvMonitor.Update(recvSize)
+		peer.resetTimeout()
+	}
+}
+
+func (peer *bpPeer) onTimeout() {
+	peer.pool.mtx.Lock()
+	defer peer.pool.mtx.Unlock()
+
+	err := errors.New("peer did not send us anything")
+	peer.pool.sendError(err, peer.id)
+	peer.logger.Error("SendTimeout", "reason", err, "timeout", peerTimeout)
+	peer.didTimeout = true
+}
+
+//-------------------------------------
+
+type bpRequester struct {
+	service.BaseService
+	pool       *BlockPool
+	height     int64
+	gotBlockCh chan struct{}
+	redoCh     chan p2p.ID // redo may send multitime, add peerId to identify repeat
+
+	mtx    tmsync.Mutex
+	peerID p2p.ID
+	block  *types.Block
+}
+
+func newBPRequester(pool *BlockPool, height int64) *bpRequester {
+	bpr := &bpRequester{
+		pool:       pool,
+		height:     height,
+		gotBlockCh: make(chan struct{}, 1),
+		redoCh:     make(chan p2p.ID, 1),
+
+		peerID: "",
+		block:  nil,
+	}
+	bpr.BaseService = *service.NewBaseService(nil, "bpRequester", bpr)
+	return bpr
+}
+
+func (bpr *bpRequester) OnStart() error {
+	go bpr.requestRoutine()
+	return nil
+}
+
+// Returns true if the peer matches and block doesn't already exist.
+func (bpr *bpRequester) setBlock(block *types.Block, peerID p2p.ID) bool {
+	bpr.mtx.Lock()
+	if bpr.block != nil || bpr.peerID != peerID {
+		bpr.mtx.Unlock()
+		return false
+	}
+	bpr.block = block
+	bpr.mtx.Unlock()
+
+	select {
+	case bpr.gotBlockCh <- struct{}{}:
+	default:
+	}
+	return true
+}
+
+func (bpr *bpRequester) getBlock() *types.Block {
+	bpr.mtx.Lock()
+	defer bpr.mtx.Unlock()
+	return bpr.block
+}
+
+func (bpr *bpRequester) getPeerID() p2p.ID {
+	bpr.mtx.Lock()
+	defer bpr.mtx.Unlock()
+	return bpr.peerID
+}
+
+// This is called from the requestRoutine, upon redo().
+func (bpr *bpRequester) reset() {
+	bpr.mtx.Lock()
+	defer bpr.mtx.Unlock()
+
+	if bpr.block != nil {
+		atomic.AddInt32(&bpr.pool.numPending, 1)
+	}
+
+	bpr.peerID = ""
+	bpr.block = nil
+}
+
+// Tells bpRequester to pick another peer and try again.
+// NOTE: Nonblocking, and does nothing if another redo
+// was already requested.
+func (bpr *bpRequester) redo(peerID p2p.ID) {
+	select {
+	case bpr.redoCh <- peerID:
+	default:
+	}
+}
+
+// Responsible for making more requests as necessary
+// Returns only when a block is found (e.g. AddBlock() is called)
+func (bpr *bpRequester) requestRoutine() {
+OUTER_LOOP:
+	for {
+		// Pick a peer to send request to.
+		var peer *bpPeer
+	PICK_PEER_LOOP:
+		for {
+			if !bpr.IsRunning() || !bpr.pool.IsRunning() {
+				return
+			}
+			peer = bpr.pool.pickIncrAvailablePeer(bpr.height)
+			if peer == nil {
+				// log.Info("No peers available", "height", height)
+				time.Sleep(requestIntervalMS * time.Millisecond)
+				continue PICK_PEER_LOOP
+			}
+			break PICK_PEER_LOOP
+		}
+		bpr.mtx.Lock()
+		bpr.peerID = peer.id
+		bpr.mtx.Unlock()
+
+		// Send request and wait.
+		bpr.pool.sendRequest(bpr.height, peer.id)
+	WAIT_LOOP:
+		for {
+			select {
+			case <-bpr.pool.Quit():
+				if err := bpr.Stop(); err != nil {
+					bpr.Logger.Error("Error stopped requester", "err", err)
+				}
+				return
+			case <-bpr.Quit():
+				return
+			case peerID := <-bpr.redoCh:
+				if peerID == bpr.peerID {
+					bpr.reset()
+					continue OUTER_LOOP
+				} else {
+					continue WAIT_LOOP
+				}
+			case <-bpr.gotBlockCh:
+				// We got a block!
+				// Continue the for-loop and wait til Quit.
+				continue WAIT_LOOP
+			}
+		}
+	}
+}
+
+// BlockRequest stores a block request identified by the block Height and the PeerID responsible for
+// delivering the block
+type BlockRequest struct {
+	Height int64
+	PeerID p2p.ID
+}

internal/blocksync/v0/pool_test.go

@@ -0,0 +1,242 @@
+package v0
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+func init() {
+	peerTimeout = 2 * time.Second
+}
+
+type testPeer struct {
+	id        p2p.ID
+	base      int64
+	height    int64
+	inputChan chan inputData // make sure each peer's data is sequential
+}
+
+type inputData struct {
+	t       *testing.T
+	pool    *BlockPool
+	request BlockRequest
+}
+
+func (p testPeer) runInputRoutine() {
+	go func() {
+		for input := range p.inputChan {
+			p.simulateInput(input)
+		}
+	}()
+}
+
+// Request desired, pretend like we got the block immediately.
+func (p testPeer) simulateInput(input inputData) {
+	block := &types.Block{Header: types.Header{Height: input.request.Height}}
+	input.pool.AddBlock(input.request.PeerID, block, 123)
+	// TODO: uncommenting this creates a race which is detected by:
+	// https://github.com/golang/go/blob/2bd767b1022dd3254bcec469f0ee164024726486/src/testing/testing.go#L854-L856
+	// see: https://github.com/tendermint/tendermint/issues/3390#issue-418379890
+	// input.t.Logf("Added block from peer %v (height: %v)", input.request.PeerID, input.request.Height)
+}
+
+type testPeers map[p2p.ID]testPeer
+
+func (ps testPeers) start() {
+	for _, v := range ps {
+		v.runInputRoutine()
+	}
+}
+
+func (ps testPeers) stop() {
+	for _, v := range ps {
+		close(v.inputChan)
+	}
+}
+
+func makePeers(numPeers int, minHeight, maxHeight int64) testPeers {
+	peers := make(testPeers, numPeers)
+	for i := 0; i < numPeers; i++ {
+		peerID := p2p.ID(tmrand.Str(12))
+		height := minHeight + tmrand.Int63n(maxHeight-minHeight)
+		base := minHeight + int64(i)
+		if base > height {
+			base = height
+		}
+		peers[peerID] = testPeer{peerID, base, height, make(chan inputData, 10)}
+	}
+	return peers
+}
+
+func TestBlockPoolBasic(t *testing.T) {
+	start := int64(42)
+	peers := makePeers(10, start+1, 1000)
+	errorsCh := make(chan peerError, 1000)
+	requestsCh := make(chan BlockRequest, 1000)
+	pool := NewBlockPool(start, requestsCh, errorsCh)
+	pool.SetLogger(log.TestingLogger())
+
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
+
+	t.Cleanup(func() {
+		if err := pool.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	peers.start()
+	defer peers.stop()
+
+	// Introduce each peer.
+	go func() {
+		for _, peer := range peers {
+			pool.SetPeerRange(peer.id, peer.base, peer.height)
+		}
+	}()
+
+	// Start a goroutine to pull blocks
+	go func() {
+		for {
+			if !pool.IsRunning() {
+				return
+			}
+			first, second := pool.PeekTwoBlocks()
+			if first != nil && second != nil {
+				pool.PopRequest()
+			} else {
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
+	// Pull from channels
+	for {
+		select {
+		case err := <-errorsCh:
+			t.Error(err)
+		case request := <-requestsCh:
+			t.Logf("Pulled new BlockRequest %v", request)
+			if request.Height == 300 {
+				return // Done!
+			}
+
+			peers[request.PeerID].inputChan <- inputData{t, pool, request}
+		}
+	}
+}
+
+func TestBlockPoolTimeout(t *testing.T) {
+	start := int64(42)
+	peers := makePeers(10, start+1, 1000)
+	errorsCh := make(chan peerError, 1000)
+	requestsCh := make(chan BlockRequest, 1000)
+	pool := NewBlockPool(start, requestsCh, errorsCh)
+	pool.SetLogger(log.TestingLogger())
+	err := pool.Start()
+	if err != nil {
+		t.Error(err)
+	}
+	t.Cleanup(func() {
+		if err := pool.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	for _, peer := range peers {
+		t.Logf("Peer %v", peer.id)
+	}
+
+	// Introduce each peer.
+	go func() {
+		for _, peer := range peers {
+			pool.SetPeerRange(peer.id, peer.base, peer.height)
+		}
+	}()
+
+	// Start a goroutine to pull blocks
+	go func() {
+		for {
+			if !pool.IsRunning() {
+				return
+			}
+			first, second := pool.PeekTwoBlocks()
+			if first != nil && second != nil {
+				pool.PopRequest()
+			} else {
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
+	// Pull from channels
+	counter := 0
+	timedOut := map[p2p.ID]struct{}{}
+	for {
+		select {
+		case err := <-errorsCh:
+			t.Log(err)
+			// consider error to be always timeout here
+			if _, ok := timedOut[err.peerID]; !ok {
+				counter++
+				if counter == len(peers) {
+					return // Done!
+				}
+			}
+		case request := <-requestsCh:
+			t.Logf("Pulled new BlockRequest %+v", request)
+		}
+	}
+}
+
+func TestBlockPoolRemovePeer(t *testing.T) {
+	peers := make(testPeers, 10)
+	for i := 0; i < 10; i++ {
+		peerID := p2p.ID(fmt.Sprintf("%d", i+1))
+		height := int64(i + 1)
+		peers[peerID] = testPeer{peerID, 0, height, make(chan inputData)}
+	}
+	requestsCh := make(chan BlockRequest)
+	errorsCh := make(chan peerError)
+
+	pool := NewBlockPool(1, requestsCh, errorsCh)
+	pool.SetLogger(log.TestingLogger())
+	err := pool.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := pool.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	// add peers
+	for peerID, peer := range peers {
+		pool.SetPeerRange(peerID, peer.base, peer.height)
+	}
+	assert.EqualValues(t, 10, pool.MaxPeerHeight())
+
+	// remove not-existing peer
+	assert.NotPanics(t, func() { pool.RemovePeer(p2p.ID("Superman")) })
+
+	// remove peer with biggest height
+	pool.RemovePeer(p2p.ID("10"))
+	assert.EqualValues(t, 9, pool.MaxPeerHeight())
+
+	// remove all peers
+	for peerID := range peers {
+		pool.RemovePeer(peerID)
+	}
+
+	assert.EqualValues(t, 0, pool.MaxPeerHeight())
+}

internal/blocksync/v0/reactor.go

@@ -0,0 +1,437 @@
+package v0
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	bc "github.com/tendermint/tendermint/blockchain"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
+	BlockchainChannel = byte(0x40)
+
+	trySyncIntervalMS = 10
+
+	// stop syncing when last block's time is
+	// within this much of the system time.
+	// stopSyncingDurationMinutes = 10
+
+	// ask for best height every 10s
+	statusUpdateIntervalSeconds = 10
+	// check if we should switch to consensus reactor
+	switchToConsensusIntervalSeconds = 1
+)
+
+type consensusReactor interface {
+	// for when we switch from blockchain reactor and fast sync to
+	// the consensus machine
+	SwitchToConsensus(state sm.State, skipWAL bool)
+}
+
+type peerError struct {
+	err    error
+	peerID p2p.ID
+}
+
+func (e peerError) Error() string {
+	return fmt.Sprintf("error with peer %v: %s", e.peerID, e.err.Error())
+}
+
+// BlockchainReactor handles long-term catchup syncing.
+type BlockchainReactor struct {
+	p2p.BaseReactor
+
+	// immutable
+	initialState sm.State
+
+	blockExec *sm.BlockExecutor
+	store     *store.BlockStore
+	pool      *BlockPool
+	fastSync  bool
+
+	requestsCh <-chan BlockRequest
+	errorsCh   <-chan peerError
+}
+
+// NewBlockchainReactor returns new reactor instance.
+func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore,
+	fastSync bool) *BlockchainReactor {
+
+	if state.LastBlockHeight != store.Height() {
+		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
+			store.Height()))
+	}
+
+	requestsCh := make(chan BlockRequest, maxTotalRequesters)
+
+	const capacity = 1000                      // must be bigger than peers count
+	errorsCh := make(chan peerError, capacity) // so we don't block in #Receive#pool.AddBlock
+
+	startHeight := store.Height() + 1
+	if startHeight == 1 {
+		startHeight = state.InitialHeight
+	}
+	pool := NewBlockPool(startHeight, requestsCh, errorsCh)
+
+	bcR := &BlockchainReactor{
+		initialState: state,
+		blockExec:    blockExec,
+		store:        store,
+		pool:         pool,
+		fastSync:     fastSync,
+		requestsCh:   requestsCh,
+		errorsCh:     errorsCh,
+	}
+	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
+	return bcR
+}
+
+// SetLogger implements service.Service by setting the logger on reactor and pool.
+func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
+	bcR.BaseService.Logger = l
+	bcR.pool.Logger = l
+}
+
+// OnStart implements service.Service.
+func (bcR *BlockchainReactor) OnStart() error {
+	if bcR.fastSync {
+		err := bcR.pool.Start()
+		if err != nil {
+			return err
+		}
+		go bcR.poolRoutine(false)
+	}
+	return nil
+}
+
+// SwitchToFastSync is called by the state sync reactor when switching to fast sync.
+func (bcR *BlockchainReactor) SwitchToFastSync(state sm.State) error {
+	bcR.fastSync = true
+	bcR.initialState = state
+
+	bcR.pool.height = state.LastBlockHeight + 1
+	err := bcR.pool.Start()
+	if err != nil {
+		return err
+	}
+	go bcR.poolRoutine(true)
+	return nil
+}
+
+// OnStop implements service.Service.
+func (bcR *BlockchainReactor) OnStop() {
+	if bcR.fastSync {
+		if err := bcR.pool.Stop(); err != nil {
+			bcR.Logger.Error("Error stopping pool", "err", err)
+		}
+	}
+}
+
+// GetChannels implements Reactor
+func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
+	return []*p2p.ChannelDescriptor{
+		{
+			ID:                  BlockchainChannel,
+			Priority:            5,
+			SendQueueCapacity:   1000,
+			RecvBufferCapacity:  50 * 4096,
+			RecvMessageCapacity: bc.MaxMsgSize,
+		},
+	}
+}
+
+// AddPeer implements Reactor by sending our state to peer.
+func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusResponse{
+		Base:   bcR.store.Base(),
+		Height: bcR.store.Height()})
+	if err != nil {
+		bcR.Logger.Error("could not convert msg to protobuf", "err", err)
+		return
+	}
+
+	peer.Send(BlockchainChannel, msgBytes)
+	// it's OK if send fails. will try later in poolRoutine
+
+	// peer is added to the pool once we receive the first
+	// bcStatusResponseMessage from the peer and call pool.SetPeerRange
+}
+
+// RemovePeer implements Reactor by removing peer from the pool.
+func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
+	bcR.pool.RemovePeer(peer.ID())
+}
+
+// respondToPeer loads a block and sends it to the requesting peer,
+// if we have it. Otherwise, we'll respond saying we don't have it.
+func (bcR *BlockchainReactor) respondToPeer(msg *bcproto.BlockRequest,
+	src p2p.Peer) (queued bool) {
+
+	block := bcR.store.LoadBlock(msg.Height)
+	if block != nil {
+		bl, err := block.ToProto()
+		if err != nil {
+			bcR.Logger.Error("could not convert msg to protobuf", "err", err)
+			return false
+		}
+
+		msgBytes, err := bc.EncodeMsg(&bcproto.BlockResponse{Block: bl})
+		if err != nil {
+			bcR.Logger.Error("could not marshal msg", "err", err)
+			return false
+		}
+
+		return src.TrySend(BlockchainChannel, msgBytes)
+	}
+
+	bcR.Logger.Info("Peer asking for a block we don't have", "src", src, "height", msg.Height)
+
+	msgBytes, err := bc.EncodeMsg(&bcproto.NoBlockResponse{Height: msg.Height})
+	if err != nil {
+		bcR.Logger.Error("could not convert msg to protobuf", "err", err)
+		return false
+	}
+
+	return src.TrySend(BlockchainChannel, msgBytes)
+}
+
+// Receive implements Reactor by handling 4 types of messages (look below).
+func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
+	msg, err := bc.DecodeMsg(msgBytes)
+	if err != nil {
+		bcR.Logger.Error("Error decoding message", "src", src, "chId", chID, "err", err)
+		bcR.Switch.StopPeerForError(src, err)
+		return
+	}
+
+	if err = bc.ValidateMsg(msg); err != nil {
+		bcR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err)
+		bcR.Switch.StopPeerForError(src, err)
+		return
+	}
+
+	bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
+
+	switch msg := msg.(type) {
+	case *bcproto.BlockRequest:
+		bcR.respondToPeer(msg, src)
+	case *bcproto.BlockResponse:
+		bi, err := types.BlockFromProto(msg.Block)
+		if err != nil {
+			bcR.Logger.Error("Block content is invalid", "err", err)
+			return
+		}
+		bcR.pool.AddBlock(src.ID(), bi, len(msgBytes))
+	case *bcproto.StatusRequest:
+		// Send peer our state.
+		msgBytes, err := bc.EncodeMsg(&bcproto.StatusResponse{
+			Height: bcR.store.Height(),
+			Base:   bcR.store.Base(),
+		})
+		if err != nil {
+			bcR.Logger.Error("could not convert msg to protobut", "err", err)
+			return
+		}
+		src.TrySend(BlockchainChannel, msgBytes)
+	case *bcproto.StatusResponse:
+		// Got a peer status. Unverified.
+		bcR.pool.SetPeerRange(src.ID(), msg.Base, msg.Height)
+	case *bcproto.NoBlockResponse:
+		bcR.Logger.Debug("Peer does not have requested block", "peer", src, "height", msg.Height)
+	default:
+		bcR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg)))
+	}
+}
+
+// Handle messages from the poolReactor telling the reactor what to do.
+// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
+func (bcR *BlockchainReactor) poolRoutine(stateSynced bool) {
+
+	trySyncTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
+	defer trySyncTicker.Stop()
+
+	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
+	defer statusUpdateTicker.Stop()
+
+	switchToConsensusTicker := time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
+	defer switchToConsensusTicker.Stop()
+
+	blocksSynced := uint64(0)
+
+	chainID := bcR.initialState.ChainID
+	state := bcR.initialState
+
+	lastHundred := time.Now()
+	lastRate := 0.0
+
+	didProcessCh := make(chan struct{}, 1)
+
+	go func() {
+		for {
+			select {
+			case <-bcR.Quit():
+				return
+			case <-bcR.pool.Quit():
+				return
+			case request := <-bcR.requestsCh:
+				peer := bcR.Switch.Peers().Get(request.PeerID)
+				if peer == nil {
+					continue
+				}
+				msgBytes, err := bc.EncodeMsg(&bcproto.BlockRequest{Height: request.Height})
+				if err != nil {
+					bcR.Logger.Error("could not convert msg to proto", "err", err)
+					continue
+				}
+
+				queued := peer.TrySend(BlockchainChannel, msgBytes)
+				if !queued {
+					bcR.Logger.Debug("Send queue is full, drop block request", "peer", peer.ID(), "height", request.Height)
+				}
+			case err := <-bcR.errorsCh:
+				peer := bcR.Switch.Peers().Get(err.peerID)
+				if peer != nil {
+					bcR.Switch.StopPeerForError(peer, err)
+				}
+
+			case <-statusUpdateTicker.C:
+				// ask for status updates
+				go bcR.BroadcastStatusRequest() // nolint: errcheck
+
+			}
+		}
+	}()
+
+FOR_LOOP:
+	for {
+		select {
+		case <-switchToConsensusTicker.C:
+			height, numPending, lenRequesters := bcR.pool.GetStatus()
+			outbound, inbound, _ := bcR.Switch.NumPeers()
+			bcR.Logger.Debug("Consensus ticker", "numPending", numPending, "total", lenRequesters,
+				"outbound", outbound, "inbound", inbound)
+			if bcR.pool.IsCaughtUp() {
+				bcR.Logger.Info("Time to switch to consensus reactor!", "height", height)
+				if err := bcR.pool.Stop(); err != nil {
+					bcR.Logger.Error("Error stopping pool", "err", err)
+				}
+				conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
+				if ok {
+					conR.SwitchToConsensus(state, blocksSynced > 0 || stateSynced)
+				}
+				// else {
+				// should only happen during testing
+				// }
+
+				break FOR_LOOP
+			}
+
+		case <-trySyncTicker.C: // chan time
+			select {
+			case didProcessCh <- struct{}{}:
+			default:
+			}
+
+		case <-didProcessCh:
+			// NOTE: It is a subtle mistake to process more than a single block
+			// at a time (e.g. 10) here, because we only TrySend 1 request per
+			// loop.  The ratio mismatch can result in starving of blocks, a
+			// sudden burst of requests and responses, and repeat.
+			// Consequently, it is better to split these routines rather than
+			// coupling them as it's written here.  TODO uncouple from request
+			// routine.
+
+			// See if there are any blocks to sync.
+			first, second := bcR.pool.PeekTwoBlocks()
+			// bcR.Logger.Info("TrySync peeked", "first", first, "second", second)
+			if first == nil || second == nil {
+				// We need both to sync the first block.
+				continue FOR_LOOP
+			} else {
+				// Try again quickly next loop.
+				didProcessCh <- struct{}{}
+			}
+
+			firstParts := first.MakePartSet(types.BlockPartSizeBytes)
+			firstPartSetHeader := firstParts.Header()
+			firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader}
+			// Finally, verify the first block using the second's commit
+			// NOTE: we can probably make this more efficient, but note that calling
+			// first.Hash() doesn't verify the tx contents, so MakePartSet() is
+			// currently necessary.
+			err := state.Validators.VerifyCommitLight(
+				chainID, firstID, first.Height, second.LastCommit)
+
+			if err == nil {
+				// validate the block before we persist it
+				err = bcR.blockExec.ValidateBlock(state, first)
+			}
+
+			if err != nil {
+				bcR.Logger.Error("Error in validation", "err", err)
+				peerID := bcR.pool.RedoRequest(first.Height)
+				peer := bcR.Switch.Peers().Get(peerID)
+				if peer != nil {
+					// NOTE: we've already removed the peer's request, but we
+					// still need to clean up the rest.
+					bcR.Switch.StopPeerForError(peer, fmt.Errorf("blockchainReactor validation error: %v", err))
+				}
+				peerID2 := bcR.pool.RedoRequest(second.Height)
+				peer2 := bcR.Switch.Peers().Get(peerID2)
+				if peer2 != nil && peer2 != peer {
+					// NOTE: we've already removed the peer's request, but we
+					// still need to clean up the rest.
+					bcR.Switch.StopPeerForError(peer2, fmt.Errorf("blockchainReactor validation error: %v", err))
+				}
+				continue FOR_LOOP
+			}
+
+			bcR.pool.PopRequest()
+
+			// TODO: batch saves so we dont persist to disk every block
+			bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+
+			// TODO: same thing for app - but we would need a way to
+			// get the hash without persisting the state
+			state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first)
+			if err != nil {
+				// TODO This is bad, are we zombie?
+				panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+			}
+			blocksSynced++
+
+			if blocksSynced%100 == 0 {
+				lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+				bcR.Logger.Info("Fast Sync Rate", "height", bcR.pool.height,
+					"max_peer_height", bcR.pool.MaxPeerHeight(), "blocks/s", lastRate)
+				lastHundred = time.Now()
+			}
+
+			continue FOR_LOOP
+
+		case <-bcR.Quit():
+			break FOR_LOOP
+		}
+	}
+}
+
+// BroadcastStatusRequest broadcasts `BlockStore` base and height.
+func (bcR *BlockchainReactor) BroadcastStatusRequest() error {
+	bm, err := bc.EncodeMsg(&bcproto.StatusRequest{})
+	if err != nil {
+		bcR.Logger.Error("could not convert msg to proto", "err", err)
+		return fmt.Errorf("could not convert msg to proto: %w", err)
+	}
+
+	bcR.Switch.Broadcast(BlockchainChannel, bm)
+
+	return nil
+}

internal/blocksync/v0/reactor_test.go

@@ -0,0 +1,328 @@
+package v0
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/mempool/mock"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+var config *cfg.Config
+
+func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
+	validators := make([]types.GenesisValidator, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
+	for i := 0; i < numValidators; i++ {
+		val, privVal := types.RandValidator(randPower, minPower)
+		validators[i] = types.GenesisValidator{
+			PubKey: val.PubKey,
+			Power:  val.VotingPower,
+		}
+		privValidators[i] = privVal
+	}
+	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
+	return &types.GenesisDoc{
+		GenesisTime: tmtime.Now(),
+		ChainID:     config.ChainID(),
+		Validators:  validators,
+	}, privValidators
+}
+
+type BlockchainReactorPair struct {
+	reactor *BlockchainReactor
+	app     proxy.AppConns
+}
+
+func newBlockchainReactor(
+	logger log.Logger,
+	genDoc *types.GenesisDoc,
+	privVals []types.PrivValidator,
+	maxBlockHeight int64) BlockchainReactorPair {
+	if len(privVals) != 1 {
+		panic("only support one validator")
+	}
+
+	app := &testApp{}
+	cc := proxy.NewLocalClientCreator(app)
+	proxyApp := proxy.NewAppConns(cc)
+	err := proxyApp.Start()
+	if err != nil {
+		panic(fmt.Errorf("error start app: %w", err))
+	}
+
+	blockDB := dbm.NewMemDB()
+	stateDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(stateDB)
+	blockStore := store.NewBlockStore(blockDB)
+
+	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+	if err != nil {
+		panic(fmt.Errorf("error constructing state from genesis file: %w", err))
+	}
+
+	// Make the BlockchainReactor itself.
+	// NOTE we have to create and commit the blocks first because
+	// pool.height is determined from the store.
+	fastSync := true
+	db := dbm.NewMemDB()
+	stateStore = sm.NewStore(db)
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
+		mock.Mempool{}, sm.EmptyEvidencePool{})
+	if err = stateStore.Save(state); err != nil {
+		panic(err)
+	}
+
+	// let's add some blocks in
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
+		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
+		if blockHeight > 1 {
+			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
+			lastBlock := blockStore.LoadBlock(blockHeight - 1)
+
+			vote, err := types.MakeVote(
+				lastBlock.Header.Height,
+				lastBlockMeta.BlockID,
+				state.Validators,
+				privVals[0],
+				lastBlock.Header.ChainID,
+				time.Now(),
+			)
+			if err != nil {
+				panic(err)
+			}
+			lastCommit = types.NewCommit(vote.Height, vote.Round,
+				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
+		}
+
+		thisBlock := makeBlock(blockHeight, state, lastCommit)
+
+		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
+		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
+
+		state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock)
+		if err != nil {
+			panic(fmt.Errorf("error apply block: %w", err))
+		}
+
+		blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
+	}
+
+	bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
+	bcReactor.SetLogger(logger.With("module", "blockchain"))
+
+	return BlockchainReactorPair{bcReactor, proxyApp}
+}
+
+func TestNoBlockResponse(t *testing.T) {
+	config = cfg.ResetTestRoot("blockchain_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
+
+	maxBlockHeight := int64(65)
+
+	reactorPairs := make([]BlockchainReactorPair, 2)
+
+	reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+
+	p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor)
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			err := r.reactor.Stop()
+			require.NoError(t, err)
+			err = r.app.Stop()
+			require.NoError(t, err)
+		}
+	}()
+
+	tests := []struct {
+		height   int64
+		existent bool
+	}{
+		{maxBlockHeight + 2, false},
+		{10, true},
+		{1, true},
+		{100, false},
+	}
+
+	for {
+		if reactorPairs[1].reactor.pool.IsCaughtUp() {
+			break
+		}
+
+		time.Sleep(10 * time.Millisecond)
+	}
+
+	assert.Equal(t, maxBlockHeight, reactorPairs[0].reactor.store.Height())
+
+	for _, tt := range tests {
+		block := reactorPairs[1].reactor.store.LoadBlock(tt.height)
+		if tt.existent {
+			assert.True(t, block != nil)
+		} else {
+			assert.True(t, block == nil)
+		}
+	}
+}
+
+// NOTE: This is too hard to test without
+// an easy way to add test peer to switch
+// or without significant refactoring of the module.
+// Alternatively we could actually dial a TCP conn but
+// that seems extreme.
+func TestBadBlockStopsPeer(t *testing.T) {
+	config = cfg.ResetTestRoot("blockchain_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
+
+	maxBlockHeight := int64(148)
+
+	// Other chain needs a different validator set
+	otherGenDoc, otherPrivVals := randGenesisDoc(1, false, 30)
+	otherChain := newBlockchainReactor(log.TestingLogger(), otherGenDoc, otherPrivVals, maxBlockHeight)
+
+	defer func() {
+		err := otherChain.reactor.Stop()
+		require.Error(t, err)
+		err = otherChain.app.Stop()
+		require.NoError(t, err)
+	}()
+
+	reactorPairs := make([]BlockchainReactorPair, 4)
+
+	reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs[2] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs[3] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+
+	switches := p2p.MakeConnectedSwitches(config.P2P, 4, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor)
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			err := r.reactor.Stop()
+			require.NoError(t, err)
+
+			err = r.app.Stop()
+			require.NoError(t, err)
+		}
+	}()
+
+	for {
+		time.Sleep(1 * time.Second)
+		caughtUp := true
+		for _, r := range reactorPairs {
+			if !r.reactor.pool.IsCaughtUp() {
+				caughtUp = false
+			}
+		}
+		if caughtUp {
+			break
+		}
+	}
+
+	// at this time, reactors[0-3] is the newest
+	assert.Equal(t, 3, reactorPairs[1].reactor.Switch.Peers().Size())
+
+	// Mark reactorPairs[3] as an invalid peer. Fiddling with .store without a mutex is a data
+	// race, but can't be easily avoided.
+	reactorPairs[3].reactor.store = otherChain.reactor.store
+
+	lastReactorPair := newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0)
+	reactorPairs = append(reactorPairs, lastReactorPair)
+
+	switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].reactor)
+		return s
+
+	}, p2p.Connect2Switches)...)
+
+	for i := 0; i < len(reactorPairs)-1; i++ {
+		p2p.Connect2Switches(switches, i, len(reactorPairs)-1)
+	}
+
+	for {
+		if lastReactorPair.reactor.pool.IsCaughtUp() || lastReactorPair.reactor.Switch.Peers().Size() == 0 {
+			break
+		}
+
+		time.Sleep(1 * time.Second)
+	}
+
+	assert.True(t, lastReactorPair.reactor.Switch.Peers().Size() < len(reactorPairs)-1)
+}
+
+//----------------------------------------------
+// utility funcs
+
+func makeTxs(height int64) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(height), byte(i)}))
+	}
+	return txs
+}
+
+func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block {
+	block, _ := state.MakeBlock(height, makeTxs(height), lastCommit, nil, state.Validators.GetProposer().Address)
+	return block
+}
+
+type testApp struct {
+	abci.BaseApplication
+}
+
+var _ abci.Application = (*testApp)(nil)
+
+func (app *testApp) Info(req abci.RequestInfo) (resInfo abci.ResponseInfo) {
+	return abci.ResponseInfo{}
+}
+
+func (app *testApp) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginBlock {
+	return abci.ResponseBeginBlock{}
+}
+
+func (app *testApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
+	return abci.ResponseEndBlock{}
+}
+
+func (app *testApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	return abci.ResponseDeliverTx{Events: []abci.Event{}}
+}
+
+func (app *testApp) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx {
+	return abci.ResponseCheckTx{}
+}
+
+func (app *testApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{}
+}
+
+func (app *testApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) {
+	return
+}

internal/blocksync/v1/peer.go

@@ -0,0 +1,211 @@
+package v1
+
+import (
+	"fmt"
+	"math"
+	"time"
+
+	flow "github.com/tendermint/tendermint/libs/flowrate"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+//--------
+// Peer
+
+// BpPeerParams stores the peer parameters that are used when creating a peer.
+type BpPeerParams struct {
+	timeout     time.Duration
+	minRecvRate int64
+	sampleRate  time.Duration
+	windowSize  time.Duration
+}
+
+// BpPeer is the datastructure associated with a fast sync peer.
+type BpPeer struct {
+	logger log.Logger
+	ID     p2p.ID
+
+	Base                    int64                  // the peer reported base
+	Height                  int64                  // the peer reported height
+	NumPendingBlockRequests int                    // number of requests still waiting for block responses
+	blocks                  map[int64]*types.Block // blocks received or expected to be received from this peer
+	blockResponseTimer      *time.Timer
+	recvMonitor             *flow.Monitor
+	params                  *BpPeerParams // parameters for timer and monitor
+
+	onErr func(err error, peerID p2p.ID) // function to call on error
+}
+
+// NewBpPeer creates a new peer.
+func NewBpPeer(peerID p2p.ID, base int64, height int64,
+	onErr func(err error, peerID p2p.ID), params *BpPeerParams) *BpPeer {
+
+	if params == nil {
+		params = BpPeerDefaultParams()
+	}
+	return &BpPeer{
+		ID:     peerID,
+		Base:   base,
+		Height: height,
+		blocks: make(map[int64]*types.Block, maxRequestsPerPeer),
+		logger: log.NewNopLogger(),
+		onErr:  onErr,
+		params: params,
+	}
+}
+
+// String returns a string representation of a peer.
+func (peer *BpPeer) String() string {
+	return fmt.Sprintf("peer: %v height: %v pending: %v", peer.ID, peer.Height, peer.NumPendingBlockRequests)
+}
+
+// SetLogger sets the logger of the peer.
+func (peer *BpPeer) SetLogger(l log.Logger) {
+	peer.logger = l
+}
+
+// Cleanup performs cleanup of the peer, removes blocks, requests, stops timer and monitor.
+func (peer *BpPeer) Cleanup() {
+	if peer.blockResponseTimer != nil {
+		peer.blockResponseTimer.Stop()
+	}
+	if peer.NumPendingBlockRequests != 0 {
+		peer.logger.Info("peer with pending requests is being cleaned", "peer", peer.ID)
+	}
+	if len(peer.blocks)-peer.NumPendingBlockRequests != 0 {
+		peer.logger.Info("peer with pending blocks is being cleaned", "peer", peer.ID)
+	}
+	for h := range peer.blocks {
+		delete(peer.blocks, h)
+	}
+	peer.NumPendingBlockRequests = 0
+	peer.recvMonitor = nil
+}
+
+// BlockAtHeight returns the block at a given height if available and errMissingBlock otherwise.
+func (peer *BpPeer) BlockAtHeight(height int64) (*types.Block, error) {
+	block, ok := peer.blocks[height]
+	if !ok {
+		return nil, errMissingBlock
+	}
+	if block == nil {
+		return nil, errMissingBlock
+	}
+	return peer.blocks[height], nil
+}
+
+// AddBlock adds a block at peer level. Block must be non-nil and recvSize a positive integer
+// The peer must have a pending request for this block.
+func (peer *BpPeer) AddBlock(block *types.Block, recvSize int) error {
+	if block == nil || recvSize < 0 {
+		panic("bad parameters")
+	}
+	existingBlock, ok := peer.blocks[block.Height]
+	if !ok {
+		peer.logger.Error("unsolicited block", "blockHeight", block.Height, "peer", peer.ID)
+		return errMissingBlock
+	}
+	if existingBlock != nil {
+		peer.logger.Error("already have a block for height", "height", block.Height)
+		return errDuplicateBlock
+	}
+	if peer.NumPendingBlockRequests == 0 {
+		panic("peer does not have pending requests")
+	}
+	peer.blocks[block.Height] = block
+	peer.NumPendingBlockRequests--
+	if peer.NumPendingBlockRequests == 0 {
+		peer.stopMonitor()
+		peer.stopBlockResponseTimer()
+	} else {
+		peer.recvMonitor.Update(recvSize)
+		peer.resetBlockResponseTimer()
+	}
+	return nil
+}
+
+// RemoveBlock removes the block of given height
+func (peer *BpPeer) RemoveBlock(height int64) {
+	delete(peer.blocks, height)
+}
+
+// RequestSent records that a request was sent, and starts the peer timer and monitor if needed.
+func (peer *BpPeer) RequestSent(height int64) {
+	peer.blocks[height] = nil
+
+	if peer.NumPendingBlockRequests == 0 {
+		peer.startMonitor()
+		peer.resetBlockResponseTimer()
+	}
+	peer.NumPendingBlockRequests++
+}
+
+// CheckRate verifies that the response rate of the peer is acceptable (higher than the minimum allowed).
+func (peer *BpPeer) CheckRate() error {
+	if peer.NumPendingBlockRequests == 0 {
+		return nil
+	}
+	curRate := peer.recvMonitor.Status().CurRate
+	// curRate can be 0 on start
+	if curRate != 0 && curRate < peer.params.minRecvRate {
+		err := errSlowPeer
+		peer.logger.Error("SendTimeout", "peer", peer,
+			"reason", err,
+			"curRate", fmt.Sprintf("%d KB/s", curRate/1024),
+			"minRate", fmt.Sprintf("%d KB/s", peer.params.minRecvRate/1024))
+		return err
+	}
+	return nil
+}
+
+func (peer *BpPeer) onTimeout() {
+	peer.onErr(errNoPeerResponse, peer.ID)
+}
+
+func (peer *BpPeer) stopMonitor() {
+	peer.recvMonitor.Done()
+	peer.recvMonitor = nil
+}
+
+func (peer *BpPeer) startMonitor() {
+	peer.recvMonitor = flow.New(peer.params.sampleRate, peer.params.windowSize)
+	initialValue := float64(peer.params.minRecvRate) * math.E
+	peer.recvMonitor.SetREMA(initialValue)
+}
+
+func (peer *BpPeer) resetBlockResponseTimer() {
+	if peer.blockResponseTimer == nil {
+		peer.blockResponseTimer = time.AfterFunc(peer.params.timeout, peer.onTimeout)
+	} else {
+		peer.blockResponseTimer.Reset(peer.params.timeout)
+	}
+}
+
+func (peer *BpPeer) stopBlockResponseTimer() bool {
+	if peer.blockResponseTimer == nil {
+		return false
+	}
+	return peer.blockResponseTimer.Stop()
+}
+
+// BpPeerDefaultParams returns the default peer parameters.
+func BpPeerDefaultParams() *BpPeerParams {
+	return &BpPeerParams{
+		// Timeout for a peer to respond to a block request.
+		timeout: 15 * time.Second,
+
+		// Minimum recv rate to ensure we're receiving blocks from a peer fast
+		// enough. If a peer is not sending data at at least that rate, we
+		// consider them to have timedout and we disconnect.
+		//
+		// Assuming a DSL connection (not a good choice) 128 Kbps (upload) ~ 15 KB/s,
+		// sending data across atlantic ~ 7.5 KB/s.
+		minRecvRate: int64(7680),
+
+		// Monitor parameters
+		sampleRate: time.Second,
+		windowSize: 40 * time.Second,
+	}
+}

internal/blocksync/v1/peer_test.go

@@ -0,0 +1,280 @@
+package v1
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestPeerMonitor(t *testing.T) {
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {},
+		nil)
+	peer.SetLogger(log.TestingLogger())
+	peer.startMonitor()
+	assert.NotNil(t, peer.recvMonitor)
+	peer.stopMonitor()
+	assert.Nil(t, peer.recvMonitor)
+}
+
+func TestPeerResetBlockResponseTimer(t *testing.T) {
+	var (
+		numErrFuncCalls int        // number of calls to the errFunc
+		lastErr         error      // last generated error
+		peerTestMtx     sync.Mutex // modifications of ^^ variables are also done from timer handler goroutine
+	)
+	params := &BpPeerParams{timeout: 20 * time.Millisecond}
+
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {
+			peerTestMtx.Lock()
+			defer peerTestMtx.Unlock()
+			lastErr = err
+			numErrFuncCalls++
+		},
+		params)
+
+	peer.SetLogger(log.TestingLogger())
+	checkByStoppingPeerTimer(t, peer, false)
+
+	// initial reset call with peer having a nil timer
+	peer.resetBlockResponseTimer()
+	assert.NotNil(t, peer.blockResponseTimer)
+	// make sure timer is running and stop it
+	checkByStoppingPeerTimer(t, peer, true)
+
+	// reset with running timer
+	peer.resetBlockResponseTimer()
+	time.Sleep(5 * time.Millisecond)
+	peer.resetBlockResponseTimer()
+	assert.NotNil(t, peer.blockResponseTimer)
+
+	// let the timer expire and ...
+	time.Sleep(50 * time.Millisecond)
+	// ... check timer is not running
+	checkByStoppingPeerTimer(t, peer, false)
+
+	peerTestMtx.Lock()
+	// ... check errNoPeerResponse has been sent
+	assert.Equal(t, 1, numErrFuncCalls)
+	assert.Equal(t, lastErr, errNoPeerResponse)
+	peerTestMtx.Unlock()
+}
+
+func TestPeerRequestSent(t *testing.T) {
+	params := &BpPeerParams{timeout: 2 * time.Millisecond}
+
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {},
+		params)
+
+	peer.SetLogger(log.TestingLogger())
+
+	peer.RequestSent(1)
+	assert.NotNil(t, peer.recvMonitor)
+	assert.NotNil(t, peer.blockResponseTimer)
+	assert.Equal(t, 1, peer.NumPendingBlockRequests)
+
+	peer.RequestSent(1)
+	assert.NotNil(t, peer.recvMonitor)
+	assert.NotNil(t, peer.blockResponseTimer)
+	assert.Equal(t, 2, peer.NumPendingBlockRequests)
+}
+
+func TestPeerGetAndRemoveBlock(t *testing.T) {
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 100,
+		func(err error, _ p2p.ID) {},
+		nil)
+
+	// Change peer height
+	peer.Height = int64(10)
+	assert.Equal(t, int64(10), peer.Height)
+
+	// request some blocks and receive few of them
+	for i := 1; i <= 10; i++ {
+		peer.RequestSent(int64(i))
+		if i > 5 {
+			// only receive blocks 1..5
+			continue
+		}
+		_ = peer.AddBlock(makeSmallBlock(i), 10)
+	}
+
+	tests := []struct {
+		name         string
+		height       int64
+		wantErr      error
+		blockPresent bool
+	}{
+		{"no request", 100, errMissingBlock, false},
+		{"no block", 6, errMissingBlock, false},
+		{"block 1 present", 1, nil, true},
+		{"block max present", 5, nil, true},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			// try to get the block
+			b, err := peer.BlockAtHeight(tt.height)
+			assert.Equal(t, tt.wantErr, err)
+			assert.Equal(t, tt.blockPresent, b != nil)
+
+			// remove the block
+			peer.RemoveBlock(tt.height)
+			_, err = peer.BlockAtHeight(tt.height)
+			assert.Equal(t, errMissingBlock, err)
+		})
+	}
+}
+
+func TestPeerAddBlock(t *testing.T) {
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 100,
+		func(err error, _ p2p.ID) {},
+		nil)
+
+	// request some blocks, receive one
+	for i := 1; i <= 10; i++ {
+		peer.RequestSent(int64(i))
+		if i == 5 {
+			// receive block 5
+			_ = peer.AddBlock(makeSmallBlock(i), 10)
+		}
+	}
+
+	tests := []struct {
+		name         string
+		height       int64
+		wantErr      error
+		blockPresent bool
+	}{
+		{"no request", 50, errMissingBlock, false},
+		{"duplicate block", 5, errDuplicateBlock, true},
+		{"block 1 successfully received", 1, nil, true},
+		{"block max successfully received", 10, nil, true},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			// try to get the block
+			err := peer.AddBlock(makeSmallBlock(int(tt.height)), 10)
+			assert.Equal(t, tt.wantErr, err)
+			_, err = peer.BlockAtHeight(tt.height)
+			assert.Equal(t, tt.blockPresent, err == nil)
+		})
+	}
+}
+
+func TestPeerOnErrFuncCalledDueToExpiration(t *testing.T) {
+
+	params := &BpPeerParams{timeout: 10 * time.Millisecond}
+	var (
+		numErrFuncCalls int        // number of calls to the onErr function
+		lastErr         error      // last generated error
+		peerTestMtx     sync.Mutex // modifications of ^^ variables are also done from timer handler goroutine
+	)
+
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {
+			peerTestMtx.Lock()
+			defer peerTestMtx.Unlock()
+			lastErr = err
+			numErrFuncCalls++
+		},
+		params)
+
+	peer.SetLogger(log.TestingLogger())
+
+	peer.RequestSent(1)
+	time.Sleep(50 * time.Millisecond)
+	// timer should have expired by now, check that the on error function was called
+	peerTestMtx.Lock()
+	assert.Equal(t, 1, numErrFuncCalls)
+	assert.Equal(t, errNoPeerResponse, lastErr)
+	peerTestMtx.Unlock()
+}
+
+func TestPeerCheckRate(t *testing.T) {
+	params := &BpPeerParams{
+		timeout:     time.Second,
+		minRecvRate: int64(100), // 100 bytes/sec exponential moving average
+	}
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {},
+		params)
+	peer.SetLogger(log.TestingLogger())
+
+	require.Nil(t, peer.CheckRate())
+
+	for i := 0; i < 40; i++ {
+		peer.RequestSent(int64(i))
+	}
+
+	// monitor starts with a higher rEMA (~ 2*minRecvRate), wait for it to go down
+	time.Sleep(900 * time.Millisecond)
+
+	// normal peer - send a bit more than 100 bytes/sec, > 10 bytes/100msec, check peer is not considered slow
+	for i := 0; i < 10; i++ {
+		_ = peer.AddBlock(makeSmallBlock(i), 11)
+		time.Sleep(100 * time.Millisecond)
+		require.Nil(t, peer.CheckRate())
+	}
+
+	// slow peer - send a bit less than 10 bytes/100msec
+	for i := 10; i < 20; i++ {
+		_ = peer.AddBlock(makeSmallBlock(i), 9)
+		time.Sleep(100 * time.Millisecond)
+	}
+	// check peer is considered slow
+	assert.Equal(t, errSlowPeer, peer.CheckRate())
+}
+
+func TestPeerCleanup(t *testing.T) {
+	params := &BpPeerParams{timeout: 2 * time.Millisecond}
+
+	peer := NewBpPeer(
+		p2p.ID(tmrand.Str(12)), 0, 10,
+		func(err error, _ p2p.ID) {},
+		params)
+	peer.SetLogger(log.TestingLogger())
+
+	assert.Nil(t, peer.blockResponseTimer)
+	peer.RequestSent(1)
+	assert.NotNil(t, peer.blockResponseTimer)
+
+	peer.Cleanup()
+	checkByStoppingPeerTimer(t, peer, false)
+}
+
+// Check if peer timer is running or not (a running timer can be successfully stopped).
+// Note: stops the timer.
+func checkByStoppingPeerTimer(t *testing.T, peer *BpPeer, running bool) {
+	assert.NotPanics(t, func() {
+		stopped := peer.stopBlockResponseTimer()
+		if running {
+			assert.True(t, stopped)
+		} else {
+			assert.False(t, stopped)
+		}
+	})
+}
+
+func makeSmallBlock(height int) *types.Block {
+	return types.MakeBlock(int64(height), []types.Tx{types.Tx("foo")}, nil, nil)
+}

internal/blocksync/v1/pool.go

@@ -0,0 +1,370 @@
+package v1
+
+import (
+	"sort"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+// BlockPool keeps track of the fast sync peers, block requests and block responses.
+type BlockPool struct {
+	logger log.Logger
+	// Set of peers that have sent status responses, with height bigger than pool.Height
+	peers map[p2p.ID]*BpPeer
+	// Set of block heights and the corresponding peers from where a block response is expected or has been received.
+	blocks map[int64]p2p.ID
+
+	plannedRequests   map[int64]struct{} // list of blocks to be assigned peers for blockRequest
+	nextRequestHeight int64              // next height to be added to plannedRequests
+
+	Height        int64 // height of next block to execute
+	MaxPeerHeight int64 // maximum height of all peers
+	toBcR         bcReactor
+}
+
+// NewBlockPool creates a new BlockPool.
+func NewBlockPool(height int64, toBcR bcReactor) *BlockPool {
+	return &BlockPool{
+		Height:            height,
+		MaxPeerHeight:     0,
+		peers:             make(map[p2p.ID]*BpPeer),
+		blocks:            make(map[int64]p2p.ID),
+		plannedRequests:   make(map[int64]struct{}),
+		nextRequestHeight: height,
+		toBcR:             toBcR,
+	}
+}
+
+// SetLogger sets the logger of the pool.
+func (pool *BlockPool) SetLogger(l log.Logger) {
+	pool.logger = l
+}
+
+// ReachedMaxHeight check if the pool has reached the maximum peer height.
+func (pool *BlockPool) ReachedMaxHeight() bool {
+	return pool.Height >= pool.MaxPeerHeight
+}
+
+func (pool *BlockPool) rescheduleRequest(peerID p2p.ID, height int64) {
+	pool.logger.Info("reschedule requests made to peer for height ", "peerID", peerID, "height", height)
+	pool.plannedRequests[height] = struct{}{}
+	delete(pool.blocks, height)
+	pool.peers[peerID].RemoveBlock(height)
+}
+
+// Updates the pool's max height. If no peers are left MaxPeerHeight is set to 0.
+func (pool *BlockPool) updateMaxPeerHeight() {
+	var newMax int64
+	for _, peer := range pool.peers {
+		peerHeight := peer.Height
+		if peerHeight > newMax {
+			newMax = peerHeight
+		}
+	}
+	pool.MaxPeerHeight = newMax
+}
+
+// UpdatePeer adds a new peer or updates an existing peer with a new base and height.
+// If a peer is short it is not added.
+func (pool *BlockPool) UpdatePeer(peerID p2p.ID, base int64, height int64) error {
+
+	peer := pool.peers[peerID]
+
+	if peer == nil {
+		if height < pool.Height {
+			pool.logger.Info("Peer height too small",
+				"peer", peerID, "height", height, "fsm_height", pool.Height)
+			return errPeerTooShort
+		}
+		// Add new peer.
+		peer = NewBpPeer(peerID, base, height, pool.toBcR.sendPeerError, nil)
+		peer.SetLogger(pool.logger.With("peer", peerID))
+		pool.peers[peerID] = peer
+		pool.logger.Info("added peer", "peerID", peerID, "base", base, "height", height, "num_peers", len(pool.peers))
+	} else {
+		// Check if peer is lowering its height. This is not allowed.
+		if height < peer.Height {
+			pool.RemovePeer(peerID, errPeerLowersItsHeight)
+			return errPeerLowersItsHeight
+		}
+		// Update existing peer.
+		peer.Base = base
+		peer.Height = height
+	}
+
+	// Update the pool's MaxPeerHeight if needed.
+	pool.updateMaxPeerHeight()
+
+	return nil
+}
+
+// Cleans and deletes the peer. Recomputes the max peer height.
+func (pool *BlockPool) deletePeer(peer *BpPeer) {
+	if peer == nil {
+		return
+	}
+	peer.Cleanup()
+	delete(pool.peers, peer.ID)
+
+	if peer.Height == pool.MaxPeerHeight {
+		pool.updateMaxPeerHeight()
+	}
+}
+
+// RemovePeer removes the blocks and requests from the peer, reschedules them and deletes the peer.
+func (pool *BlockPool) RemovePeer(peerID p2p.ID, err error) {
+	peer := pool.peers[peerID]
+	if peer == nil {
+		return
+	}
+	pool.logger.Info("removing peer", "peerID", peerID, "error", err)
+
+	// Reschedule the block requests made to the peer, or received and not processed yet.
+	// Note that some of the requests may be removed further down.
+	for h := range pool.peers[peerID].blocks {
+		pool.rescheduleRequest(peerID, h)
+	}
+
+	oldMaxPeerHeight := pool.MaxPeerHeight
+	// Delete the peer. This operation may result in the pool's MaxPeerHeight being lowered.
+	pool.deletePeer(peer)
+
+	// Check if the pool's MaxPeerHeight has been lowered.
+	// This may happen if the tallest peer has been removed.
+	if oldMaxPeerHeight > pool.MaxPeerHeight {
+		// Remove any planned requests for heights over the new MaxPeerHeight.
+		for h := range pool.plannedRequests {
+			if h > pool.MaxPeerHeight {
+				delete(pool.plannedRequests, h)
+			}
+		}
+		// Adjust the nextRequestHeight to the new max plus one.
+		if pool.nextRequestHeight > pool.MaxPeerHeight {
+			pool.nextRequestHeight = pool.MaxPeerHeight + 1
+		}
+	}
+}
+
+func (pool *BlockPool) removeShortPeers() {
+	for _, peer := range pool.peers {
+		if peer.Height < pool.Height {
+			pool.RemovePeer(peer.ID, nil)
+		}
+	}
+}
+
+func (pool *BlockPool) removeBadPeers() {
+	pool.removeShortPeers()
+	for _, peer := range pool.peers {
+		if err := peer.CheckRate(); err != nil {
+			pool.RemovePeer(peer.ID, err)
+			pool.toBcR.sendPeerError(err, peer.ID)
+		}
+	}
+}
+
+// MakeNextRequests creates more requests if the block pool is running low.
+func (pool *BlockPool) MakeNextRequests(maxNumRequests int) {
+	heights := pool.makeRequestBatch(maxNumRequests)
+	if len(heights) != 0 {
+		pool.logger.Info("makeNextRequests will make following requests",
+			"number", len(heights), "heights", heights)
+	}
+
+	for _, height := range heights {
+		h := int64(height)
+		if !pool.sendRequest(h) {
+			// If a good peer was not found for sending the request at height h then return,
+			// as it shouldn't be possible to find a peer for h+1.
+			return
+		}
+		delete(pool.plannedRequests, h)
+	}
+}
+
+// Makes a batch of requests sorted by height such that the block pool has up to maxNumRequests entries.
+func (pool *BlockPool) makeRequestBatch(maxNumRequests int) []int {
+	pool.removeBadPeers()
+	// At this point pool.requests may include heights for requests to be redone due to removal of peers:
+	// - peers timed out or were removed by switch
+	// - FSM timed out on waiting to advance the block execution due to missing blocks at h or h+1
+	// Determine the number of requests needed by subtracting the number of requests already made from the maximum
+	// allowed
+	numNeeded := maxNumRequests - len(pool.blocks)
+	for len(pool.plannedRequests) < numNeeded {
+		if pool.nextRequestHeight > pool.MaxPeerHeight {
+			break
+		}
+		pool.plannedRequests[pool.nextRequestHeight] = struct{}{}
+		pool.nextRequestHeight++
+	}
+
+	heights := make([]int, 0, len(pool.plannedRequests))
+	for k := range pool.plannedRequests {
+		heights = append(heights, int(k))
+	}
+	sort.Ints(heights)
+	return heights
+}
+
+func (pool *BlockPool) sendRequest(height int64) bool {
+	for _, peer := range pool.peers {
+		if peer.NumPendingBlockRequests >= maxRequestsPerPeer {
+			continue
+		}
+		if peer.Base > height || peer.Height < height {
+			continue
+		}
+
+		err := pool.toBcR.sendBlockRequest(peer.ID, height)
+		if err == errNilPeerForBlockRequest {
+			// Switch does not have this peer, remove it and continue to look for another peer.
+			pool.logger.Error("switch does not have peer..removing peer selected for height", "peer",
+				peer.ID, "height", height)
+			pool.RemovePeer(peer.ID, err)
+			continue
+		}
+
+		if err == errSendQueueFull {
+			pool.logger.Error("peer queue is full", "peer", peer.ID, "height", height)
+			continue
+		}
+
+		pool.logger.Info("assigned request to peer", "peer", peer.ID, "height", height)
+
+		pool.blocks[height] = peer.ID
+		peer.RequestSent(height)
+
+		return true
+	}
+	pool.logger.Error("could not find peer to send request for block at height", "height", height)
+	return false
+}
+
+// AddBlock validates that the block comes from the peer it was expected from and stores it in the 'blocks' map.
+func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int) error {
+	peer, ok := pool.peers[peerID]
+	if !ok {
+		pool.logger.Error("block from unknown peer", "height", block.Height, "peer", peerID)
+		return errBadDataFromPeer
+	}
+	if wantPeerID, ok := pool.blocks[block.Height]; ok && wantPeerID != peerID {
+		pool.logger.Error("block received from wrong peer", "height", block.Height,
+			"peer", peerID, "expected_peer", wantPeerID)
+		return errBadDataFromPeer
+	}
+
+	return peer.AddBlock(block, blockSize)
+}
+
+// BlockData stores the peer responsible to deliver a block and the actual block if delivered.
+type BlockData struct {
+	block *types.Block
+	peer  *BpPeer
+}
+
+// BlockAndPeerAtHeight retrieves the block and delivery peer at specified height.
+// Returns errMissingBlock if a block was not found
+func (pool *BlockPool) BlockAndPeerAtHeight(height int64) (bData *BlockData, err error) {
+	peerID := pool.blocks[height]
+	peer := pool.peers[peerID]
+	if peer == nil {
+		return nil, errMissingBlock
+	}
+
+	block, err := peer.BlockAtHeight(height)
+	if err != nil {
+		return nil, err
+	}
+
+	return &BlockData{peer: peer, block: block}, nil
+
+}
+
+// FirstTwoBlocksAndPeers returns the blocks and the delivery peers at pool's height H and H+1.
+func (pool *BlockPool) FirstTwoBlocksAndPeers() (first, second *BlockData, err error) {
+	first, err = pool.BlockAndPeerAtHeight(pool.Height)
+	second, err2 := pool.BlockAndPeerAtHeight(pool.Height + 1)
+	if err == nil {
+		err = err2
+	}
+	return
+}
+
+// InvalidateFirstTwoBlocks removes the peers that sent us the first two blocks, blocks are removed by RemovePeer().
+func (pool *BlockPool) InvalidateFirstTwoBlocks(err error) {
+	first, err1 := pool.BlockAndPeerAtHeight(pool.Height)
+	second, err2 := pool.BlockAndPeerAtHeight(pool.Height + 1)
+
+	if err1 == nil {
+		pool.RemovePeer(first.peer.ID, err)
+	}
+	if err2 == nil {
+		pool.RemovePeer(second.peer.ID, err)
+	}
+}
+
+// ProcessedCurrentHeightBlock performs cleanup after a block is processed. It removes block at pool height and
+// the peers that are now short.
+func (pool *BlockPool) ProcessedCurrentHeightBlock() {
+	peerID, peerOk := pool.blocks[pool.Height]
+	if peerOk {
+		pool.peers[peerID].RemoveBlock(pool.Height)
+	}
+	delete(pool.blocks, pool.Height)
+	pool.logger.Debug("removed block at height", "height", pool.Height)
+	pool.Height++
+	pool.removeShortPeers()
+}
+
+// RemovePeerAtCurrentHeights checks if a block at pool's height H exists and if not, it removes the
+// delivery peer and returns. If a block at height H exists then the check and peer removal is done for H+1.
+// This function is called when the FSM is not able to make progress for some time.
+// This happens if either the block H or H+1 have not been delivered.
+func (pool *BlockPool) RemovePeerAtCurrentHeights(err error) {
+	peerID := pool.blocks[pool.Height]
+	peer, ok := pool.peers[peerID]
+	if ok {
+		if _, err := peer.BlockAtHeight(pool.Height); err != nil {
+			pool.logger.Info("remove peer that hasn't sent block at pool.Height",
+				"peer", peerID, "height", pool.Height)
+			pool.RemovePeer(peerID, err)
+			return
+		}
+	}
+	peerID = pool.blocks[pool.Height+1]
+	peer, ok = pool.peers[peerID]
+	if ok {
+		if _, err := peer.BlockAtHeight(pool.Height + 1); err != nil {
+			pool.logger.Info("remove peer that hasn't sent block at pool.Height+1",
+				"peer", peerID, "height", pool.Height+1)
+			pool.RemovePeer(peerID, err)
+			return
+		}
+	}
+}
+
+// Cleanup performs pool and peer cleanup
+func (pool *BlockPool) Cleanup() {
+	for id, peer := range pool.peers {
+		peer.Cleanup()
+		delete(pool.peers, id)
+	}
+	pool.plannedRequests = make(map[int64]struct{})
+	pool.blocks = make(map[int64]p2p.ID)
+	pool.nextRequestHeight = 0
+	pool.Height = 0
+	pool.MaxPeerHeight = 0
+}
+
+// NumPeers returns the number of peers in the pool
+func (pool *BlockPool) NumPeers() int {
+	return len(pool.peers)
+}
+
+// NeedsBlocks returns true if more blocks are required.
+func (pool *BlockPool) NeedsBlocks() bool {
+	return len(pool.blocks) < maxNumRequests
+}

internal/blocksync/v1/pool_test.go

@@ -0,0 +1,691 @@
+package v1
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+type testPeer struct {
+	id     p2p.ID
+	base   int64
+	height int64
+}
+
+type testBcR struct {
+	logger log.Logger
+}
+
+type testValues struct {
+	numRequestsSent int
+}
+
+var testResults testValues
+
+func resetPoolTestResults() {
+	testResults.numRequestsSent = 0
+}
+
+func (testR *testBcR) sendPeerError(err error, peerID p2p.ID) {
+}
+
+func (testR *testBcR) sendStatusRequest() {
+}
+
+func (testR *testBcR) sendBlockRequest(peerID p2p.ID, height int64) error {
+	testResults.numRequestsSent++
+	return nil
+}
+
+func (testR *testBcR) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) {
+}
+
+func (testR *testBcR) switchToConsensus() {
+
+}
+
+func newTestBcR() *testBcR {
+	testBcR := &testBcR{logger: log.TestingLogger()}
+	return testBcR
+}
+
+type tPBlocks struct {
+	id     p2p.ID
+	create bool
+}
+
+// Makes a block pool with specified current height, list of peers, block requests and block responses
+func makeBlockPool(bcr *testBcR, height int64, peers []BpPeer, blocks map[int64]tPBlocks) *BlockPool {
+	bPool := NewBlockPool(height, bcr)
+	bPool.SetLogger(bcr.logger)
+
+	txs := []types.Tx{types.Tx("foo"), types.Tx("bar")}
+
+	var maxH int64
+	for _, p := range peers {
+		if p.Height > maxH {
+			maxH = p.Height
+		}
+		bPool.peers[p.ID] = NewBpPeer(p.ID, p.Base, p.Height, bcr.sendPeerError, nil)
+		bPool.peers[p.ID].SetLogger(bcr.logger)
+
+	}
+	bPool.MaxPeerHeight = maxH
+	for h, p := range blocks {
+		bPool.blocks[h] = p.id
+		bPool.peers[p.id].RequestSent(h)
+		if p.create {
+			// simulate that a block at height h has been received
+			_ = bPool.peers[p.id].AddBlock(types.MakeBlock(h, txs, nil, nil), 100)
+		}
+	}
+	return bPool
+}
+
+func assertPeerSetsEquivalent(t *testing.T, set1 map[p2p.ID]*BpPeer, set2 map[p2p.ID]*BpPeer) {
+	assert.Equal(t, len(set1), len(set2))
+	for peerID, peer1 := range set1 {
+		peer2 := set2[peerID]
+		assert.NotNil(t, peer2)
+		assert.Equal(t, peer1.NumPendingBlockRequests, peer2.NumPendingBlockRequests)
+		assert.Equal(t, peer1.Height, peer2.Height)
+		assert.Equal(t, peer1.Base, peer2.Base)
+		assert.Equal(t, len(peer1.blocks), len(peer2.blocks))
+		for h, block1 := range peer1.blocks {
+			block2 := peer2.blocks[h]
+			// block1 and block2 could be nil if a request was made but no block was received
+			assert.Equal(t, block1, block2)
+		}
+	}
+}
+
+func assertBlockPoolEquivalent(t *testing.T, poolWanted, pool *BlockPool) {
+	assert.Equal(t, poolWanted.blocks, pool.blocks)
+	assertPeerSetsEquivalent(t, poolWanted.peers, pool.peers)
+	assert.Equal(t, poolWanted.MaxPeerHeight, pool.MaxPeerHeight)
+	assert.Equal(t, poolWanted.Height, pool.Height)
+
+}
+
+func TestBlockPoolUpdatePeer(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		args       testPeer
+		poolWanted *BlockPool
+		errWanted  error
+	}{
+		{
+			name:       "add a first short peer",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+			args:       testPeer{"P1", 0, 50},
+			errWanted:  errPeerTooShort,
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name:       "add a first good peer",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+			args:       testPeer{"P1", 0, 101},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 101}}, map[int64]tPBlocks{}),
+		},
+		{
+			name:       "add a first good peer with base",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+			args:       testPeer{"P1", 10, 101},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Base: 10, Height: 101}}, map[int64]tPBlocks{}),
+		},
+		{
+			name:       "increase the height of P1 from 120 to 123",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}),
+			args:       testPeer{"P1", 0, 123},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 123}}, map[int64]tPBlocks{}),
+		},
+		{
+			name:       "decrease the height of P1 from 120 to 110",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}),
+			args:       testPeer{"P1", 0, 110},
+			errWanted:  errPeerLowersItsHeight,
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "decrease the height of P1 from 105 to 102 with blocks",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 105}},
+				map[int64]tPBlocks{
+					100: {"P1", true}, 101: {"P1", true}, 102: {"P1", true}}),
+			args:      testPeer{"P1", 0, 102},
+			errWanted: errPeerLowersItsHeight,
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{},
+				map[int64]tPBlocks{}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			pool := tt.pool
+			err := pool.UpdatePeer(tt.args.id, tt.args.base, tt.args.height)
+			assert.Equal(t, tt.errWanted, err)
+			assert.Equal(t, tt.poolWanted.blocks, tt.pool.blocks)
+			assertPeerSetsEquivalent(t, tt.poolWanted.peers, tt.pool.peers)
+			assert.Equal(t, tt.poolWanted.MaxPeerHeight, tt.pool.MaxPeerHeight)
+		})
+	}
+}
+
+func TestBlockPoolRemovePeer(t *testing.T) {
+	testBcR := newTestBcR()
+
+	type args struct {
+		peerID p2p.ID
+		err    error
+	}
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		args       args
+		poolWanted *BlockPool
+	}{
+		{
+			name:       "attempt to delete non-existing peer",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}),
+			args:       args{"P99", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}),
+		},
+		{
+			name:       "delete the only peer without blocks",
+			pool:       makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}}, map[int64]tPBlocks{}),
+			args:       args{"P1", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "delete the shortest of two peers without blocks",
+			pool: makeBlockPool(
+				testBcR,
+				100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 120}},
+				map[int64]tPBlocks{}),
+			args:       args{"P1", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P2", Height: 120}}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "delete the tallest of two peers without blocks",
+			pool: makeBlockPool(
+				testBcR,
+				100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 120}},
+				map[int64]tPBlocks{}),
+			args:       args{"P2", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "delete the only peer with block requests sent and blocks received",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}},
+				map[int64]tPBlocks{100: {"P1", true}, 101: {"P1", false}}),
+			args:       args{"P1", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "delete the shortest of two peers with block requests sent and blocks received",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 200}},
+				map[int64]tPBlocks{100: {"P1", true}, 101: {"P1", false}}),
+			args:       args{"P1", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P2", Height: 200}}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "delete the tallest of two peers with block requests sent and blocks received",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 110}},
+				map[int64]tPBlocks{100: {"P1", true}, 101: {"P1", false}}),
+			args:       args{"P1", nil},
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P2", Height: 110}}, map[int64]tPBlocks{}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			tt.pool.RemovePeer(tt.args.peerID, tt.args.err)
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}
+
+func TestBlockPoolRemoveShortPeers(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		poolWanted *BlockPool
+	}{
+		{
+			name: "no short peers",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 110}, {ID: "P3", Height: 120}}, map[int64]tPBlocks{}),
+			poolWanted: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 110}, {ID: "P3", Height: 120}}, map[int64]tPBlocks{}),
+		},
+
+		{
+			name: "one short peer",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 90}, {ID: "P3", Height: 120}}, map[int64]tPBlocks{}),
+			poolWanted: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P3", Height: 120}}, map[int64]tPBlocks{}),
+		},
+
+		{
+			name: "all short peers",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 90}, {ID: "P2", Height: 91}, {ID: "P3", Height: 92}}, map[int64]tPBlocks{}),
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			pool := tt.pool
+			pool.removeShortPeers()
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}
+
+func TestBlockPoolSendRequestBatch(t *testing.T) {
+	type testPeerResult struct {
+		id                      p2p.ID
+		numPendingBlockRequests int
+	}
+
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name               string
+		pool               *BlockPool
+		maxRequestsPerPeer int
+		expRequests        map[int64]bool
+		expRequestsSent    int
+		expPeerResults     []testPeerResult
+	}{
+		{
+			name:               "one peer - send up to maxRequestsPerPeer block requests",
+			pool:               makeBlockPool(testBcR, 10, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}),
+			maxRequestsPerPeer: 2,
+			expRequests:        map[int64]bool{10: true, 11: true},
+			expRequestsSent:    2,
+			expPeerResults:     []testPeerResult{{id: "P1", numPendingBlockRequests: 2}},
+		},
+		{
+			name: "multiple peers - stops at gap between height and base",
+			pool: makeBlockPool(testBcR, 10, []BpPeer{
+				{ID: "P1", Base: 1, Height: 12},
+				{ID: "P2", Base: 15, Height: 100},
+			}, map[int64]tPBlocks{}),
+			maxRequestsPerPeer: 10,
+			expRequests:        map[int64]bool{10: true, 11: true, 12: true},
+			expRequestsSent:    3,
+			expPeerResults: []testPeerResult{
+				{id: "P1", numPendingBlockRequests: 3},
+				{id: "P2", numPendingBlockRequests: 0},
+			},
+		},
+		{
+			name: "n peers - send n*maxRequestsPerPeer block requests",
+			pool: makeBlockPool(
+				testBcR,
+				10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{}),
+			maxRequestsPerPeer: 2,
+			expRequests:        map[int64]bool{10: true, 11: true},
+			expRequestsSent:    4,
+			expPeerResults: []testPeerResult{
+				{id: "P1", numPendingBlockRequests: 2},
+				{id: "P2", numPendingBlockRequests: 2}},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			resetPoolTestResults()
+
+			var pool = tt.pool
+			maxRequestsPerPeer = tt.maxRequestsPerPeer
+			pool.MakeNextRequests(10)
+
+			assert.Equal(t, tt.expRequestsSent, testResults.numRequestsSent)
+			for _, tPeer := range tt.expPeerResults {
+				var peer = pool.peers[tPeer.id]
+				assert.NotNil(t, peer)
+				assert.Equal(t, tPeer.numPendingBlockRequests, peer.NumPendingBlockRequests)
+			}
+		})
+	}
+}
+
+func TestBlockPoolAddBlock(t *testing.T) {
+	testBcR := newTestBcR()
+	txs := []types.Tx{types.Tx("foo"), types.Tx("bar")}
+
+	type args struct {
+		peerID    p2p.ID
+		block     *types.Block
+		blockSize int
+	}
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		args       args
+		poolWanted *BlockPool
+		errWanted  error
+	}{
+		{name: "block from unknown peer",
+			pool: makeBlockPool(testBcR, 10, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}),
+			args: args{
+				peerID:    "P2",
+				block:     types.MakeBlock(int64(10), txs, nil, nil),
+				blockSize: 100,
+			},
+			poolWanted: makeBlockPool(testBcR, 10, []BpPeer{{ID: "P1", Height: 100}}, map[int64]tPBlocks{}),
+			errWanted:  errBadDataFromPeer,
+		},
+		{name: "unexpected block 11 from known peer - waiting for 10",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", false}}),
+			args: args{
+				peerID:    "P1",
+				block:     types.MakeBlock(int64(11), txs, nil, nil),
+				blockSize: 100,
+			},
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", false}}),
+			errWanted: errMissingBlock,
+		},
+		{name: "unexpected block 10 from known peer - already have 10",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", true}, 11: {"P1", false}}),
+			args: args{
+				peerID:    "P1",
+				block:     types.MakeBlock(int64(10), txs, nil, nil),
+				blockSize: 100,
+			},
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", true}, 11: {"P1", false}}),
+			errWanted: errDuplicateBlock,
+		},
+		{name: "unexpected block 10 from known peer P2 - expected 10 to come from P1",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", false}}),
+			args: args{
+				peerID:    "P2",
+				block:     types.MakeBlock(int64(10), txs, nil, nil),
+				blockSize: 100,
+			},
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", false}}),
+			errWanted: errBadDataFromPeer,
+		},
+		{name: "expected block from known peer",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", false}}),
+			args: args{
+				peerID:    "P1",
+				block:     types.MakeBlock(int64(10), txs, nil, nil),
+				blockSize: 100,
+			},
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", true}}),
+			errWanted: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			err := tt.pool.AddBlock(tt.args.peerID, tt.args.block, tt.args.blockSize)
+			assert.Equal(t, tt.errWanted, err)
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}
+
+func TestBlockPoolFirstTwoBlocksAndPeers(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name         string
+		pool         *BlockPool
+		firstWanted  int64
+		secondWanted int64
+		errWanted    error
+	}{
+		{
+			name: "both blocks missing",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{15: {"P1", true}, 16: {"P2", true}}),
+			errWanted: errMissingBlock,
+		},
+		{
+			name: "second block missing",
+			pool: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{15: {"P1", true}, 18: {"P2", true}}),
+			firstWanted: 15,
+			errWanted:   errMissingBlock,
+		},
+		{
+			name: "first block missing",
+			pool: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{16: {"P2", true}, 18: {"P2", true}}),
+			secondWanted: 16,
+			errWanted:    errMissingBlock,
+		},
+		{
+			name: "both blocks present",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", true}, 11: {"P2", true}}),
+			firstWanted:  10,
+			secondWanted: 11,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			pool := tt.pool
+			gotFirst, gotSecond, err := pool.FirstTwoBlocksAndPeers()
+			assert.Equal(t, tt.errWanted, err)
+
+			if tt.firstWanted != 0 {
+				peer := pool.blocks[tt.firstWanted]
+				block := pool.peers[peer].blocks[tt.firstWanted]
+				assert.Equal(t, block, gotFirst.block,
+					"BlockPool.FirstTwoBlocksAndPeers() gotFirst = %v, want %v",
+					tt.firstWanted, gotFirst.block.Height)
+			}
+
+			if tt.secondWanted != 0 {
+				peer := pool.blocks[tt.secondWanted]
+				block := pool.peers[peer].blocks[tt.secondWanted]
+				assert.Equal(t, block, gotSecond.block,
+					"BlockPool.FirstTwoBlocksAndPeers() gotFirst = %v, want %v",
+					tt.secondWanted, gotSecond.block.Height)
+			}
+		})
+	}
+}
+
+func TestBlockPoolInvalidateFirstTwoBlocks(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		poolWanted *BlockPool
+	}{
+		{
+			name: "both blocks missing",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{15: {"P1", true}, 16: {"P2", true}}),
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{15: {"P1", true}, 16: {"P2", true}}),
+		},
+		{
+			name: "second block missing",
+			pool: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{15: {"P1", true}, 18: {"P2", true}}),
+			poolWanted: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P2", Height: 100}},
+				map[int64]tPBlocks{18: {"P2", true}}),
+		},
+		{
+			name: "first block missing",
+			pool: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{18: {"P1", true}, 16: {"P2", true}}),
+			poolWanted: makeBlockPool(testBcR, 15,
+				[]BpPeer{{ID: "P1", Height: 100}},
+				map[int64]tPBlocks{18: {"P1", true}}),
+		},
+		{
+			name: "both blocks present",
+			pool: makeBlockPool(testBcR, 10,
+				[]BpPeer{{ID: "P1", Height: 100}, {ID: "P2", Height: 100}},
+				map[int64]tPBlocks{10: {"P1", true}, 11: {"P2", true}}),
+			poolWanted: makeBlockPool(testBcR, 10,
+				[]BpPeer{},
+				map[int64]tPBlocks{}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			tt.pool.InvalidateFirstTwoBlocks(errNoPeerResponse)
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}
+
+func TestProcessedCurrentHeightBlock(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		poolWanted *BlockPool
+	}{
+		{
+			name: "one peer",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}},
+				map[int64]tPBlocks{100: {"P1", true}, 101: {"P1", true}}),
+			poolWanted: makeBlockPool(testBcR, 101, []BpPeer{{ID: "P1", Height: 120}},
+				map[int64]tPBlocks{101: {"P1", true}}),
+		},
+		{
+			name: "multiple peers",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					100: {"P1", true}, 104: {"P1", true}, 105: {"P1", false},
+					101: {"P2", true}, 103: {"P2", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+			poolWanted: makeBlockPool(testBcR, 101,
+				[]BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					104: {"P1", true}, 105: {"P1", false},
+					101: {"P2", true}, 103: {"P2", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			tt.pool.ProcessedCurrentHeightBlock()
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}
+
+func TestRemovePeerAtCurrentHeight(t *testing.T) {
+	testBcR := newTestBcR()
+
+	tests := []struct {
+		name       string
+		pool       *BlockPool
+		poolWanted *BlockPool
+	}{
+		{
+			name: "one peer, remove peer for block at H",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}},
+				map[int64]tPBlocks{100: {"P1", false}, 101: {"P1", true}}),
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "one peer, remove peer for block at H+1",
+			pool: makeBlockPool(testBcR, 100, []BpPeer{{ID: "P1", Height: 120}},
+				map[int64]tPBlocks{100: {"P1", true}, 101: {"P1", false}}),
+			poolWanted: makeBlockPool(testBcR, 100, []BpPeer{}, map[int64]tPBlocks{}),
+		},
+		{
+			name: "multiple peers, remove peer for block at H",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					100: {"P1", false}, 104: {"P1", true}, 105: {"P1", false},
+					101: {"P2", true}, 103: {"P2", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+			poolWanted: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P2", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					101: {"P2", true}, 103: {"P2", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+		},
+		{
+			name: "multiple peers, remove peer for block at H+1",
+			pool: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 120}, {ID: "P2", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					100: {"P1", true}, 104: {"P1", true}, 105: {"P1", false},
+					101: {"P2", false}, 103: {"P2", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+			poolWanted: makeBlockPool(testBcR, 100,
+				[]BpPeer{{ID: "P1", Height: 120}, {ID: "P3", Height: 130}},
+				map[int64]tPBlocks{
+					100: {"P1", true}, 104: {"P1", true}, 105: {"P1", false},
+					102: {"P3", true}, 106: {"P3", true}}),
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			tt.pool.RemovePeerAtCurrentHeights(errNoPeerResponse)
+			assertBlockPoolEquivalent(t, tt.poolWanted, tt.pool)
+		})
+	}
+}

internal/blocksync/v1/reactor.go

@@ -0,0 +1,569 @@
+package v1
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/tendermint/tendermint/behaviour"
+	bc "github.com/tendermint/tendermint/blockchain"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
+	BlockchainChannel = byte(0x40)
+	trySyncIntervalMS = 10
+	trySendIntervalMS = 10
+
+	// ask for best height every 10s
+	statusUpdateIntervalSeconds = 10
+)
+
+var (
+	// Maximum number of requests that can be pending per peer, i.e. for which requests have been sent but blocks
+	// have not been received.
+	maxRequestsPerPeer = 20
+	// Maximum number of block requests for the reactor, pending or for which blocks have been received.
+	maxNumRequests = 64
+)
+
+type consensusReactor interface {
+	// for when we switch from blockchain reactor and fast sync to
+	// the consensus machine
+	SwitchToConsensus(state sm.State, skipWAL bool)
+}
+
+// BlockchainReactor handles long-term catchup syncing.
+type BlockchainReactor struct {
+	p2p.BaseReactor
+
+	initialState sm.State // immutable
+	state        sm.State
+
+	blockExec *sm.BlockExecutor
+	store     *store.BlockStore
+
+	fastSync    bool
+	stateSynced bool
+
+	fsm          *BcReactorFSM
+	blocksSynced uint64
+
+	// Receive goroutine forwards messages to this channel to be processed in the context of the poolRoutine.
+	messagesForFSMCh chan bcReactorMessage
+
+	// Switch goroutine may send RemovePeer to the blockchain reactor. This is an error message that is relayed
+	// to this channel to be processed in the context of the poolRoutine.
+	errorsForFSMCh chan bcReactorMessage
+
+	// This channel is used by the FSM and indirectly the block pool to report errors to the blockchain reactor and
+	// the switch.
+	eventsFromFSMCh chan bcFsmMessage
+
+	swReporter *behaviour.SwitchReporter
+}
+
+// NewBlockchainReactor returns new reactor instance.
+func NewBlockchainReactor(state sm.State, blockExec *sm.BlockExecutor, store *store.BlockStore,
+	fastSync bool) *BlockchainReactor {
+
+	if state.LastBlockHeight != store.Height() {
+		panic(fmt.Sprintf("state (%v) and store (%v) height mismatch", state.LastBlockHeight,
+			store.Height()))
+	}
+
+	const capacity = 1000
+	eventsFromFSMCh := make(chan bcFsmMessage, capacity)
+	messagesForFSMCh := make(chan bcReactorMessage, capacity)
+	errorsForFSMCh := make(chan bcReactorMessage, capacity)
+
+	startHeight := store.Height() + 1
+	if startHeight == 1 {
+		startHeight = state.InitialHeight
+	}
+	bcR := &BlockchainReactor{
+		initialState:     state,
+		state:            state,
+		blockExec:        blockExec,
+		fastSync:         fastSync,
+		store:            store,
+		messagesForFSMCh: messagesForFSMCh,
+		eventsFromFSMCh:  eventsFromFSMCh,
+		errorsForFSMCh:   errorsForFSMCh,
+	}
+	fsm := NewFSM(startHeight, bcR)
+	bcR.fsm = fsm
+	bcR.BaseReactor = *p2p.NewBaseReactor("BlockchainReactor", bcR)
+	// bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch)
+
+	return bcR
+}
+
+// bcReactorMessage is used by the reactor to send messages to the FSM.
+type bcReactorMessage struct {
+	event bReactorEvent
+	data  bReactorEventData
+}
+
+type bFsmEvent uint
+
+const (
+	// message type events
+	peerErrorEv = iota + 1
+	syncFinishedEv
+)
+
+type bFsmEventData struct {
+	peerID p2p.ID
+	err    error
+}
+
+// bcFsmMessage is used by the FSM to send messages to the reactor
+type bcFsmMessage struct {
+	event bFsmEvent
+	data  bFsmEventData
+}
+
+// SetLogger implements service.Service by setting the logger on reactor and pool.
+func (bcR *BlockchainReactor) SetLogger(l log.Logger) {
+	bcR.BaseService.Logger = l
+	bcR.fsm.SetLogger(l)
+}
+
+// OnStart implements service.Service.
+func (bcR *BlockchainReactor) OnStart() error {
+	bcR.swReporter = behaviour.NewSwitchReporter(bcR.BaseReactor.Switch)
+	if bcR.fastSync {
+		go bcR.poolRoutine()
+	}
+	return nil
+}
+
+// OnStop implements service.Service.
+func (bcR *BlockchainReactor) OnStop() {
+	_ = bcR.Stop()
+}
+
+// SwitchToFastSync is called by the state sync reactor when switching to fast sync.
+func (bcR *BlockchainReactor) SwitchToFastSync(state sm.State) error {
+	bcR.fastSync = true
+	bcR.initialState = state
+	bcR.state = state
+	bcR.stateSynced = true
+
+	bcR.fsm = NewFSM(state.LastBlockHeight+1, bcR)
+	bcR.fsm.SetLogger(bcR.Logger)
+	go bcR.poolRoutine()
+	return nil
+}
+
+// GetChannels implements Reactor
+func (bcR *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
+	return []*p2p.ChannelDescriptor{
+		{
+			ID:                  BlockchainChannel,
+			Priority:            10,
+			SendQueueCapacity:   2000,
+			RecvBufferCapacity:  50 * 4096,
+			RecvMessageCapacity: bc.MaxMsgSize,
+		},
+	}
+}
+
+// AddPeer implements Reactor by sending our state to peer.
+func (bcR *BlockchainReactor) AddPeer(peer p2p.Peer) {
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusResponse{
+		Base:   bcR.store.Base(),
+		Height: bcR.store.Height(),
+	})
+	if err != nil {
+		bcR.Logger.Error("could not convert msg to protobuf", "err", err)
+		return
+	}
+	peer.Send(BlockchainChannel, msgBytes)
+	// it's OK if send fails. will try later in poolRoutine
+
+	// peer is added to the pool once we receive the first
+	// bcStatusResponseMessage from the peer and call pool.updatePeer()
+}
+
+// sendBlockToPeer loads a block and sends it to the requesting peer.
+// If the block doesn't exist a bcNoBlockResponseMessage is sent.
+// If all nodes are honest, no node should be requesting for a block that doesn't exist.
+func (bcR *BlockchainReactor) sendBlockToPeer(msg *bcproto.BlockRequest,
+	src p2p.Peer) (queued bool) {
+
+	block := bcR.store.LoadBlock(msg.Height)
+	if block != nil {
+		pbbi, err := block.ToProto()
+		if err != nil {
+			bcR.Logger.Error("Could not send block message to peer", "err", err)
+			return false
+		}
+		msgBytes, err := bc.EncodeMsg(&bcproto.BlockResponse{Block: pbbi})
+		if err != nil {
+			bcR.Logger.Error("unable to marshal msg", "err", err)
+			return false
+		}
+		return src.TrySend(BlockchainChannel, msgBytes)
+	}
+
+	bcR.Logger.Info("peer asking for a block we don't have", "src", src, "height", msg.Height)
+
+	msgBytes, err := bc.EncodeMsg(&bcproto.NoBlockResponse{Height: msg.Height})
+	if err != nil {
+		bcR.Logger.Error("unable to marshal msg", "err", err)
+		return false
+	}
+	return src.TrySend(BlockchainChannel, msgBytes)
+}
+
+func (bcR *BlockchainReactor) sendStatusResponseToPeer(msg *bcproto.StatusRequest, src p2p.Peer) (queued bool) {
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusResponse{
+		Base:   bcR.store.Base(),
+		Height: bcR.store.Height(),
+	})
+	if err != nil {
+		bcR.Logger.Error("unable to marshal msg", "err", err)
+		return false
+	}
+
+	return src.TrySend(BlockchainChannel, msgBytes)
+}
+
+// RemovePeer implements Reactor by removing peer from the pool.
+func (bcR *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
+	msgData := bcReactorMessage{
+		event: peerRemoveEv,
+		data: bReactorEventData{
+			peerID: peer.ID(),
+			err:    errSwitchRemovesPeer,
+		},
+	}
+	bcR.errorsForFSMCh <- msgData
+}
+
+// Receive implements Reactor by handling 4 types of messages (look below).
+func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
+	msg, err := bc.DecodeMsg(msgBytes)
+	if err != nil {
+		bcR.Logger.Error("error decoding message", "src", src, "chId", chID, "err", err)
+		_ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
+		return
+	}
+
+	if err = bc.ValidateMsg(msg); err != nil {
+		bcR.Logger.Error("peer sent us invalid msg", "peer", src, "msg", msg, "err", err)
+		_ = bcR.swReporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
+		return
+	}
+
+	bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg)
+
+	switch msg := msg.(type) {
+	case *bcproto.BlockRequest:
+		if queued := bcR.sendBlockToPeer(msg, src); !queued {
+			// Unfortunately not queued since the queue is full.
+			bcR.Logger.Error("Could not send block message to peer", "src", src, "height", msg.Height)
+		}
+
+	case *bcproto.StatusRequest:
+		// Send peer our state.
+		if queued := bcR.sendStatusResponseToPeer(msg, src); !queued {
+			// Unfortunately not queued since the queue is full.
+			bcR.Logger.Error("Could not send status message to peer", "src", src)
+		}
+
+	case *bcproto.BlockResponse:
+		bi, err := types.BlockFromProto(msg.Block)
+		if err != nil {
+			bcR.Logger.Error("error transition block from protobuf", "err", err)
+			return
+		}
+		msgForFSM := bcReactorMessage{
+			event: blockResponseEv,
+			data: bReactorEventData{
+				peerID: src.ID(),
+				height: bi.Height,
+				block:  bi,
+				length: len(msgBytes),
+			},
+		}
+		bcR.Logger.Info("Received", "src", src, "height", bi.Height)
+		bcR.messagesForFSMCh <- msgForFSM
+	case *bcproto.NoBlockResponse:
+		msgForFSM := bcReactorMessage{
+			event: noBlockResponseEv,
+			data: bReactorEventData{
+				peerID: src.ID(),
+				height: msg.Height,
+			},
+		}
+		bcR.Logger.Debug("Peer does not have requested block", "peer", src, "height", msg.Height)
+		bcR.messagesForFSMCh <- msgForFSM
+
+	case *bcproto.StatusResponse:
+		// Got a peer status. Unverified.
+		msgForFSM := bcReactorMessage{
+			event: statusResponseEv,
+			data: bReactorEventData{
+				peerID: src.ID(),
+				height: msg.Height,
+				length: len(msgBytes),
+			},
+		}
+		bcR.messagesForFSMCh <- msgForFSM
+
+	default:
+		bcR.Logger.Error(fmt.Sprintf("unknown message type %v", reflect.TypeOf(msg)))
+	}
+}
+
+// processBlocksRoutine processes blocks until signlaed to stop over the stopProcessing channel
+func (bcR *BlockchainReactor) processBlocksRoutine(stopProcessing chan struct{}) {
+
+	processReceivedBlockTicker := time.NewTicker(trySyncIntervalMS * time.Millisecond)
+	doProcessBlockCh := make(chan struct{}, 1)
+
+	lastHundred := time.Now()
+	lastRate := 0.0
+
+ForLoop:
+	for {
+		select {
+		case <-stopProcessing:
+			bcR.Logger.Info("finishing block execution")
+			break ForLoop
+		case <-processReceivedBlockTicker.C: // try to execute blocks
+			select {
+			case doProcessBlockCh <- struct{}{}:
+			default:
+			}
+		case <-doProcessBlockCh:
+			for {
+				err := bcR.processBlock()
+				if err == errMissingBlock {
+					break
+				}
+				// Notify FSM of block processing result.
+				msgForFSM := bcReactorMessage{
+					event: processedBlockEv,
+					data: bReactorEventData{
+						err: err,
+					},
+				}
+				_ = bcR.fsm.Handle(&msgForFSM)
+
+				if err != nil {
+					break
+				}
+
+				bcR.blocksSynced++
+				if bcR.blocksSynced%100 == 0 {
+					lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+					height, maxPeerHeight := bcR.fsm.Status()
+					bcR.Logger.Info("Fast Sync Rate", "height", height,
+						"max_peer_height", maxPeerHeight, "blocks/s", lastRate)
+					lastHundred = time.Now()
+				}
+			}
+		}
+	}
+}
+
+// poolRoutine receives and handles messages from the Receive() routine and from the FSM.
+func (bcR *BlockchainReactor) poolRoutine() {
+
+	bcR.fsm.Start()
+
+	sendBlockRequestTicker := time.NewTicker(trySendIntervalMS * time.Millisecond)
+	statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
+
+	stopProcessing := make(chan struct{}, 1)
+	go bcR.processBlocksRoutine(stopProcessing)
+
+ForLoop:
+	for {
+		select {
+
+		case <-sendBlockRequestTicker.C:
+			if !bcR.fsm.NeedsBlocks() {
+				continue
+			}
+			_ = bcR.fsm.Handle(&bcReactorMessage{
+				event: makeRequestsEv,
+				data: bReactorEventData{
+					maxNumRequests: maxNumRequests}})
+
+		case <-statusUpdateTicker.C:
+			// Ask for status updates.
+			go bcR.sendStatusRequest()
+
+		case msg := <-bcR.messagesForFSMCh:
+			// Sent from the Receive() routine when status (statusResponseEv) and
+			// block (blockResponseEv) response events are received
+			_ = bcR.fsm.Handle(&msg)
+
+		case msg := <-bcR.errorsForFSMCh:
+			// Sent from the switch.RemovePeer() routine (RemovePeerEv) and
+			// FSM state timer expiry routine (stateTimeoutEv).
+			_ = bcR.fsm.Handle(&msg)
+
+		case msg := <-bcR.eventsFromFSMCh:
+			switch msg.event {
+			case syncFinishedEv:
+				stopProcessing <- struct{}{}
+				// Sent from the FSM when it enters finished state.
+				break ForLoop
+			case peerErrorEv:
+				// Sent from the FSM when it detects peer error
+				bcR.reportPeerErrorToSwitch(msg.data.err, msg.data.peerID)
+				if msg.data.err == errNoPeerResponse {
+					// Sent from the peer timeout handler routine
+					_ = bcR.fsm.Handle(&bcReactorMessage{
+						event: peerRemoveEv,
+						data: bReactorEventData{
+							peerID: msg.data.peerID,
+							err:    msg.data.err,
+						},
+					})
+				}
+				// else {
+				// For slow peers, or errors due to blocks received from wrong peer
+				// the FSM had already removed the peers
+				// }
+			default:
+				bcR.Logger.Error("Event from FSM not supported", "type", msg.event)
+			}
+
+		case <-bcR.Quit():
+			break ForLoop
+		}
+	}
+}
+
+func (bcR *BlockchainReactor) reportPeerErrorToSwitch(err error, peerID p2p.ID) {
+	peer := bcR.Switch.Peers().Get(peerID)
+	if peer != nil {
+		_ = bcR.swReporter.Report(behaviour.BadMessage(peerID, err.Error()))
+	}
+}
+
+func (bcR *BlockchainReactor) processBlock() error {
+
+	first, second, err := bcR.fsm.FirstTwoBlocks()
+	if err != nil {
+		// We need both to sync the first block.
+		return err
+	}
+
+	chainID := bcR.initialState.ChainID
+
+	firstParts := first.MakePartSet(types.BlockPartSizeBytes)
+	firstPartSetHeader := firstParts.Header()
+	firstID := types.BlockID{Hash: first.Hash(), PartSetHeader: firstPartSetHeader}
+	// Finally, verify the first block using the second's commit
+	// NOTE: we can probably make this more efficient, but note that calling
+	// first.Hash() doesn't verify the tx contents, so MakePartSet() is
+	// currently necessary.
+	err = bcR.state.Validators.VerifyCommitLight(chainID, firstID, first.Height, second.LastCommit)
+	if err != nil {
+		bcR.Logger.Error("error during commit verification", "err", err,
+			"first", first.Height, "second", second.Height)
+		return errBlockVerificationFailure
+	}
+
+	bcR.store.SaveBlock(first, firstParts, second.LastCommit)
+
+	bcR.state, _, err = bcR.blockExec.ApplyBlock(bcR.state, firstID, first)
+	if err != nil {
+		panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+	}
+
+	return nil
+}
+
+// Implements bcRNotifier
+// sendStatusRequest broadcasts `BlockStore` height.
+func (bcR *BlockchainReactor) sendStatusRequest() {
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusRequest{})
+	if err != nil {
+		panic(err)
+	}
+	bcR.Switch.Broadcast(BlockchainChannel, msgBytes)
+}
+
+// Implements bcRNotifier
+// BlockRequest sends `BlockRequest` height.
+func (bcR *BlockchainReactor) sendBlockRequest(peerID p2p.ID, height int64) error {
+	peer := bcR.Switch.Peers().Get(peerID)
+	if peer == nil {
+		return errNilPeerForBlockRequest
+	}
+
+	msgBytes, err := bc.EncodeMsg(&bcproto.BlockRequest{Height: height})
+	if err != nil {
+		return err
+	}
+	queued := peer.TrySend(BlockchainChannel, msgBytes)
+	if !queued {
+		return errSendQueueFull
+	}
+	return nil
+}
+
+// Implements bcRNotifier
+func (bcR *BlockchainReactor) switchToConsensus() {
+	conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor)
+	if ok {
+		conR.SwitchToConsensus(bcR.state, bcR.blocksSynced > 0 || bcR.stateSynced)
+		bcR.eventsFromFSMCh <- bcFsmMessage{event: syncFinishedEv}
+	}
+	// else {
+	// Should only happen during testing.
+	// }
+}
+
+// Implements bcRNotifier
+// Called by FSM and pool:
+// - pool calls when it detects slow peer or when peer times out
+// - FSM calls when:
+//    - adding a block (addBlock) fails
+//    - reactor processing of a block reports failure and FSM sends back the peers of first and second blocks
+func (bcR *BlockchainReactor) sendPeerError(err error, peerID p2p.ID) {
+	bcR.Logger.Info("sendPeerError:", "peer", peerID, "error", err)
+	msgData := bcFsmMessage{
+		event: peerErrorEv,
+		data: bFsmEventData{
+			peerID: peerID,
+			err:    err,
+		},
+	}
+	bcR.eventsFromFSMCh <- msgData
+}
+
+// Implements bcRNotifier
+func (bcR *BlockchainReactor) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) {
+	if timer == nil {
+		panic("nil timer pointer parameter")
+	}
+	if *timer == nil {
+		*timer = time.AfterFunc(timeout, func() {
+			msg := bcReactorMessage{
+				event: stateTimeoutEv,
+				data: bReactorEventData{
+					stateName: name,
+				},
+			}
+			bcR.errorsForFSMCh <- msg
+		})
+	} else {
+		(*timer).Reset(timeout)
+	}
+}

internal/blocksync/v1/reactor_fsm.go

@@ -0,0 +1,462 @@
+package v1
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Blockchain Reactor State
+type bcReactorFSMState struct {
+	name string
+
+	// called when transitioning out of current state
+	handle func(*BcReactorFSM, bReactorEvent, bReactorEventData) (next *bcReactorFSMState, err error)
+	// called when entering the state
+	enter func(fsm *BcReactorFSM)
+
+	// timeout to ensure FSM is not stuck in a state forever
+	// the timer is owned and run by the fsm instance
+	timeout time.Duration
+}
+
+func (s *bcReactorFSMState) String() string {
+	return s.name
+}
+
+// BcReactorFSM is the datastructure for the Blockchain Reactor State Machine
+type BcReactorFSM struct {
+	logger log.Logger
+	mtx    sync.Mutex
+
+	startTime time.Time
+
+	state      *bcReactorFSMState
+	stateTimer *time.Timer
+	pool       *BlockPool
+
+	// interface used to call the Blockchain reactor to send StatusRequest, BlockRequest, reporting errors, etc.
+	toBcR bcReactor
+}
+
+// NewFSM creates a new reactor FSM.
+func NewFSM(height int64, toBcR bcReactor) *BcReactorFSM {
+	return &BcReactorFSM{
+		state:     unknown,
+		startTime: time.Now(),
+		pool:      NewBlockPool(height, toBcR),
+		toBcR:     toBcR,
+	}
+}
+
+// bReactorEventData is part of the message sent by the reactor to the FSM and used by the state handlers.
+type bReactorEventData struct {
+	peerID         p2p.ID
+	err            error        // for peer error: timeout, slow; for processed block event if error occurred
+	base           int64        // for status response
+	height         int64        // for status response; for processed block event
+	block          *types.Block // for block response
+	stateName      string       // for state timeout events
+	length         int          // for block response event, length of received block, used to detect slow peers
+	maxNumRequests int          // for request needed event, maximum number of pending requests
+}
+
+// Blockchain Reactor Events (the input to the state machine)
+type bReactorEvent uint
+
+const (
+	// message type events
+	startFSMEv = iota + 1
+	statusResponseEv
+	blockResponseEv
+	noBlockResponseEv
+	processedBlockEv
+	makeRequestsEv
+	stopFSMEv
+
+	// other events
+	peerRemoveEv = iota + 256
+	stateTimeoutEv
+)
+
+func (msg *bcReactorMessage) String() string {
+	var dataStr string
+
+	switch msg.event {
+	case startFSMEv:
+		dataStr = ""
+	case statusResponseEv:
+		dataStr = fmt.Sprintf("peer=%v base=%v height=%v", msg.data.peerID, msg.data.base, msg.data.height)
+	case blockResponseEv:
+		dataStr = fmt.Sprintf("peer=%v block.height=%v length=%v",
+			msg.data.peerID, msg.data.block.Height, msg.data.length)
+	case noBlockResponseEv:
+		dataStr = fmt.Sprintf("peer=%v requested height=%v",
+			msg.data.peerID, msg.data.height)
+	case processedBlockEv:
+		dataStr = fmt.Sprintf("error=%v", msg.data.err)
+	case makeRequestsEv:
+		dataStr = ""
+	case stopFSMEv:
+		dataStr = ""
+	case peerRemoveEv:
+		dataStr = fmt.Sprintf("peer: %v is being removed by the switch", msg.data.peerID)
+	case stateTimeoutEv:
+		dataStr = fmt.Sprintf("state=%v", msg.data.stateName)
+	default:
+		dataStr = "cannot interpret message data"
+	}
+
+	return fmt.Sprintf("%v: %v", msg.event, dataStr)
+}
+
+func (ev bReactorEvent) String() string {
+	switch ev {
+	case startFSMEv:
+		return "startFSMEv"
+	case statusResponseEv:
+		return "statusResponseEv"
+	case blockResponseEv:
+		return "blockResponseEv"
+	case noBlockResponseEv:
+		return "noBlockResponseEv"
+	case processedBlockEv:
+		return "processedBlockEv"
+	case makeRequestsEv:
+		return "makeRequestsEv"
+	case stopFSMEv:
+		return "stopFSMEv"
+	case peerRemoveEv:
+		return "peerRemoveEv"
+	case stateTimeoutEv:
+		return "stateTimeoutEv"
+	default:
+		return "event unknown"
+	}
+
+}
+
+// states
+var (
+	unknown      *bcReactorFSMState
+	waitForPeer  *bcReactorFSMState
+	waitForBlock *bcReactorFSMState
+	finished     *bcReactorFSMState
+)
+
+// timeouts for state timers
+const (
+	waitForPeerTimeout                 = 3 * time.Second
+	waitForBlockAtCurrentHeightTimeout = 10 * time.Second
+)
+
+// errors
+var (
+	// internal to the package
+	errNoErrorFinished        = errors.New("fast sync is finished")
+	errInvalidEvent           = errors.New("invalid event in current state")
+	errMissingBlock           = errors.New("missing blocks")
+	errNilPeerForBlockRequest = errors.New("peer for block request does not exist in the switch")
+	errSendQueueFull          = errors.New("block request not made, send-queue is full")
+	errPeerTooShort           = errors.New("peer height too low, old peer removed/ new peer not added")
+	errSwitchRemovesPeer      = errors.New("switch is removing peer")
+	errTimeoutEventWrongState = errors.New("timeout event for a state different than the current one")
+	errNoTallerPeer           = errors.New("fast sync timed out on waiting for a peer taller than this node")
+
+	// reported eventually to the switch
+	// handle return
+	errPeerLowersItsHeight = errors.New("fast sync peer reports a height lower than previous")
+	// handle return
+	errNoPeerResponseForCurrentHeights = errors.New("fast sync timed out on peer block response for current heights")
+	errNoPeerResponse                  = errors.New("fast sync timed out on peer block response")               // xx
+	errBadDataFromPeer                 = errors.New("fast sync received block from wrong peer or block is bad") // xx
+	errDuplicateBlock                  = errors.New("fast sync received duplicate block from peer")
+	errBlockVerificationFailure        = errors.New("fast sync block verification failure")              // xx
+	errSlowPeer                        = errors.New("fast sync peer is not sending us data fast enough") // xx
+
+)
+
+func init() {
+	unknown = &bcReactorFSMState{
+		name: "unknown",
+		handle: func(fsm *BcReactorFSM, ev bReactorEvent, data bReactorEventData) (*bcReactorFSMState, error) {
+			switch ev {
+			case startFSMEv:
+				// Broadcast Status message. Currently doesn't return non-nil error.
+				fsm.toBcR.sendStatusRequest()
+				return waitForPeer, nil
+
+			case stopFSMEv:
+				return finished, errNoErrorFinished
+
+			default:
+				return unknown, errInvalidEvent
+			}
+		},
+	}
+
+	waitForPeer = &bcReactorFSMState{
+		name:    "waitForPeer",
+		timeout: waitForPeerTimeout,
+		enter: func(fsm *BcReactorFSM) {
+			// Stop when leaving the state.
+			fsm.resetStateTimer()
+		},
+		handle: func(fsm *BcReactorFSM, ev bReactorEvent, data bReactorEventData) (*bcReactorFSMState, error) {
+			switch ev {
+			case stateTimeoutEv:
+				if data.stateName != "waitForPeer" {
+					fsm.logger.Error("received a state timeout event for different state",
+						"state", data.stateName)
+					return waitForPeer, errTimeoutEventWrongState
+				}
+				// There was no statusResponse received from any peer.
+				// Should we send status request again?
+				return finished, errNoTallerPeer
+
+			case statusResponseEv:
+				if err := fsm.pool.UpdatePeer(data.peerID, data.base, data.height); err != nil {
+					if fsm.pool.NumPeers() == 0 {
+						return waitForPeer, err
+					}
+				}
+				if fsm.stateTimer != nil {
+					fsm.stateTimer.Stop()
+				}
+				return waitForBlock, nil
+
+			case stopFSMEv:
+				if fsm.stateTimer != nil {
+					fsm.stateTimer.Stop()
+				}
+				return finished, errNoErrorFinished
+
+			default:
+				return waitForPeer, errInvalidEvent
+			}
+		},
+	}
+
+	waitForBlock = &bcReactorFSMState{
+		name:    "waitForBlock",
+		timeout: waitForBlockAtCurrentHeightTimeout,
+		enter: func(fsm *BcReactorFSM) {
+			// Stop when leaving the state.
+			fsm.resetStateTimer()
+		},
+		handle: func(fsm *BcReactorFSM, ev bReactorEvent, data bReactorEventData) (*bcReactorFSMState, error) {
+			switch ev {
+
+			case statusResponseEv:
+				err := fsm.pool.UpdatePeer(data.peerID, data.base, data.height)
+				if fsm.pool.NumPeers() == 0 {
+					return waitForPeer, err
+				}
+				if fsm.pool.ReachedMaxHeight() {
+					return finished, err
+				}
+				return waitForBlock, err
+
+			case blockResponseEv:
+				fsm.logger.Debug("blockResponseEv", "H", data.block.Height)
+				err := fsm.pool.AddBlock(data.peerID, data.block, data.length)
+				if err != nil {
+					// A block was received that was unsolicited, from unexpected peer, or that we already have it.
+					// Ignore block, remove peer and send error to switch.
+					fsm.pool.RemovePeer(data.peerID, err)
+					fsm.toBcR.sendPeerError(err, data.peerID)
+				}
+				if fsm.pool.NumPeers() == 0 {
+					return waitForPeer, err
+				}
+				return waitForBlock, err
+			case noBlockResponseEv:
+				fsm.logger.Error("peer does not have requested block", "peer", data.peerID)
+
+				return waitForBlock, nil
+			case processedBlockEv:
+				if data.err != nil {
+					first, second, _ := fsm.pool.FirstTwoBlocksAndPeers()
+					fsm.logger.Error("error processing block", "err", data.err,
+						"first", first.block.Height, "second", second.block.Height)
+					fsm.logger.Error("send peer error for", "peer", first.peer.ID)
+					fsm.toBcR.sendPeerError(data.err, first.peer.ID)
+					fsm.logger.Error("send peer error for", "peer", second.peer.ID)
+					fsm.toBcR.sendPeerError(data.err, second.peer.ID)
+					// Remove the first two blocks. This will also remove the peers
+					fsm.pool.InvalidateFirstTwoBlocks(data.err)
+				} else {
+					fsm.pool.ProcessedCurrentHeightBlock()
+					// Since we advanced one block reset the state timer
+					fsm.resetStateTimer()
+				}
+
+				// Both cases above may result in achieving maximum height.
+				if fsm.pool.ReachedMaxHeight() {
+					return finished, nil
+				}
+
+				return waitForBlock, data.err
+
+			case peerRemoveEv:
+				// This event is sent by the switch to remove disconnected and errored peers.
+				fsm.pool.RemovePeer(data.peerID, data.err)
+				if fsm.pool.NumPeers() == 0 {
+					return waitForPeer, nil
+				}
+				if fsm.pool.ReachedMaxHeight() {
+					return finished, nil
+				}
+				return waitForBlock, nil
+
+			case makeRequestsEv:
+				fsm.makeNextRequests(data.maxNumRequests)
+				return waitForBlock, nil
+
+			case stateTimeoutEv:
+				if data.stateName != "waitForBlock" {
+					fsm.logger.Error("received a state timeout event for different state",
+						"state", data.stateName)
+					return waitForBlock, errTimeoutEventWrongState
+				}
+				// We haven't received the block at current height or height+1. Remove peer.
+				fsm.pool.RemovePeerAtCurrentHeights(errNoPeerResponseForCurrentHeights)
+				fsm.resetStateTimer()
+				if fsm.pool.NumPeers() == 0 {
+					return waitForPeer, errNoPeerResponseForCurrentHeights
+				}
+				if fsm.pool.ReachedMaxHeight() {
+					return finished, nil
+				}
+				return waitForBlock, errNoPeerResponseForCurrentHeights
+
+			case stopFSMEv:
+				if fsm.stateTimer != nil {
+					fsm.stateTimer.Stop()
+				}
+				return finished, errNoErrorFinished
+
+			default:
+				return waitForBlock, errInvalidEvent
+			}
+		},
+	}
+
+	finished = &bcReactorFSMState{
+		name: "finished",
+		enter: func(fsm *BcReactorFSM) {
+			fsm.logger.Info("Time to switch to consensus reactor!", "height", fsm.pool.Height)
+			fsm.toBcR.switchToConsensus()
+			fsm.cleanup()
+		},
+		handle: func(fsm *BcReactorFSM, ev bReactorEvent, data bReactorEventData) (*bcReactorFSMState, error) {
+			return finished, nil
+		},
+	}
+}
+
+// Interface used by FSM for sending Block and Status requests,
+// informing of peer errors and state timeouts
+// Implemented by BlockchainReactor and tests
+type bcReactor interface {
+	sendStatusRequest()
+	sendBlockRequest(peerID p2p.ID, height int64) error
+	sendPeerError(err error, peerID p2p.ID)
+	resetStateTimer(name string, timer **time.Timer, timeout time.Duration)
+	switchToConsensus()
+}
+
+// SetLogger sets the FSM logger.
+func (fsm *BcReactorFSM) SetLogger(l log.Logger) {
+	fsm.logger = l
+	fsm.pool.SetLogger(l)
+}
+
+// Start starts the FSM.
+func (fsm *BcReactorFSM) Start() {
+	_ = fsm.Handle(&bcReactorMessage{event: startFSMEv})
+}
+
+// Handle processes messages and events sent to the FSM.
+func (fsm *BcReactorFSM) Handle(msg *bcReactorMessage) error {
+	fsm.mtx.Lock()
+	defer fsm.mtx.Unlock()
+	fsm.logger.Debug("FSM received", "event", msg, "state", fsm.state)
+
+	if fsm.state == nil {
+		fsm.state = unknown
+	}
+	next, err := fsm.state.handle(fsm, msg.event, msg.data)
+	if err != nil {
+		fsm.logger.Error("FSM event handler returned", "err", err,
+			"state", fsm.state, "event", msg.event)
+	}
+
+	oldState := fsm.state.name
+	fsm.transition(next)
+	if oldState != fsm.state.name {
+		fsm.logger.Info("FSM changed state", "new_state", fsm.state)
+	}
+	return err
+}
+
+func (fsm *BcReactorFSM) transition(next *bcReactorFSMState) {
+	if next == nil {
+		return
+	}
+	if fsm.state != next {
+		fsm.state = next
+		if next.enter != nil {
+			next.enter(fsm)
+		}
+	}
+}
+
+// Called when entering an FSM state in order to detect lack of progress in the state machine.
+// Note the use of the 'bcr' interface to facilitate testing without timer expiring.
+func (fsm *BcReactorFSM) resetStateTimer() {
+	fsm.toBcR.resetStateTimer(fsm.state.name, &fsm.stateTimer, fsm.state.timeout)
+}
+
+func (fsm *BcReactorFSM) isCaughtUp() bool {
+	return fsm.state == finished
+}
+
+func (fsm *BcReactorFSM) makeNextRequests(maxNumRequests int) {
+	fsm.pool.MakeNextRequests(maxNumRequests)
+}
+
+func (fsm *BcReactorFSM) cleanup() {
+	fsm.pool.Cleanup()
+}
+
+// NeedsBlocks checks if more block requests are required.
+func (fsm *BcReactorFSM) NeedsBlocks() bool {
+	fsm.mtx.Lock()
+	defer fsm.mtx.Unlock()
+	return fsm.state.name == "waitForBlock" && fsm.pool.NeedsBlocks()
+}
+
+// FirstTwoBlocks returns the two blocks at pool height and height+1
+func (fsm *BcReactorFSM) FirstTwoBlocks() (first, second *types.Block, err error) {
+	fsm.mtx.Lock()
+	defer fsm.mtx.Unlock()
+	firstBP, secondBP, err := fsm.pool.FirstTwoBlocksAndPeers()
+	if err == nil {
+		first = firstBP.block
+		second = secondBP.block
+	}
+	return
+}
+
+// Status returns the pool's height and the maximum peer height.
+func (fsm *BcReactorFSM) Status() (height, maxPeerHeight int64) {
+	fsm.mtx.Lock()
+	defer fsm.mtx.Unlock()
+	return fsm.pool.Height, fsm.pool.MaxPeerHeight
+}

internal/blocksync/v1/reactor_fsm_test.go

@@ -0,0 +1,944 @@
+package v1
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/libs/log"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+type lastBlockRequestT struct {
+	peerID p2p.ID
+	height int64
+}
+
+type lastPeerErrorT struct {
+	peerID p2p.ID
+	err    error
+}
+
+// reactor for FSM testing
+type testReactor struct {
+	logger            log.Logger
+	fsm               *BcReactorFSM
+	numStatusRequests int
+	numBlockRequests  int
+	lastBlockRequest  lastBlockRequestT
+	lastPeerError     lastPeerErrorT
+	stateTimerStarts  map[string]int
+}
+
+func sendEventToFSM(fsm *BcReactorFSM, ev bReactorEvent, data bReactorEventData) error {
+	return fsm.Handle(&bcReactorMessage{event: ev, data: data})
+}
+
+type fsmStepTestValues struct {
+	currentState string
+	event        bReactorEvent
+	data         bReactorEventData
+
+	wantErr           error
+	wantState         string
+	wantStatusReqSent bool
+	wantReqIncreased  bool
+	wantNewBlocks     []int64
+	wantRemovedPeers  []p2p.ID
+}
+
+// ---------------------------------------------------------------------------
+// helper test function for different FSM events, state and expected behavior
+func sStopFSMEv(current, expected string) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        stopFSMEv,
+		wantState:    expected,
+		wantErr:      errNoErrorFinished}
+}
+
+func sUnknownFSMEv(current string) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        1234,
+		wantState:    current,
+		wantErr:      errInvalidEvent}
+}
+
+func sStartFSMEv() fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState:      "unknown",
+		event:             startFSMEv,
+		wantState:         "waitForPeer",
+		wantStatusReqSent: true}
+}
+
+func sStateTimeoutEv(current, expected string, timedoutState string, wantErr error) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        stateTimeoutEv,
+		data: bReactorEventData{
+			stateName: timedoutState,
+		},
+		wantState: expected,
+		wantErr:   wantErr,
+	}
+}
+
+func sProcessedBlockEv(current, expected string, reactorError error) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        processedBlockEv,
+		data: bReactorEventData{
+			err: reactorError,
+		},
+		wantState: expected,
+		wantErr:   reactorError,
+	}
+}
+
+func sStatusEv(current, expected string, peerID p2p.ID, height int64, err error) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        statusResponseEv,
+		data:         bReactorEventData{peerID: peerID, height: height},
+		wantState:    expected,
+		wantErr:      err}
+}
+
+func sMakeRequestsEv(current, expected string, maxPendingRequests int) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState:     current,
+		event:            makeRequestsEv,
+		data:             bReactorEventData{maxNumRequests: maxPendingRequests},
+		wantState:        expected,
+		wantReqIncreased: true,
+	}
+}
+
+func sMakeRequestsEvErrored(current, expected string,
+	maxPendingRequests int, err error, peersRemoved []p2p.ID) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState:     current,
+		event:            makeRequestsEv,
+		data:             bReactorEventData{maxNumRequests: maxPendingRequests},
+		wantState:        expected,
+		wantErr:          err,
+		wantRemovedPeers: peersRemoved,
+		wantReqIncreased: true,
+	}
+}
+
+func sBlockRespEv(current, expected string, peerID p2p.ID, height int64, prevBlocks []int64) fsmStepTestValues {
+	txs := []types.Tx{types.Tx("foo"), types.Tx("bar")}
+	return fsmStepTestValues{
+		currentState: current,
+		event:        blockResponseEv,
+		data: bReactorEventData{
+			peerID: peerID,
+			height: height,
+			block:  types.MakeBlock(height, txs, nil, nil),
+			length: 100},
+		wantState:     expected,
+		wantNewBlocks: append(prevBlocks, height),
+	}
+}
+
+func sBlockRespEvErrored(current, expected string,
+	peerID p2p.ID, height int64, prevBlocks []int64, wantErr error, peersRemoved []p2p.ID) fsmStepTestValues {
+	txs := []types.Tx{types.Tx("foo"), types.Tx("bar")}
+
+	return fsmStepTestValues{
+		currentState: current,
+		event:        blockResponseEv,
+		data: bReactorEventData{
+			peerID: peerID,
+			height: height,
+			block:  types.MakeBlock(height, txs, nil, nil),
+			length: 100},
+		wantState:        expected,
+		wantErr:          wantErr,
+		wantRemovedPeers: peersRemoved,
+		wantNewBlocks:    prevBlocks,
+	}
+}
+
+func sPeerRemoveEv(current, expected string, peerID p2p.ID, err error, peersRemoved []p2p.ID) fsmStepTestValues {
+	return fsmStepTestValues{
+		currentState: current,
+		event:        peerRemoveEv,
+		data: bReactorEventData{
+			peerID: peerID,
+			err:    err,
+		},
+		wantState:        expected,
+		wantRemovedPeers: peersRemoved,
+	}
+}
+
+// --------------------------------------------
+
+func newTestReactor(height int64) *testReactor {
+	testBcR := &testReactor{logger: log.TestingLogger(), stateTimerStarts: make(map[string]int)}
+	testBcR.fsm = NewFSM(height, testBcR)
+	testBcR.fsm.SetLogger(testBcR.logger)
+	return testBcR
+}
+
+func fixBlockResponseEvStep(step *fsmStepTestValues, testBcR *testReactor) {
+	// There is currently no good way to know to which peer a block request was sent.
+	// So in some cases where it does not matter, before we simulate a block response
+	// we cheat and look where it is expected from.
+	if step.event == blockResponseEv {
+		height := step.data.height
+		peerID, ok := testBcR.fsm.pool.blocks[height]
+		if ok {
+			step.data.peerID = peerID
+		}
+	}
+}
+
+type testFields struct {
+	name               string
+	startingHeight     int64
+	maxRequestsPerPeer int
+	maxPendingRequests int
+	steps              []fsmStepTestValues
+}
+
+func executeFSMTests(t *testing.T, tests []testFields, matchRespToReq bool) {
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			// Create test reactor
+			testBcR := newTestReactor(tt.startingHeight)
+
+			if tt.maxRequestsPerPeer != 0 {
+				maxRequestsPerPeer = tt.maxRequestsPerPeer
+			}
+
+			for _, step := range tt.steps {
+				step := step
+				assert.Equal(t, step.currentState, testBcR.fsm.state.name)
+
+				var heightBefore int64
+				if step.event == processedBlockEv && step.data.err == errBlockVerificationFailure {
+					heightBefore = testBcR.fsm.pool.Height
+				}
+				oldNumStatusRequests := testBcR.numStatusRequests
+				oldNumBlockRequests := testBcR.numBlockRequests
+				if matchRespToReq {
+					fixBlockResponseEvStep(&step, testBcR)
+				}
+
+				fsmErr := sendEventToFSM(testBcR.fsm, step.event, step.data)
+				assert.Equal(t, step.wantErr, fsmErr)
+
+				if step.wantStatusReqSent {
+					assert.Equal(t, oldNumStatusRequests+1, testBcR.numStatusRequests)
+				} else {
+					assert.Equal(t, oldNumStatusRequests, testBcR.numStatusRequests)
+				}
+
+				if step.wantReqIncreased {
+					assert.True(t, oldNumBlockRequests < testBcR.numBlockRequests)
+				} else {
+					assert.Equal(t, oldNumBlockRequests, testBcR.numBlockRequests)
+				}
+
+				for _, height := range step.wantNewBlocks {
+					_, err := testBcR.fsm.pool.BlockAndPeerAtHeight(height)
+					assert.Nil(t, err)
+				}
+				if step.event == processedBlockEv && step.data.err == errBlockVerificationFailure {
+					heightAfter := testBcR.fsm.pool.Height
+					assert.Equal(t, heightBefore, heightAfter)
+					firstAfter, err1 := testBcR.fsm.pool.BlockAndPeerAtHeight(testBcR.fsm.pool.Height)
+					secondAfter, err2 := testBcR.fsm.pool.BlockAndPeerAtHeight(testBcR.fsm.pool.Height + 1)
+					assert.NotNil(t, err1)
+					assert.NotNil(t, err2)
+					assert.Nil(t, firstAfter)
+					assert.Nil(t, secondAfter)
+				}
+
+				assert.Equal(t, step.wantState, testBcR.fsm.state.name)
+
+				if step.wantState == "finished" {
+					assert.True(t, testBcR.fsm.isCaughtUp())
+				}
+			}
+		})
+	}
+}
+
+func TestFSMBasic(t *testing.T) {
+	tests := []testFields{
+		{
+			name:               "one block, one peer - TS2",
+			startingHeight:     1,
+			maxRequestsPerPeer: 2,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 2, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 2, []int64{1}),
+				sProcessedBlockEv("waitForBlock", "finished", nil),
+			},
+		},
+		{
+			name:               "multi block, multi peer - TS2",
+			startingHeight:     1,
+			maxRequestsPerPeer: 2,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 4, nil),
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 4, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 2, []int64{1}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 3, []int64{1, 2}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 4, []int64{1, 2, 3}),
+
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+				sProcessedBlockEv("waitForBlock", "finished", nil),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, true)
+}
+
+func TestFSMBlockVerificationFailure(t *testing.T) {
+	tests := []testFields{
+		{
+			name:               "block verification failure - TS2 variant",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+
+				// add P1 and get blocks 1-3 from it
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 2, []int64{1}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 3, []int64{1, 2}),
+
+				// add P2
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, nil),
+
+				// process block failure, should remove P1 and all blocks
+				sProcessedBlockEv("waitForBlock", "waitForBlock", errBlockVerificationFailure),
+
+				// get blocks 1-3 from P2
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 2, []int64{1}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 3, []int64{1, 2}),
+
+				// finish after processing blocks 1 and 2
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+				sProcessedBlockEv("waitForBlock", "finished", nil),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, false)
+}
+
+func TestFSMBadBlockFromPeer(t *testing.T) {
+	tests := []testFields{
+		{
+			name:               "block we haven't asked for",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1 and ask for blocks 1-3
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 300, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				// blockResponseEv for height 100 should cause an error
+				sBlockRespEvErrored("waitForBlock", "waitForPeer",
+					"P1", 100, []int64{}, errMissingBlock, []p2p.ID{}),
+			},
+		},
+		{
+			name:               "block we already have",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1 and get block 1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 100, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock",
+					"P1", 1, []int64{}),
+
+				// Get block 1 again. Since peer is removed together with block 1,
+				// the blocks present in the pool should be {}
+				sBlockRespEvErrored("waitForBlock", "waitForPeer",
+					"P1", 1, []int64{}, errDuplicateBlock, []p2p.ID{"P1"}),
+			},
+		},
+		{
+			name:               "block from unknown peer",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1 and get block 1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+
+				// get block 1 from unknown peer P2
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEvErrored("waitForBlock", "waitForBlock",
+					"P2", 1, []int64{}, errBadDataFromPeer, []p2p.ID{"P2"}),
+			},
+		},
+		{
+			name:               "block from wrong peer",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1, make requests for blocks 1-3 to P1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				// add P2
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, nil),
+
+				// receive block 1 from P2
+				sBlockRespEvErrored("waitForBlock", "waitForBlock",
+					"P2", 1, []int64{}, errBadDataFromPeer, []p2p.ID{"P2"}),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, false)
+}
+
+func TestFSMBlockAtCurrentHeightDoesNotArriveInTime(t *testing.T) {
+	tests := []testFields{
+		{
+			name:               "block at current height undelivered - TS5",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1, get blocks 1 and 2, process block 1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock",
+					"P1", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock",
+					"P1", 2, []int64{1}),
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+
+				// add P2
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, nil),
+
+				// timeout on block 3, P1 should be removed
+				sStateTimeoutEv("waitForBlock", "waitForBlock", "waitForBlock", errNoPeerResponseForCurrentHeights),
+
+				// make requests and finish by receiving blocks 2 and 3 from P2
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 2, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P2", 3, []int64{2}),
+				sProcessedBlockEv("waitForBlock", "finished", nil),
+			},
+		},
+		{
+			name:               "block at current height undelivered, at maxPeerHeight after peer removal - TS3",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1, request blocks 1-3 from P1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				// add P2 (tallest)
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 30, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				// receive blocks 1-3 from P1
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 1, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 2, []int64{1}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 3, []int64{1, 2}),
+
+				// process blocks at heights 1 and 2
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+
+				// timeout on block at height 4
+				sStateTimeoutEv("waitForBlock", "finished", "waitForBlock", nil),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, true)
+}
+
+func TestFSMPeerRelatedEvents(t *testing.T) {
+	tests := []testFields{
+		{
+			name:           "peer remove event with no blocks",
+			startingHeight: 1,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1, P2, P3
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, nil),
+				sStatusEv("waitForBlock", "waitForBlock", "P3", 3, nil),
+
+				// switch removes P2
+				sPeerRemoveEv("waitForBlock", "waitForBlock", "P2", errSwitchRemovesPeer, []p2p.ID{"P2"}),
+			},
+		},
+		{
+			name:           "only peer removed while in waitForBlock state",
+			startingHeight: 100,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 200, nil),
+
+				// switch removes P1
+				sPeerRemoveEv("waitForBlock", "waitForPeer", "P1", errSwitchRemovesPeer, []p2p.ID{"P1"}),
+			},
+		},
+		{
+			name:               "highest peer removed while in waitForBlock state, node reaches maxPeerHeight - TS4 ",
+			startingHeight:     100,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1 and make requests
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 101, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				// add P2
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 200, nil),
+
+				// get blocks 100 and 101 from P1 and process block at height 100
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 100, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 101, []int64{100}),
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+
+				// switch removes peer P1, should be finished
+				sPeerRemoveEv("waitForBlock", "finished", "P2", errSwitchRemovesPeer, []p2p.ID{"P2"}),
+			},
+		},
+		{
+			name:               "highest peer lowers its height in waitForBlock state, node reaches maxPeerHeight - TS4",
+			startingHeight:     100,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1 and make requests
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 101, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+
+				// add P2
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 200, nil),
+
+				// get blocks 100 and 101 from P1
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 100, []int64{}),
+				sBlockRespEv("waitForBlock", "waitForBlock", "P1", 101, []int64{100}),
+
+				// processed block at heights 100
+				sProcessedBlockEv("waitForBlock", "waitForBlock", nil),
+
+				// P2 becomes short
+				sStatusEv("waitForBlock", "finished", "P2", 100, errPeerLowersItsHeight),
+			},
+		},
+		{
+			name:           "new short peer while in waitForPeer state",
+			startingHeight: 100,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForPeer", "P1", 3, errPeerTooShort),
+			},
+		},
+		{
+			name:           "new short peer while in waitForBlock state",
+			startingHeight: 100,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 200, nil),
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, errPeerTooShort),
+			},
+		},
+		{
+			name:           "only peer updated with low height while in waitForBlock state",
+			startingHeight: 100,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 200, nil),
+				sStatusEv("waitForBlock", "waitForPeer", "P1", 3, errPeerLowersItsHeight),
+			},
+		},
+		{
+			name:               "peer does not exist in the switch",
+			startingHeight:     9999999,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				// add P1
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 20000000, nil),
+				// send request for block 9999999
+				// Note: For this block request the "switch missing the peer" error is simulated,
+				// see implementation of bcReactor interface, sendBlockRequest(), in this file.
+				sMakeRequestsEvErrored("waitForBlock", "waitForBlock",
+					maxNumRequests, nil, []p2p.ID{"P1"}),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, true)
+}
+
+func TestFSMStopFSM(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "stopFSMEv in unknown",
+			steps: []fsmStepTestValues{
+				sStopFSMEv("unknown", "finished"),
+			},
+		},
+		{
+			name:           "stopFSMEv in waitForPeer",
+			startingHeight: 1,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStopFSMEv("waitForPeer", "finished"),
+			},
+		},
+		{
+			name:           "stopFSMEv in waitForBlock",
+			startingHeight: 1,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sStopFSMEv("waitForBlock", "finished"),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, false)
+}
+
+func TestFSMUnknownElements(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "unknown event for state unknown",
+			steps: []fsmStepTestValues{
+				sUnknownFSMEv("unknown"),
+			},
+		},
+		{
+			name: "unknown event for state waitForPeer",
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sUnknownFSMEv("waitForPeer"),
+			},
+		},
+		{
+			name:           "unknown event for state waitForBlock",
+			startingHeight: 1,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sUnknownFSMEv("waitForBlock"),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, false)
+}
+
+func TestFSMPeerStateTimeoutEvent(t *testing.T) {
+	tests := []testFields{
+		{
+			name:               "timeout event for state waitForPeer while in state waitForPeer - TS1",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStateTimeoutEv("waitForPeer", "finished", "waitForPeer", errNoTallerPeer),
+			},
+		},
+		{
+			name:               "timeout event for state waitForPeer while in a state != waitForPeer",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStateTimeoutEv("waitForPeer", "waitForPeer", "waitForBlock", errTimeoutEventWrongState),
+			},
+		},
+		{
+			name:               "timeout event for state waitForBlock while in state waitForBlock ",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sStateTimeoutEv("waitForBlock", "waitForPeer", "waitForBlock", errNoPeerResponseForCurrentHeights),
+			},
+		},
+		{
+			name:               "timeout event for state waitForBlock while in a state != waitForBlock",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sStateTimeoutEv("waitForBlock", "waitForBlock", "waitForPeer", errTimeoutEventWrongState),
+			},
+		},
+		{
+			name:               "timeout event for state waitForBlock with multiple peers",
+			startingHeight:     1,
+			maxRequestsPerPeer: 3,
+			steps: []fsmStepTestValues{
+				sStartFSMEv(),
+				sStatusEv("waitForPeer", "waitForBlock", "P1", 3, nil),
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+				sStatusEv("waitForBlock", "waitForBlock", "P2", 3, nil),
+				sStateTimeoutEv("waitForBlock", "waitForBlock", "waitForBlock", errNoPeerResponseForCurrentHeights),
+			},
+		},
+	}
+
+	executeFSMTests(t, tests, false)
+}
+
+func makeCorrectTransitionSequence(startingHeight int64, numBlocks int64, numPeers int, randomPeerHeights bool,
+	maxRequestsPerPeer int, maxPendingRequests int) testFields {
+
+	// Generate numPeers peers with random or numBlocks heights according to the randomPeerHeights flag.
+	peerHeights := make([]int64, numPeers)
+	for i := 0; i < numPeers; i++ {
+		if i == 0 {
+			peerHeights[0] = numBlocks
+			continue
+		}
+		if randomPeerHeights {
+			peerHeights[i] = int64(tmmath.MaxInt(tmrand.Intn(int(numBlocks)), int(startingHeight)+1))
+		} else {
+			peerHeights[i] = numBlocks
+		}
+	}
+
+	// Approximate the slice capacity to save time for appends.
+	testSteps := make([]fsmStepTestValues, 0, 3*numBlocks+int64(numPeers))
+
+	testName := fmt.Sprintf("%v-blocks %v-startingHeight %v-peers %v-maxRequestsPerPeer %v-maxNumRequests",
+		numBlocks, startingHeight, numPeers, maxRequestsPerPeer, maxPendingRequests)
+
+	// Add startFSMEv step.
+	testSteps = append(testSteps, sStartFSMEv())
+
+	// For each peer, add statusResponseEv step.
+	for i := 0; i < numPeers; i++ {
+		peerName := fmt.Sprintf("P%d", i)
+		if i == 0 {
+			testSteps = append(
+				testSteps,
+				sStatusEv("waitForPeer", "waitForBlock", p2p.ID(peerName), peerHeights[i], nil))
+		} else {
+			testSteps = append(testSteps,
+				sStatusEv("waitForBlock", "waitForBlock", p2p.ID(peerName), peerHeights[i], nil))
+		}
+	}
+
+	height := startingHeight
+	numBlocksReceived := 0
+	prevBlocks := make([]int64, 0, maxPendingRequests)
+
+forLoop:
+	for i := 0; i < int(numBlocks); i++ {
+
+		// Add the makeRequestEv step periodically.
+		if i%maxRequestsPerPeer == 0 {
+			testSteps = append(
+				testSteps,
+				sMakeRequestsEv("waitForBlock", "waitForBlock", maxNumRequests),
+			)
+		}
+
+		// Add the blockRespEv step
+		testSteps = append(
+			testSteps,
+			sBlockRespEv("waitForBlock", "waitForBlock",
+				"P0", height, prevBlocks))
+		prevBlocks = append(prevBlocks, height)
+		height++
+		numBlocksReceived++
+
+		// Add the processedBlockEv step periodically.
+		if numBlocksReceived >= maxRequestsPerPeer || height >= numBlocks {
+			for j := int(height) - numBlocksReceived; j < int(height); j++ {
+				if j >= int(numBlocks) {
+					// This is the last block that is processed, we should be in "finished" state.
+					testSteps = append(
+						testSteps,
+						sProcessedBlockEv("waitForBlock", "finished", nil))
+					break forLoop
+				}
+				testSteps = append(
+					testSteps,
+					sProcessedBlockEv("waitForBlock", "waitForBlock", nil))
+			}
+			numBlocksReceived = 0
+			prevBlocks = make([]int64, 0, maxPendingRequests)
+		}
+	}
+
+	return testFields{
+		name:               testName,
+		startingHeight:     startingHeight,
+		maxRequestsPerPeer: maxRequestsPerPeer,
+		maxPendingRequests: maxPendingRequests,
+		steps:              testSteps,
+	}
+}
+
+const (
+	maxStartingHeightTest       = 100
+	maxRequestsPerPeerTest      = 20
+	maxTotalPendingRequestsTest = 600
+	maxNumPeersTest             = 1000
+	maxNumBlocksInChainTest     = 10000 // should be smaller than 9999999
+)
+
+func makeCorrectTransitionSequenceWithRandomParameters() testFields {
+	// Generate a starting height for fast sync.
+	startingHeight := int64(tmrand.Intn(maxStartingHeightTest) + 1)
+
+	// Generate the number of requests per peer.
+	maxRequestsPerPeer := tmrand.Intn(maxRequestsPerPeerTest) + 1
+
+	// Generate the maximum number of total pending requests, >= maxRequestsPerPeer.
+	maxPendingRequests := tmrand.Intn(maxTotalPendingRequestsTest-maxRequestsPerPeer) + maxRequestsPerPeer
+
+	// Generate the number of blocks to be synced.
+	numBlocks := int64(tmrand.Intn(maxNumBlocksInChainTest)) + startingHeight
+
+	// Generate a number of peers.
+	numPeers := tmrand.Intn(maxNumPeersTest) + 1
+
+	return makeCorrectTransitionSequence(startingHeight, numBlocks, numPeers, true, maxRequestsPerPeer, maxPendingRequests)
+}
+
+func shouldApplyProcessedBlockEvStep(step *fsmStepTestValues, testBcR *testReactor) bool {
+	if step.event == processedBlockEv {
+		_, err := testBcR.fsm.pool.BlockAndPeerAtHeight(testBcR.fsm.pool.Height)
+		if err == errMissingBlock {
+			return false
+		}
+		_, err = testBcR.fsm.pool.BlockAndPeerAtHeight(testBcR.fsm.pool.Height + 1)
+		if err == errMissingBlock {
+			return false
+		}
+	}
+	return true
+}
+
+func TestFSMCorrectTransitionSequences(t *testing.T) {
+
+	tests := []testFields{
+		makeCorrectTransitionSequence(1, 100, 10, true, 10, 40),
+		makeCorrectTransitionSequenceWithRandomParameters(),
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			// Create test reactor
+			testBcR := newTestReactor(tt.startingHeight)
+
+			if tt.maxRequestsPerPeer != 0 {
+				maxRequestsPerPeer = tt.maxRequestsPerPeer
+			}
+
+			for _, step := range tt.steps {
+				step := step
+				assert.Equal(t, step.currentState, testBcR.fsm.state.name)
+
+				oldNumStatusRequests := testBcR.numStatusRequests
+				fixBlockResponseEvStep(&step, testBcR)
+				if !shouldApplyProcessedBlockEvStep(&step, testBcR) {
+					continue
+				}
+
+				fsmErr := sendEventToFSM(testBcR.fsm, step.event, step.data)
+				assert.Equal(t, step.wantErr, fsmErr)
+
+				if step.wantStatusReqSent {
+					assert.Equal(t, oldNumStatusRequests+1, testBcR.numStatusRequests)
+				} else {
+					assert.Equal(t, oldNumStatusRequests, testBcR.numStatusRequests)
+				}
+
+				assert.Equal(t, step.wantState, testBcR.fsm.state.name)
+				if step.wantState == "finished" {
+					assert.True(t, testBcR.fsm.isCaughtUp())
+				}
+			}
+
+		})
+	}
+}
+
+// ----------------------------------------
+// implements the bcRNotifier
+func (testR *testReactor) sendPeerError(err error, peerID p2p.ID) {
+	testR.logger.Info("Reactor received sendPeerError call from FSM", "peer", peerID, "err", err)
+	testR.lastPeerError.peerID = peerID
+	testR.lastPeerError.err = err
+}
+
+func (testR *testReactor) sendStatusRequest() {
+	testR.logger.Info("Reactor received sendStatusRequest call from FSM")
+	testR.numStatusRequests++
+}
+
+func (testR *testReactor) sendBlockRequest(peerID p2p.ID, height int64) error {
+	testR.logger.Info("Reactor received sendBlockRequest call from FSM", "peer", peerID, "height", height)
+	testR.numBlockRequests++
+	testR.lastBlockRequest.peerID = peerID
+	testR.lastBlockRequest.height = height
+	if height == 9999999 {
+		// simulate switch does not have peer
+		return errNilPeerForBlockRequest
+	}
+	return nil
+}
+
+func (testR *testReactor) resetStateTimer(name string, timer **time.Timer, timeout time.Duration) {
+	testR.logger.Info("Reactor received resetStateTimer call from FSM", "state", name, "timeout", timeout)
+	if _, ok := testR.stateTimerStarts[name]; !ok {
+		testR.stateTimerStarts[name] = 1
+	} else {
+		testR.stateTimerStarts[name]++
+	}
+}
+
+func (testR *testReactor) switchToConsensus() {
+}
+
+// ----------------------------------------

internal/blocksync/v1/reactor_test.go

@@ -0,0 +1,365 @@
+package v1
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/mempool/mock"
+	"github.com/tendermint/tendermint/p2p"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+var config *cfg.Config
+
+func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
+	validators := make([]types.GenesisValidator, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
+	for i := 0; i < numValidators; i++ {
+		val, privVal := types.RandValidator(randPower, minPower)
+		validators[i] = types.GenesisValidator{
+			PubKey: val.PubKey,
+			Power:  val.VotingPower,
+		}
+		privValidators[i] = privVal
+	}
+	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
+	return &types.GenesisDoc{
+		GenesisTime: tmtime.Now(),
+		ChainID:     config.ChainID(),
+		Validators:  validators,
+	}, privValidators
+}
+
+func makeVote(
+	t *testing.T,
+	header *types.Header,
+	blockID types.BlockID,
+	valset *types.ValidatorSet,
+	privVal types.PrivValidator) *types.Vote {
+
+	pubKey, err := privVal.GetPubKey()
+	require.NoError(t, err)
+
+	valIdx, _ := valset.GetByAddress(pubKey.Address())
+	vote := &types.Vote{
+		ValidatorAddress: pubKey.Address(),
+		ValidatorIndex:   valIdx,
+		Height:           header.Height,
+		Round:            1,
+		Timestamp:        tmtime.Now(),
+		Type:             tmproto.PrecommitType,
+		BlockID:          blockID,
+	}
+
+	vpb := vote.ToProto()
+
+	_ = privVal.SignVote(header.ChainID, vpb)
+	vote.Signature = vpb.Signature
+
+	return vote
+}
+
+type BlockchainReactorPair struct {
+	bcR  *BlockchainReactor
+	conR *consensusReactorTest
+}
+
+func newBlockchainReactor(
+	t *testing.T,
+	logger log.Logger,
+	genDoc *types.GenesisDoc,
+	privVals []types.PrivValidator,
+	maxBlockHeight int64) *BlockchainReactor {
+	if len(privVals) != 1 {
+		panic("only support one validator")
+	}
+
+	app := &testApp{}
+	cc := proxy.NewLocalClientCreator(app)
+	proxyApp := proxy.NewAppConns(cc)
+	err := proxyApp.Start()
+	if err != nil {
+		panic(fmt.Errorf("error start app: %w", err))
+	}
+
+	blockDB := dbm.NewMemDB()
+	stateDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(stateDB)
+	blockStore := store.NewBlockStore(blockDB)
+
+	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+	if err != nil {
+		panic(fmt.Errorf("error constructing state from genesis file: %w", err))
+	}
+
+	// Make the BlockchainReactor itself.
+	// NOTE we have to create and commit the blocks first because
+	// pool.height is determined from the store.
+	fastSync := true
+	db := dbm.NewMemDB()
+	stateStore = sm.NewStore(db)
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
+		mock.Mempool{}, sm.EmptyEvidencePool{})
+	if err = stateStore.Save(state); err != nil {
+		panic(err)
+	}
+
+	// let's add some blocks in
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
+		lastCommit := types.NewCommit(blockHeight-1, 1, types.BlockID{}, nil)
+		if blockHeight > 1 {
+			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
+			lastBlock := blockStore.LoadBlock(blockHeight - 1)
+
+			vote := makeVote(t, &lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0])
+			lastCommit = types.NewCommit(vote.Height, vote.Round, lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
+		}
+
+		thisBlock := makeBlock(blockHeight, state, lastCommit)
+
+		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
+		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
+
+		state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock)
+		if err != nil {
+			panic(fmt.Errorf("error apply block: %w", err))
+		}
+
+		blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
+	}
+
+	bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync)
+	bcReactor.SetLogger(logger.With("module", "blockchain"))
+
+	return bcReactor
+}
+
+func newBlockchainReactorPair(
+	t *testing.T,
+	logger log.Logger,
+	genDoc *types.GenesisDoc,
+	privVals []types.PrivValidator,
+	maxBlockHeight int64) BlockchainReactorPair {
+
+	consensusReactor := &consensusReactorTest{}
+	consensusReactor.BaseReactor = *p2p.NewBaseReactor("Consensus reactor", consensusReactor)
+
+	return BlockchainReactorPair{
+		newBlockchainReactor(t, logger, genDoc, privVals, maxBlockHeight),
+		consensusReactor}
+}
+
+type consensusReactorTest struct {
+	p2p.BaseReactor     // BaseService + p2p.Switch
+	switchedToConsensus bool
+	mtx                 sync.Mutex
+}
+
+func (conR *consensusReactorTest) SwitchToConsensus(state sm.State, blocksSynced bool) {
+	conR.mtx.Lock()
+	defer conR.mtx.Unlock()
+	conR.switchedToConsensus = true
+}
+
+func TestFastSyncNoBlockResponse(t *testing.T) {
+
+	config = cfg.ResetTestRoot("blockchain_new_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
+
+	maxBlockHeight := int64(65)
+
+	reactorPairs := make([]BlockchainReactorPair, 2)
+
+	logger := log.TestingLogger()
+	reactorPairs[0] = newBlockchainReactorPair(t, logger, genDoc, privVals, maxBlockHeight)
+	reactorPairs[1] = newBlockchainReactorPair(t, logger, genDoc, privVals, 0)
+
+	p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].bcR)
+		s.AddReactor("CONSENSUS", reactorPairs[i].conR)
+		moduleName := fmt.Sprintf("blockchain-%v", i)
+		reactorPairs[i].bcR.SetLogger(logger.With("module", moduleName))
+
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			_ = r.bcR.Stop()
+			_ = r.conR.Stop()
+		}
+	}()
+
+	tests := []struct {
+		height   int64
+		existent bool
+	}{
+		{maxBlockHeight + 2, false},
+		{10, true},
+		{1, true},
+		{maxBlockHeight + 100, false},
+	}
+
+	for {
+		time.Sleep(10 * time.Millisecond)
+		reactorPairs[1].conR.mtx.Lock()
+		if reactorPairs[1].conR.switchedToConsensus {
+			reactorPairs[1].conR.mtx.Unlock()
+			break
+		}
+		reactorPairs[1].conR.mtx.Unlock()
+	}
+
+	assert.Equal(t, maxBlockHeight, reactorPairs[0].bcR.store.Height())
+
+	for _, tt := range tests {
+		block := reactorPairs[1].bcR.store.LoadBlock(tt.height)
+		if tt.existent {
+			assert.True(t, block != nil)
+		} else {
+			assert.True(t, block == nil)
+		}
+	}
+}
+
+// NOTE: This is too hard to test without
+// an easy way to add test peer to switch
+// or without significant refactoring of the module.
+// Alternatively we could actually dial a TCP conn but
+// that seems extreme.
+func TestFastSyncBadBlockStopsPeer(t *testing.T) {
+	numNodes := 4
+	maxBlockHeight := int64(148)
+
+	config = cfg.ResetTestRoot("blockchain_reactor_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(1, false, 30)
+
+	otherChain := newBlockchainReactorPair(t, log.TestingLogger(), genDoc, privVals, maxBlockHeight)
+	defer func() {
+		_ = otherChain.bcR.Stop()
+		_ = otherChain.conR.Stop()
+	}()
+
+	reactorPairs := make([]BlockchainReactorPair, numNodes)
+	logger := make([]log.Logger, numNodes)
+
+	for i := 0; i < numNodes; i++ {
+		logger[i] = log.TestingLogger()
+		height := int64(0)
+		if i == 0 {
+			height = maxBlockHeight
+		}
+		reactorPairs[i] = newBlockchainReactorPair(t, logger[i], genDoc, privVals, height)
+	}
+
+	switches := p2p.MakeConnectedSwitches(config.P2P, numNodes, func(i int, s *p2p.Switch) *p2p.Switch {
+		reactorPairs[i].conR.mtx.Lock()
+		s.AddReactor("BLOCKCHAIN", reactorPairs[i].bcR)
+		s.AddReactor("CONSENSUS", reactorPairs[i].conR)
+		moduleName := fmt.Sprintf("blockchain-%v", i)
+		reactorPairs[i].bcR.SetLogger(logger[i].With("module", moduleName))
+		reactorPairs[i].conR.mtx.Unlock()
+		return s
+
+	}, p2p.Connect2Switches)
+
+	defer func() {
+		for _, r := range reactorPairs {
+			_ = r.bcR.Stop()
+			_ = r.conR.Stop()
+		}
+	}()
+
+outerFor:
+	for {
+		time.Sleep(10 * time.Millisecond)
+		for i := 0; i < numNodes; i++ {
+			reactorPairs[i].conR.mtx.Lock()
+			if !reactorPairs[i].conR.switchedToConsensus {
+				reactorPairs[i].conR.mtx.Unlock()
+				continue outerFor
+			}
+			reactorPairs[i].conR.mtx.Unlock()
+		}
+		break
+	}
+
+	// at this time, reactors[0-3] is the newest
+	assert.Equal(t, numNodes-1, reactorPairs[1].bcR.Switch.Peers().Size())
+
+	// mark last reactorPair as an invalid peer
+	reactorPairs[numNodes-1].bcR.store = otherChain.bcR.store
+
+	lastLogger := log.TestingLogger()
+	lastReactorPair := newBlockchainReactorPair(t, lastLogger, genDoc, privVals, 0)
+	reactorPairs = append(reactorPairs, lastReactorPair)
+
+	switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].bcR)
+		s.AddReactor("CONSENSUS", reactorPairs[len(reactorPairs)-1].conR)
+		moduleName := fmt.Sprintf("blockchain-%v", len(reactorPairs)-1)
+		reactorPairs[len(reactorPairs)-1].bcR.SetLogger(lastLogger.With("module", moduleName))
+		return s
+
+	}, p2p.Connect2Switches)...)
+
+	for i := 0; i < len(reactorPairs)-1; i++ {
+		p2p.Connect2Switches(switches, i, len(reactorPairs)-1)
+	}
+
+	for {
+		time.Sleep(1 * time.Second)
+		lastReactorPair.conR.mtx.Lock()
+		if lastReactorPair.conR.switchedToConsensus {
+			lastReactorPair.conR.mtx.Unlock()
+			break
+		}
+		lastReactorPair.conR.mtx.Unlock()
+
+		if lastReactorPair.bcR.Switch.Peers().Size() == 0 {
+			break
+		}
+	}
+
+	assert.True(t, lastReactorPair.bcR.Switch.Peers().Size() < len(reactorPairs)-1)
+}
+
+//----------------------------------------------
+// utility funcs
+
+func makeTxs(height int64) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(height), byte(i)}))
+	}
+	return txs
+}
+
+func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block {
+	block, _ := state.MakeBlock(height, makeTxs(height), lastCommit, nil, state.Validators.GetProposer().Address)
+	return block
+}
+
+type testApp struct {
+	abci.BaseApplication
+}

internal/blocksync/v2/io.go

@@ -0,0 +1,140 @@
+package v2
+
+import (
+	"fmt"
+
+	bc "github.com/tendermint/tendermint/blockchain"
+	"github.com/tendermint/tendermint/p2p"
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+type iIO interface {
+	sendBlockRequest(peerID p2p.ID, height int64) error
+	sendBlockToPeer(block *types.Block, peerID p2p.ID) error
+	sendBlockNotFound(height int64, peerID p2p.ID) error
+	sendStatusResponse(base, height int64, peerID p2p.ID) error
+
+	broadcastStatusRequest() error
+
+	trySwitchToConsensus(state state.State, skipWAL bool) bool
+}
+
+type switchIO struct {
+	sw *p2p.Switch
+}
+
+func newSwitchIo(sw *p2p.Switch) *switchIO {
+	return &switchIO{
+		sw: sw,
+	}
+}
+
+const (
+	// BlockchainChannel is a channel for blocks and status updates (`BlockStore` height)
+	BlockchainChannel = byte(0x40)
+)
+
+type consensusReactor interface {
+	// for when we switch from blockchain reactor and fast sync to
+	// the consensus machine
+	SwitchToConsensus(state state.State, skipWAL bool)
+}
+
+func (sio *switchIO) sendBlockRequest(peerID p2p.ID, height int64) error {
+	peer := sio.sw.Peers().Get(peerID)
+	if peer == nil {
+		return fmt.Errorf("peer not found")
+	}
+	msgBytes, err := bc.EncodeMsg(&bcproto.BlockRequest{Height: height})
+	if err != nil {
+		return err
+	}
+
+	queued := peer.TrySend(BlockchainChannel, msgBytes)
+	if !queued {
+		return fmt.Errorf("send queue full")
+	}
+	return nil
+}
+
+func (sio *switchIO) sendStatusResponse(base int64, height int64, peerID p2p.ID) error {
+	peer := sio.sw.Peers().Get(peerID)
+	if peer == nil {
+		return fmt.Errorf("peer not found")
+	}
+
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusResponse{Height: height, Base: base})
+	if err != nil {
+		return err
+	}
+
+	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
+		return fmt.Errorf("peer queue full")
+	}
+
+	return nil
+}
+
+func (sio *switchIO) sendBlockToPeer(block *types.Block, peerID p2p.ID) error {
+	peer := sio.sw.Peers().Get(peerID)
+	if peer == nil {
+		return fmt.Errorf("peer not found")
+	}
+	if block == nil {
+		panic("trying to send nil block")
+	}
+
+	bpb, err := block.ToProto()
+	if err != nil {
+		return err
+	}
+
+	msgBytes, err := bc.EncodeMsg(&bcproto.BlockResponse{Block: bpb})
+	if err != nil {
+		return err
+	}
+	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
+		return fmt.Errorf("peer queue full")
+	}
+
+	return nil
+}
+
+func (sio *switchIO) sendBlockNotFound(height int64, peerID p2p.ID) error {
+	peer := sio.sw.Peers().Get(peerID)
+	if peer == nil {
+		return fmt.Errorf("peer not found")
+	}
+	msgBytes, err := bc.EncodeMsg(&bcproto.NoBlockResponse{Height: height})
+	if err != nil {
+		return err
+	}
+
+	if queued := peer.TrySend(BlockchainChannel, msgBytes); !queued {
+		return fmt.Errorf("peer queue full")
+	}
+
+	return nil
+}
+
+func (sio *switchIO) trySwitchToConsensus(state state.State, skipWAL bool) bool {
+	conR, ok := sio.sw.Reactor("CONSENSUS").(consensusReactor)
+	if ok {
+		conR.SwitchToConsensus(state, skipWAL)
+	}
+	return ok
+}
+
+func (sio *switchIO) broadcastStatusRequest() error {
+	msgBytes, err := bc.EncodeMsg(&bcproto.StatusRequest{})
+	if err != nil {
+		return err
+	}
+
+	// XXX: maybe we should use an io specific peer list here
+	sio.sw.Broadcast(BlockchainChannel, msgBytes)
+
+	return nil
+}

internal/blocksync/v2/metrics.go

@@ -0,0 +1,125 @@
+package v2
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "blockchain"
+)
+
+// Metrics contains metrics exposed by this package.
+type Metrics struct {
+	// events_in
+	EventsIn metrics.Counter
+	// events_in
+	EventsHandled metrics.Counter
+	// events_out
+	EventsOut metrics.Counter
+	// errors_in
+	ErrorsIn metrics.Counter
+	// errors_handled
+	ErrorsHandled metrics.Counter
+	// errors_out
+	ErrorsOut metrics.Counter
+	// events_shed
+	EventsShed metrics.Counter
+	// events_sent
+	EventsSent metrics.Counter
+	// errors_sent
+	ErrorsSent metrics.Counter
+	// errors_shed
+	ErrorsShed metrics.Counter
+}
+
+// PrometheusMetrics returns metrics for in and out events, errors, etc. handled by routines.
+// Can we burn in the routine name here?
+func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(labelsAndValues); i += 2 {
+		labels = append(labels, labelsAndValues[i])
+	}
+	return &Metrics{
+		EventsIn: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "events_in",
+			Help:      "Events read from the channel.",
+		}, labels).With(labelsAndValues...),
+		EventsHandled: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "events_handled",
+			Help:      "Events handled",
+		}, labels).With(labelsAndValues...),
+		EventsOut: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "events_out",
+			Help:      "Events output from routine.",
+		}, labels).With(labelsAndValues...),
+		ErrorsIn: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "errors_in",
+			Help:      "Errors read from the channel.",
+		}, labels).With(labelsAndValues...),
+		ErrorsHandled: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "errors_handled",
+			Help:      "Errors handled.",
+		}, labels).With(labelsAndValues...),
+		ErrorsOut: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "errors_out",
+			Help:      "Errors output from routine.",
+		}, labels).With(labelsAndValues...),
+		ErrorsSent: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "errors_sent",
+			Help:      "Errors sent to routine.",
+		}, labels).With(labelsAndValues...),
+		ErrorsShed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "errors_shed",
+			Help:      "Errors dropped from sending.",
+		}, labels).With(labelsAndValues...),
+		EventsSent: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "events_sent",
+			Help:      "Events sent to routine.",
+		}, labels).With(labelsAndValues...),
+		EventsShed: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "events_shed",
+			Help:      "Events dropped from sending.",
+		}, labels).With(labelsAndValues...),
+	}
+}
+
+// NopMetrics returns no-op Metrics.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		EventsIn:      discard.NewCounter(),
+		EventsHandled: discard.NewCounter(),
+		EventsOut:     discard.NewCounter(),
+		ErrorsIn:      discard.NewCounter(),
+		ErrorsHandled: discard.NewCounter(),
+		ErrorsOut:     discard.NewCounter(),
+		EventsShed:    discard.NewCounter(),
+		EventsSent:    discard.NewCounter(),
+		ErrorsSent:    discard.NewCounter(),
+		ErrorsShed:    discard.NewCounter(),
+	}
+}

internal/blocksync/v2/processor.go

@@ -0,0 +1,192 @@
+package v2
+
+import (
+	"fmt"
+
+	"github.com/tendermint/tendermint/p2p"
+	tmState "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Events generated by the processor:
+// block execution failure, event will indicate the peer(s) that caused the error
+type pcBlockVerificationFailure struct {
+	priorityNormal
+	height       int64
+	firstPeerID  p2p.ID
+	secondPeerID p2p.ID
+}
+
+func (e pcBlockVerificationFailure) String() string {
+	return fmt.Sprintf("pcBlockVerificationFailure{%d 1st peer: %v, 2nd peer: %v}",
+		e.height, e.firstPeerID, e.secondPeerID)
+}
+
+// successful block execution
+type pcBlockProcessed struct {
+	priorityNormal
+	height int64
+	peerID p2p.ID
+}
+
+func (e pcBlockProcessed) String() string {
+	return fmt.Sprintf("pcBlockProcessed{%d peer: %v}", e.height, e.peerID)
+}
+
+// processor has finished
+type pcFinished struct {
+	priorityNormal
+	blocksSynced int
+	tmState      tmState.State
+}
+
+func (p pcFinished) Error() string {
+	return "finished"
+}
+
+type queueItem struct {
+	block  *types.Block
+	peerID p2p.ID
+}
+
+type blockQueue map[int64]queueItem
+
+type pcState struct {
+	// blocks waiting to be processed
+	queue blockQueue
+
+	// draining indicates that the next rProcessBlock event with a queue miss constitutes completion
+	draining bool
+
+	// the number of blocks successfully synced by the processor
+	blocksSynced int
+
+	// the processorContext which contains the processor dependencies
+	context processorContext
+}
+
+func (state *pcState) String() string {
+	return fmt.Sprintf("height: %d queue length: %d draining: %v blocks synced: %d",
+		state.height(), len(state.queue), state.draining, state.blocksSynced)
+}
+
+// newPcState returns a pcState initialized with the last verified block enqueued
+func newPcState(context processorContext) *pcState {
+	return &pcState{
+		queue:        blockQueue{},
+		draining:     false,
+		blocksSynced: 0,
+		context:      context,
+	}
+}
+
+// nextTwo returns the next two unverified blocks
+func (state *pcState) nextTwo() (queueItem, queueItem, error) {
+	if first, ok := state.queue[state.height()+1]; ok {
+		if second, ok := state.queue[state.height()+2]; ok {
+			return first, second, nil
+		}
+	}
+	return queueItem{}, queueItem{}, fmt.Errorf("not found")
+}
+
+// synced returns true when at most the last verified block remains in the queue
+func (state *pcState) synced() bool {
+	return len(state.queue) <= 1
+}
+
+func (state *pcState) enqueue(peerID p2p.ID, block *types.Block, height int64) {
+	if item, ok := state.queue[height]; ok {
+		panic(fmt.Sprintf(
+			"duplicate block %d (%X) enqueued by processor (sent by %v; existing block %X from %v)",
+			height, block.Hash(), peerID, item.block.Hash(), item.peerID))
+	}
+
+	state.queue[height] = queueItem{block: block, peerID: peerID}
+}
+
+func (state *pcState) height() int64 {
+	return state.context.tmState().LastBlockHeight
+}
+
+// purgePeer moves all unprocessed blocks from the queue
+func (state *pcState) purgePeer(peerID p2p.ID) {
+	// what if height is less than state.height?
+	for height, item := range state.queue {
+		if item.peerID == peerID {
+			delete(state.queue, height)
+		}
+	}
+}
+
+// handle processes FSM events
+func (state *pcState) handle(event Event) (Event, error) {
+	switch event := event.(type) {
+	case bcResetState:
+		state.context.setState(event.state)
+		return noOp, nil
+
+	case scFinishedEv:
+		if state.synced() {
+			return pcFinished{tmState: state.context.tmState(), blocksSynced: state.blocksSynced}, nil
+		}
+		state.draining = true
+		return noOp, nil
+
+	case scPeerError:
+		state.purgePeer(event.peerID)
+		return noOp, nil
+
+	case scBlockReceived:
+		if event.block == nil {
+			return noOp, nil
+		}
+
+		// enqueue block if height is higher than state height, else ignore it
+		if event.block.Height > state.height() {
+			state.enqueue(event.peerID, event.block, event.block.Height)
+		}
+		return noOp, nil
+
+	case rProcessBlock:
+		tmState := state.context.tmState()
+		firstItem, secondItem, err := state.nextTwo()
+		if err != nil {
+			if state.draining {
+				return pcFinished{tmState: tmState, blocksSynced: state.blocksSynced}, nil
+			}
+			return noOp, nil
+		}
+
+		var (
+			first, second = firstItem.block, secondItem.block
+			firstParts    = first.MakePartSet(types.BlockPartSizeBytes)
+			firstID       = types.BlockID{Hash: first.Hash(), PartSetHeader: firstParts.Header()}
+		)
+
+		// verify if +second+ last commit "confirms" +first+ block
+		err = state.context.verifyCommit(tmState.ChainID, firstID, first.Height, second.LastCommit)
+		if err != nil {
+			state.purgePeer(firstItem.peerID)
+			if firstItem.peerID != secondItem.peerID {
+				state.purgePeer(secondItem.peerID)
+			}
+			return pcBlockVerificationFailure{
+					height: first.Height, firstPeerID: firstItem.peerID, secondPeerID: secondItem.peerID},
+				nil
+		}
+
+		state.context.saveBlock(first, firstParts, second.LastCommit)
+
+		if err := state.context.applyBlock(firstID, first); err != nil {
+			panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err))
+		}
+
+		delete(state.queue, first.Height)
+		state.blocksSynced++
+
+		return pcBlockProcessed{height: first.Height, peerID: firstItem.peerID}, nil
+	}
+
+	return noOp, nil
+}

internal/blocksync/v2/processor_context.go

@@ -0,0 +1,100 @@
+package v2
+
+import (
+	"fmt"
+
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+type processorContext interface {
+	applyBlock(blockID types.BlockID, block *types.Block) error
+	verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error
+	saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit)
+	tmState() state.State
+	setState(state.State)
+}
+
+type pContext struct {
+	store   blockStore
+	applier blockApplier
+	state   state.State
+}
+
+func newProcessorContext(st blockStore, ex blockApplier, s state.State) *pContext {
+	return &pContext{
+		store:   st,
+		applier: ex,
+		state:   s,
+	}
+}
+
+func (pc *pContext) applyBlock(blockID types.BlockID, block *types.Block) error {
+	newState, _, err := pc.applier.ApplyBlock(pc.state, blockID, block)
+	pc.state = newState
+	return err
+}
+
+func (pc pContext) tmState() state.State {
+	return pc.state
+}
+
+func (pc *pContext) setState(state state.State) {
+	pc.state = state
+}
+
+func (pc pContext) verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error {
+	return pc.state.Validators.VerifyCommitLight(chainID, blockID, height, commit)
+}
+
+func (pc *pContext) saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+	pc.store.SaveBlock(block, blockParts, seenCommit)
+}
+
+type mockPContext struct {
+	applicationBL  []int64
+	verificationBL []int64
+	state          state.State
+}
+
+func newMockProcessorContext(
+	state state.State,
+	verificationBlackList []int64,
+	applicationBlackList []int64) *mockPContext {
+	return &mockPContext{
+		applicationBL:  applicationBlackList,
+		verificationBL: verificationBlackList,
+		state:          state,
+	}
+}
+
+func (mpc *mockPContext) applyBlock(blockID types.BlockID, block *types.Block) error {
+	for _, h := range mpc.applicationBL {
+		if h == block.Height {
+			return fmt.Errorf("generic application error")
+		}
+	}
+	mpc.state.LastBlockHeight = block.Height
+	return nil
+}
+
+func (mpc *mockPContext) verifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error {
+	for _, h := range mpc.verificationBL {
+		if h == height {
+			return fmt.Errorf("generic verification error")
+		}
+	}
+	return nil
+}
+
+func (mpc *mockPContext) saveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+
+}
+
+func (mpc *mockPContext) setState(state state.State) {
+	mpc.state = state
+}
+
+func (mpc *mockPContext) tmState() state.State {
+	return mpc.state
+}

internal/blocksync/v2/processor_test.go

@@ -0,0 +1,306 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/p2p"
+	tmState "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+// pcBlock is a test helper structure with simple types. Its purpose is to help with test readability.
+type pcBlock struct {
+	pid    string
+	height int64
+}
+
+// params is a test structure used to create processor state.
+type params struct {
+	height       int64
+	items        []pcBlock
+	blocksSynced int
+	verBL        []int64
+	appBL        []int64
+	draining     bool
+}
+
+// makePcBlock makes an empty block.
+func makePcBlock(height int64) *types.Block {
+	return &types.Block{Header: types.Header{Height: height}}
+}
+
+// makeState takes test parameters and creates a specific processor state.
+func makeState(p *params) *pcState {
+	var (
+		tmState = tmState.State{LastBlockHeight: p.height}
+		context = newMockProcessorContext(tmState, p.verBL, p.appBL)
+	)
+	state := newPcState(context)
+
+	for _, item := range p.items {
+		state.enqueue(p2p.ID(item.pid), makePcBlock(item.height), item.height)
+	}
+
+	state.blocksSynced = p.blocksSynced
+	state.draining = p.draining
+	return state
+}
+
+func mBlockResponse(peerID p2p.ID, height int64) scBlockReceived {
+	return scBlockReceived{
+		peerID: peerID,
+		block:  makePcBlock(height),
+	}
+}
+
+type pcFsmMakeStateValues struct {
+	currentState  *params
+	event         Event
+	wantState     *params
+	wantNextEvent Event
+	wantErr       error
+	wantPanic     bool
+}
+
+type testFields struct {
+	name  string
+	steps []pcFsmMakeStateValues
+}
+
+func executeProcessorTests(t *testing.T, tests []testFields) {
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			var state *pcState
+			for _, step := range tt.steps {
+				defer func() {
+					r := recover()
+					if (r != nil) != step.wantPanic {
+						t.Errorf("recover = %v, wantPanic = %v", r, step.wantPanic)
+					}
+				}()
+
+				// First step must always initialise the currentState as state.
+				if step.currentState != nil {
+					state = makeState(step.currentState)
+				}
+				if state == nil {
+					panic("Bad (initial?) step")
+				}
+
+				nextEvent, err := state.handle(step.event)
+				t.Log(state)
+				assert.Equal(t, step.wantErr, err)
+				assert.Equal(t, makeState(step.wantState), state)
+				assert.Equal(t, step.wantNextEvent, nextEvent)
+				// Next step may use the wantedState as their currentState.
+				state = makeState(step.wantState)
+			}
+		})
+	}
+}
+
+func TestRProcessPeerError(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "error for existing peer",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
+					event:         scPeerError{peerID: "P2"},
+					wantState:     &params{items: []pcBlock{{"P1", 1}}},
+					wantNextEvent: noOp,
+				},
+			},
+		},
+		{
+			name: "error for unknown peer",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
+					event:         scPeerError{peerID: "P3"},
+					wantState:     &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}},
+					wantNextEvent: noOp,
+				},
+			},
+		},
+	}
+
+	executeProcessorTests(t, tests)
+}
+
+func TestPcBlockResponse(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "add one block",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{}, event: mBlockResponse("P1", 1),
+					wantState: &params{items: []pcBlock{{"P1", 1}}}, wantNextEvent: noOp,
+				},
+			},
+		},
+
+		{
+			name: "add two blocks",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{}, event: mBlockResponse("P1", 3),
+					wantState: &params{items: []pcBlock{{"P1", 3}}}, wantNextEvent: noOp,
+				},
+				{ // use previous wantState as currentState,
+					event:     mBlockResponse("P1", 4),
+					wantState: &params{items: []pcBlock{{"P1", 3}, {"P1", 4}}}, wantNextEvent: noOp,
+				},
+			},
+		},
+	}
+
+	executeProcessorTests(t, tests)
+}
+
+func TestRProcessBlockSuccess(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "noop - no blocks over current height",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{}, event: rProcessBlock{},
+					wantState: &params{}, wantNextEvent: noOp,
+				},
+			},
+		},
+		{
+			name: "noop - high new blocks",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{height: 5, items: []pcBlock{{"P1", 30}, {"P2", 31}}}, event: rProcessBlock{},
+					wantState: &params{height: 5, items: []pcBlock{{"P1", 30}, {"P2", 31}}}, wantNextEvent: noOp,
+				},
+			},
+		},
+		{
+			name: "blocks H+1 and H+2 present",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}}, event: rProcessBlock{},
+					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}}, blocksSynced: 1},
+					wantNextEvent: pcBlockProcessed{height: 1, peerID: "P1"},
+				},
+			},
+		},
+		{
+			name: "blocks H+1 and H+2 present after draining",
+			steps: []pcFsmMakeStateValues{
+				{ // some contiguous blocks - on stop check draining is set
+					currentState:  &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P1", 4}}},
+					event:         scFinishedEv{},
+					wantState:     &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P1", 4}}, draining: true},
+					wantNextEvent: noOp,
+				},
+				{
+					event:         rProcessBlock{},
+					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}, {"P1", 4}}, blocksSynced: 1, draining: true},
+					wantNextEvent: pcBlockProcessed{height: 1, peerID: "P1"},
+				},
+				{ // finish when H+1 or/and H+2 are missing
+					event:         rProcessBlock{},
+					wantState:     &params{height: 1, items: []pcBlock{{"P2", 2}, {"P1", 4}}, blocksSynced: 1, draining: true},
+					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 1}, blocksSynced: 1},
+				},
+			},
+		},
+	}
+
+	executeProcessorTests(t, tests)
+}
+
+func TestRProcessBlockFailures(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "blocks H+1 and H+2 present from different peers - H+1 verification fails ",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}, verBL: []int64{1}}, event: rProcessBlock{},
+					wantState:     &params{items: []pcBlock{}, verBL: []int64{1}},
+					wantNextEvent: pcBlockVerificationFailure{height: 1, firstPeerID: "P1", secondPeerID: "P2"},
+				},
+			},
+		},
+		{
+			name: "blocks H+1 and H+2 present from same peer - H+1 applyBlock fails ",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}}, appBL: []int64{1}}, event: rProcessBlock{},
+					wantState: &params{items: []pcBlock{}, appBL: []int64{1}}, wantPanic: true,
+				},
+			},
+		},
+		{
+			name: "blocks H+1 and H+2 present from same peers - H+1 verification fails ",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{height: 0, items: []pcBlock{{"P1", 1}, {"P1", 2}, {"P2", 3}},
+						verBL: []int64{1}}, event: rProcessBlock{},
+					wantState:     &params{height: 0, items: []pcBlock{{"P2", 3}}, verBL: []int64{1}},
+					wantNextEvent: pcBlockVerificationFailure{height: 1, firstPeerID: "P1", secondPeerID: "P1"},
+				},
+			},
+		},
+		{
+			name: "blocks H+1 and H+2 present from different peers - H+1 applyBlock fails ",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{items: []pcBlock{{"P1", 1}, {"P2", 2}, {"P2", 3}}, appBL: []int64{1}},
+					event:        rProcessBlock{},
+					wantState:    &params{items: []pcBlock{{"P2", 3}}, appBL: []int64{1}}, wantPanic: true,
+				},
+			},
+		},
+	}
+
+	executeProcessorTests(t, tests)
+}
+
+func TestScFinishedEv(t *testing.T) {
+	tests := []testFields{
+		{
+			name: "no blocks",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{height: 100, items: []pcBlock{}, blocksSynced: 100}, event: scFinishedEv{},
+					wantState:     &params{height: 100, items: []pcBlock{}, blocksSynced: 100},
+					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 100}, blocksSynced: 100},
+				},
+			},
+		},
+		{
+			name: "maxHeight+1 block present",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{height: 100, items: []pcBlock{
+						{"P1", 101}}, blocksSynced: 100}, event: scFinishedEv{},
+					wantState:     &params{height: 100, items: []pcBlock{{"P1", 101}}, blocksSynced: 100},
+					wantNextEvent: pcFinished{tmState: tmState.State{LastBlockHeight: 100}, blocksSynced: 100},
+				},
+			},
+		},
+		{
+			name: "more blocks present",
+			steps: []pcFsmMakeStateValues{
+				{
+					currentState: &params{height: 100, items: []pcBlock{
+						{"P1", 101}, {"P1", 102}}, blocksSynced: 100}, event: scFinishedEv{},
+					wantState: &params{height: 100, items: []pcBlock{
+						{"P1", 101}, {"P1", 102}}, blocksSynced: 100, draining: true},
+					wantNextEvent: noOp,
+					wantErr:       nil,
+				},
+			},
+		},
+	}
+
+	executeProcessorTests(t, tests)
+}

internal/blocksync/v2/reactor.go

@@ -0,0 +1,564 @@
+package v2
+
+import (
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/behaviour"
+	bc "github.com/tendermint/tendermint/blockchain"
+	"github.com/tendermint/tendermint/libs/log"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	"github.com/tendermint/tendermint/p2p"
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	"github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// chBufferSize is the buffer size of all event channels.
+	chBufferSize int = 1000
+)
+
+type blockStore interface {
+	LoadBlock(height int64) *types.Block
+	SaveBlock(*types.Block, *types.PartSet, *types.Commit)
+	Base() int64
+	Height() int64
+}
+
+// BlockchainReactor handles fast sync protocol.
+type BlockchainReactor struct {
+	p2p.BaseReactor
+
+	fastSync    bool // if true, enable fast sync on start
+	stateSynced bool // set to true when SwitchToFastSync is called by state sync
+	scheduler   *Routine
+	processor   *Routine
+	logger      log.Logger
+
+	mtx           tmsync.RWMutex
+	maxPeerHeight int64
+	syncHeight    int64
+	events        chan Event // non-nil during a fast sync
+
+	reporter behaviour.Reporter
+	io       iIO
+	store    blockStore
+}
+
+//nolint:unused,deadcode
+type blockVerifier interface {
+	VerifyCommit(chainID string, blockID types.BlockID, height int64, commit *types.Commit) error
+}
+
+type blockApplier interface {
+	ApplyBlock(state state.State, blockID types.BlockID, block *types.Block) (state.State, int64, error)
+}
+
+// XXX: unify naming in this package around tmState
+func newReactor(state state.State, store blockStore, reporter behaviour.Reporter,
+	blockApplier blockApplier, fastSync bool) *BlockchainReactor {
+	initHeight := state.LastBlockHeight + 1
+	if initHeight == 1 {
+		initHeight = state.InitialHeight
+	}
+	scheduler := newScheduler(initHeight, time.Now())
+	pContext := newProcessorContext(store, blockApplier, state)
+	// TODO: Fix naming to just newProcesssor
+	// newPcState requires a processorContext
+	processor := newPcState(pContext)
+
+	return &BlockchainReactor{
+		scheduler: newRoutine("scheduler", scheduler.handle, chBufferSize),
+		processor: newRoutine("processor", processor.handle, chBufferSize),
+		store:     store,
+		reporter:  reporter,
+		logger:    log.NewNopLogger(),
+		fastSync:  fastSync,
+	}
+}
+
+// NewBlockchainReactor creates a new reactor instance.
+func NewBlockchainReactor(
+	state state.State,
+	blockApplier blockApplier,
+	store blockStore,
+	fastSync bool) *BlockchainReactor {
+	reporter := behaviour.NewMockReporter()
+	return newReactor(state, store, reporter, blockApplier, fastSync)
+}
+
+// SetSwitch implements Reactor interface.
+func (r *BlockchainReactor) SetSwitch(sw *p2p.Switch) {
+	r.Switch = sw
+	if sw != nil {
+		r.io = newSwitchIo(sw)
+	} else {
+		r.io = nil
+	}
+}
+
+func (r *BlockchainReactor) setMaxPeerHeight(height int64) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	if height > r.maxPeerHeight {
+		r.maxPeerHeight = height
+	}
+}
+
+func (r *BlockchainReactor) setSyncHeight(height int64) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	r.syncHeight = height
+}
+
+// SyncHeight returns the height to which the BlockchainReactor has synced.
+func (r *BlockchainReactor) SyncHeight() int64 {
+	r.mtx.RLock()
+	defer r.mtx.RUnlock()
+	return r.syncHeight
+}
+
+// SetLogger sets the logger of the reactor.
+func (r *BlockchainReactor) SetLogger(logger log.Logger) {
+	r.logger = logger
+	r.scheduler.setLogger(logger)
+	r.processor.setLogger(logger)
+}
+
+// Start implements cmn.Service interface
+func (r *BlockchainReactor) Start() error {
+	r.reporter = behaviour.NewSwitchReporter(r.BaseReactor.Switch)
+	if r.fastSync {
+		err := r.startSync(nil)
+		if err != nil {
+			return fmt.Errorf("failed to start fast sync: %w", err)
+		}
+	}
+	return nil
+}
+
+// startSync begins a fast sync, signalled by r.events being non-nil. If state is non-nil,
+// the scheduler and processor is updated with this state on startup.
+func (r *BlockchainReactor) startSync(state *state.State) error {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	if r.events != nil {
+		return errors.New("fast sync already in progress")
+	}
+	r.events = make(chan Event, chBufferSize)
+	go r.scheduler.start()
+	go r.processor.start()
+	if state != nil {
+		<-r.scheduler.ready()
+		<-r.processor.ready()
+		r.scheduler.send(bcResetState{state: *state})
+		r.processor.send(bcResetState{state: *state})
+	}
+	go r.demux(r.events)
+	return nil
+}
+
+// endSync ends a fast sync
+func (r *BlockchainReactor) endSync() {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	if r.events != nil {
+		close(r.events)
+	}
+	r.events = nil
+	r.scheduler.stop()
+	r.processor.stop()
+}
+
+// SwitchToFastSync is called by the state sync reactor when switching to fast sync.
+func (r *BlockchainReactor) SwitchToFastSync(state state.State) error {
+	r.stateSynced = true
+	state = state.Copy()
+	return r.startSync(&state)
+}
+
+// reactor generated ticker events:
+// ticker for cleaning peers
+type rTryPrunePeer struct {
+	priorityHigh
+	time time.Time
+}
+
+func (e rTryPrunePeer) String() string {
+	return fmt.Sprintf("rTryPrunePeer{%v}", e.time)
+}
+
+// ticker event for scheduling block requests
+type rTrySchedule struct {
+	priorityHigh
+	time time.Time
+}
+
+func (e rTrySchedule) String() string {
+	return fmt.Sprintf("rTrySchedule{%v}", e.time)
+}
+
+// ticker for block processing
+type rProcessBlock struct {
+	priorityNormal
+}
+
+func (e rProcessBlock) String() string {
+	return "rProcessBlock"
+}
+
+// reactor generated events based on blockchain related messages from peers:
+// blockResponse message received from a peer
+type bcBlockResponse struct {
+	priorityNormal
+	time   time.Time
+	peerID p2p.ID
+	size   int64
+	block  *types.Block
+}
+
+func (resp bcBlockResponse) String() string {
+	return fmt.Sprintf("bcBlockResponse{%d#%X (size: %d bytes) from %v at %v}",
+		resp.block.Height, resp.block.Hash(), resp.size, resp.peerID, resp.time)
+}
+
+// blockNoResponse message received from a peer
+type bcNoBlockResponse struct {
+	priorityNormal
+	time   time.Time
+	peerID p2p.ID
+	height int64
+}
+
+func (resp bcNoBlockResponse) String() string {
+	return fmt.Sprintf("bcNoBlockResponse{%v has no block at height %d at %v}",
+		resp.peerID, resp.height, resp.time)
+}
+
+// statusResponse message received from a peer
+type bcStatusResponse struct {
+	priorityNormal
+	time   time.Time
+	peerID p2p.ID
+	base   int64
+	height int64
+}
+
+func (resp bcStatusResponse) String() string {
+	return fmt.Sprintf("bcStatusResponse{%v is at height %d (base: %d) at %v}",
+		resp.peerID, resp.height, resp.base, resp.time)
+}
+
+// new peer is connected
+type bcAddNewPeer struct {
+	priorityNormal
+	peerID p2p.ID
+}
+
+func (resp bcAddNewPeer) String() string {
+	return fmt.Sprintf("bcAddNewPeer{%v}", resp.peerID)
+}
+
+// existing peer is removed
+type bcRemovePeer struct {
+	priorityHigh
+	peerID p2p.ID
+	reason interface{}
+}
+
+func (resp bcRemovePeer) String() string {
+	return fmt.Sprintf("bcRemovePeer{%v due to %v}", resp.peerID, resp.reason)
+}
+
+// resets the scheduler and processor state, e.g. following a switch from state syncing
+type bcResetState struct {
+	priorityHigh
+	state state.State
+}
+
+func (e bcResetState) String() string {
+	return fmt.Sprintf("bcResetState{%v}", e.state)
+}
+
+// Takes the channel as a parameter to avoid race conditions on r.events.
+func (r *BlockchainReactor) demux(events <-chan Event) {
+	var lastRate = 0.0
+	var lastHundred = time.Now()
+
+	var (
+		processBlockFreq = 20 * time.Millisecond
+		doProcessBlockCh = make(chan struct{}, 1)
+		doProcessBlockTk = time.NewTicker(processBlockFreq)
+	)
+	defer doProcessBlockTk.Stop()
+
+	var (
+		prunePeerFreq = 1 * time.Second
+		doPrunePeerCh = make(chan struct{}, 1)
+		doPrunePeerTk = time.NewTicker(prunePeerFreq)
+	)
+	defer doPrunePeerTk.Stop()
+
+	var (
+		scheduleFreq = 20 * time.Millisecond
+		doScheduleCh = make(chan struct{}, 1)
+		doScheduleTk = time.NewTicker(scheduleFreq)
+	)
+	defer doScheduleTk.Stop()
+
+	var (
+		statusFreq = 10 * time.Second
+		doStatusCh = make(chan struct{}, 1)
+		doStatusTk = time.NewTicker(statusFreq)
+	)
+	defer doStatusTk.Stop()
+	doStatusCh <- struct{}{} // immediately broadcast to get status of existing peers
+
+	// XXX: Extract timers to make testing atemporal
+	for {
+		select {
+		// Pacers: send at most per frequency but don't saturate
+		case <-doProcessBlockTk.C:
+			select {
+			case doProcessBlockCh <- struct{}{}:
+			default:
+			}
+		case <-doPrunePeerTk.C:
+			select {
+			case doPrunePeerCh <- struct{}{}:
+			default:
+			}
+		case <-doScheduleTk.C:
+			select {
+			case doScheduleCh <- struct{}{}:
+			default:
+			}
+		case <-doStatusTk.C:
+			select {
+			case doStatusCh <- struct{}{}:
+			default:
+			}
+
+		// Tickers: perform tasks periodically
+		case <-doScheduleCh:
+			r.scheduler.send(rTrySchedule{time: time.Now()})
+		case <-doPrunePeerCh:
+			r.scheduler.send(rTryPrunePeer{time: time.Now()})
+		case <-doProcessBlockCh:
+			r.processor.send(rProcessBlock{})
+		case <-doStatusCh:
+			if err := r.io.broadcastStatusRequest(); err != nil {
+				r.logger.Error("Error broadcasting status request", "err", err)
+			}
+
+		// Events from peers. Closing the channel signals event loop termination.
+		case event, ok := <-events:
+			if !ok {
+				r.logger.Info("Stopping event processing")
+				return
+			}
+			switch event := event.(type) {
+			case bcStatusResponse:
+				r.setMaxPeerHeight(event.height)
+				r.scheduler.send(event)
+			case bcAddNewPeer, bcRemovePeer, bcBlockResponse, bcNoBlockResponse:
+				r.scheduler.send(event)
+			default:
+				r.logger.Error("Received unexpected event", "event", fmt.Sprintf("%T", event))
+			}
+
+		// Incremental events from scheduler
+		case event := <-r.scheduler.next():
+			switch event := event.(type) {
+			case scBlockReceived:
+				r.processor.send(event)
+			case scPeerError:
+				r.processor.send(event)
+				if err := r.reporter.Report(behaviour.BadMessage(event.peerID, "scPeerError")); err != nil {
+					r.logger.Error("Error reporting peer", "err", err)
+				}
+			case scBlockRequest:
+				if err := r.io.sendBlockRequest(event.peerID, event.height); err != nil {
+					r.logger.Error("Error sending block request", "err", err)
+				}
+			case scFinishedEv:
+				r.processor.send(event)
+				r.scheduler.stop()
+			case scSchedulerFail:
+				r.logger.Error("Scheduler failure", "err", event.reason.Error())
+			case scPeersPruned:
+				// Remove peers from the processor.
+				for _, peerID := range event.peers {
+					r.processor.send(scPeerError{peerID: peerID, reason: errors.New("peer was pruned")})
+				}
+				r.logger.Debug("Pruned peers", "count", len(event.peers))
+			case noOpEvent:
+			default:
+				r.logger.Error("Received unexpected scheduler event", "event", fmt.Sprintf("%T", event))
+			}
+
+		// Incremental events from processor
+		case event := <-r.processor.next():
+			switch event := event.(type) {
+			case pcBlockProcessed:
+				r.setSyncHeight(event.height)
+				if r.syncHeight%100 == 0 {
+					lastRate = 0.9*lastRate + 0.1*(100/time.Since(lastHundred).Seconds())
+					r.logger.Info("Fast Sync Rate", "height", r.syncHeight,
+						"max_peer_height", r.maxPeerHeight, "blocks/s", lastRate)
+					lastHundred = time.Now()
+				}
+				r.scheduler.send(event)
+			case pcBlockVerificationFailure:
+				r.scheduler.send(event)
+			case pcFinished:
+				r.logger.Info("Fast sync complete, switching to consensus")
+				if !r.io.trySwitchToConsensus(event.tmState, event.blocksSynced > 0 || r.stateSynced) {
+					r.logger.Error("Failed to switch to consensus reactor")
+				}
+				r.endSync()
+				return
+			case noOpEvent:
+			default:
+				r.logger.Error("Received unexpected processor event", "event", fmt.Sprintf("%T", event))
+			}
+
+		// Terminal event from scheduler
+		case err := <-r.scheduler.final():
+			switch err {
+			case nil:
+				r.logger.Info("Scheduler stopped")
+			default:
+				r.logger.Error("Scheduler aborted with error", "err", err)
+			}
+
+		// Terminal event from processor
+		case err := <-r.processor.final():
+			switch err {
+			case nil:
+				r.logger.Info("Processor stopped")
+			default:
+				r.logger.Error("Processor aborted with error", "err", err)
+			}
+		}
+	}
+}
+
+// Stop implements cmn.Service interface.
+func (r *BlockchainReactor) Stop() error {
+	r.logger.Info("reactor stopping")
+	r.endSync()
+	r.logger.Info("reactor stopped")
+	return nil
+}
+
+// Receive implements Reactor by handling different message types.
+func (r *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) {
+	msg, err := bc.DecodeMsg(msgBytes)
+	if err != nil {
+		r.logger.Error("error decoding message",
+			"src", src.ID(), "chId", chID, "msg", msg, "err", err)
+		_ = r.reporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
+		return
+	}
+
+	if err = bc.ValidateMsg(msg); err != nil {
+		r.logger.Error("peer sent us invalid msg", "peer", src, "msg", msg, "err", err)
+		_ = r.reporter.Report(behaviour.BadMessage(src.ID(), err.Error()))
+		return
+	}
+
+	r.logger.Debug("Receive", "src", src.ID(), "chID", chID, "msg", msg)
+
+	switch msg := msg.(type) {
+	case *bcproto.StatusRequest:
+		if err := r.io.sendStatusResponse(r.store.Base(), r.store.Height(), src.ID()); err != nil {
+			r.logger.Error("Could not send status message to peer", "src", src)
+		}
+
+	case *bcproto.BlockRequest:
+		block := r.store.LoadBlock(msg.Height)
+		if block != nil {
+			if err = r.io.sendBlockToPeer(block, src.ID()); err != nil {
+				r.logger.Error("Could not send block message to peer: ", err)
+			}
+		} else {
+			r.logger.Info("peer asking for a block we don't have", "src", src, "height", msg.Height)
+			peerID := src.ID()
+			if err = r.io.sendBlockNotFound(msg.Height, peerID); err != nil {
+				r.logger.Error("Couldn't send block not found: ", err)
+			}
+		}
+
+	case *bcproto.StatusResponse:
+		r.mtx.RLock()
+		if r.events != nil {
+			r.events <- bcStatusResponse{peerID: src.ID(), base: msg.Base, height: msg.Height}
+		}
+		r.mtx.RUnlock()
+
+	case *bcproto.BlockResponse:
+		bi, err := types.BlockFromProto(msg.Block)
+		if err != nil {
+			r.logger.Error("error transitioning block from protobuf", "err", err)
+			return
+		}
+		r.mtx.RLock()
+		if r.events != nil {
+			r.events <- bcBlockResponse{
+				peerID: src.ID(),
+				block:  bi,
+				size:   int64(len(msgBytes)),
+				time:   time.Now(),
+			}
+		}
+		r.mtx.RUnlock()
+
+	case *bcproto.NoBlockResponse:
+		r.mtx.RLock()
+		if r.events != nil {
+			r.events <- bcNoBlockResponse{peerID: src.ID(), height: msg.Height, time: time.Now()}
+		}
+		r.mtx.RUnlock()
+	}
+}
+
+// AddPeer implements Reactor interface
+func (r *BlockchainReactor) AddPeer(peer p2p.Peer) {
+	err := r.io.sendStatusResponse(r.store.Base(), r.store.Height(), peer.ID())
+	if err != nil {
+		r.logger.Error("Could not send status message to peer new", "src", peer.ID, "height", r.SyncHeight())
+	}
+	r.mtx.RLock()
+	defer r.mtx.RUnlock()
+	if r.events != nil {
+		r.events <- bcAddNewPeer{peerID: peer.ID()}
+	}
+}
+
+// RemovePeer implements Reactor interface.
+func (r *BlockchainReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
+	r.mtx.RLock()
+	defer r.mtx.RUnlock()
+	if r.events != nil {
+		r.events <- bcRemovePeer{
+			peerID: peer.ID(),
+			reason: reason,
+		}
+	}
+}
+
+// GetChannels implements Reactor
+func (r *BlockchainReactor) GetChannels() []*p2p.ChannelDescriptor {
+	return []*p2p.ChannelDescriptor{
+		{
+			ID:                  BlockchainChannel,
+			Priority:            5,
+			SendQueueCapacity:   2000,
+			RecvBufferCapacity:  50 * 4096,
+			RecvMessageCapacity: bc.MaxMsgSize,
+		},
+	}
+}

internal/blocksync/v2/reactor_test.go

@@ -0,0 +1,555 @@
+package v2
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	dbm "github.com/tendermint/tm-db"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/behaviour"
+	bc "github.com/tendermint/tendermint/blockchain"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/service"
+	"github.com/tendermint/tendermint/mempool/mock"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/p2p/conn"
+	bcproto "github.com/tendermint/tendermint/proto/tendermint/blockchain"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+type mockPeer struct {
+	service.Service
+	id p2p.ID
+}
+
+func (mp mockPeer) FlushStop()           {}
+func (mp mockPeer) ID() p2p.ID           { return mp.id }
+func (mp mockPeer) RemoteIP() net.IP     { return net.IP{} }
+func (mp mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.RemoteIP(), Port: 8800} }
+
+func (mp mockPeer) IsOutbound() bool   { return true }
+func (mp mockPeer) IsPersistent() bool { return true }
+func (mp mockPeer) CloseConn() error   { return nil }
+
+func (mp mockPeer) NodeInfo() p2p.NodeInfo {
+	return p2p.DefaultNodeInfo{
+		DefaultNodeID: "",
+		ListenAddr:    "",
+	}
+}
+func (mp mockPeer) Status() conn.ConnectionStatus { return conn.ConnectionStatus{} }
+func (mp mockPeer) SocketAddr() *p2p.NetAddress   { return &p2p.NetAddress{} }
+
+func (mp mockPeer) Send(byte, []byte) bool    { return true }
+func (mp mockPeer) TrySend(byte, []byte) bool { return true }
+
+func (mp mockPeer) Set(string, interface{}) {}
+func (mp mockPeer) Get(string) interface{}  { return struct{}{} }
+
+// nolint:unused // ignore
+type mockBlockStore struct {
+	blocks map[int64]*types.Block
+}
+
+// nolint:unused // ignore
+func (ml *mockBlockStore) Height() int64 {
+	return int64(len(ml.blocks))
+}
+
+// nolint:unused // ignore
+func (ml *mockBlockStore) LoadBlock(height int64) *types.Block {
+	return ml.blocks[height]
+}
+
+// nolint:unused // ignore
+func (ml *mockBlockStore) SaveBlock(block *types.Block, part *types.PartSet, commit *types.Commit) {
+	ml.blocks[block.Height] = block
+}
+
+type mockBlockApplier struct {
+}
+
+// XXX: Add whitelist/blacklist?
+func (mba *mockBlockApplier) ApplyBlock(
+	state sm.State, blockID types.BlockID, block *types.Block,
+) (sm.State, int64, error) {
+	state.LastBlockHeight++
+	return state, 0, nil
+}
+
+type mockSwitchIo struct {
+	mtx                 sync.Mutex
+	switchedToConsensus bool
+	numStatusResponse   int
+	numBlockResponse    int
+	numNoBlockResponse  int
+}
+
+func (sio *mockSwitchIo) sendBlockRequest(peerID p2p.ID, height int64) error {
+	return nil
+}
+
+func (sio *mockSwitchIo) sendStatusResponse(base, height int64, peerID p2p.ID) error {
+	sio.mtx.Lock()
+	defer sio.mtx.Unlock()
+	sio.numStatusResponse++
+	return nil
+}
+
+func (sio *mockSwitchIo) sendBlockToPeer(block *types.Block, peerID p2p.ID) error {
+	sio.mtx.Lock()
+	defer sio.mtx.Unlock()
+	sio.numBlockResponse++
+	return nil
+}
+
+func (sio *mockSwitchIo) sendBlockNotFound(height int64, peerID p2p.ID) error {
+	sio.mtx.Lock()
+	defer sio.mtx.Unlock()
+	sio.numNoBlockResponse++
+	return nil
+}
+
+func (sio *mockSwitchIo) trySwitchToConsensus(state sm.State, skipWAL bool) bool {
+	sio.mtx.Lock()
+	defer sio.mtx.Unlock()
+	sio.switchedToConsensus = true
+	return true
+}
+
+func (sio *mockSwitchIo) broadcastStatusRequest() error {
+	return nil
+}
+
+type testReactorParams struct {
+	logger      log.Logger
+	genDoc      *types.GenesisDoc
+	privVals    []types.PrivValidator
+	startHeight int64
+	mockA       bool
+}
+
+func newTestReactor(p testReactorParams) *BlockchainReactor {
+	store, state, _ := newReactorStore(p.genDoc, p.privVals, p.startHeight)
+	reporter := behaviour.NewMockReporter()
+
+	var appl blockApplier
+
+	if p.mockA {
+		appl = &mockBlockApplier{}
+	} else {
+		app := &testApp{}
+		cc := proxy.NewLocalClientCreator(app)
+		proxyApp := proxy.NewAppConns(cc)
+		err := proxyApp.Start()
+		if err != nil {
+			panic(fmt.Errorf("error start app: %w", err))
+		}
+		db := dbm.NewMemDB()
+		stateStore := sm.NewStore(db)
+		appl = sm.NewBlockExecutor(stateStore, p.logger, proxyApp.Consensus(), mock.Mempool{}, sm.EmptyEvidencePool{})
+		if err = stateStore.Save(state); err != nil {
+			panic(err)
+		}
+	}
+
+	r := newReactor(state, store, reporter, appl, true)
+	logger := log.TestingLogger()
+	r.SetLogger(logger.With("module", "blockchain"))
+
+	return r
+}
+
+// This test is left here and not deleted to retain the termination cases for
+// future improvement in [#4482](https://github.com/tendermint/tendermint/issues/4482).
+// func TestReactorTerminationScenarios(t *testing.T) {
+
+// 	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
+// 	defer os.RemoveAll(config.RootDir)
+// 	genDoc, privVals := randGenesisDoc(config.ChainID(), 1, false, 30)
+// 	refStore, _, _ := newReactorStore(genDoc, privVals, 20)
+
+// 	params := testReactorParams{
+// 		logger:      log.TestingLogger(),
+// 		genDoc:      genDoc,
+// 		privVals:    privVals,
+// 		startHeight: 10,
+// 		bufferSize:  100,
+// 		mockA:       true,
+// 	}
+
+// 	type testEvent struct {
+// 		evType string
+// 		peer   string
+// 		height int64
+// 	}
+
+// 	tests := []struct {
+// 		name   string
+// 		params testReactorParams
+// 		msgs   []testEvent
+// 	}{
+// 		{
+// 			name:   "simple termination on max peer height - one peer",
+// 			params: params,
+// 			msgs: []testEvent{
+// 				{evType: "AddPeer", peer: "P1"},
+// 				{evType: "ReceiveS", peer: "P1", height: 13},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P1", height: 11},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P1", height: 12},
+// 				{evType: "Process"},
+// 				{evType: "ReceiveB", peer: "P1", height: 13},
+// 				{evType: "Process"},
+// 			},
+// 		},
+// 		{
+// 			name:   "simple termination on max peer height - two peers",
+// 			params: params,
+// 			msgs: []testEvent{
+// 				{evType: "AddPeer", peer: "P1"},
+// 				{evType: "AddPeer", peer: "P2"},
+// 				{evType: "ReceiveS", peer: "P1", height: 13},
+// 				{evType: "ReceiveS", peer: "P2", height: 15},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P1", height: 11},
+// 				{evType: "ReceiveB", peer: "P2", height: 12},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P1", height: 13},
+// 				{evType: "Process"},
+// 				{evType: "ReceiveB", peer: "P2", height: 14},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 15},
+// 				{evType: "Process"},
+// 			},
+// 		},
+// 		{
+// 			name:   "termination on max peer height - two peers, noBlock error",
+// 			params: params,
+// 			msgs: []testEvent{
+// 				{evType: "AddPeer", peer: "P1"},
+// 				{evType: "AddPeer", peer: "P2"},
+// 				{evType: "ReceiveS", peer: "P1", height: 13},
+// 				{evType: "ReceiveS", peer: "P2", height: 15},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveNB", peer: "P1", height: 11},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 12},
+// 				{evType: "ReceiveB", peer: "P2", height: 11},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 13},
+// 				{evType: "Process"},
+// 				{evType: "ReceiveB", peer: "P2", height: 14},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 15},
+// 				{evType: "Process"},
+// 			},
+// 		},
+// 		{
+// 			name:   "termination on max peer height - two peers, remove one peer",
+// 			params: params,
+// 			msgs: []testEvent{
+// 				{evType: "AddPeer", peer: "P1"},
+// 				{evType: "AddPeer", peer: "P2"},
+// 				{evType: "ReceiveS", peer: "P1", height: 13},
+// 				{evType: "ReceiveS", peer: "P2", height: 15},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "RemovePeer", peer: "P1"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 12},
+// 				{evType: "ReceiveB", peer: "P2", height: 11},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 13},
+// 				{evType: "Process"},
+// 				{evType: "ReceiveB", peer: "P2", height: 14},
+// 				{evType: "Process"},
+// 				{evType: "BlockReq"},
+// 				{evType: "ReceiveB", peer: "P2", height: 15},
+// 				{evType: "Process"},
+// 			},
+// 		},
+// 	}
+
+// 	for _, tt := range tests {
+// 		tt := tt
+// 		t.Run(tt.name, func(t *testing.T) {
+// 			reactor := newTestReactor(params)
+// 			reactor.Start()
+// 			reactor.reporter = behaviour.NewMockReporter()
+// 			mockSwitch := &mockSwitchIo{switchedToConsensus: false}
+// 			reactor.io = mockSwitch
+// 			// time for go routines to start
+// 			time.Sleep(time.Millisecond)
+
+// 			for _, step := range tt.msgs {
+// 				switch step.evType {
+// 				case "AddPeer":
+// 					reactor.scheduler.send(bcAddNewPeer{peerID: p2p.ID(step.peer)})
+// 				case "RemovePeer":
+// 					reactor.scheduler.send(bcRemovePeer{peerID: p2p.ID(step.peer)})
+// 				case "ReceiveS":
+// 					reactor.scheduler.send(bcStatusResponse{
+// 						peerID: p2p.ID(step.peer),
+// 						height: step.height,
+// 						time:   time.Now(),
+// 					})
+// 				case "ReceiveB":
+// 					reactor.scheduler.send(bcBlockResponse{
+// 						peerID: p2p.ID(step.peer),
+// 						block:  refStore.LoadBlock(step.height),
+// 						size:   10,
+// 						time:   time.Now(),
+// 					})
+// 				case "ReceiveNB":
+// 					reactor.scheduler.send(bcNoBlockResponse{
+// 						peerID: p2p.ID(step.peer),
+// 						height: step.height,
+// 						time:   time.Now(),
+// 					})
+// 				case "BlockReq":
+// 					reactor.scheduler.send(rTrySchedule{time: time.Now()})
+// 				case "Process":
+// 					reactor.processor.send(rProcessBlock{})
+// 				}
+// 				// give time for messages to propagate between routines
+// 				time.Sleep(time.Millisecond)
+// 			}
+
+// 			// time for processor to finish and reactor to switch to consensus
+// 			time.Sleep(20 * time.Millisecond)
+// 			assert.True(t, mockSwitch.hasSwitchedToConsensus())
+// 			reactor.Stop()
+// 		})
+// 	}
+// }
+
+func TestReactorHelperMode(t *testing.T) {
+	var (
+		channelID = byte(0x40)
+	)
+
+	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(config.ChainID(), 1, false, 30)
+
+	params := testReactorParams{
+		logger:      log.TestingLogger(),
+		genDoc:      genDoc,
+		privVals:    privVals,
+		startHeight: 20,
+		mockA:       true,
+	}
+
+	type testEvent struct {
+		peer  string
+		event interface{}
+	}
+
+	tests := []struct {
+		name   string
+		params testReactorParams
+		msgs   []testEvent
+	}{
+		{
+			name:   "status request",
+			params: params,
+			msgs: []testEvent{
+				{"P1", bcproto.StatusRequest{}},
+				{"P1", bcproto.BlockRequest{Height: 13}},
+				{"P1", bcproto.BlockRequest{Height: 20}},
+				{"P1", bcproto.BlockRequest{Height: 22}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			reactor := newTestReactor(params)
+			mockSwitch := &mockSwitchIo{switchedToConsensus: false}
+			reactor.io = mockSwitch
+			err := reactor.Start()
+			require.NoError(t, err)
+
+			for i := 0; i < len(tt.msgs); i++ {
+				step := tt.msgs[i]
+				switch ev := step.event.(type) {
+				case bcproto.StatusRequest:
+					old := mockSwitch.numStatusResponse
+					msg, err := bc.EncodeMsg(&ev)
+					assert.NoError(t, err)
+					reactor.Receive(channelID, mockPeer{id: p2p.ID(step.peer)}, msg)
+					assert.Equal(t, old+1, mockSwitch.numStatusResponse)
+				case bcproto.BlockRequest:
+					if ev.Height > params.startHeight {
+						old := mockSwitch.numNoBlockResponse
+						msg, err := bc.EncodeMsg(&ev)
+						assert.NoError(t, err)
+						reactor.Receive(channelID, mockPeer{id: p2p.ID(step.peer)}, msg)
+						assert.Equal(t, old+1, mockSwitch.numNoBlockResponse)
+					} else {
+						old := mockSwitch.numBlockResponse
+						msg, err := bc.EncodeMsg(&ev)
+						assert.NoError(t, err)
+						assert.NoError(t, err)
+						reactor.Receive(channelID, mockPeer{id: p2p.ID(step.peer)}, msg)
+						assert.Equal(t, old+1, mockSwitch.numBlockResponse)
+					}
+				}
+			}
+			err = reactor.Stop()
+			require.NoError(t, err)
+		})
+	}
+}
+
+func TestReactorSetSwitchNil(t *testing.T) {
+	config := cfg.ResetTestRoot("blockchain_reactor_v2_test")
+	defer os.RemoveAll(config.RootDir)
+	genDoc, privVals := randGenesisDoc(config.ChainID(), 1, false, 30)
+
+	reactor := newTestReactor(testReactorParams{
+		logger:   log.TestingLogger(),
+		genDoc:   genDoc,
+		privVals: privVals,
+	})
+	reactor.SetSwitch(nil)
+
+	assert.Nil(t, reactor.Switch)
+	assert.Nil(t, reactor.io)
+}
+
+//----------------------------------------------
+// utility funcs
+
+func makeTxs(height int64) (txs []types.Tx) {
+	for i := 0; i < 10; i++ {
+		txs = append(txs, types.Tx([]byte{byte(height), byte(i)}))
+	}
+	return txs
+}
+
+func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block {
+	block, _ := state.MakeBlock(height, makeTxs(height), lastCommit, nil, state.Validators.GetProposer().Address)
+	return block
+}
+
+type testApp struct {
+	abci.BaseApplication
+}
+
+func randGenesisDoc(chainID string, numValidators int, randPower bool, minPower int64) (
+	*types.GenesisDoc, []types.PrivValidator) {
+	validators := make([]types.GenesisValidator, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
+	for i := 0; i < numValidators; i++ {
+		val, privVal := types.RandValidator(randPower, minPower)
+		validators[i] = types.GenesisValidator{
+			PubKey: val.PubKey,
+			Power:  val.VotingPower,
+		}
+		privValidators[i] = privVal
+	}
+	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
+	return &types.GenesisDoc{
+		GenesisTime: tmtime.Now(),
+		ChainID:     chainID,
+		Validators:  validators,
+	}, privValidators
+}
+
+// Why are we importing the entire blockExecutor dependency graph here
+// when we have the facilities to
+func newReactorStore(
+	genDoc *types.GenesisDoc,
+	privVals []types.PrivValidator,
+	maxBlockHeight int64) (*store.BlockStore, sm.State, *sm.BlockExecutor) {
+	if len(privVals) != 1 {
+		panic("only support one validator")
+	}
+	app := &testApp{}
+	cc := proxy.NewLocalClientCreator(app)
+	proxyApp := proxy.NewAppConns(cc)
+	err := proxyApp.Start()
+	if err != nil {
+		panic(fmt.Errorf("error start app: %w", err))
+	}
+
+	stateDB := dbm.NewMemDB()
+	blockStore := store.NewBlockStore(dbm.NewMemDB())
+	stateStore := sm.NewStore(stateDB)
+	state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+	if err != nil {
+		panic(fmt.Errorf("error constructing state from genesis file: %w", err))
+	}
+
+	db := dbm.NewMemDB()
+	stateStore = sm.NewStore(db)
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(),
+		mock.Mempool{}, sm.EmptyEvidencePool{})
+	if err = stateStore.Save(state); err != nil {
+		panic(err)
+	}
+
+	// add blocks in
+	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
+		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
+		if blockHeight > 1 {
+			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
+			lastBlock := blockStore.LoadBlock(blockHeight - 1)
+			vote, err := types.MakeVote(
+				lastBlock.Header.Height,
+				lastBlockMeta.BlockID,
+				state.Validators,
+				privVals[0],
+				lastBlock.Header.ChainID,
+				time.Now(),
+			)
+			if err != nil {
+				panic(err)
+			}
+			lastCommit = types.NewCommit(vote.Height, vote.Round,
+				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
+		}
+
+		thisBlock := makeBlock(blockHeight, state, lastCommit)
+
+		thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes)
+		blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()}
+
+		state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock)
+		if err != nil {
+			panic(fmt.Errorf("error apply block: %w", err))
+		}
+
+		blockStore.SaveBlock(thisBlock, thisParts, lastCommit)
+	}
+	return blockStore, state, blockExec
+}

internal/blocksync/v2/routine.go

@@ -0,0 +1,166 @@
+package v2
+
+import (
+	"fmt"
+	"strings"
+	"sync/atomic"
+
+	"github.com/Workiva/go-datastructures/queue"
+
+	"github.com/tendermint/tendermint/libs/log"
+)
+
+type handleFunc = func(event Event) (Event, error)
+
+const historySize = 25
+
+// Routine is a structure that models a finite state machine as serialized
+// stream of events processed by a handle function. This Routine structure
+// handles the concurrency and messaging guarantees. Events are sent via
+// `send` are handled by the `handle` function to produce an iterator
+// `next()`. Calling `stop()` on a routine will conclude processing of all
+// sent events and produce `final()` event representing the terminal state.
+type Routine struct {
+	name    string
+	handle  handleFunc
+	queue   *queue.PriorityQueue
+	history []Event
+	out     chan Event
+	fin     chan error
+	rdy     chan struct{}
+	running *uint32
+	logger  log.Logger
+	metrics *Metrics
+}
+
+func newRoutine(name string, handleFunc handleFunc, bufferSize int) *Routine {
+	return &Routine{
+		name:    name,
+		handle:  handleFunc,
+		queue:   queue.NewPriorityQueue(bufferSize, true),
+		history: make([]Event, 0, historySize),
+		out:     make(chan Event, bufferSize),
+		rdy:     make(chan struct{}, 1),
+		fin:     make(chan error, 1),
+		running: new(uint32),
+		logger:  log.NewNopLogger(),
+		metrics: NopMetrics(),
+	}
+}
+
+func (rt *Routine) setLogger(logger log.Logger) {
+	rt.logger = logger
+}
+
+// nolint:unused
+func (rt *Routine) setMetrics(metrics *Metrics) {
+	rt.metrics = metrics
+}
+
+func (rt *Routine) start() {
+	rt.logger.Info("routine start", "msg", log.NewLazySprintf("%s: run", rt.name))
+	running := atomic.CompareAndSwapUint32(rt.running, uint32(0), uint32(1))
+	if !running {
+		panic(fmt.Sprintf("%s is already running", rt.name))
+	}
+	close(rt.rdy)
+	defer func() {
+		if r := recover(); r != nil {
+			var (
+				b strings.Builder
+				j int
+			)
+			for i := len(rt.history) - 1; i >= 0; i-- {
+				fmt.Fprintf(&b, "%d: %+v\n", j, rt.history[i])
+				j++
+			}
+			panic(fmt.Sprintf("%v\nlast events:\n%v", r, b.String()))
+		}
+		stopped := atomic.CompareAndSwapUint32(rt.running, uint32(1), uint32(0))
+		if !stopped {
+			panic(fmt.Sprintf("%s is failed to stop", rt.name))
+		}
+	}()
+
+	for {
+		events, err := rt.queue.Get(1)
+		if err == queue.ErrDisposed {
+			rt.terminate(nil)
+			return
+		} else if err != nil {
+			rt.terminate(err)
+			return
+		}
+		oEvent, err := rt.handle(events[0].(Event))
+		rt.metrics.EventsHandled.With("routine", rt.name).Add(1)
+		if err != nil {
+			rt.terminate(err)
+			return
+		}
+		rt.metrics.EventsOut.With("routine", rt.name).Add(1)
+		rt.logger.Debug("routine start", "msg", log.NewLazySprintf("%s: produced %T %+v", rt.name, oEvent, oEvent))
+
+		// Skip rTrySchedule and rProcessBlock events as they clutter the history
+		// due to their frequency.
+		switch events[0].(type) {
+		case rTrySchedule:
+		case rProcessBlock:
+		default:
+			rt.history = append(rt.history, events[0].(Event))
+			if len(rt.history) > historySize {
+				rt.history = rt.history[1:]
+			}
+		}
+
+		rt.out <- oEvent
+	}
+}
+
+// XXX: look into returning OpError in the net package
+func (rt *Routine) send(event Event) bool {
+	rt.logger.Debug("routine send", "msg", log.NewLazySprintf("%s: received %T %+v", rt.name, event, event))
+	if !rt.isRunning() {
+		return false
+	}
+	err := rt.queue.Put(event)
+	if err != nil {
+		rt.metrics.EventsShed.With("routine", rt.name).Add(1)
+		rt.logger.Error(fmt.Sprintf("%s: send failed, queue was full/stopped", rt.name))
+		return false
+	}
+
+	rt.metrics.EventsSent.With("routine", rt.name).Add(1)
+	return true
+}
+
+func (rt *Routine) isRunning() bool {
+	return atomic.LoadUint32(rt.running) == 1
+}
+
+func (rt *Routine) next() chan Event {
+	return rt.out
+}
+
+func (rt *Routine) ready() chan struct{} {
+	return rt.rdy
+}
+
+func (rt *Routine) stop() {
+	if !rt.isRunning() { // XXX: this should check rt.queue.Disposed()
+		return
+	}
+
+	rt.logger.Info("routine stop", "msg", log.NewLazySprintf("%s: stop", rt.name))
+	rt.queue.Dispose() // this should block until all queue items are free?
+}
+
+func (rt *Routine) final() chan error {
+	return rt.fin
+}
+
+// XXX: Maybe get rid of this
+func (rt *Routine) terminate(reason error) {
+	// We don't close the rt.out channel here, to avoid spinning on the closed channel
+	// in the event loop.
+	rt.fin <- reason
+}

internal/blocksync/v2/routine_test.go

@@ -0,0 +1,163 @@
+package v2
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type eventA struct {
+	priorityNormal
+}
+
+var errDone = fmt.Errorf("done")
+
+func simpleHandler(event Event) (Event, error) {
+	if _, ok := event.(eventA); ok {
+		return noOp, errDone
+	}
+	return noOp, nil
+}
+
+func TestRoutineFinal(t *testing.T) {
+	var (
+		bufferSize = 10
+		routine    = newRoutine("simpleRoutine", simpleHandler, bufferSize)
+	)
+
+	assert.False(t, routine.isRunning(),
+		"expected an initialized routine to not be running")
+	go routine.start()
+	<-routine.ready()
+	assert.True(t, routine.isRunning(),
+		"expected an started routine")
+
+	assert.True(t, routine.send(eventA{}),
+		"expected sending to a ready routine to succeed")
+
+	assert.Equal(t, errDone, <-routine.final(),
+		"expected the final event to be done")
+
+	assert.False(t, routine.isRunning(),
+		"expected an completed routine to no longer be running")
+}
+
+func TestRoutineStop(t *testing.T) {
+	var (
+		bufferSize = 10
+		routine    = newRoutine("simpleRoutine", simpleHandler, bufferSize)
+	)
+
+	assert.False(t, routine.send(eventA{}),
+		"expected sending to an unstarted routine to fail")
+
+	go routine.start()
+	<-routine.ready()
+
+	assert.True(t, routine.send(eventA{}),
+		"expected sending to a running routine to succeed")
+
+	routine.stop()
+
+	assert.False(t, routine.send(eventA{}),
+		"expected sending to a stopped routine to fail")
+}
+
+type finalCount struct {
+	count int
+}
+
+func (f finalCount) Error() string {
+	return "end"
+}
+
+func genStatefulHandler(maxCount int) handleFunc {
+	counter := 0
+	return func(event Event) (Event, error) {
+		if _, ok := event.(eventA); ok {
+			counter++
+			if counter >= maxCount {
+				return noOp, finalCount{counter}
+			}
+
+			return eventA{}, nil
+		}
+		return noOp, nil
+	}
+}
+
+func feedback(r *Routine) {
+	for event := range r.next() {
+		r.send(event)
+	}
+}
+
+func TestStatefulRoutine(t *testing.T) {
+	var (
+		count      = 10
+		handler    = genStatefulHandler(count)
+		bufferSize = 20
+		routine    = newRoutine("statefulRoutine", handler, bufferSize)
+	)
+
+	go routine.start()
+	go feedback(routine)
+	<-routine.ready()
+
+	assert.True(t, routine.send(eventA{}),
+		"expected sending to a started routine to succeed")
+
+	final := <-routine.final()
+	if fnl, ok := final.(finalCount); ok {
+		assert.Equal(t, count, fnl.count,
+			"expected the routine to count to 10")
+	} else {
+		t.Fail()
+	}
+}
+
+type lowPriorityEvent struct {
+	priorityLow
+}
+
+type highPriorityEvent struct {
+	priorityHigh
+}
+
+func handleWithPriority(event Event) (Event, error) {
+	switch event.(type) {
+	case lowPriorityEvent:
+		return noOp, nil
+	case highPriorityEvent:
+		return noOp, errDone
+	}
+	return noOp, nil
+}
+
+func TestPriority(t *testing.T) {
+	var (
+		bufferSize = 20
+		routine    = newRoutine("priorityRoutine", handleWithPriority, bufferSize)
+	)
+
+	go routine.start()
+	<-routine.ready()
+	go func() {
+		for {
+			routine.send(lowPriorityEvent{})
+			time.Sleep(1 * time.Millisecond)
+		}
+	}()
+	time.Sleep(10 * time.Millisecond)
+
+	assert.True(t, routine.isRunning(),
+		"expected an started routine")
+	assert.True(t, routine.send(highPriorityEvent{}),
+		"expected send to succeed even when saturated")
+
+	assert.Equal(t, errDone, <-routine.final())
+	assert.False(t, routine.isRunning(),
+		"expected an started routine")
+}

internal/blocksync/v2/scheduler.go

@@ -0,0 +1,712 @@
+package v2
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"time"
+
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Events generated by the scheduler:
+// all blocks have been processed
+type scFinishedEv struct {
+	priorityNormal
+	reason string
+}
+
+func (e scFinishedEv) String() string {
+	return fmt.Sprintf("scFinishedEv{%v}", e.reason)
+}
+
+// send a blockRequest message
+type scBlockRequest struct {
+	priorityNormal
+	peerID p2p.ID
+	height int64
+}
+
+func (e scBlockRequest) String() string {
+	return fmt.Sprintf("scBlockRequest{%d from %v}", e.height, e.peerID)
+}
+
+// a block has been received and validated by the scheduler
+type scBlockReceived struct {
+	priorityNormal
+	peerID p2p.ID
+	block  *types.Block
+}
+
+func (e scBlockReceived) String() string {
+	return fmt.Sprintf("scBlockReceived{%d#%X from %v}", e.block.Height, e.block.Hash(), e.peerID)
+}
+
+// scheduler detected a peer error
+type scPeerError struct {
+	priorityHigh
+	peerID p2p.ID
+	reason error
+}
+
+func (e scPeerError) String() string {
+	return fmt.Sprintf("scPeerError{%v errored with %v}", e.peerID, e.reason)
+}
+
+// scheduler removed a set of peers (timed out or slow peer)
+type scPeersPruned struct {
+	priorityHigh
+	peers []p2p.ID
+}
+
+func (e scPeersPruned) String() string {
+	return fmt.Sprintf("scPeersPruned{%v}", e.peers)
+}
+
+// XXX: make this fatal?
+// scheduler encountered a fatal error
+type scSchedulerFail struct {
+	priorityHigh
+	reason error
+}
+
+func (e scSchedulerFail) String() string {
+	return fmt.Sprintf("scSchedulerFail{%v}", e.reason)
+}
+
+type blockState int
+
+const (
+	blockStateUnknown   blockState = iota + 1 // no known peer has this block
+	blockStateNew                             // indicates that a peer has reported having this block
+	blockStatePending                         // indicates that this block has been requested from a peer
+	blockStateReceived                        // indicates that this block has been received by a peer
+	blockStateProcessed                       // indicates that this block has been applied
+)
+
+func (e blockState) String() string {
+	switch e {
+	case blockStateUnknown:
+		return "Unknown"
+	case blockStateNew:
+		return "New"
+	case blockStatePending:
+		return "Pending"
+	case blockStateReceived:
+		return "Received"
+	case blockStateProcessed:
+		return "Processed"
+	default:
+		return fmt.Sprintf("invalid blockState: %d", e)
+	}
+}
+
+type peerState int
+
+const (
+	peerStateNew = iota + 1
+	peerStateReady
+	peerStateRemoved
+)
+
+func (e peerState) String() string {
+	switch e {
+	case peerStateNew:
+		return "New"
+	case peerStateReady:
+		return "Ready"
+	case peerStateRemoved:
+		return "Removed"
+	default:
+		panic(fmt.Sprintf("unknown peerState: %d", e))
+	}
+}
+
+type scPeer struct {
+	peerID p2p.ID
+
+	// initialized as New when peer is added, updated to Ready when statusUpdate is received,
+	// updated to Removed when peer is removed
+	state peerState
+
+	base        int64 // updated when statusResponse is received
+	height      int64 // updated when statusResponse is received
+	lastTouched time.Time
+	lastRate    int64 // last receive rate in bytes
+}
+
+func (p scPeer) String() string {
+	return fmt.Sprintf("{state %v, base %d, height %d, lastTouched %v, lastRate %d, id %v}",
+		p.state, p.base, p.height, p.lastTouched, p.lastRate, p.peerID)
+}
+
+func newScPeer(peerID p2p.ID) *scPeer {
+	return &scPeer{
+		peerID:      peerID,
+		state:       peerStateNew,
+		base:        -1,
+		height:      -1,
+		lastTouched: time.Time{},
+	}
+}
+
+// The scheduler keep track of the state of each block and each peer. The
+// scheduler will attempt to schedule new block requests with `trySchedule`
+// events and remove slow peers with `tryPrune` events.
+type scheduler struct {
+	initHeight int64
+
+	// next block that needs to be processed. All blocks with smaller height are
+	// in Processed state.
+	height int64
+
+	// lastAdvance tracks the last time a block execution happened.
+	// syncTimeout is the maximum time the scheduler waits to advance in the fast sync process before finishing.
+	// This covers the cases where there are no peers or all peers have a lower height.
+	lastAdvance time.Time
+	syncTimeout time.Duration
+
+	// a map of peerID to scheduler specific peer struct `scPeer` used to keep
+	// track of peer specific state
+	peers       map[p2p.ID]*scPeer
+	peerTimeout time.Duration // maximum response time from a peer otherwise prune
+	minRecvRate int64         // minimum receive rate from peer otherwise prune
+
+	// the maximum number of blocks that should be New, Received or Pending at any point
+	// in time. This is used to enforce a limit on the blockStates map.
+	targetPending int
+	// a list of blocks to be scheduled (New), Pending or Received. Its length should be
+	// smaller than targetPending.
+	blockStates map[int64]blockState
+
+	// a map of heights to the peer we are waiting a response from
+	pendingBlocks map[int64]p2p.ID
+
+	// the time at which a block was put in blockStatePending
+	pendingTime map[int64]time.Time
+
+	// a map of heights to the peers that put the block in blockStateReceived
+	receivedBlocks map[int64]p2p.ID
+}
+
+func (sc scheduler) String() string {
+	return fmt.Sprintf("ih: %d, bst: %v, peers: %v, pblks: %v, ptm %v, rblks: %v",
+		sc.initHeight, sc.blockStates, sc.peers, sc.pendingBlocks, sc.pendingTime, sc.receivedBlocks)
+}
+
+func newScheduler(initHeight int64, startTime time.Time) *scheduler {
+	sc := scheduler{
+		initHeight:     initHeight,
+		lastAdvance:    startTime,
+		syncTimeout:    60 * time.Second,
+		height:         initHeight,
+		blockStates:    make(map[int64]blockState),
+		peers:          make(map[p2p.ID]*scPeer),
+		pendingBlocks:  make(map[int64]p2p.ID),
+		pendingTime:    make(map[int64]time.Time),
+		receivedBlocks: make(map[int64]p2p.ID),
+		targetPending:  10,               // TODO - pass as param
+		peerTimeout:    15 * time.Second, // TODO - pass as param
+		minRecvRate:    0,                // int64(7680), TODO - pass as param
+	}
+
+	return &sc
+}
+
+func (sc *scheduler) ensurePeer(peerID p2p.ID) *scPeer {
+	if _, ok := sc.peers[peerID]; !ok {
+		sc.peers[peerID] = newScPeer(peerID)
+	}
+	return sc.peers[peerID]
+}
+
+func (sc *scheduler) touchPeer(peerID p2p.ID, time time.Time) error {
+	peer, ok := sc.peers[peerID]
+	if !ok {
+		return fmt.Errorf("couldn't find peer %s", peerID)
+	}
+
+	if peer.state != peerStateReady {
+		return fmt.Errorf("tried to touch peer in state %s, must be Ready", peer.state)
+	}
+
+	peer.lastTouched = time
+
+	return nil
+}
+
+func (sc *scheduler) removePeer(peerID p2p.ID) {
+	peer, ok := sc.peers[peerID]
+	if !ok {
+		return
+	}
+	if peer.state == peerStateRemoved {
+		return
+	}
+
+	for height, pendingPeerID := range sc.pendingBlocks {
+		if pendingPeerID == peerID {
+			sc.setStateAtHeight(height, blockStateNew)
+			delete(sc.pendingTime, height)
+			delete(sc.pendingBlocks, height)
+		}
+	}
+
+	for height, rcvPeerID := range sc.receivedBlocks {
+		if rcvPeerID == peerID {
+			sc.setStateAtHeight(height, blockStateNew)
+			delete(sc.receivedBlocks, height)
+		}
+	}
+
+	// remove the blocks from blockStates if the peer removal causes the max peer height to be lower.
+	peer.state = peerStateRemoved
+	maxPeerHeight := int64(0)
+	for _, otherPeer := range sc.peers {
+		if otherPeer.state != peerStateReady {
+			continue
+		}
+		if otherPeer.peerID != peer.peerID && otherPeer.height > maxPeerHeight {
+			maxPeerHeight = otherPeer.height
+		}
+	}
+	for h := range sc.blockStates {
+		if h > maxPeerHeight {
+			delete(sc.blockStates, h)
+		}
+	}
+}
+
+// check if the blockPool is running low and add new blocks in New state to be requested.
+// This function is called when there is an increase in the maximum peer height or when
+// blocks are processed.
+func (sc *scheduler) addNewBlocks() {
+	if len(sc.blockStates) >= sc.targetPending {
+		return
+	}
+
+	for i := sc.height; i < int64(sc.targetPending)+sc.height; i++ {
+		if i > sc.maxHeight() {
+			break
+		}
+		if sc.getStateAtHeight(i) == blockStateUnknown {
+			sc.setStateAtHeight(i, blockStateNew)
+		}
+	}
+}
+
+func (sc *scheduler) setPeerRange(peerID p2p.ID, base int64, height int64) error {
+	peer := sc.ensurePeer(peerID)
+
+	if peer.state == peerStateRemoved {
+		return nil // noop
+	}
+
+	if height < peer.height {
+		sc.removePeer(peerID)
+		return fmt.Errorf("cannot move peer height lower. from %d to %d", peer.height, height)
+	}
+
+	if base > height {
+		sc.removePeer(peerID)
+		return fmt.Errorf("cannot set peer base higher than its height")
+	}
+
+	peer.base = base
+	peer.height = height
+	peer.state = peerStateReady
+
+	sc.addNewBlocks()
+	return nil
+}
+
+func (sc *scheduler) getStateAtHeight(height int64) blockState {
+	if height < sc.height {
+		return blockStateProcessed
+	} else if state, ok := sc.blockStates[height]; ok {
+		return state
+	} else {
+		return blockStateUnknown
+	}
+}
+
+func (sc *scheduler) getPeersWithHeight(height int64) []p2p.ID {
+	peers := make([]p2p.ID, 0)
+	for _, peer := range sc.peers {
+		if peer.state != peerStateReady {
+			continue
+		}
+		if peer.base <= height && peer.height >= height {
+			peers = append(peers, peer.peerID)
+		}
+	}
+	return peers
+}
+
+func (sc *scheduler) prunablePeers(peerTimout time.Duration, minRecvRate int64, now time.Time) []p2p.ID {
+	prunable := make([]p2p.ID, 0)
+	for peerID, peer := range sc.peers {
+		if peer.state != peerStateReady {
+			continue
+		}
+		if now.Sub(peer.lastTouched) > peerTimout || peer.lastRate < minRecvRate {
+			prunable = append(prunable, peerID)
+		}
+	}
+	// Tests for handleTryPrunePeer() may fail without sort due to range non-determinism
+	sort.Sort(PeerByID(prunable))
+	return prunable
+}
+
+func (sc *scheduler) setStateAtHeight(height int64, state blockState) {
+	sc.blockStates[height] = state
+}
+
+// CONTRACT: peer exists and in Ready state.
+func (sc *scheduler) markReceived(peerID p2p.ID, height int64, size int64, now time.Time) error {
+	peer := sc.peers[peerID]
+
+	if state := sc.getStateAtHeight(height); state != blockStatePending || sc.pendingBlocks[height] != peerID {
+		return fmt.Errorf("received block %d from peer %s without being requested", height, peerID)
+	}
+
+	pendingTime, ok := sc.pendingTime[height]
+	if !ok || now.Sub(pendingTime) <= 0 {
+		return fmt.Errorf("clock error: block %d received at %s but requested at %s",
+			height, pendingTime, now)
+	}
+
+	peer.lastRate = size / now.Sub(pendingTime).Nanoseconds()
+
+	sc.setStateAtHeight(height, blockStateReceived)
+	delete(sc.pendingBlocks, height)
+	delete(sc.pendingTime, height)
+
+	sc.receivedBlocks[height] = peerID
+
+	return nil
+}
+
+func (sc *scheduler) markPending(peerID p2p.ID, height int64, time time.Time) error {
+	state := sc.getStateAtHeight(height)
+	if state != blockStateNew {
+		return fmt.Errorf("block %d should be in blockStateNew but is %s", height, state)
+	}
+
+	peer, ok := sc.peers[peerID]
+	if !ok {
+		return fmt.Errorf("cannot find peer %s", peerID)
+	}
+
+	if peer.state != peerStateReady {
+		return fmt.Errorf("cannot schedule %d from %s in %s", height, peerID, peer.state)
+	}
+
+	if height > peer.height {
+		return fmt.Errorf("cannot request height %d from peer %s that is at height %d",
+			height, peerID, peer.height)
+	}
+
+	if height < peer.base {
+		return fmt.Errorf("cannot request height %d for peer %s with base %d",
+			height, peerID, peer.base)
+	}
+
+	sc.setStateAtHeight(height, blockStatePending)
+	sc.pendingBlocks[height] = peerID
+	sc.pendingTime[height] = time
+
+	return nil
+}
+
+func (sc *scheduler) markProcessed(height int64) error {
+	// It is possible that a peer error or timeout is handled after the processor
+	// has processed the block but before the scheduler received this event, so
+	// when pcBlockProcessed event is received, the block had been requested
+	// again => don't check the block state.
+	sc.lastAdvance = time.Now()
+	sc.height = height + 1
+	delete(sc.pendingBlocks, height)
+	delete(sc.pendingTime, height)
+	delete(sc.receivedBlocks, height)
+	delete(sc.blockStates, height)
+	sc.addNewBlocks()
+	return nil
+}
+
+func (sc *scheduler) allBlocksProcessed() bool {
+	if len(sc.peers) == 0 {
+		return false
+	}
+	return sc.height >= sc.maxHeight()
+}
+
+// returns max peer height or the last processed block, i.e. sc.height
+func (sc *scheduler) maxHeight() int64 {
+	max := sc.height - 1
+	for _, peer := range sc.peers {
+		if peer.state != peerStateReady {
+			continue
+		}
+		if max < peer.height {
+			max = peer.height
+		}
+	}
+	return max
+}
+
+// lowest block in sc.blockStates with state == blockStateNew or -1 if no new blocks
+func (sc *scheduler) nextHeightToSchedule() int64 {
+	var min int64 = math.MaxInt64
+	for height, state := range sc.blockStates {
+		if state == blockStateNew && height < min {
+			min = height
+		}
+	}
+	if min == math.MaxInt64 {
+		min = -1
+	}
+	return min
+}
+
+func (sc *scheduler) pendingFrom(peerID p2p.ID) []int64 {
+	var heights []int64
+	for height, pendingPeerID := range sc.pendingBlocks {
+		if pendingPeerID == peerID {
+			heights = append(heights, height)
+		}
+	}
+	return heights
+}
+
+func (sc *scheduler) selectPeer(height int64) (p2p.ID, error) {
+	peers := sc.getPeersWithHeight(height)
+	if len(peers) == 0 {
+		return "", fmt.Errorf("cannot find peer for height %d", height)
+	}
+
+	// create a map from number of pending requests to a list
+	// of peers having that number of pending requests.
+	pendingFrom := make(map[int][]p2p.ID)
+	for _, peerID := range peers {
+		numPending := len(sc.pendingFrom(peerID))
+		pendingFrom[numPending] = append(pendingFrom[numPending], peerID)
+	}
+
+	// find the set of peers with minimum number of pending requests.
+	var minPending int64 = math.MaxInt64
+	for mp := range pendingFrom {
+		if int64(mp) < minPending {
+			minPending = int64(mp)
+		}
+	}
+
+	sort.Sort(PeerByID(pendingFrom[int(minPending)]))
+	return pendingFrom[int(minPending)][0], nil
+}
+
+// PeerByID is a list of peers sorted by peerID.
+type PeerByID []p2p.ID
+
+func (peers PeerByID) Len() int {
+	return len(peers)
+}
+func (peers PeerByID) Less(i, j int) bool {
+	return bytes.Compare([]byte(peers[i]), []byte(peers[j])) == -1
+}
+
+func (peers PeerByID) Swap(i, j int) {
+	peers[i], peers[j] = peers[j], peers[i]
+}
+
+// Handlers
+
+// This handler gets the block, performs some validation and then passes it on to the processor.
+func (sc *scheduler) handleBlockResponse(event bcBlockResponse) (Event, error) {
+	err := sc.touchPeer(event.peerID, event.time)
+	if err != nil {
+		// peer does not exist OR not ready
+		return noOp, nil
+	}
+
+	err = sc.markReceived(event.peerID, event.block.Height, event.size, event.time)
+	if err != nil {
+		sc.removePeer(event.peerID)
+		return scPeerError{peerID: event.peerID, reason: err}, nil
+	}
+
+	return scBlockReceived{peerID: event.peerID, block: event.block}, nil
+}
+
+func (sc *scheduler) handleNoBlockResponse(event bcNoBlockResponse) (Event, error) {
+	// No such peer or peer was removed.
+	peer, ok := sc.peers[event.peerID]
+	if !ok || peer.state == peerStateRemoved {
+		return noOp, nil
+	}
+
+	// The peer may have been just removed due to errors, low speed or timeouts.
+	sc.removePeer(event.peerID)
+
+	return scPeerError{peerID: event.peerID,
+		reason: fmt.Errorf("peer %v with base %d height %d claims no block for %d",
+			event.peerID, peer.base, peer.height, event.height)}, nil
+}
+
+func (sc *scheduler) handleBlockProcessed(event pcBlockProcessed) (Event, error) {
+	if event.height != sc.height {
+		panic(fmt.Sprintf("processed height %d, but expected height %d", event.height, sc.height))
+	}
+
+	err := sc.markProcessed(event.height)
+	if err != nil {
+		return scSchedulerFail{reason: err}, nil
+	}
+
+	if sc.allBlocksProcessed() {
+		return scFinishedEv{reason: "processed all blocks"}, nil
+	}
+
+	return noOp, nil
+}
+
+// Handles an error from the processor. The processor had already cleaned the blocks from
+// the peers included in this event. Just attempt to remove the peers.
+func (sc *scheduler) handleBlockProcessError(event pcBlockVerificationFailure) (Event, error) {
+	// The peers may have been just removed due to errors, low speed or timeouts.
+	sc.removePeer(event.firstPeerID)
+	if event.firstPeerID != event.secondPeerID {
+		sc.removePeer(event.secondPeerID)
+	}
+
+	if sc.allBlocksProcessed() {
+		return scFinishedEv{reason: "error on last block"}, nil
+	}
+
+	return noOp, nil
+}
+
+func (sc *scheduler) handleAddNewPeer(event bcAddNewPeer) (Event, error) {
+	sc.ensurePeer(event.peerID)
+	return noOp, nil
+}
+
+func (sc *scheduler) handleRemovePeer(event bcRemovePeer) (Event, error) {
+	sc.removePeer(event.peerID)
+
+	if sc.allBlocksProcessed() {
+		return scFinishedEv{reason: "removed peer"}, nil
+	}
+
+	// Return scPeerError so the peer (and all associated blocks) is removed from
+	// the processor.
+	return scPeerError{peerID: event.peerID, reason: errors.New("peer was stopped")}, nil
+}
+
+func (sc *scheduler) handleTryPrunePeer(event rTryPrunePeer) (Event, error) {
+	// Check behavior of peer responsible to deliver block at sc.height.
+	timeHeightAsked, ok := sc.pendingTime[sc.height]
+	if ok && time.Since(timeHeightAsked) > sc.peerTimeout {
+		// A request was sent to a peer for block at sc.height but a response was not received
+		// from that peer within sc.peerTimeout. Remove the peer. This is to ensure that a peer
+		// will be timed out even if it sends blocks at higher heights but prevents progress by
+		// not sending the block at current height.
+		sc.removePeer(sc.pendingBlocks[sc.height])
+	}
+
+	prunablePeers := sc.prunablePeers(sc.peerTimeout, sc.minRecvRate, event.time)
+	if len(prunablePeers) == 0 {
+		return noOp, nil
+	}
+	for _, peerID := range prunablePeers {
+		sc.removePeer(peerID)
+	}
+
+	// If all blocks are processed we should finish.
+	if sc.allBlocksProcessed() {
+		return scFinishedEv{reason: "after try prune"}, nil
+	}
+
+	return scPeersPruned{peers: prunablePeers}, nil
+}
+
+func (sc *scheduler) handleResetState(event bcResetState) (Event, error) {
+	initHeight := event.state.LastBlockHeight + 1
+	if initHeight == 1 {
+		initHeight = event.state.InitialHeight
+	}
+	sc.initHeight = initHeight
+	sc.height = initHeight
+	sc.lastAdvance = time.Now()
+	sc.addNewBlocks()
+	return noOp, nil
+}
+
+func (sc *scheduler) handleTrySchedule(event rTrySchedule) (Event, error) {
+	if time.Since(sc.lastAdvance) > sc.syncTimeout {
+		return scFinishedEv{reason: "timeout, no advance"}, nil
+	}
+
+	nextHeight := sc.nextHeightToSchedule()
+	if nextHeight == -1 {
+		return noOp, nil
+	}
+
+	bestPeerID, err := sc.selectPeer(nextHeight)
+	if err != nil {
+		return scSchedulerFail{reason: err}, nil
+	}
+	if err := sc.markPending(bestPeerID, nextHeight, event.time); err != nil {
+		return scSchedulerFail{reason: err}, nil // XXX: peerError might be more appropriate
+	}
+	return scBlockRequest{peerID: bestPeerID, height: nextHeight}, nil
+
+}
+
+func (sc *scheduler) handleStatusResponse(event bcStatusResponse) (Event, error) {
+	err := sc.setPeerRange(event.peerID, event.base, event.height)
+	if err != nil {
+		return scPeerError{peerID: event.peerID, reason: err}, nil
+	}
+	return noOp, nil
+}
+
+func (sc *scheduler) handle(event Event) (Event, error) {
+	switch event := event.(type) {
+	case bcResetState:
+		nextEvent, err := sc.handleResetState(event)
+		return nextEvent, err
+	case bcStatusResponse:
+		nextEvent, err := sc.handleStatusResponse(event)
+		return nextEvent, err
+	case bcBlockResponse:
+		nextEvent, err := sc.handleBlockResponse(event)
+		return nextEvent, err
+	case bcNoBlockResponse:
+		nextEvent, err := sc.handleNoBlockResponse(event)
+		return nextEvent, err
+	case rTrySchedule:
+		nextEvent, err := sc.handleTrySchedule(event)
+		return nextEvent, err
+	case bcAddNewPeer:
+		nextEvent, err := sc.handleAddNewPeer(event)
+		return nextEvent, err
+	case bcRemovePeer:
+		nextEvent, err := sc.handleRemovePeer(event)
+		return nextEvent, err
+	case rTryPrunePeer:
+		nextEvent, err := sc.handleTryPrunePeer(event)
+		return nextEvent, err
+	case pcBlockProcessed:
+		nextEvent, err := sc.handleBlockProcessed(event)
+		return nextEvent, err
+	case pcBlockVerificationFailure:
+		nextEvent, err := sc.handleBlockProcessError(event)
+		return nextEvent, err
+	default:
+		return scSchedulerFail{reason: fmt.Errorf("unknown event %v", event)}, nil
+	}
+}

internal/blocksync/v2/types.go

@@ -0,0 +1,65 @@
+package v2
+
+import (
+	"github.com/Workiva/go-datastructures/queue"
+)
+
+// Event is the type that can be added to the priority queue.
+type Event queue.Item
+
+type priority interface {
+	Compare(other queue.Item) int
+	Priority() int
+}
+
+type priorityLow struct{}
+type priorityNormal struct{}
+type priorityHigh struct{}
+
+func (p priorityLow) Priority() int {
+	return 1
+}
+
+func (p priorityNormal) Priority() int {
+	return 2
+}
+
+func (p priorityHigh) Priority() int {
+	return 3
+}
+
+func (p priorityLow) Compare(other queue.Item) int {
+	op := other.(priority)
+	if p.Priority() > op.Priority() {
+		return 1
+	} else if p.Priority() == op.Priority() {
+		return 0
+	}
+	return -1
+}
+
+func (p priorityNormal) Compare(other queue.Item) int {
+	op := other.(priority)
+	if p.Priority() > op.Priority() {
+		return 1
+	} else if p.Priority() == op.Priority() {
+		return 0
+	}
+	return -1
+}
+
+func (p priorityHigh) Compare(other queue.Item) int {
+	op := other.(priority)
+	if p.Priority() > op.Priority() {
+		return 1
+	} else if p.Priority() == op.Priority() {
+		return 0
+	}
+	return -1
+}
+
+type noOpEvent struct {
+	priorityLow
+}
+
+var noOp = noOpEvent{}

internal/consensus/README.md

@@ -0,0 +1,3 @@
+# Consensus 
+
+See the [consensus spec](https://github.com/tendermint/spec/tree/master/spec/consensus) and the [reactor consensus spec](https://github.com/tendermint/spec/tree/master/spec/reactors/consensus) for more information.

internal/consensus/byzantine_test.go

@@ -0,0 +1,575 @@
+package consensus
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/evidence"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/service"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	mempl "github.com/tendermint/tendermint/mempool"
+
+	cfg "github.com/tendermint/tendermint/config"
+	mempoolv0 "github.com/tendermint/tendermint/mempool/v0"
+	mempoolv1 "github.com/tendermint/tendermint/mempool/v1"
+	"github.com/tendermint/tendermint/p2p"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+//----------------------------------------------
+// byzantine failures
+
+// Byzantine node sends two different prevotes (nil and blockID) to the same validator
+func TestByzantinePrevoteEquivocation(t *testing.T) {
+	const nValidators = 4
+	const byzantineNode = 0
+	const prevoteHeight = int64(2)
+	testName := "consensus_byzantine_test"
+	tickerFunc := newMockTickerFunc(true)
+	appFunc := newCounter
+
+	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
+	css := make([]*State, nValidators)
+
+	for i := 0; i < nValidators; i++ {
+		logger := consensusLogger().With("test", "byzantine", "validator", i)
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		defer os.RemoveAll(thisConfig.RootDir)
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		app := appFunc()
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		blockDB := dbm.NewMemDB()
+		blockStore := store.NewBlockStore(blockDB)
+
+		mtx := new(tmsync.Mutex)
+		// one for mempool, one for consensus
+		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+		proxyAppConnConMem := abcicli.NewLocalClient(mtx, app)
+
+		// Make Mempool
+		var mempool mempl.Mempool
+
+		switch thisConfig.Mempool.Version {
+		case cfg.MempoolV0:
+			mempool = mempoolv0.NewCListMempool(config.Mempool,
+				proxyAppConnConMem,
+				state.LastBlockHeight,
+				mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
+				mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
+		case cfg.MempoolV1:
+			mempool = mempoolv1.NewTxMempool(logger,
+				config.Mempool,
+				proxyAppConnConMem,
+				state.LastBlockHeight,
+				mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
+				mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
+			)
+		}
+
+		if thisConfig.Consensus.WaitForTxs() {
+			mempool.EnableTxsAvailable()
+		}
+
+		// Make a full instance of the evidence pool
+		evidenceDB := dbm.NewMemDB()
+		evpool, err := evidence.NewPool(evidenceDB, stateStore, blockStore)
+		require.NoError(t, err)
+		evpool.SetLogger(logger.With("module", "evidence"))
+
+		// Make State
+		blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+		cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+		cs.SetLogger(cs.Logger)
+		// set private validator
+		pv := privVals[i]
+		cs.SetPrivValidator(pv)
+
+		eventBus := types.NewEventBus()
+		eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+		err = eventBus.Start()
+		require.NoError(t, err)
+		cs.SetEventBus(eventBus)
+
+		cs.SetTimeoutTicker(tickerFunc())
+		cs.SetLogger(logger)
+
+		css[i] = cs
+	}
+
+	// initialize the reactors for each of the validators
+	reactors := make([]*Reactor, nValidators)
+	blocksSubs := make([]types.Subscription, 0)
+	eventBuses := make([]*types.EventBus, nValidators)
+	for i := 0; i < nValidators; i++ {
+		reactors[i] = NewReactor(css[i], true) // so we dont start the consensus states
+		reactors[i].SetLogger(css[i].Logger)
+
+		// eventBus is already started with the cs
+		eventBuses[i] = css[i].eventBus
+		reactors[i].SetEventBus(eventBuses[i])
+
+		blocksSub, err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, 100)
+		require.NoError(t, err)
+		blocksSubs = append(blocksSubs, blocksSub)
+
+		if css[i].state.LastBlockHeight == 0 { // simulate handle initChain in handshake
+			err = css[i].blockExec.Store().Save(css[i].state)
+			require.NoError(t, err)
+		}
+	}
+	// make connected switches and start all reactors
+	p2p.MakeConnectedSwitches(config.P2P, nValidators, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("CONSENSUS", reactors[i])
+		s.SetLogger(reactors[i].conS.Logger.With("module", "p2p"))
+		return s
+	}, p2p.Connect2Switches)
+
+	// create byzantine validator
+	bcs := css[byzantineNode]
+
+	// alter prevote so that the byzantine node double votes when height is 2
+	bcs.doPrevote = func(height int64, round int32) {
+		// allow first height to happen normally so that byzantine validator is no longer proposer
+		if height == prevoteHeight {
+			bcs.Logger.Info("Sending two votes")
+			prevote1, err := bcs.signVote(tmproto.PrevoteType, bcs.ProposalBlock.Hash(), bcs.ProposalBlockParts.Header())
+			require.NoError(t, err)
+			prevote2, err := bcs.signVote(tmproto.PrevoteType, nil, types.PartSetHeader{})
+			require.NoError(t, err)
+			peerList := reactors[byzantineNode].Switch.Peers().List()
+			bcs.Logger.Info("Getting peer list", "peers", peerList)
+			// send two votes to all peers (1st to one half, 2nd to another half)
+			for i, peer := range peerList {
+				if i < len(peerList)/2 {
+					bcs.Logger.Info("Signed and pushed vote", "vote", prevote1, "peer", peer)
+					peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote1}))
+				} else {
+					bcs.Logger.Info("Signed and pushed vote", "vote", prevote2, "peer", peer)
+					peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote2}))
+				}
+			}
+		} else {
+			bcs.Logger.Info("Behaving normally")
+			bcs.defaultDoPrevote(height, round)
+		}
+	}
+
+	// introducing a lazy proposer means that the time of the block committed is different to the
+	// timestamp that the other nodes have. This tests to ensure that the evidence that finally gets
+	// proposed will have a valid timestamp
+	lazyProposer := css[1]
+
+	lazyProposer.decideProposal = func(height int64, round int32) {
+		lazyProposer.Logger.Info("Lazy Proposer proposing condensed commit")
+		if lazyProposer.privValidator == nil {
+			panic("entered createProposalBlock with privValidator being nil")
+		}
+
+		var commit *types.Commit
+		switch {
+		case lazyProposer.Height == lazyProposer.state.InitialHeight:
+			// We're creating a proposal for the first block.
+			// The commit is empty, but not nil.
+			commit = types.NewCommit(0, 0, types.BlockID{}, nil)
+		case lazyProposer.LastCommit.HasTwoThirdsMajority():
+			// Make the commit from LastCommit
+			commit = lazyProposer.LastCommit.MakeCommit()
+		default: // This shouldn't happen.
+			lazyProposer.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block")
+			return
+		}
+
+		// omit the last signature in the commit
+		commit.Signatures[len(commit.Signatures)-1] = types.NewCommitSigAbsent()
+
+		if lazyProposer.privValidatorPubKey == nil {
+			// If this node is a validator & proposer in the current round, it will
+			// miss the opportunity to create a block.
+			lazyProposer.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet))
+			return
+		}
+		proposerAddr := lazyProposer.privValidatorPubKey.Address()
+
+		block, blockParts := lazyProposer.blockExec.CreateProposalBlock(
+			lazyProposer.Height, lazyProposer.state, commit, proposerAddr,
+		)
+
+		// Flush the WAL. Otherwise, we may not recompute the same proposal to sign,
+		// and the privValidator will refuse to sign anything.
+		if err := lazyProposer.wal.FlushAndSync(); err != nil {
+			lazyProposer.Logger.Error("Error flushing to disk")
+		}
+
+		// Make proposal
+		propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
+		proposal := types.NewProposal(height, round, lazyProposer.ValidRound, propBlockID)
+		p := proposal.ToProto()
+		if err := lazyProposer.privValidator.SignProposal(lazyProposer.state.ChainID, p); err == nil {
+			proposal.Signature = p.Signature
+
+			// send proposal and block parts on internal msg queue
+			lazyProposer.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
+			for i := 0; i < int(blockParts.Total()); i++ {
+				part := blockParts.GetPart(i)
+				lazyProposer.sendInternalMessage(msgInfo{&BlockPartMessage{lazyProposer.Height, lazyProposer.Round, part}, ""})
+			}
+			lazyProposer.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
+			lazyProposer.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block))
+		} else if !lazyProposer.replayMode {
+			lazyProposer.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err)
+		}
+	}
+
+	// start the consensus reactors
+	for i := 0; i < nValidators; i++ {
+		s := reactors[i].conS.GetState()
+		reactors[i].SwitchToConsensus(s, false)
+	}
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// Evidence should be submitted and committed at the third height but
+	// we will check the first six just in case
+	evidenceFromEachValidator := make([]types.Evidence, nValidators)
+
+	wg := new(sync.WaitGroup)
+	for i := 0; i < nValidators; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			for msg := range blocksSubs[i].Out() {
+				block := msg.Data().(types.EventDataNewBlock).Block
+				if len(block.Evidence.Evidence) != 0 {
+					evidenceFromEachValidator[i] = block.Evidence.Evidence[0]
+					return
+				}
+			}
+		}(i)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	pubkey, err := bcs.privValidator.GetPubKey()
+	require.NoError(t, err)
+
+	select {
+	case <-done:
+		for idx, ev := range evidenceFromEachValidator {
+			if assert.NotNil(t, ev, idx) {
+				ev, ok := ev.(*types.DuplicateVoteEvidence)
+				assert.True(t, ok)
+				assert.Equal(t, pubkey.Address(), ev.VoteA.ValidatorAddress)
+				assert.Equal(t, prevoteHeight, ev.Height())
+			}
+		}
+	case <-time.After(20 * time.Second):
+		for i, reactor := range reactors {
+			t.Logf("Consensus Reactor %d\n%v", i, reactor)
+		}
+		t.Fatalf("Timed out waiting for validators to commit evidence")
+	}
+}
+
+// 4 validators. 1 is byzantine. The other three are partitioned into A (1 val) and B (2 vals).
+// byzantine validator sends conflicting proposals into A and B,
+// and prevotes/precommits on both of them.
+// B sees a commit, A doesn't.
+// Heal partition and ensure A sees the commit
+func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
+	N := 4
+	logger := consensusLogger().With("test", "byzantine")
+	app := newCounter
+	css, cleanup := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), app)
+	defer cleanup()
+
+	// give the byzantine validator a normal ticker
+	ticker := NewTimeoutTicker()
+	ticker.SetLogger(css[0].Logger)
+	css[0].SetTimeoutTicker(ticker)
+
+	switches := make([]*p2p.Switch, N)
+	p2pLogger := logger.With("module", "p2p")
+	for i := 0; i < N; i++ {
+		switches[i] = p2p.MakeSwitch(
+			config.P2P,
+			i,
+			"foo", "1.0.0",
+			func(i int, sw *p2p.Switch) *p2p.Switch {
+				return sw
+			})
+		switches[i].SetLogger(p2pLogger.With("validator", i))
+	}
+
+	blocksSubs := make([]types.Subscription, N)
+	reactors := make([]p2p.Reactor, N)
+	for i := 0; i < N; i++ {
+
+		// enable txs so we can create different proposals
+		assertMempool(css[i].txNotifier).EnableTxsAvailable()
+		// make first val byzantine
+		if i == 0 {
+			// NOTE: Now, test validators are MockPV, which by default doesn't
+			// do any safety checks.
+			css[i].privValidator.(types.MockPV).DisableChecks()
+			css[i].decideProposal = func(j int32) func(int64, int32) {
+				return func(height int64, round int32) {
+					byzantineDecideProposalFunc(t, height, round, css[j], switches[j])
+				}
+			}(int32(i))
+			// We are setting the prevote function to do nothing because the prevoting
+			// and precommitting are done alongside the proposal.
+			css[i].doPrevote = func(height int64, round int32) {}
+		}
+
+		eventBus := css[i].eventBus
+		eventBus.SetLogger(logger.With("module", "events", "validator", i))
+
+		var err error
+		blocksSubs[i], err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
+		require.NoError(t, err)
+
+		conR := NewReactor(css[i], true) // so we don't start the consensus states
+		conR.SetLogger(logger.With("validator", i))
+		conR.SetEventBus(eventBus)
+
+		var conRI p2p.Reactor = conR
+
+		// make first val byzantine
+		if i == 0 {
+			conRI = NewByzantineReactor(conR)
+		}
+
+		reactors[i] = conRI
+		err = css[i].blockExec.Store().Save(css[i].state) // for save height 1's validators info
+		require.NoError(t, err)
+	}
+
+	defer func() {
+		for _, r := range reactors {
+			if rr, ok := r.(*ByzantineReactor); ok {
+				err := rr.reactor.Switch.Stop()
+				require.NoError(t, err)
+			} else {
+				err := r.(*Reactor).Switch.Stop()
+				require.NoError(t, err)
+			}
+		}
+	}()
+
+	p2p.MakeConnectedSwitches(config.P2P, N, func(i int, s *p2p.Switch) *p2p.Switch {
+		// ignore new switch s, we already made ours
+		switches[i].AddReactor("CONSENSUS", reactors[i])
+		return switches[i]
+	}, func(sws []*p2p.Switch, i, j int) {
+		// the network starts partitioned with globally active adversary
+		if i != 0 {
+			return
+		}
+		p2p.Connect2Switches(sws, i, j)
+	})
+
+	// start the non-byz state machines.
+	// note these must be started before the byz
+	for i := 1; i < N; i++ {
+		cr := reactors[i].(*Reactor)
+		cr.SwitchToConsensus(cr.conS.GetState(), false)
+	}
+
+	// start the byzantine state machine
+	byzR := reactors[0].(*ByzantineReactor)
+	s := byzR.reactor.conS.GetState()
+	byzR.reactor.SwitchToConsensus(s, false)
+
+	// byz proposer sends one block to peers[0]
+	// and the other block to peers[1] and peers[2].
+	// note peers and switches order don't match.
+	peers := switches[0].Peers().List()
+
+	// partition A
+	ind0 := getSwitchIndex(switches, peers[0])
+
+	// partition B
+	ind1 := getSwitchIndex(switches, peers[1])
+	ind2 := getSwitchIndex(switches, peers[2])
+	p2p.Connect2Switches(switches, ind1, ind2)
+
+	// wait for someone in the big partition (B) to make a block
+	<-blocksSubs[ind2].Out()
+
+	t.Log("A block has been committed. Healing partition")
+	p2p.Connect2Switches(switches, ind0, ind1)
+	p2p.Connect2Switches(switches, ind0, ind2)
+
+	// wait till everyone makes the first new block
+	// (one of them already has)
+	wg := new(sync.WaitGroup)
+	for i := 1; i < N-1; i++ {
+		wg.Add(1)
+		go func(j int) {
+			<-blocksSubs[j].Out()
+			wg.Done()
+		}(i)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	tick := time.NewTicker(time.Second * 10)
+	select {
+	case <-done:
+	case <-tick.C:
+		for i, reactor := range reactors {
+			t.Log(fmt.Sprintf("Consensus Reactor %v", i))
+			t.Log(fmt.Sprintf("%v", reactor))
+		}
+		t.Fatalf("Timed out waiting for all validators to commit first block")
+	}
+}
+
+//-------------------------------
+// byzantine consensus functions
+
+func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch) {
+	// byzantine user should create two proposals and try to split the vote.
+	// Avoid sending on internalMsgQueue and running consensus state.
+
+	// Create a new proposal block from state/txs from the mempool.
+	block1, blockParts1 := cs.createProposalBlock()
+	polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()}
+	proposal1 := types.NewProposal(height, round, polRound, propBlockID)
+	p1 := proposal1.ToProto()
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil {
+		t.Error(err)
+	}
+
+	proposal1.Signature = p1.Signature
+
+	// some new transactions come in (this ensures that the proposals are different)
+	deliverTxsRange(cs, 0, 1)
+
+	// Create a new proposal block from state/txs from the mempool.
+	block2, blockParts2 := cs.createProposalBlock()
+	polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()}
+	proposal2 := types.NewProposal(height, round, polRound, propBlockID)
+	p2 := proposal2.ToProto()
+	if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil {
+		t.Error(err)
+	}
+
+	proposal2.Signature = p2.Signature
+
+	block1Hash := block1.Hash()
+	block2Hash := block2.Hash()
+
+	// broadcast conflicting proposals/block parts to peers
+	peers := sw.Peers().List()
+	t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers))
+	for i, peer := range peers {
+		if i < len(peers)/2 {
+			go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1)
+		} else {
+			go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2)
+		}
+	}
+}
+
+func sendProposalAndParts(
+	height int64,
+	round int32,
+	cs *State,
+	peer p2p.Peer,
+	proposal *types.Proposal,
+	blockHash []byte,
+	parts *types.PartSet,
+) {
+	// proposal
+	msg := &ProposalMessage{Proposal: proposal}
+	peer.Send(DataChannel, MustEncode(msg))
+
+	// parts
+	for i := 0; i < int(parts.Total()); i++ {
+		part := parts.GetPart(i)
+		msg := &BlockPartMessage{
+			Height: height, // This tells peer that this part applies to us.
+			Round:  round,  // This tells peer that this part applies to us.
+			Part:   part,
+		}
+		peer.Send(DataChannel, MustEncode(msg))
+	}
+
+	// votes
+	cs.mtx.Lock()
+	prevote, _ := cs.signVote(tmproto.PrevoteType, blockHash, parts.Header())
+	precommit, _ := cs.signVote(tmproto.PrecommitType, blockHash, parts.Header())
+	cs.mtx.Unlock()
+
+	peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote}))
+	peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit}))
+}
+
+//----------------------------------------
+// byzantine consensus reactor
+
+type ByzantineReactor struct {
+	service.Service
+	reactor *Reactor
+}
+
+func NewByzantineReactor(conR *Reactor) *ByzantineReactor {
+	return &ByzantineReactor{
+		Service: conR,
+		reactor: conR,
+	}
+}
+
+func (br *ByzantineReactor) SetSwitch(s *p2p.Switch)               { br.reactor.SetSwitch(s) }
+func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() }
+func (br *ByzantineReactor) AddPeer(peer p2p.Peer) {
+	if !br.reactor.IsRunning() {
+		return
+	}
+
+	// Create peerState for peer
+	peerState := NewPeerState(peer).SetLogger(br.reactor.Logger)
+	peer.Set(types.PeerStateKey, peerState)
+
+	// Send our state to peer.
+	// If we're syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
+	if !br.reactor.waitSync {
+		br.reactor.sendNewRoundStepMessage(peer)
+	}
+}
+func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
+	br.reactor.RemovePeer(peer, reason)
+}
+func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) {
+	br.reactor.Receive(chID, peer, msgBytes)
+}
+func (br *ByzantineReactor) InitPeer(peer p2p.Peer) p2p.Peer { return peer }

internal/consensus/common_test.go

@@ -0,0 +1,919 @@
+package consensus
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log/term"
+	"github.com/stretchr/testify/require"
+
+	"path"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/example/counter"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	cstypes "github.com/tendermint/tendermint/consensus/types"
+	tmbytes "github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	mempl "github.com/tendermint/tendermint/mempool"
+	mempoolv0 "github.com/tendermint/tendermint/mempool/v0"
+	mempoolv1 "github.com/tendermint/tendermint/mempool/v1"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	testSubscriber = "test-client"
+)
+
+// A cleanupFunc cleans up any config / test files created for a particular
+// test.
+type cleanupFunc func()
+
+// genesis, chain_id, priv_val
+var (
+	config                *cfg.Config // NOTE: must be reset for each _test.go file
+	consensusReplayConfig *cfg.Config
+	ensureTimeout         = time.Millisecond * 200
+)
+
+func ensureDir(dir string, mode os.FileMode) {
+	if err := tmos.EnsureDir(dir, mode); err != nil {
+		panic(err)
+	}
+}
+
+func ResetConfig(name string) *cfg.Config {
+	return cfg.ResetTestRoot(name)
+}
+
+//-------------------------------------------------------------------------------
+// validator stub (a kvstore consensus peer we control)
+
+type validatorStub struct {
+	Index  int32 // Validator index. NOTE: we don't assume validator set changes.
+	Height int64
+	Round  int32
+	types.PrivValidator
+	VotingPower int64
+	lastVote    *types.Vote
+}
+
+var testMinPower int64 = 10
+
+func newValidatorStub(privValidator types.PrivValidator, valIndex int32) *validatorStub {
+	return &validatorStub{
+		Index:         valIndex,
+		PrivValidator: privValidator,
+		VotingPower:   testMinPower,
+	}
+}
+
+func (vs *validatorStub) signVote(
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader) (*types.Vote, error) {
+
+	pubKey, err := vs.PrivValidator.GetPubKey()
+	if err != nil {
+		return nil, fmt.Errorf("can't get pubkey: %w", err)
+	}
+
+	vote := &types.Vote{
+		ValidatorIndex:   vs.Index,
+		ValidatorAddress: pubKey.Address(),
+		Height:           vs.Height,
+		Round:            vs.Round,
+		Timestamp:        tmtime.Now(),
+		Type:             voteType,
+		BlockID:          types.BlockID{Hash: hash, PartSetHeader: header},
+	}
+	v := vote.ToProto()
+	if err := vs.PrivValidator.SignVote(config.ChainID(), v); err != nil {
+		return nil, fmt.Errorf("sign vote failed: %w", err)
+	}
+
+	// ref: signVote in FilePV, the vote should use the privious vote info when the sign data is the same.
+	if signDataIsEqual(vs.lastVote, v) {
+		v.Signature = vs.lastVote.Signature
+		v.Timestamp = vs.lastVote.Timestamp
+	}
+
+	vote.Signature = v.Signature
+	vote.Timestamp = v.Timestamp
+
+	return vote, err
+}
+
+// Sign vote for type/hash/header
+func signVote(vs *validatorStub, voteType tmproto.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote {
+	v, err := vs.signVote(voteType, hash, header)
+	if err != nil {
+		panic(fmt.Errorf("failed to sign vote: %v", err))
+	}
+
+	vs.lastVote = v
+
+	return v
+}
+
+func signVotes(
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader,
+	vss ...*validatorStub) []*types.Vote {
+	votes := make([]*types.Vote, len(vss))
+	for i, vs := range vss {
+		votes[i] = signVote(vs, voteType, hash, header)
+	}
+	return votes
+}
+
+func incrementHeight(vss ...*validatorStub) {
+	for _, vs := range vss {
+		vs.Height++
+	}
+}
+
+func incrementRound(vss ...*validatorStub) {
+	for _, vs := range vss {
+		vs.Round++
+	}
+}
+
+type ValidatorStubsByPower []*validatorStub
+
+func (vss ValidatorStubsByPower) Len() int {
+	return len(vss)
+}
+
+func (vss ValidatorStubsByPower) Less(i, j int) bool {
+	vssi, err := vss[i].GetPubKey()
+	if err != nil {
+		panic(err)
+	}
+	vssj, err := vss[j].GetPubKey()
+	if err != nil {
+		panic(err)
+	}
+
+	if vss[i].VotingPower == vss[j].VotingPower {
+		return bytes.Compare(vssi.Address(), vssj.Address()) == -1
+	}
+	return vss[i].VotingPower > vss[j].VotingPower
+}
+
+func (vss ValidatorStubsByPower) Swap(i, j int) {
+	it := vss[i]
+	vss[i] = vss[j]
+	vss[i].Index = int32(i)
+	vss[j] = it
+	vss[j].Index = int32(j)
+}
+
+//-------------------------------------------------------------------------------
+// Functions for transitioning the consensus state
+
+func startTestRound(cs *State, height int64, round int32) {
+	cs.enterNewRound(height, round)
+	cs.startRoutines(0)
+}
+
+// Create proposal block from cs1 but sign it with vs.
+func decideProposal(
+	cs1 *State,
+	vs *validatorStub,
+	height int64,
+	round int32,
+) (proposal *types.Proposal, block *types.Block) {
+	cs1.mtx.Lock()
+	block, blockParts := cs1.createProposalBlock()
+	validRound := cs1.ValidRound
+	chainID := cs1.state.ChainID
+	cs1.mtx.Unlock()
+	if block == nil {
+		panic("Failed to createProposalBlock. Did you forget to add commit for previous block?")
+	}
+
+	// Make proposal
+	polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
+	proposal = types.NewProposal(height, round, polRound, propBlockID)
+	p := proposal.ToProto()
+	if err := vs.SignProposal(chainID, p); err != nil {
+		panic(err)
+	}
+
+	proposal.Signature = p.Signature
+
+	return
+}
+
+func addVotes(to *State, votes ...*types.Vote) {
+	for _, vote := range votes {
+		to.peerMsgQueue <- msgInfo{Msg: &VoteMessage{vote}}
+	}
+}
+
+func signAddVotes(
+	to *State,
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader,
+	vss ...*validatorStub,
+) {
+	votes := signVotes(voteType, hash, header, vss...)
+	addVotes(to, votes...)
+}
+
+func validatePrevote(t *testing.T, cs *State, round int32, privVal *validatorStub, blockHash []byte) {
+	prevotes := cs.Votes.Prevotes(round)
+	pubKey, err := privVal.GetPubKey()
+	require.NoError(t, err)
+	address := pubKey.Address()
+	var vote *types.Vote
+	if vote = prevotes.GetByAddress(address); vote == nil {
+		panic("Failed to find prevote from validator")
+	}
+	if blockHash == nil {
+		if vote.BlockID.Hash != nil {
+			panic(fmt.Sprintf("Expected prevote to be for nil, got %X", vote.BlockID.Hash))
+		}
+	} else {
+		if !bytes.Equal(vote.BlockID.Hash, blockHash) {
+			panic(fmt.Sprintf("Expected prevote to be for %X, got %X", blockHash, vote.BlockID.Hash))
+		}
+	}
+}
+
+func validateLastPrecommit(t *testing.T, cs *State, privVal *validatorStub, blockHash []byte) {
+	votes := cs.LastCommit
+	pv, err := privVal.GetPubKey()
+	require.NoError(t, err)
+	address := pv.Address()
+	var vote *types.Vote
+	if vote = votes.GetByAddress(address); vote == nil {
+		panic("Failed to find precommit from validator")
+	}
+	if !bytes.Equal(vote.BlockID.Hash, blockHash) {
+		panic(fmt.Sprintf("Expected precommit to be for %X, got %X", blockHash, vote.BlockID.Hash))
+	}
+}
+
+func validatePrecommit(
+	t *testing.T,
+	cs *State,
+	thisRound,
+	lockRound int32,
+	privVal *validatorStub,
+	votedBlockHash,
+	lockedBlockHash []byte,
+) {
+	precommits := cs.Votes.Precommits(thisRound)
+	pv, err := privVal.GetPubKey()
+	require.NoError(t, err)
+	address := pv.Address()
+	var vote *types.Vote
+	if vote = precommits.GetByAddress(address); vote == nil {
+		panic("Failed to find precommit from validator")
+	}
+
+	if votedBlockHash == nil {
+		if vote.BlockID.Hash != nil {
+			panic("Expected precommit to be for nil")
+		}
+	} else {
+		if !bytes.Equal(vote.BlockID.Hash, votedBlockHash) {
+			panic("Expected precommit to be for proposal block")
+		}
+	}
+
+	if lockedBlockHash == nil {
+		if cs.LockedRound != lockRound || cs.LockedBlock != nil {
+			panic(fmt.Sprintf(
+				"Expected to be locked on nil at round %d. Got locked at round %d with block %v",
+				lockRound,
+				cs.LockedRound,
+				cs.LockedBlock))
+		}
+	} else {
+		if cs.LockedRound != lockRound || !bytes.Equal(cs.LockedBlock.Hash(), lockedBlockHash) {
+			panic(fmt.Sprintf(
+				"Expected block to be locked on round %d, got %d. Got locked block %X, expected %X",
+				lockRound,
+				cs.LockedRound,
+				cs.LockedBlock.Hash(),
+				lockedBlockHash))
+		}
+	}
+}
+
+func validatePrevoteAndPrecommit(
+	t *testing.T,
+	cs *State,
+	thisRound,
+	lockRound int32,
+	privVal *validatorStub,
+	votedBlockHash,
+	lockedBlockHash []byte,
+) {
+	// verify the prevote
+	validatePrevote(t, cs, thisRound, privVal, votedBlockHash)
+	// verify precommit
+	cs.mtx.Lock()
+	validatePrecommit(t, cs, thisRound, lockRound, privVal, votedBlockHash, lockedBlockHash)
+	cs.mtx.Unlock()
+}
+
+func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message {
+	votesSub, err := cs.eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, types.EventQueryVote)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
+	}
+	ch := make(chan tmpubsub.Message)
+	go func() {
+		for msg := range votesSub.Out() {
+			vote := msg.Data().(types.EventDataVote)
+			// we only fire for our own votes
+			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
+				ch <- msg
+			}
+		}
+	}()
+	return ch
+}
+
+//-------------------------------------------------------------------------------
+// consensus states
+
+func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State {
+	config := cfg.ResetTestRoot("consensus_state_test")
+	return newStateWithConfig(config, state, pv, app)
+}
+
+func newStateWithConfig(
+	thisConfig *cfg.Config,
+	state sm.State,
+	pv types.PrivValidator,
+	app abci.Application,
+) *State {
+	blockDB := dbm.NewMemDB()
+	return newStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB)
+}
+
+func newStateWithConfigAndBlockStore(
+	thisConfig *cfg.Config,
+	state sm.State,
+	pv types.PrivValidator,
+	app abci.Application,
+	blockDB dbm.DB,
+) *State {
+	// Get BlockStore
+	blockStore := store.NewBlockStore(blockDB)
+
+	// one for mempool, one for consensus
+	mtx := new(tmsync.Mutex)
+
+	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+	proxyAppConnConMem := abcicli.NewLocalClient(mtx, app)
+	// Make Mempool
+	memplMetrics := mempl.NopMetrics()
+
+	// Make Mempool
+	var mempool mempl.Mempool
+
+	switch config.Mempool.Version {
+	case cfg.MempoolV0:
+		mempool = mempoolv0.NewCListMempool(config.Mempool,
+			proxyAppConnConMem,
+			state.LastBlockHeight,
+			mempoolv0.WithMetrics(memplMetrics),
+			mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
+			mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
+	case cfg.MempoolV1:
+		logger := consensusLogger()
+		mempool = mempoolv1.NewTxMempool(logger,
+			config.Mempool,
+			proxyAppConnConMem,
+			state.LastBlockHeight,
+			mempoolv1.WithMetrics(memplMetrics),
+			mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
+			mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
+		)
+	}
+	if thisConfig.Consensus.WaitForTxs() {
+		mempool.EnableTxsAvailable()
+	}
+
+	evpool := sm.EmptyEvidencePool{}
+
+	// Make State
+	stateDB := blockDB
+	stateStore := sm.NewStore(stateDB)
+	if err := stateStore.Save(state); err != nil { // for save height 1's validators info
+		panic(err)
+	}
+
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+	cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+	cs.SetLogger(log.TestingLogger().With("module", "consensus"))
+	cs.SetPrivValidator(pv)
+
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	err := eventBus.Start()
+	if err != nil {
+		panic(err)
+	}
+	cs.SetEventBus(eventBus)
+	return cs
+}
+
+func loadPrivValidator(config *cfg.Config) *privval.FilePV {
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	ensureDir(filepath.Dir(privValidatorKeyFile), 0700)
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
+	privValidator.Reset()
+	return privValidator
+}
+
+func randState(nValidators int) (*State, []*validatorStub) {
+	// Get State
+	state, privVals := randGenesisState(nValidators, false, 10)
+
+	vss := make([]*validatorStub, nValidators)
+
+	cs := newState(state, privVals[0], counter.NewApplication(true))
+
+	for i := 0; i < nValidators; i++ {
+		vss[i] = newValidatorStub(privVals[i], int32(i))
+	}
+	// since cs1 starts at 1
+	incrementHeight(vss[1:]...)
+
+	return cs, vss
+}
+
+//-------------------------------------------------------------------------------
+
+func ensureNoNewEvent(ch <-chan tmpubsub.Message, timeout time.Duration,
+	errorMessage string) {
+	select {
+	case <-time.After(timeout):
+		break
+	case <-ch:
+		panic(errorMessage)
+	}
+}
+
+func ensureNoNewEventOnChannel(ch <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		ch,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving new event on the channel")
+}
+
+func ensureNoNewRoundStep(stepCh <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		stepCh,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving NewRoundStep event")
+}
+
+func ensureNoNewUnlock(unlockCh <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		unlockCh,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving Unlock event")
+}
+
+func ensureNoNewTimeout(stepCh <-chan tmpubsub.Message, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
+	ensureNoNewEvent(
+		stepCh,
+		timeoutDuration,
+		"We should be stuck waiting, not receiving NewTimeout event")
+}
+
+func ensureNewEvent(ch <-chan tmpubsub.Message, height int64, round int32, timeout time.Duration, errorMessage string) {
+	select {
+	case <-time.After(timeout):
+		panic(errorMessage)
+	case msg := <-ch:
+		roundStateEvent, ok := msg.Data().(types.EventDataRoundState)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataRoundState, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if roundStateEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, roundStateEvent.Height))
+		}
+		if roundStateEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, roundStateEvent.Round))
+		}
+		// TODO: We could check also for a step at this point!
+	}
+}
+
+func ensureNewRound(roundCh <-chan tmpubsub.Message, height int64, round int32) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewRound event")
+	case msg := <-roundCh:
+		newRoundEvent, ok := msg.Data().(types.EventDataNewRound)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewRound, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if newRoundEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, newRoundEvent.Height))
+		}
+		if newRoundEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, newRoundEvent.Round))
+		}
+	}
+}
+
+func ensureNewTimeout(timeoutCh <-chan tmpubsub.Message, height int64, round int32, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
+	ensureNewEvent(timeoutCh, height, round, timeoutDuration,
+		"Timeout expired while waiting for NewTimeout event")
+}
+
+func ensureNewProposal(proposalCh <-chan tmpubsub.Message, height int64, round int32) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+	}
+}
+
+func ensureNewValidBlock(validBlockCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureNewEvent(validBlockCh, height, round, ensureTimeout,
+		"Timeout expired while waiting for NewValidBlock event")
+}
+
+func ensureNewBlock(blockCh <-chan tmpubsub.Message, height int64) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewBlock event")
+	case msg := <-blockCh:
+		blockEvent, ok := msg.Data().(types.EventDataNewBlock)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewBlock, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if blockEvent.Block.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockEvent.Block.Height))
+		}
+	}
+}
+
+func ensureNewBlockHeader(blockCh <-chan tmpubsub.Message, height int64, blockHash tmbytes.HexBytes) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewBlockHeader event")
+	case msg := <-blockCh:
+		blockHeaderEvent, ok := msg.Data().(types.EventDataNewBlockHeader)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewBlockHeader, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if blockHeaderEvent.Header.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockHeaderEvent.Header.Height))
+		}
+		if !bytes.Equal(blockHeaderEvent.Header.Hash(), blockHash) {
+			panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeaderEvent.Header.Hash()))
+		}
+	}
+}
+
+func ensureNewUnlock(unlockCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureNewEvent(unlockCh, height, round, ensureTimeout,
+		"Timeout expired while waiting for NewUnlock event")
+}
+
+func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int32, propID types.BlockID) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+		if !proposalEvent.BlockID.Equals(propID) {
+			panic(fmt.Sprintf("Proposed block does not match expected block (%v != %v)", proposalEvent.BlockID, propID))
+		}
+	}
+}
+
+func ensurePrecommit(voteCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureVote(voteCh, height, round, tmproto.PrecommitType)
+}
+
+func ensurePrevote(voteCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureVote(voteCh, height, round, tmproto.PrevoteType)
+}
+
+func ensureVote(voteCh <-chan tmpubsub.Message, height int64, round int32,
+	voteType tmproto.SignedMsgType) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewVote event")
+	case msg := <-voteCh:
+		voteEvent, ok := msg.Data().(types.EventDataVote)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataVote, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		vote := voteEvent.Vote
+		if vote.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, vote.Height))
+		}
+		if vote.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, vote.Round))
+		}
+		if vote.Type != voteType {
+			panic(fmt.Sprintf("expected type %v, got %v", voteType, vote.Type))
+		}
+	}
+}
+
+func ensurePrecommitTimeout(ch <-chan tmpubsub.Message) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for the Precommit to Timeout")
+	case <-ch:
+	}
+}
+
+func ensureNewEventOnChannel(ch <-chan tmpubsub.Message) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for new activity on the channel")
+	case <-ch:
+	}
+}
+
+//-------------------------------------------------------------------------------
+// consensus nets
+
+// consensusLogger is a TestingLogger which uses a different
+// color for each validator ("validator" key must exist).
+func consensusLogger() log.Logger {
+	return log.TestingLoggerWithColorFn(func(keyvals ...interface{}) term.FgBgColor {
+		for i := 0; i < len(keyvals)-1; i += 2 {
+			if keyvals[i] == "validator" {
+				return term.FgBgColor{Fg: term.Color(uint8(keyvals[i+1].(int) + 1))}
+			}
+		}
+		return term.FgBgColor{}
+	}).With("module", "consensus")
+}
+
+func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker,
+	appFunc func() abci.Application, configOpts ...func(*cfg.Config)) ([]*State, cleanupFunc) {
+	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
+	css := make([]*State, nValidators)
+	logger := consensusLogger()
+	configRootDirs := make([]string, 0, nValidators)
+	for i := 0; i < nValidators; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
+		for _, opt := range configOpts {
+			opt(thisConfig)
+		}
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		app := appFunc()
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newStateWithConfigAndBlockStore(thisConfig, state, privVals[i], app, stateDB)
+		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+	}
+	return css, func() {
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
+}
+
+// nPeers = nValidators + nNotValidator
+func randConsensusNetWithPeers(
+	nValidators,
+	nPeers int,
+	testName string,
+	tickerFunc func() TimeoutTicker,
+	appFunc func(string) abci.Application,
+) ([]*State, *types.GenesisDoc, *cfg.Config, cleanupFunc) {
+	genDoc, privVals := randGenesisDoc(nValidators, false, testMinPower)
+	css := make([]*State, nPeers)
+	logger := consensusLogger()
+	var peer0Config *cfg.Config
+	configRootDirs := make([]string, 0, nPeers)
+	for i := 0; i < nPeers; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		if i == 0 {
+			peer0Config = thisConfig
+		}
+		var privVal types.PrivValidator
+		if i < nValidators {
+			privVal = privVals[i]
+		} else {
+			tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_")
+			if err != nil {
+				panic(err)
+			}
+			tempStateFile, err := ioutil.TempFile("", "priv_validator_state_")
+			if err != nil {
+				panic(err)
+			}
+
+			privVal = privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name())
+		}
+
+		app := appFunc(path.Join(config.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		if _, ok := app.(*kvstore.PersistentKVStoreApplication); ok {
+			// simulate handshake, receive app version. If don't do this, replay test will fail
+			state.Version.Consensus.App = kvstore.ProtocolVersion
+		}
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+		// sm.SaveState(stateDB,state)	//height 1's validatorsInfo already saved in LoadStateFromDBOrGenesisDoc above
+
+		css[i] = newStateWithConfig(thisConfig, state, privVal, app)
+		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+	}
+	return css, genDoc, peer0Config, func() {
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
+}
+
+func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int {
+	for i, s := range switches {
+		if peer.NodeInfo().ID() == s.NodeInfo().ID() {
+			return i
+		}
+	}
+	panic("didnt find peer in switches")
+}
+
+//-------------------------------------------------------------------------------
+// genesis
+
+func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) {
+	validators := make([]types.GenesisValidator, numValidators)
+	privValidators := make([]types.PrivValidator, numValidators)
+	for i := 0; i < numValidators; i++ {
+		val, privVal := types.RandValidator(randPower, minPower)
+		validators[i] = types.GenesisValidator{
+			PubKey: val.PubKey,
+			Power:  val.VotingPower,
+		}
+		privValidators[i] = privVal
+	}
+	sort.Sort(types.PrivValidatorsByAddress(privValidators))
+
+	return &types.GenesisDoc{
+		GenesisTime:   tmtime.Now(),
+		InitialHeight: 1,
+		ChainID:       config.ChainID(),
+		Validators:    validators,
+	}, privValidators
+}
+
+func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) {
+	genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower)
+	s0, _ := sm.MakeGenesisState(genDoc)
+	return s0, privValidators
+}
+
+//------------------------------------
+// mock ticker
+
+func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker {
+	return func() TimeoutTicker {
+		return &mockTicker{
+			c:        make(chan timeoutInfo, 10),
+			onlyOnce: onlyOnce,
+		}
+	}
+}
+
+// mock ticker only fires on RoundStepNewHeight
+// and only once if onlyOnce=true
+type mockTicker struct {
+	c chan timeoutInfo
+
+	mtx      sync.Mutex
+	onlyOnce bool
+	fired    bool
+}
+
+func (m *mockTicker) Start() error {
+	return nil
+}
+
+func (m *mockTicker) Stop() error {
+	return nil
+}
+
+func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	if m.onlyOnce && m.fired {
+		return
+	}
+	if ti.Step == cstypes.RoundStepNewHeight {
+		m.c <- ti
+		m.fired = true
+	}
+}
+
+func (m *mockTicker) Chan() <-chan timeoutInfo {
+	return m.c
+}
+
+func (*mockTicker) SetLogger(log.Logger) {}
+
+//------------------------------------
+
+func newCounter() abci.Application {
+	return counter.NewApplication(true)
+}
+
+func newPersistentKVStore() abci.Application {
+	dir, err := ioutil.TempDir("", "persistent-kvstore")
+	if err != nil {
+		panic(err)
+	}
+	return kvstore.NewPersistentKVStoreApplication(dir)
+}
+
+func newPersistentKVStoreWithPath(dbDir string) abci.Application {
+	return kvstore.NewPersistentKVStoreApplication(dbDir)
+}
+
+func signDataIsEqual(v1 *types.Vote, v2 *tmproto.Vote) bool {
+	if v1 == nil || v2 == nil {
+		return false
+	}
+
+	return v1.Type == v2.Type &&
+		bytes.Equal(v1.BlockID.Hash, v2.BlockID.GetHash()) &&
+		v1.Height == v2.GetHeight() &&
+		v1.Round == v2.Round &&
+		bytes.Equal(v1.ValidatorAddress.Bytes(), v2.GetValidatorAddress()) &&
+		v1.ValidatorIndex == v2.GetValidatorIndex()
+}

internal/consensus/invalid_test.go

@@ -0,0 +1,100 @@
+package consensus
+
+import (
+	"testing"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/p2p"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+//----------------------------------------------
+// byzantine failures
+
+// one byz val sends a precommit for a random block at each height
+// Ensure a testnet makes blocks
+func TestReactorInvalidPrecommit(t *testing.T) {
+	N := 4
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+
+	for i := 0; i < 4; i++ {
+		ticker := NewTimeoutTicker()
+		ticker.SetLogger(css[i].Logger)
+		css[i].SetTimeoutTicker(ticker)
+
+	}
+
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
+
+	// this val sends a random precommit at each height
+	byzValIdx := 0
+	byzVal := css[byzValIdx]
+	byzR := reactors[byzValIdx]
+
+	// update the doPrevote function to just send a valid precommit for a random block
+	// and otherwise disable the priv validator
+	byzVal.mtx.Lock()
+	pv := byzVal.privValidator
+	byzVal.doPrevote = func(height int64, round int32) {
+		invalidDoPrevoteFunc(t, height, round, byzVal, byzR.Switch, pv)
+	}
+	byzVal.mtx.Unlock()
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// wait for a bunch of blocks
+	// TODO: make this tighter by ensuring the halt happens by block 2
+	for i := 0; i < 10; i++ {
+		timeoutWaitGroup(t, N, func(j int) {
+			<-blocksSubs[j].Out()
+		}, css)
+	}
+}
+
+func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch, pv types.PrivValidator) {
+	// routine to:
+	// - precommit for a random block
+	// - send precommit to all peers
+	// - disable privValidator (so we don't do normal precommits)
+	go func() {
+		cs.mtx.Lock()
+		cs.privValidator = pv
+		pubKey, err := cs.privValidator.GetPubKey()
+		if err != nil {
+			panic(err)
+		}
+		addr := pubKey.Address()
+		valIndex, _ := cs.Validators.GetByAddress(addr)
+
+		// precommit a random block
+		blockHash := bytes.HexBytes(tmrand.Bytes(32))
+		precommit := &types.Vote{
+			ValidatorAddress: addr,
+			ValidatorIndex:   valIndex,
+			Height:           cs.Height,
+			Round:            cs.Round,
+			Timestamp:        cs.voteTime(),
+			Type:             tmproto.PrecommitType,
+			BlockID: types.BlockID{
+				Hash:          blockHash,
+				PartSetHeader: types.PartSetHeader{Total: 1, Hash: tmrand.Bytes(32)}},
+		}
+		p := precommit.ToProto()
+		err = cs.privValidator.SignVote(cs.state.ChainID, p)
+		if err != nil {
+			t.Error(err)
+		}
+		precommit.Signature = p.Signature
+		cs.privValidator = nil // disable priv val so we don't do normal votes
+		cs.mtx.Unlock()
+
+		peers := sw.Peers().List()
+		for _, peer := range peers {
+			cs.Logger.Info("Sending bad vote", "block", blockHash, "peer", peer)
+			peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit}))
+		}
+	}()
+}

internal/consensus/mempool_test.go

@@ -0,0 +1,258 @@
+package consensus
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/abci/example/code"
+	abci "github.com/tendermint/tendermint/abci/types"
+	mempl "github.com/tendermint/tendermint/mempool"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+// for testing
+func assertMempool(txn txNotifier) mempl.Mempool {
+	return txn.(mempl.Mempool)
+}
+
+func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
+	config.Consensus.CreateEmptyBlocks = false
+	state, privVals := randGenesisState(1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+	height, round := cs.Height, cs.Round
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	startTestRound(cs, height, round)
+
+	ensureNewEventOnChannel(newBlockCh) // first block gets committed
+	ensureNoNewEventOnChannel(newBlockCh)
+	deliverTxsRange(cs, 0, 1)
+	ensureNewEventOnChannel(newBlockCh) // commit txs
+	ensureNewEventOnChannel(newBlockCh) // commit updated app hash
+	ensureNoNewEventOnChannel(newBlockCh)
+}
+
+func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
+
+	config.Consensus.CreateEmptyBlocksInterval = ensureTimeout
+	state, privVals := randGenesisState(1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	startTestRound(cs, cs.Height, cs.Round)
+
+	ensureNewEventOnChannel(newBlockCh)   // first block gets committed
+	ensureNoNewEventOnChannel(newBlockCh) // then we dont make a block ...
+	ensureNewEventOnChannel(newBlockCh)   // until the CreateEmptyBlocksInterval has passed
+}
+
+func TestMempoolProgressInHigherRound(t *testing.T) {
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	defer os.RemoveAll(config.RootDir)
+	config.Consensus.CreateEmptyBlocks = false
+	state, privVals := randGenesisState(1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+	height, round := cs.Height, cs.Round
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
+	cs.setProposal = func(proposal *types.Proposal) error {
+		if cs.Height == 2 && cs.Round == 0 {
+			// dont set the proposal in round 0 so we timeout and
+			// go to next round
+			cs.Logger.Info("Ignoring set proposal at height 2, round 0")
+			return nil
+		}
+		return cs.defaultSetProposal(proposal)
+	}
+	startTestRound(cs, height, round)
+
+	ensureNewRound(newRoundCh, height, round) // first round at first height
+	ensureNewEventOnChannel(newBlockCh)       // first block gets committed
+
+	height++ // moving to the next height
+	round = 0
+
+	ensureNewRound(newRoundCh, height, round) // first round at next height
+	deliverTxsRange(cs, 0, 1)                 // we deliver txs, but dont set a proposal so we get the next round
+	ensureNewTimeout(timeoutCh, height, round, cs.config.TimeoutPropose.Nanoseconds())
+
+	round++                                   // moving to the next round
+	ensureNewRound(newRoundCh, height, round) // wait for the next round
+	ensureNewEventOnChannel(newBlockCh)       // now we can commit the block
+}
+
+func deliverTxsRange(cs *State, start, end int) {
+	// Deliver some txs.
+	for i := start; i < end; i++ {
+		txBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(txBytes, uint64(i))
+		err := assertMempool(cs.txNotifier).CheckTx(txBytes, nil, mempl.TxInfo{})
+		if err != nil {
+			panic(fmt.Sprintf("Error after CheckTx: %v", err))
+		}
+	}
+}
+
+func TestMempoolTxConcurrentWithCommit(t *testing.T) {
+	state, privVals := randGenesisState(1, false, 10)
+	blockDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(blockDB)
+	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB)
+	err := stateStore.Save(state)
+	require.NoError(t, err)
+	newBlockHeaderCh := subscribe(cs.eventBus, types.EventQueryNewBlockHeader)
+
+	const numTxs int64 = 3000
+	go deliverTxsRange(cs, 0, int(numTxs))
+
+	startTestRound(cs, cs.Height, cs.Round)
+	for n := int64(0); n < numTxs; {
+		select {
+		case msg := <-newBlockHeaderCh:
+			headerEvent := msg.Data().(types.EventDataNewBlockHeader)
+			n += headerEvent.NumTxs
+		case <-time.After(30 * time.Second):
+			t.Fatal("Timed out waiting 30s to commit blocks with transactions")
+		}
+	}
+}
+
+func TestMempoolRmBadTx(t *testing.T) {
+	state, privVals := randGenesisState(1, false, 10)
+	app := NewCounterApplication()
+	blockDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(blockDB)
+	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], app, blockDB)
+	err := stateStore.Save(state)
+	require.NoError(t, err)
+
+	// increment the counter by 1
+	txBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(txBytes, uint64(0))
+
+	resDeliver := app.DeliverTx(abci.RequestDeliverTx{Tx: txBytes})
+	assert.False(t, resDeliver.IsErr(), fmt.Sprintf("expected no error. got %v", resDeliver))
+
+	resCommit := app.Commit()
+	assert.True(t, len(resCommit.Data) > 0)
+
+	emptyMempoolCh := make(chan struct{})
+	checkTxRespCh := make(chan struct{})
+	go func() {
+		// Try to send the tx through the mempool.
+		// CheckTx should not err, but the app should return a bad abci code
+		// and the tx should get removed from the pool
+		err := assertMempool(cs.txNotifier).CheckTx(txBytes, func(r *abci.Response) {
+			if r.GetCheckTx().Code != code.CodeTypeBadNonce {
+				t.Errorf("expected checktx to return bad nonce, got %v", r)
+				return
+			}
+			checkTxRespCh <- struct{}{}
+		}, mempl.TxInfo{})
+		if err != nil {
+			t.Errorf("error after CheckTx: %v", err)
+			return
+		}
+
+		// check for the tx
+		for {
+			txs := assertMempool(cs.txNotifier).ReapMaxBytesMaxGas(int64(len(txBytes)), -1)
+			if len(txs) == 0 {
+				emptyMempoolCh <- struct{}{}
+				return
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}()
+
+	// Wait until the tx returns
+	ticker := time.After(time.Second * 5)
+	select {
+	case <-checkTxRespCh:
+		// success
+	case <-ticker:
+		t.Errorf("timed out waiting for tx to return")
+		return
+	}
+
+	// Wait until the tx is removed
+	ticker = time.After(time.Second * 5)
+	select {
+	case <-emptyMempoolCh:
+		// success
+	case <-ticker:
+		t.Errorf("timed out waiting for tx to be removed")
+		return
+	}
+}
+
+// CounterApplication that maintains a mempool state and resets it upon commit
+type CounterApplication struct {
+	abci.BaseApplication
+
+	txCount        int
+	mempoolTxCount int
+}
+
+func NewCounterApplication() *CounterApplication {
+	return &CounterApplication{}
+}
+
+func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo {
+	return abci.ResponseInfo{Data: fmt.Sprintf("txs:%v", app.txCount)}
+}
+
+func (app *CounterApplication) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	txValue := txAsUint64(req.Tx)
+	if txValue != uint64(app.txCount) {
+		return abci.ResponseDeliverTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)}
+	}
+	app.txCount++
+	return abci.ResponseDeliverTx{Code: code.CodeTypeOK}
+}
+
+func (app *CounterApplication) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx {
+	txValue := txAsUint64(req.Tx)
+	if txValue != uint64(app.mempoolTxCount) {
+		return abci.ResponseCheckTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)}
+	}
+	app.mempoolTxCount++
+	return abci.ResponseCheckTx{Code: code.CodeTypeOK}
+}
+
+func txAsUint64(tx []byte) uint64 {
+	tx8 := make([]byte, 8)
+	copy(tx8[len(tx8)-len(tx):], tx)
+	return binary.BigEndian.Uint64(tx8)
+}
+
+func (app *CounterApplication) Commit() abci.ResponseCommit {
+	app.mempoolTxCount = app.txCount
+	if app.txCount == 0 {
+		return abci.ResponseCommit{}
+	}
+	hash := make([]byte, 8)
+	binary.BigEndian.PutUint64(hash, uint64(app.txCount))
+	return abci.ResponseCommit{Data: hash}
+}

internal/consensus/metrics.go

@@ -0,0 +1,252 @@
+package consensus
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+
+	prometheus "github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "consensus"
+)
+
+// Metrics contains metrics exposed by this package.
+type Metrics struct {
+	// Height of the chain.
+	Height metrics.Gauge
+
+	// ValidatorLastSignedHeight of a validator.
+	ValidatorLastSignedHeight metrics.Gauge
+
+	// Number of rounds.
+	Rounds metrics.Gauge
+
+	// Number of validators.
+	Validators metrics.Gauge
+	// Total power of all validators.
+	ValidatorsPower metrics.Gauge
+	// Power of a validator.
+	ValidatorPower metrics.Gauge
+	// Amount of blocks missed by a validator.
+	ValidatorMissedBlocks metrics.Gauge
+	// Number of validators who did not sign.
+	MissingValidators metrics.Gauge
+	// Total power of the missing validators.
+	MissingValidatorsPower metrics.Gauge
+	// Number of validators who tried to double sign.
+	ByzantineValidators metrics.Gauge
+	// Total power of the byzantine validators.
+	ByzantineValidatorsPower metrics.Gauge
+
+	// Time between this and the last block.
+	BlockIntervalSeconds metrics.Histogram
+
+	// Number of transactions.
+	NumTxs metrics.Gauge
+	// Size of the block.
+	BlockSizeBytes metrics.Gauge
+	// Total number of transactions.
+	TotalTxs metrics.Gauge
+	// The latest block height.
+	CommittedHeight metrics.Gauge
+	// Whether or not a node is fast syncing. 1 if yes, 0 if no.
+	FastSyncing metrics.Gauge
+	// Whether or not a node is state syncing. 1 if yes, 0 if no.
+	StateSyncing metrics.Gauge
+
+	// Number of blockparts transmitted by peer.
+	BlockParts metrics.Counter
+
+	// QuroumPrevoteMessageDelay is the interval in seconds between the proposal
+	// timestamp and the timestamp of the earliest prevote that achieved a quorum
+	// during the prevote step.
+	//
+	// To compute it, sum the voting power over each prevote received, in increasing
+	// order of timestamp. The timestamp of the first prevote to increase the sum to
+	// be above 2/3 of the total voting power of the network defines the endpoint
+	// the endpoint of the interval. Subtract the proposal timestamp from this endpoint
+	// to obtain the quorum delay.
+	QuorumPrevoteMessageDelay metrics.Gauge
+
+	// FullPrevoteMessageDelay is the interval in seconds between the proposal
+	// timestamp and the timestamp of the latest prevote in a round where 100%
+	// of the voting power on the network issued prevotes.
+	FullPrevoteMessageDelay metrics.Gauge
+}
+
+// PrometheusMetrics returns Metrics build using Prometheus client library.
+// Optionally, labels can be provided along with their values ("foo",
+// "fooValue").
+func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(labelsAndValues); i += 2 {
+		labels = append(labels, labelsAndValues[i])
+	}
+	return &Metrics{
+		Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "height",
+			Help:      "Height of the chain.",
+		}, labels).With(labelsAndValues...),
+		Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "rounds",
+			Help:      "Number of rounds.",
+		}, labels).With(labelsAndValues...),
+
+		Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validators",
+			Help:      "Number of validators.",
+		}, labels).With(labelsAndValues...),
+		ValidatorLastSignedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_last_signed_height",
+			Help:      "Last signed height for a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		ValidatorMissedBlocks: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_missed_blocks",
+			Help:      "Total missed blocks for a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validators_power",
+			Help:      "Total power of all validators.",
+		}, labels).With(labelsAndValues...),
+		ValidatorPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_power",
+			Help:      "Power of a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "missing_validators",
+			Help:      "Number of validators who did not sign.",
+		}, labels).With(labelsAndValues...),
+		MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "missing_validators_power",
+			Help:      "Total power of the missing validators.",
+		}, labels).With(labelsAndValues...),
+		ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "byzantine_validators",
+			Help:      "Number of validators who tried to double sign.",
+		}, labels).With(labelsAndValues...),
+		ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "byzantine_validators_power",
+			Help:      "Total power of the byzantine validators.",
+		}, labels).With(labelsAndValues...),
+		BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_interval_seconds",
+			Help:      "Time between this and the last block.",
+		}, labels).With(labelsAndValues...),
+		NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "num_txs",
+			Help:      "Number of transactions.",
+		}, labels).With(labelsAndValues...),
+		BlockSizeBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_size_bytes",
+			Help:      "Size of the block.",
+		}, labels).With(labelsAndValues...),
+		TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "total_txs",
+			Help:      "Total number of transactions.",
+		}, labels).With(labelsAndValues...),
+		CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "latest_block_height",
+			Help:      "The latest block height.",
+		}, labels).With(labelsAndValues...),
+		FastSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "fast_syncing",
+			Help:      "Whether or not a node is fast syncing. 1 if yes, 0 if no.",
+		}, labels).With(labelsAndValues...),
+		StateSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "state_syncing",
+			Help:      "Whether or not a node is state syncing. 1 if yes, 0 if no.",
+		}, labels).With(labelsAndValues...),
+		BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_parts",
+			Help:      "Number of blockparts transmitted by peer.",
+		}, append(labels, "peer_id")).With(labelsAndValues...),
+		QuorumPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "quorum_prevote_message_delay",
+			Help: "Difference in seconds between the proposal timestamp and the timestamp " +
+				"of the latest prevote that achieved a quorum in the prevote step.",
+		}, labels).With(labelsAndValues...),
+		FullPrevoteMessageDelay: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "full_prevote_message_delay",
+			Help: "Difference in seconds between the proposal timestamp and the timestamp " +
+				"of the latest prevote that achieved 100% of the voting power in the prevote step.",
+		}, labels).With(labelsAndValues...),
+	}
+}
+
+// NopMetrics returns no-op Metrics.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		Height: discard.NewGauge(),
+
+		ValidatorLastSignedHeight: discard.NewGauge(),
+
+		Rounds: discard.NewGauge(),
+
+		Validators:               discard.NewGauge(),
+		ValidatorsPower:          discard.NewGauge(),
+		ValidatorPower:           discard.NewGauge(),
+		ValidatorMissedBlocks:    discard.NewGauge(),
+		MissingValidators:        discard.NewGauge(),
+		MissingValidatorsPower:   discard.NewGauge(),
+		ByzantineValidators:      discard.NewGauge(),
+		ByzantineValidatorsPower: discard.NewGauge(),
+
+		BlockIntervalSeconds: discard.NewHistogram(),
+
+		NumTxs:                    discard.NewGauge(),
+		BlockSizeBytes:            discard.NewGauge(),
+		TotalTxs:                  discard.NewGauge(),
+		CommittedHeight:           discard.NewGauge(),
+		FastSyncing:               discard.NewGauge(),
+		StateSyncing:              discard.NewGauge(),
+		BlockParts:                discard.NewCounter(),
+		QuorumPrevoteMessageDelay: discard.NewGauge(),
+		FullPrevoteMessageDelay:   discard.NewGauge(),
+	}
+}

internal/consensus/msgs.go

@@ -0,0 +1,377 @@
+package consensus
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/gogo/protobuf/proto"
+
+	cstypes "github.com/tendermint/tendermint/consensus/types"
+	"github.com/tendermint/tendermint/libs/bits"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	"github.com/tendermint/tendermint/p2p"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// MsgToProto takes a consensus message type and returns the proto defined consensus message
+func MsgToProto(msg Message) (*tmcons.Message, error) {
+	if msg == nil {
+		return nil, errors.New("consensus: message is nil")
+	}
+	var pb tmcons.Message
+
+	switch msg := msg.(type) {
+	case *NewRoundStepMessage:
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_NewRoundStep{
+				NewRoundStep: &tmcons.NewRoundStep{
+					Height:                msg.Height,
+					Round:                 msg.Round,
+					Step:                  uint32(msg.Step),
+					SecondsSinceStartTime: msg.SecondsSinceStartTime,
+					LastCommitRound:       msg.LastCommitRound,
+				},
+			},
+		}
+	case *NewValidBlockMessage:
+		pbPartSetHeader := msg.BlockPartSetHeader.ToProto()
+		pbBits := msg.BlockParts.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_NewValidBlock{
+				NewValidBlock: &tmcons.NewValidBlock{
+					Height:             msg.Height,
+					Round:              msg.Round,
+					BlockPartSetHeader: pbPartSetHeader,
+					BlockParts:         pbBits,
+					IsCommit:           msg.IsCommit,
+				},
+			},
+		}
+	case *ProposalMessage:
+		pbP := msg.Proposal.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_Proposal{
+				Proposal: &tmcons.Proposal{
+					Proposal: *pbP,
+				},
+			},
+		}
+	case *ProposalPOLMessage:
+		pbBits := msg.ProposalPOL.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_ProposalPol{
+				ProposalPol: &tmcons.ProposalPOL{
+					Height:           msg.Height,
+					ProposalPolRound: msg.ProposalPOLRound,
+					ProposalPol:      *pbBits,
+				},
+			},
+		}
+	case *BlockPartMessage:
+		parts, err := msg.Part.ToProto()
+		if err != nil {
+			return nil, fmt.Errorf("msg to proto error: %w", err)
+		}
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_BlockPart{
+				BlockPart: &tmcons.BlockPart{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Part:   *parts,
+				},
+			},
+		}
+	case *VoteMessage:
+		vote := msg.Vote.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_Vote{
+				Vote: &tmcons.Vote{
+					Vote: vote,
+				},
+			},
+		}
+	case *HasVoteMessage:
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_HasVote{
+				HasVote: &tmcons.HasVote{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Type:   msg.Type,
+					Index:  msg.Index,
+				},
+			},
+		}
+	case *VoteSetMaj23Message:
+		bi := msg.BlockID.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_VoteSetMaj23{
+				VoteSetMaj23: &tmcons.VoteSetMaj23{
+					Height:  msg.Height,
+					Round:   msg.Round,
+					Type:    msg.Type,
+					BlockID: bi,
+				},
+			},
+		}
+	case *VoteSetBitsMessage:
+		bi := msg.BlockID.ToProto()
+		bits := msg.Votes.ToProto()
+
+		vsb := &tmcons.Message_VoteSetBits{
+			VoteSetBits: &tmcons.VoteSetBits{
+				Height:  msg.Height,
+				Round:   msg.Round,
+				Type:    msg.Type,
+				BlockID: bi,
+			},
+		}
+
+		if bits != nil {
+			vsb.VoteSetBits.Votes = *bits
+		}
+
+		pb = tmcons.Message{
+			Sum: vsb,
+		}
+
+	default:
+		return nil, fmt.Errorf("consensus: message not recognized: %T", msg)
+	}
+
+	return &pb, nil
+}
+
+// MsgFromProto takes a consensus proto message and returns the native go type
+func MsgFromProto(msg *tmcons.Message) (Message, error) {
+	if msg == nil {
+		return nil, errors.New("consensus: nil message")
+	}
+	var pb Message
+
+	switch msg := msg.Sum.(type) {
+	case *tmcons.Message_NewRoundStep:
+		rs, err := tmmath.SafeConvertUint8(int64(msg.NewRoundStep.Step))
+		// deny message based on possible overflow
+		if err != nil {
+			return nil, fmt.Errorf("denying message due to possible overflow: %w", err)
+		}
+		pb = &NewRoundStepMessage{
+			Height:                msg.NewRoundStep.Height,
+			Round:                 msg.NewRoundStep.Round,
+			Step:                  cstypes.RoundStepType(rs),
+			SecondsSinceStartTime: msg.NewRoundStep.SecondsSinceStartTime,
+			LastCommitRound:       msg.NewRoundStep.LastCommitRound,
+		}
+	case *tmcons.Message_NewValidBlock:
+		pbPartSetHeader, err := types.PartSetHeaderFromProto(&msg.NewValidBlock.BlockPartSetHeader)
+		if err != nil {
+			return nil, fmt.Errorf("parts to proto error: %w", err)
+		}
+
+		pbBits := new(bits.BitArray)
+		pbBits.FromProto(msg.NewValidBlock.BlockParts)
+
+		pb = &NewValidBlockMessage{
+			Height:             msg.NewValidBlock.Height,
+			Round:              msg.NewValidBlock.Round,
+			BlockPartSetHeader: *pbPartSetHeader,
+			BlockParts:         pbBits,
+			IsCommit:           msg.NewValidBlock.IsCommit,
+		}
+	case *tmcons.Message_Proposal:
+		pbP, err := types.ProposalFromProto(&msg.Proposal.Proposal)
+		if err != nil {
+			return nil, fmt.Errorf("proposal msg to proto error: %w", err)
+		}
+
+		pb = &ProposalMessage{
+			Proposal: pbP,
+		}
+	case *tmcons.Message_ProposalPol:
+		pbBits := new(bits.BitArray)
+		pbBits.FromProto(&msg.ProposalPol.ProposalPol)
+		pb = &ProposalPOLMessage{
+			Height:           msg.ProposalPol.Height,
+			ProposalPOLRound: msg.ProposalPol.ProposalPolRound,
+			ProposalPOL:      pbBits,
+		}
+	case *tmcons.Message_BlockPart:
+		parts, err := types.PartFromProto(&msg.BlockPart.Part)
+		if err != nil {
+			return nil, fmt.Errorf("blockpart msg to proto error: %w", err)
+		}
+		pb = &BlockPartMessage{
+			Height: msg.BlockPart.Height,
+			Round:  msg.BlockPart.Round,
+			Part:   parts,
+		}
+	case *tmcons.Message_Vote:
+		vote, err := types.VoteFromProto(msg.Vote.Vote)
+		if err != nil {
+			return nil, fmt.Errorf("vote msg to proto error: %w", err)
+		}
+
+		pb = &VoteMessage{
+			Vote: vote,
+		}
+	case *tmcons.Message_HasVote:
+		pb = &HasVoteMessage{
+			Height: msg.HasVote.Height,
+			Round:  msg.HasVote.Round,
+			Type:   msg.HasVote.Type,
+			Index:  msg.HasVote.Index,
+		}
+	case *tmcons.Message_VoteSetMaj23:
+		bi, err := types.BlockIDFromProto(&msg.VoteSetMaj23.BlockID)
+		if err != nil {
+			return nil, fmt.Errorf("voteSetMaj23 msg to proto error: %w", err)
+		}
+		pb = &VoteSetMaj23Message{
+			Height:  msg.VoteSetMaj23.Height,
+			Round:   msg.VoteSetMaj23.Round,
+			Type:    msg.VoteSetMaj23.Type,
+			BlockID: *bi,
+		}
+	case *tmcons.Message_VoteSetBits:
+		bi, err := types.BlockIDFromProto(&msg.VoteSetBits.BlockID)
+		if err != nil {
+			return nil, fmt.Errorf("voteSetBits msg to proto error: %w", err)
+		}
+		bits := new(bits.BitArray)
+		bits.FromProto(&msg.VoteSetBits.Votes)
+
+		pb = &VoteSetBitsMessage{
+			Height:  msg.VoteSetBits.Height,
+			Round:   msg.VoteSetBits.Round,
+			Type:    msg.VoteSetBits.Type,
+			BlockID: *bi,
+			Votes:   bits,
+		}
+	default:
+		return nil, fmt.Errorf("consensus: message not recognized: %T", msg)
+	}
+
+	if err := pb.ValidateBasic(); err != nil {
+		return nil, err
+	}
+
+	return pb, nil
+}
+
+// MustEncode takes the reactors msg, makes it proto and marshals it
+// this mimics `MustMarshalBinaryBare` in that is panics on error
+func MustEncode(msg Message) []byte {
+	pb, err := MsgToProto(msg)
+	if err != nil {
+		panic(err)
+	}
+	enc, err := proto.Marshal(pb)
+	if err != nil {
+		panic(err)
+	}
+	return enc
+}
+
+// WALToProto takes a WAL message and return a proto walMessage and error
+func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) {
+	var pb tmcons.WALMessage
+
+	switch msg := msg.(type) {
+	case types.EventDataRoundState:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EventDataRoundState{
+				EventDataRoundState: &tmproto.EventDataRoundState{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Step:   msg.Step,
+				},
+			},
+		}
+	case msgInfo:
+		consMsg, err := MsgToProto(msg.Msg)
+		if err != nil {
+			return nil, err
+		}
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_MsgInfo{
+				MsgInfo: &tmcons.MsgInfo{
+					Msg:    *consMsg,
+					PeerID: string(msg.PeerID),
+				},
+			},
+		}
+	case timeoutInfo:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_TimeoutInfo{
+				TimeoutInfo: &tmcons.TimeoutInfo{
+					Duration: msg.Duration,
+					Height:   msg.Height,
+					Round:    msg.Round,
+					Step:     uint32(msg.Step),
+				},
+			},
+		}
+	case EndHeightMessage:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EndHeight{
+				EndHeight: &tmcons.EndHeight{
+					Height: msg.Height,
+				},
+			},
+		}
+	default:
+		return nil, fmt.Errorf("to proto: wal message not recognized: %T", msg)
+	}
+
+	return &pb, nil
+}
+
+// WALFromProto takes a proto wal message and return a consensus walMessage and error
+func WALFromProto(msg *tmcons.WALMessage) (WALMessage, error) {
+	if msg == nil {
+		return nil, errors.New("nil WAL message")
+	}
+	var pb WALMessage
+
+	switch msg := msg.Sum.(type) {
+	case *tmcons.WALMessage_EventDataRoundState:
+		pb = types.EventDataRoundState{
+			Height: msg.EventDataRoundState.Height,
+			Round:  msg.EventDataRoundState.Round,
+			Step:   msg.EventDataRoundState.Step,
+		}
+	case *tmcons.WALMessage_MsgInfo:
+		walMsg, err := MsgFromProto(&msg.MsgInfo.Msg)
+		if err != nil {
+			return nil, fmt.Errorf("msgInfo from proto error: %w", err)
+		}
+		pb = msgInfo{
+			Msg:    walMsg,
+			PeerID: p2p.ID(msg.MsgInfo.PeerID),
+		}
+
+	case *tmcons.WALMessage_TimeoutInfo:
+		tis, err := tmmath.SafeConvertUint8(int64(msg.TimeoutInfo.Step))
+		// deny message based on possible overflow
+		if err != nil {
+			return nil, fmt.Errorf("denying message due to possible overflow: %w", err)
+		}
+		pb = timeoutInfo{
+			Duration: msg.TimeoutInfo.Duration,
+			Height:   msg.TimeoutInfo.Height,
+			Round:    msg.TimeoutInfo.Round,
+			Step:     cstypes.RoundStepType(tis),
+		}
+		return pb, nil
+	case *tmcons.WALMessage_EndHeight:
+		pb := EndHeightMessage{
+			Height: msg.EndHeight.Height,
+		}
+		return pb, nil
+	default:
+		return nil, fmt.Errorf("from proto: wal message not recognized: %T", msg)
+	}
+	return pb, nil
+}

internal/consensus/msgs_test.go

@@ -0,0 +1,427 @@
+package consensus
+
+import (
+	"encoding/hex"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/libs/bits"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/p2p"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestMsgToProto(t *testing.T) {
+	psh := types.PartSetHeader{
+		Total: 1,
+		Hash:  tmrand.Bytes(32),
+	}
+	pbPsh := psh.ToProto()
+	bi := types.BlockID{
+		Hash:          tmrand.Bytes(32),
+		PartSetHeader: psh,
+	}
+	pbBi := bi.ToProto()
+	bits := bits.NewBitArray(1)
+	pbBits := bits.ToProto()
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: tmrand.Bytes(32),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	proposal := types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     1,
+		POLRound:  1,
+		BlockID:   bi,
+		Timestamp: time.Now(),
+		Signature: tmrand.Bytes(20),
+	}
+	pbProposal := proposal.ToProto()
+
+	pv := types.NewMockPV()
+	pk, err := pv.GetPubKey()
+	require.NoError(t, err)
+	val := types.NewValidator(pk, 100)
+
+	vote, err := types.MakeVote(
+		1, types.BlockID{}, &types.ValidatorSet{Proposer: val, Validators: []*types.Validator{val}},
+		pv, "chainID", time.Now())
+	require.NoError(t, err)
+	pbVote := vote.ToProto()
+
+	testsCases := []struct {
+		testName string
+		msg      Message
+		want     *tmcons.Message
+		wantErr  bool
+	}{
+		{"successful NewRoundStepMessage", &NewRoundStepMessage{
+			Height:                2,
+			Round:                 1,
+			Step:                  1,
+			SecondsSinceStartTime: 1,
+			LastCommitRound:       2,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_NewRoundStep{
+				NewRoundStep: &tmcons.NewRoundStep{
+					Height:                2,
+					Round:                 1,
+					Step:                  1,
+					SecondsSinceStartTime: 1,
+					LastCommitRound:       2,
+				},
+			},
+		}, false},
+
+		{"successful NewValidBlockMessage", &NewValidBlockMessage{
+			Height:             1,
+			Round:              1,
+			BlockPartSetHeader: psh,
+			BlockParts:         bits,
+			IsCommit:           false,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_NewValidBlock{
+				NewValidBlock: &tmcons.NewValidBlock{
+					Height:             1,
+					Round:              1,
+					BlockPartSetHeader: pbPsh,
+					BlockParts:         pbBits,
+					IsCommit:           false,
+				},
+			},
+		}, false},
+		{"successful BlockPartMessage", &BlockPartMessage{
+			Height: 100,
+			Round:  1,
+			Part:   &parts,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_BlockPart{
+				BlockPart: &tmcons.BlockPart{
+					Height: 100,
+					Round:  1,
+					Part:   *pbParts,
+				},
+			},
+		}, false},
+		{"successful ProposalPOLMessage", &ProposalPOLMessage{
+			Height:           1,
+			ProposalPOLRound: 1,
+			ProposalPOL:      bits,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_ProposalPol{
+				ProposalPol: &tmcons.ProposalPOL{
+					Height:           1,
+					ProposalPolRound: 1,
+					ProposalPol:      *pbBits,
+				},
+			}}, false},
+		{"successful ProposalMessage", &ProposalMessage{
+			Proposal: &proposal,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_Proposal{
+				Proposal: &tmcons.Proposal{
+					Proposal: *pbProposal,
+				},
+			},
+		}, false},
+		{"successful VoteMessage", &VoteMessage{
+			Vote: vote,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_Vote{
+				Vote: &tmcons.Vote{
+					Vote: pbVote,
+				},
+			},
+		}, false},
+		{"successful VoteSetMaj23", &VoteSetMaj23Message{
+			Height:  1,
+			Round:   1,
+			Type:    1,
+			BlockID: bi,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_VoteSetMaj23{
+				VoteSetMaj23: &tmcons.VoteSetMaj23{
+					Height:  1,
+					Round:   1,
+					Type:    1,
+					BlockID: pbBi,
+				},
+			},
+		}, false},
+		{"successful VoteSetBits", &VoteSetBitsMessage{
+			Height:  1,
+			Round:   1,
+			Type:    1,
+			BlockID: bi,
+			Votes:   bits,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_VoteSetBits{
+				VoteSetBits: &tmcons.VoteSetBits{
+					Height:  1,
+					Round:   1,
+					Type:    1,
+					BlockID: pbBi,
+					Votes:   *pbBits,
+				},
+			},
+		}, false},
+		{"failure", nil, &tmcons.Message{}, true},
+	}
+	for _, tt := range testsCases {
+		tt := tt
+		t.Run(tt.testName, func(t *testing.T) {
+			pb, err := MsgToProto(tt.msg)
+			if tt.wantErr == true {
+				assert.Equal(t, err != nil, tt.wantErr)
+				return
+			}
+			assert.EqualValues(t, tt.want, pb, tt.testName)
+
+			msg, err := MsgFromProto(pb)
+
+			if !tt.wantErr {
+				require.NoError(t, err)
+				bcm := assert.Equal(t, tt.msg, msg, tt.testName)
+				assert.True(t, bcm, tt.testName)
+			} else {
+				require.Error(t, err, tt.testName)
+			}
+		})
+	}
+}
+
+func TestWALMsgProto(t *testing.T) {
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: tmrand.Bytes(32),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	testsCases := []struct {
+		testName string
+		msg      WALMessage
+		want     *tmcons.WALMessage
+		wantErr  bool
+	}{
+		{"successful EventDataRoundState", types.EventDataRoundState{
+			Height: 2,
+			Round:  1,
+			Step:   "ronies",
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EventDataRoundState{
+				EventDataRoundState: &tmproto.EventDataRoundState{
+					Height: 2,
+					Round:  1,
+					Step:   "ronies",
+				},
+			},
+		}, false},
+		{"successful msgInfo", msgInfo{
+			Msg: &BlockPartMessage{
+				Height: 100,
+				Round:  1,
+				Part:   &parts,
+			},
+			PeerID: p2p.ID("string"),
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_MsgInfo{
+				MsgInfo: &tmcons.MsgInfo{
+					Msg: tmcons.Message{
+						Sum: &tmcons.Message_BlockPart{
+							BlockPart: &tmcons.BlockPart{
+								Height: 100,
+								Round:  1,
+								Part:   *pbParts,
+							},
+						},
+					},
+					PeerID: "string",
+				},
+			},
+		}, false},
+		{"successful timeoutInfo", timeoutInfo{
+			Duration: time.Duration(100),
+			Height:   1,
+			Round:    1,
+			Step:     1,
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_TimeoutInfo{
+				TimeoutInfo: &tmcons.TimeoutInfo{
+					Duration: time.Duration(100),
+					Height:   1,
+					Round:    1,
+					Step:     1,
+				},
+			},
+		}, false},
+		{"successful EndHeightMessage", EndHeightMessage{
+			Height: 1,
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EndHeight{
+				EndHeight: &tmcons.EndHeight{
+					Height: 1,
+				},
+			},
+		}, false},
+		{"failure", nil, &tmcons.WALMessage{}, true},
+	}
+	for _, tt := range testsCases {
+		tt := tt
+		t.Run(tt.testName, func(t *testing.T) {
+			pb, err := WALToProto(tt.msg)
+			if tt.wantErr == true {
+				assert.Equal(t, err != nil, tt.wantErr)
+				return
+			}
+			assert.EqualValues(t, tt.want, pb, tt.testName)
+
+			msg, err := WALFromProto(pb)
+
+			if !tt.wantErr {
+				require.NoError(t, err)
+				assert.Equal(t, tt.msg, msg, tt.testName) // need the concrete type as WAL Message is a empty interface
+			} else {
+				require.Error(t, err, tt.testName)
+			}
+		})
+	}
+}
+
+// nolint:lll //ignore line length for tests
+func TestConsMsgsVectors(t *testing.T) {
+	date := time.Date(2018, 8, 30, 12, 0, 0, 0, time.UTC)
+	psh := types.PartSetHeader{
+		Total: 1,
+		Hash:  []byte("add_more_exclamation_marks_code-"),
+	}
+	pbPsh := psh.ToProto()
+
+	bi := types.BlockID{
+		Hash:          []byte("add_more_exclamation_marks_code-"),
+		PartSetHeader: psh,
+	}
+	pbBi := bi.ToProto()
+	bits := bits.NewBitArray(1)
+	pbBits := bits.ToProto()
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: []byte("add_more_exclamation_marks_code-"),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	proposal := types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     1,
+		POLRound:  1,
+		BlockID:   bi,
+		Timestamp: date,
+		Signature: []byte("add_more_exclamation"),
+	}
+	pbProposal := proposal.ToProto()
+
+	v := &types.Vote{
+		ValidatorAddress: []byte("add_more_exclamation"),
+		ValidatorIndex:   1,
+		Height:           1,
+		Round:            0,
+		Timestamp:        date,
+		Type:             tmproto.PrecommitType,
+		BlockID:          bi,
+	}
+	vpb := v.ToProto()
+
+	testCases := []struct {
+		testName string
+		cMsg     proto.Message
+		expBytes string
+	}{
+		{"NewRoundStep", &tmcons.Message{Sum: &tmcons.Message_NewRoundStep{NewRoundStep: &tmcons.NewRoundStep{
+			Height:                1,
+			Round:                 1,
+			Step:                  1,
+			SecondsSinceStartTime: 1,
+			LastCommitRound:       1,
+		}}}, "0a0a08011001180120012801"},
+		{"NewRoundStep Max", &tmcons.Message{Sum: &tmcons.Message_NewRoundStep{NewRoundStep: &tmcons.NewRoundStep{
+			Height:                math.MaxInt64,
+			Round:                 math.MaxInt32,
+			Step:                  math.MaxUint32,
+			SecondsSinceStartTime: math.MaxInt64,
+			LastCommitRound:       math.MaxInt32,
+		}}}, "0a2608ffffffffffffffff7f10ffffffff0718ffffffff0f20ffffffffffffffff7f28ffffffff07"},
+		{"NewValidBlock", &tmcons.Message{Sum: &tmcons.Message_NewValidBlock{
+			NewValidBlock: &tmcons.NewValidBlock{
+				Height: 1, Round: 1, BlockPartSetHeader: pbPsh, BlockParts: pbBits, IsCommit: false}}},
+			"1231080110011a24080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d22050801120100"},
+		{"Proposal", &tmcons.Message{Sum: &tmcons.Message_Proposal{Proposal: &tmcons.Proposal{Proposal: *pbProposal}}},
+			"1a720a7008201001180120012a480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d320608c0b89fdc053a146164645f6d6f72655f6578636c616d6174696f6e"},
+		{"ProposalPol", &tmcons.Message{Sum: &tmcons.Message_ProposalPol{
+			ProposalPol: &tmcons.ProposalPOL{Height: 1, ProposalPolRound: 1}}},
+			"2206080110011a00"},
+		{"BlockPart", &tmcons.Message{Sum: &tmcons.Message_BlockPart{
+			BlockPart: &tmcons.BlockPart{Height: 1, Round: 1, Part: *pbParts}}},
+			"2a36080110011a3008011204746573741a26080110011a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d"},
+		{"Vote", &tmcons.Message{Sum: &tmcons.Message_Vote{
+			Vote: &tmcons.Vote{Vote: vpb}}},
+			"32700a6e0802100122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d2a0608c0b89fdc0532146164645f6d6f72655f6578636c616d6174696f6e3801"},
+		{"HasVote", &tmcons.Message{Sum: &tmcons.Message_HasVote{
+			HasVote: &tmcons.HasVote{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 1}}},
+			"3a080801100118012001"},
+		{"HasVote", &tmcons.Message{Sum: &tmcons.Message_HasVote{
+			HasVote: &tmcons.HasVote{Height: math.MaxInt64, Round: math.MaxInt32,
+				Type: tmproto.PrevoteType, Index: math.MaxInt32}}},
+			"3a1808ffffffffffffffff7f10ffffffff07180120ffffffff07"},
+		{"VoteSetMaj23", &tmcons.Message{Sum: &tmcons.Message_VoteSetMaj23{
+			VoteSetMaj23: &tmcons.VoteSetMaj23{Height: 1, Round: 1, Type: tmproto.PrevoteType, BlockID: pbBi}}},
+			"425008011001180122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d"},
+		{"VoteSetBits", &tmcons.Message{Sum: &tmcons.Message_VoteSetBits{
+			VoteSetBits: &tmcons.VoteSetBits{Height: 1, Round: 1, Type: tmproto.PrevoteType, BlockID: pbBi, Votes: *pbBits}}},
+			"4a5708011001180122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d2a050801120100"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			bz, err := proto.Marshal(tc.cMsg)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expBytes, hex.EncodeToString(bz))
+		})
+	}
+}

internal/consensus/reactor_test.go

@@ -0,0 +1,995 @@
+package consensus
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	cstypes "github.com/tendermint/tendermint/consensus/types"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/libs/bits"
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/log"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	mempl "github.com/tendermint/tendermint/mempool"
+	mempoolv0 "github.com/tendermint/tendermint/mempool/v0"
+	mempoolv1 "github.com/tendermint/tendermint/mempool/v1"
+	"github.com/tendermint/tendermint/p2p"
+	p2pmock "github.com/tendermint/tendermint/p2p/mock"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	sm "github.com/tendermint/tendermint/state"
+	statemocks "github.com/tendermint/tendermint/state/mocks"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+//----------------------------------------------
+// in-process testnets
+
+var defaultTestTime = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
+
+func startConsensusNet(t *testing.T, css []*State, n int) (
+	[]*Reactor,
+	[]types.Subscription,
+	[]*types.EventBus,
+) {
+	reactors := make([]*Reactor, n)
+	blocksSubs := make([]types.Subscription, 0)
+	eventBuses := make([]*types.EventBus, n)
+	for i := 0; i < n; i++ {
+		/*logger, err := tmflags.ParseLogLevel("consensus:info,*:error", logger, "info")
+		if err != nil {	t.Fatal(err)}*/
+		reactors[i] = NewReactor(css[i], true) // so we dont start the consensus states
+		reactors[i].SetLogger(css[i].Logger)
+
+		// eventBus is already started with the cs
+		eventBuses[i] = css[i].eventBus
+		reactors[i].SetEventBus(eventBuses[i])
+
+		blocksSub, err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
+		require.NoError(t, err)
+		blocksSubs = append(blocksSubs, blocksSub)
+
+		if css[i].state.LastBlockHeight == 0 { // simulate handle initChain in handshake
+			if err := css[i].blockExec.Store().Save(css[i].state); err != nil {
+				t.Error(err)
+			}
+
+		}
+	}
+	// make connected switches and start all reactors
+	p2p.MakeConnectedSwitches(config.P2P, n, func(i int, s *p2p.Switch) *p2p.Switch {
+		s.AddReactor("CONSENSUS", reactors[i])
+		s.SetLogger(reactors[i].conS.Logger.With("module", "p2p"))
+		return s
+	}, p2p.Connect2Switches)
+
+	// now that everyone is connected,  start the state machines
+	// If we started the state machines before everyone was connected,
+	// we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors
+	// TODO: is this still true with new pubsub?
+	for i := 0; i < n; i++ {
+		s := reactors[i].conS.GetState()
+		reactors[i].SwitchToConsensus(s, false)
+	}
+	return reactors, blocksSubs, eventBuses
+}
+
+func stopConsensusNet(logger log.Logger, reactors []*Reactor, eventBuses []*types.EventBus) {
+	logger.Info("stopConsensusNet", "n", len(reactors))
+	for i, r := range reactors {
+		logger.Info("stopConsensusNet: Stopping Reactor", "i", i)
+		if err := r.Switch.Stop(); err != nil {
+			logger.Error("error trying to stop switch", "error", err)
+		}
+	}
+	for i, b := range eventBuses {
+		logger.Info("stopConsensusNet: Stopping eventBus", "i", i)
+		if err := b.Stop(); err != nil {
+			logger.Error("error trying to stop eventbus", "error", err)
+		}
+	}
+	logger.Info("stopConsensusNet: DONE", "n", len(reactors))
+}
+
+// Ensure a testnet makes blocks
+func TestReactorBasic(t *testing.T) {
+	N := 4
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+	// wait till everyone makes the first new block
+	timeoutWaitGroup(t, N, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+}
+
+// Ensure we can process blocks with evidence
+func TestReactorWithEvidence(t *testing.T) {
+	nValidators := 4
+	testName := "consensus_reactor_test"
+	tickerFunc := newMockTickerFunc(true)
+	appFunc := newCounter
+
+	// heed the advice from https://www.sandimetz.com/blog/2016/1/20/the-wrong-abstraction
+	// to unroll unwieldy abstractions. Here we duplicate the code from:
+	// css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+
+	genDoc, privVals := randGenesisDoc(nValidators, false, 30)
+	css := make([]*State, nValidators)
+	logger := consensusLogger()
+	for i := 0; i < nValidators; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		defer os.RemoveAll(thisConfig.RootDir)
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		app := appFunc()
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		pv := privVals[i]
+		// duplicate code from:
+		// css[i] = newStateWithConfig(thisConfig, state, privVals[i], app)
+
+		blockDB := dbm.NewMemDB()
+		blockStore := store.NewBlockStore(blockDB)
+
+		mtx := new(tmsync.Mutex)
+		memplMetrics := mempl.NopMetrics()
+		// one for mempool, one for consensus
+		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+		proxyAppConnConMem := abcicli.NewLocalClient(mtx, app)
+
+		// Make Mempool
+		var mempool mempl.Mempool
+
+		switch config.Mempool.Version {
+		case cfg.MempoolV0:
+			mempool = mempoolv0.NewCListMempool(config.Mempool,
+				proxyAppConnConMem,
+				state.LastBlockHeight,
+				mempoolv0.WithMetrics(memplMetrics),
+				mempoolv0.WithPreCheck(sm.TxPreCheck(state)),
+				mempoolv0.WithPostCheck(sm.TxPostCheck(state)))
+		case cfg.MempoolV1:
+			mempool = mempoolv1.NewTxMempool(logger,
+				config.Mempool,
+				proxyAppConnConMem,
+				state.LastBlockHeight,
+				mempoolv1.WithMetrics(memplMetrics),
+				mempoolv1.WithPreCheck(sm.TxPreCheck(state)),
+				mempoolv1.WithPostCheck(sm.TxPostCheck(state)),
+			)
+		}
+		if thisConfig.Consensus.WaitForTxs() {
+			mempool.EnableTxsAvailable()
+		}
+
+		// mock the evidence pool
+		// everyone includes evidence of another double signing
+		vIdx := (i + 1) % nValidators
+		ev := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID())
+		evpool := &statemocks.EvidencePool{}
+		evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil)
+		evpool.On("PendingEvidence", mock.AnythingOfType("int64")).Return([]types.Evidence{
+			ev}, int64(len(ev.Bytes())))
+		evpool.On("Update", mock.AnythingOfType("state.State"), mock.AnythingOfType("types.EvidenceList")).Return()
+
+		evpool2 := sm.EmptyEvidencePool{}
+
+		// Make State
+		blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+		cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2)
+		cs.SetLogger(log.TestingLogger().With("module", "consensus"))
+		cs.SetPrivValidator(pv)
+
+		eventBus := types.NewEventBus()
+		eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+		err := eventBus.Start()
+		require.NoError(t, err)
+		cs.SetEventBus(eventBus)
+
+		cs.SetTimeoutTicker(tickerFunc())
+		cs.SetLogger(logger.With("validator", i, "module", "consensus"))
+
+		css[i] = cs
+	}
+
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nValidators)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// we expect for each validator that is the proposer to propose one piece of evidence.
+	for i := 0; i < nValidators; i++ {
+		timeoutWaitGroup(t, nValidators, func(j int) {
+			msg := <-blocksSubs[j].Out()
+			block := msg.Data().(types.EventDataNewBlock).Block
+			assert.Len(t, block.Evidence.Evidence, 1)
+		}, css)
+	}
+}
+
+//------------------------------------
+
+// Ensure a testnet makes blocks when there are txs
+func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
+	N := 4
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter,
+		func(c *cfg.Config) {
+			c.Consensus.CreateEmptyBlocks = false
+		})
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// send a tx
+	if err := assertMempool(css[3].txNotifier).CheckTx([]byte{1, 2, 3}, nil, mempl.TxInfo{}); err != nil {
+		t.Error(err)
+	}
+
+	// wait till everyone makes the first new block
+	timeoutWaitGroup(t, N, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+}
+
+func TestReactorReceiveDoesNotPanicIfAddPeerHasntBeenCalledYet(t *testing.T) {
+	N := 1
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, _, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	var (
+		reactor = reactors[0]
+		peer    = p2pmock.NewPeer(nil)
+		msg     = MustEncode(&HasVoteMessage{Height: 1,
+			Round: 1, Index: 1, Type: tmproto.PrevoteType})
+	)
+
+	reactor.InitPeer(peer)
+
+	// simulate switch calling Receive before AddPeer
+	assert.NotPanics(t, func() {
+		reactor.Receive(StateChannel, peer, msg)
+		reactor.AddPeer(peer)
+	})
+}
+
+func TestReactorReceivePanicsIfInitPeerHasntBeenCalledYet(t *testing.T) {
+	N := 1
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, _, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	var (
+		reactor = reactors[0]
+		peer    = p2pmock.NewPeer(nil)
+		msg     = MustEncode(&HasVoteMessage{Height: 1,
+			Round: 1, Index: 1, Type: tmproto.PrevoteType})
+	)
+
+	// we should call InitPeer here
+
+	// simulate switch calling Receive before AddPeer
+	assert.Panics(t, func() {
+		reactor.Receive(StateChannel, peer, msg)
+	})
+}
+
+// Test we record stats about votes and block parts from other peers.
+func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
+	N := 4
+	css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// wait till everyone makes the first new block
+	timeoutWaitGroup(t, N, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+
+	// Get peer
+	peer := reactors[1].Switch.Peers().List()[0]
+	// Get peer state
+	ps := peer.Get(types.PeerStateKey).(*PeerState)
+
+	assert.Equal(t, true, ps.VotesSent() > 0, "number of votes sent should have increased")
+	assert.Equal(t, true, ps.BlockPartsSent() > 0, "number of votes sent should have increased")
+}
+
+//-------------------------------------------------------------
+// ensure we can make blocks despite cycling a validator set
+
+func TestReactorVotingPowerChange(t *testing.T) {
+	nVals := 4
+	logger := log.TestingLogger()
+	css, cleanup := randConsensusNet(
+		nVals,
+		"consensus_voting_power_changes_test",
+		newMockTickerFunc(true),
+		newPersistentKVStore)
+	defer cleanup()
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nVals)
+	defer stopConsensusNet(logger, reactors, eventBuses)
+
+	// map of active validators
+	activeVals := make(map[string]struct{})
+	for i := 0; i < nVals; i++ {
+		pubKey, err := css[i].privValidator.GetPubKey()
+		require.NoError(t, err)
+		addr := pubKey.Address()
+		activeVals[string(addr)] = struct{}{}
+	}
+
+	// wait till everyone makes block 1
+	timeoutWaitGroup(t, nVals, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+
+	//---------------------------------------------------------------------------
+	logger.Debug("---------------------------- Testing changing the voting power of one validator a few times")
+
+	val1PubKey, err := css[0].privValidator.GetPubKey()
+	require.NoError(t, err)
+
+	val1PubKeyABCI, err := cryptoenc.PubKeyToProto(val1PubKey)
+	require.NoError(t, err)
+	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
+	previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf(
+			"expected voting power to change (before: %d, after: %d)",
+			previousTotalVotingPower,
+			css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2)
+	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf(
+			"expected voting power to change (before: %d, after: %d)",
+			previousTotalVotingPower,
+			css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26)
+	previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+	waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css)
+
+	if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Fatalf(
+			"expected voting power to change (before: %d, after: %d)",
+			previousTotalVotingPower,
+			css[0].GetRoundState().LastValidators.TotalVotingPower())
+	}
+}
+
+func TestReactorValidatorSetChanges(t *testing.T) {
+	nPeers := 7
+	nVals := 4
+	css, _, _, cleanup := randConsensusNetWithPeers(
+		nVals,
+		nPeers,
+		"consensus_val_set_changes_test",
+		newMockTickerFunc(true),
+		newPersistentKVStoreWithPath)
+
+	defer cleanup()
+	logger := log.TestingLogger()
+
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nPeers)
+	defer stopConsensusNet(logger, reactors, eventBuses)
+
+	// map of active validators
+	activeVals := make(map[string]struct{})
+	for i := 0; i < nVals; i++ {
+		pubKey, err := css[i].privValidator.GetPubKey()
+		require.NoError(t, err)
+		activeVals[string(pubKey.Address())] = struct{}{}
+	}
+
+	// wait till everyone makes block 1
+	timeoutWaitGroup(t, nPeers, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+
+	//---------------------------------------------------------------------------
+	logger.Info("---------------------------- Testing adding one validator")
+
+	newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey()
+	assert.NoError(t, err)
+	valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1)
+	assert.NoError(t, err)
+	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
+
+	// wait till everyone makes block 2
+	// ensure the commit includes all validators
+	// send newValTx to change vals in block 3
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1)
+
+	// wait till everyone makes block 3.
+	// it includes the commit for block 2, which is by the original validator set
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1)
+
+	// wait till everyone makes block 4.
+	// it includes the commit for block 3, which is by the original validator set
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
+
+	// the commits for block 4 should be with the updated validator set
+	activeVals[string(newValidatorPubKey1.Address())] = struct{}{}
+
+	// wait till everyone makes block 5
+	// it includes the commit for block 4, which should have the updated validator set
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
+
+	//---------------------------------------------------------------------------
+	logger.Info("---------------------------- Testing changing the voting power of one validator")
+
+	updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey()
+	require.NoError(t, err)
+	updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1)
+	require.NoError(t, err)
+	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
+	previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
+
+	if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower {
+		t.Errorf(
+			"expected voting power to change (before: %d, after: %d)",
+			previousTotalVotingPower,
+			css[nVals].GetRoundState().LastValidators.TotalVotingPower())
+	}
+
+	//---------------------------------------------------------------------------
+	logger.Info("---------------------------- Testing adding two validators at once")
+
+	newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey()
+	require.NoError(t, err)
+	newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2)
+	require.NoError(t, err)
+	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
+
+	newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey()
+	require.NoError(t, err)
+	newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3)
+	require.NoError(t, err)
+	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
+	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
+	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
+
+	//---------------------------------------------------------------------------
+	logger.Info("---------------------------- Testing removing two validators at once")
+
+	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
+	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css)
+	delete(activeVals, string(newValidatorPubKey2.Address()))
+	delete(activeVals, string(newValidatorPubKey3.Address()))
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css)
+}
+
+// Check we can make blocks with skip_timeout_commit=false
+func TestReactorWithTimeoutCommit(t *testing.T) {
+	N := 4
+	css, cleanup := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter)
+	defer cleanup()
+	// override default SkipTimeoutCommit == true for tests
+	for i := 0; i < N; i++ {
+		css[i].config.SkipTimeoutCommit = false
+	}
+
+	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N-1)
+	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
+
+	// wait till everyone makes the first new block
+	timeoutWaitGroup(t, N-1, func(j int) {
+		<-blocksSubs[j].Out()
+	}, css)
+}
+
+func waitForAndValidateBlock(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*State,
+	txs ...[]byte,
+) {
+	timeoutWaitGroup(t, n, func(j int) {
+		css[j].Logger.Debug("waitForAndValidateBlock")
+		msg := <-blocksSubs[j].Out()
+		newBlock := msg.Data().(types.EventDataNewBlock).Block
+		css[j].Logger.Debug("waitForAndValidateBlock: Got block", "height", newBlock.Height)
+		err := validateBlock(newBlock, activeVals)
+		assert.Nil(t, err)
+		for _, tx := range txs {
+			err := assertMempool(css[j].txNotifier).CheckTx(tx, nil, mempl.TxInfo{})
+			assert.Nil(t, err)
+		}
+	}, css)
+}
+
+func waitForAndValidateBlockWithTx(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*State,
+	txs ...[]byte,
+) {
+	timeoutWaitGroup(t, n, func(j int) {
+		ntxs := 0
+	BLOCK_TX_LOOP:
+		for {
+			css[j].Logger.Debug("waitForAndValidateBlockWithTx", "ntxs", ntxs)
+			msg := <-blocksSubs[j].Out()
+			newBlock := msg.Data().(types.EventDataNewBlock).Block
+			css[j].Logger.Debug("waitForAndValidateBlockWithTx: Got block", "height", newBlock.Height)
+			err := validateBlock(newBlock, activeVals)
+			assert.Nil(t, err)
+
+			// check that txs match the txs we're waiting for.
+			// note they could be spread over multiple blocks,
+			// but they should be in order.
+			for _, tx := range newBlock.Data.Txs {
+				assert.EqualValues(t, txs[ntxs], tx)
+				ntxs++
+			}
+
+			if ntxs == len(txs) {
+				break BLOCK_TX_LOOP
+			}
+		}
+
+	}, css)
+}
+
+func waitForBlockWithUpdatedValsAndValidateIt(
+	t *testing.T,
+	n int,
+	updatedVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*State,
+) {
+	timeoutWaitGroup(t, n, func(j int) {
+
+		var newBlock *types.Block
+	LOOP:
+		for {
+			css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt")
+			msg := <-blocksSubs[j].Out()
+			newBlock = msg.Data().(types.EventDataNewBlock).Block
+			if newBlock.LastCommit.Size() == len(updatedVals) {
+				css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block", "height", newBlock.Height)
+				break LOOP
+			} else {
+				css[j].Logger.Debug(
+					"waitForBlockWithUpdatedValsAndValidateIt: Got block with no new validators. Skipping",
+					"height",
+					newBlock.Height)
+			}
+		}
+
+		err := validateBlock(newBlock, updatedVals)
+		assert.Nil(t, err)
+	}, css)
+}
+
+// expects high synchrony!
+func validateBlock(block *types.Block, activeVals map[string]struct{}) error {
+	if block.LastCommit.Size() != len(activeVals) {
+		return fmt.Errorf(
+			"commit size doesn't match number of active validators. Got %d, expected %d",
+			block.LastCommit.Size(),
+			len(activeVals))
+	}
+
+	for _, commitSig := range block.LastCommit.Signatures {
+		if _, ok := activeVals[string(commitSig.ValidatorAddress)]; !ok {
+			return fmt.Errorf("found vote for inactive validator %X", commitSig.ValidatorAddress)
+		}
+	}
+	return nil
+}
+
+func timeoutWaitGroup(t *testing.T, n int, f func(int), css []*State) {
+	wg := new(sync.WaitGroup)
+	wg.Add(n)
+	for i := 0; i < n; i++ {
+		go func(j int) {
+			f(j)
+			wg.Done()
+		}(i)
+	}
+
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	// we're running many nodes in-process, possibly in in a virtual machine,
+	// and spewing debug messages - making a block could take a while,
+	timeout := time.Second * 120
+
+	select {
+	case <-done:
+	case <-time.After(timeout):
+		for i, cs := range css {
+			t.Log("#################")
+			t.Log("Validator", i)
+			t.Log(cs.GetRoundState())
+			t.Log("")
+		}
+		os.Stdout.Write([]byte("pprof.Lookup('goroutine'):\n"))
+		err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
+		require.NoError(t, err)
+		capture()
+		panic("Timed out waiting for all validators to commit a block")
+	}
+}
+
+func capture() {
+	trace := make([]byte, 10240000)
+	count := runtime.Stack(trace, true)
+	fmt.Printf("Stack of %d bytes: %s\n", count, trace)
+}
+
+//-------------------------------------------------------------
+// Ensure basic validation of structs is functioning
+
+func TestNewRoundStepMessageValidateBasic(t *testing.T) {
+	testCases := []struct { // nolint: maligned
+		expectErr              bool
+		messageRound           int32
+		messageLastCommitRound int32
+		messageHeight          int64
+		testName               string
+		messageStep            cstypes.RoundStepType
+	}{
+		{false, 0, 0, 0, "Valid Message", cstypes.RoundStepNewHeight},
+		{true, -1, 0, 0, "Negative round", cstypes.RoundStepNewHeight},
+		{true, 0, 0, -1, "Negative height", cstypes.RoundStepNewHeight},
+		{true, 0, 0, 0, "Invalid Step", cstypes.RoundStepCommit + 1},
+		// The following cases will be handled by ValidateHeight
+		{false, 0, 0, 1, "H == 1 but LCR != -1 ", cstypes.RoundStepNewHeight},
+		{false, 0, -1, 2, "H > 1 but LCR < 0", cstypes.RoundStepNewHeight},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := NewRoundStepMessage{
+				Height:          tc.messageHeight,
+				Round:           tc.messageRound,
+				Step:            tc.messageStep,
+				LastCommitRound: tc.messageLastCommitRound,
+			}
+
+			err := message.ValidateBasic()
+			if tc.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestNewRoundStepMessageValidateHeight(t *testing.T) {
+	initialHeight := int64(10)
+	testCases := []struct { // nolint: maligned
+		expectErr              bool
+		messageLastCommitRound int32
+		messageHeight          int64
+		testName               string
+	}{
+		{false, 0, 11, "Valid Message"},
+		{true, 0, -1, "Negative height"},
+		{true, 0, 0, "Zero height"},
+		{true, 0, 10, "Initial height but LCR != -1 "},
+		{true, -1, 11, "Normal height but LCR < 0"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := NewRoundStepMessage{
+				Height:          tc.messageHeight,
+				Round:           0,
+				Step:            cstypes.RoundStepNewHeight,
+				LastCommitRound: tc.messageLastCommitRound,
+			}
+
+			err := message.ValidateHeight(initialHeight)
+			if tc.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestNewValidBlockMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*NewValidBlockMessage)
+		expErr     string
+	}{
+		{func(msg *NewValidBlockMessage) {}, ""},
+		{func(msg *NewValidBlockMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *NewValidBlockMessage) { msg.Round = -1 }, "negative Round"},
+		{
+			func(msg *NewValidBlockMessage) { msg.BlockPartSetHeader.Total = 2 },
+			"blockParts bit array size 1 not equal to BlockPartSetHeader.Total 2",
+		},
+		{
+			func(msg *NewValidBlockMessage) {
+				msg.BlockPartSetHeader.Total = 0
+				msg.BlockParts = bits.NewBitArray(0)
+			},
+			"empty blockParts",
+		},
+		{
+			func(msg *NewValidBlockMessage) { msg.BlockParts = bits.NewBitArray(int(types.MaxBlockPartsCount) + 1) },
+			"blockParts bit array size 1602 not equal to BlockPartSetHeader.Total 1",
+		},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &NewValidBlockMessage{
+				Height: 1,
+				Round:  0,
+				BlockPartSetHeader: types.PartSetHeader{
+					Total: 1,
+				},
+				BlockParts: bits.NewBitArray(1),
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}
+
+func TestProposalPOLMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*ProposalPOLMessage)
+		expErr     string
+	}{
+		{func(msg *ProposalPOLMessage) {}, ""},
+		{func(msg *ProposalPOLMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOLRound = -1 }, "negative ProposalPOLRound"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(0) }, "empty ProposalPOL bit array"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(types.MaxVotesCount + 1) },
+			"proposalPOL bit array is too big: 10001, max: 10000"},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &ProposalPOLMessage{
+				Height:           1,
+				ProposalPOLRound: 1,
+				ProposalPOL:      bits.NewBitArray(1),
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}
+
+func TestBlockPartMessageValidateBasic(t *testing.T) {
+	testPart := new(types.Part)
+	testPart.Proof.LeafHash = tmhash.Sum([]byte("leaf"))
+	testCases := []struct {
+		testName      string
+		messageHeight int64
+		messageRound  int32
+		messagePart   *types.Part
+		expectErr     bool
+	}{
+		{"Valid Message", 0, 0, testPart, false},
+		{"Invalid Message", -1, 0, testPart, true},
+		{"Invalid Message", 0, -1, testPart, true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := BlockPartMessage{
+				Height: tc.messageHeight,
+				Round:  tc.messageRound,
+				Part:   tc.messagePart,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+
+	message := BlockPartMessage{Height: 0, Round: 0, Part: new(types.Part)}
+	message.Part.Index = 1
+
+	assert.Equal(t, true, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+}
+
+func TestHasVoteMessageValidateBasic(t *testing.T) {
+	const (
+		validSignedMsgType   tmproto.SignedMsgType = 0x01
+		invalidSignedMsgType tmproto.SignedMsgType = 0x03
+	)
+
+	testCases := []struct { // nolint: maligned
+		expectErr     bool
+		messageRound  int32
+		messageIndex  int32
+		messageHeight int64
+		testName      string
+		messageType   tmproto.SignedMsgType
+	}{
+		{false, 0, 0, 0, "Valid Message", validSignedMsgType},
+		{true, -1, 0, 0, "Invalid Message", validSignedMsgType},
+		{true, 0, -1, 0, "Invalid Message", validSignedMsgType},
+		{true, 0, 0, 0, "Invalid Message", invalidSignedMsgType},
+		{true, 0, 0, -1, "Invalid Message", validSignedMsgType},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := HasVoteMessage{
+				Height: tc.messageHeight,
+				Round:  tc.messageRound,
+				Type:   tc.messageType,
+				Index:  tc.messageIndex,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+func TestVoteSetMaj23MessageValidateBasic(t *testing.T) {
+	const (
+		validSignedMsgType   tmproto.SignedMsgType = 0x01
+		invalidSignedMsgType tmproto.SignedMsgType = 0x03
+	)
+
+	validBlockID := types.BlockID{}
+	invalidBlockID := types.BlockID{
+		Hash: bytes.HexBytes{},
+		PartSetHeader: types.PartSetHeader{
+			Total: 1,
+			Hash:  []byte{0},
+		},
+	}
+
+	testCases := []struct { // nolint: maligned
+		expectErr      bool
+		messageRound   int32
+		messageHeight  int64
+		testName       string
+		messageType    tmproto.SignedMsgType
+		messageBlockID types.BlockID
+	}{
+		{false, 0, 0, "Valid Message", validSignedMsgType, validBlockID},
+		{true, -1, 0, "Invalid Message", validSignedMsgType, validBlockID},
+		{true, 0, -1, "Invalid Message", validSignedMsgType, validBlockID},
+		{true, 0, 0, "Invalid Message", invalidSignedMsgType, validBlockID},
+		{true, 0, 0, "Invalid Message", validSignedMsgType, invalidBlockID},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := VoteSetMaj23Message{
+				Height:  tc.messageHeight,
+				Round:   tc.messageRound,
+				Type:    tc.messageType,
+				BlockID: tc.messageBlockID,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+func TestVoteSetBitsMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*VoteSetBitsMessage)
+		expErr     string
+	}{
+		{func(msg *VoteSetBitsMessage) {}, ""},
+		{func(msg *VoteSetBitsMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *VoteSetBitsMessage) { msg.Type = 0x03 }, "invalid Type"},
+		{func(msg *VoteSetBitsMessage) {
+			msg.BlockID = types.BlockID{
+				Hash: bytes.HexBytes{},
+				PartSetHeader: types.PartSetHeader{
+					Total: 1,
+					Hash:  []byte{0},
+				},
+			}
+		}, "wrong BlockID: wrong PartSetHeader: wrong Hash:"},
+		{func(msg *VoteSetBitsMessage) { msg.Votes = bits.NewBitArray(types.MaxVotesCount + 1) },
+			"votes bit array is too big: 10001, max: 10000"},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &VoteSetBitsMessage{
+				Height:  1,
+				Round:   0,
+				Type:    0x01,
+				Votes:   bits.NewBitArray(1),
+				BlockID: types.BlockID{},
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}

internal/consensus/replay.go

@@ -0,0 +1,533 @@
+package consensus
+
+import (
+	"bytes"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"reflect"
+	"time"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+var crc32c = crc32.MakeTable(crc32.Castagnoli)
+
+// Functionality to replay blocks and messages on recovery from a crash.
+// There are two general failure scenarios:
+//
+//  1. failure during consensus
+//  2. failure while applying the block
+//
+// The former is handled by the WAL, the latter by the proxyApp Handshake on
+// restart, which ultimately hands off the work to the WAL.
+
+//-----------------------------------------
+// 1. Recover from failure during consensus
+// (by replaying messages from the WAL)
+//-----------------------------------------
+
+// Unmarshal and apply a single message to the consensus state as if it were
+// received in receiveRoutine.  Lines that start with "#" are ignored.
+// NOTE: receiveRoutine should not be running.
+func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error {
+	// Skip meta messages which exist for demarcating boundaries.
+	if _, ok := msg.Msg.(EndHeightMessage); ok {
+		return nil
+	}
+
+	// for logging
+	switch m := msg.Msg.(type) {
+	case types.EventDataRoundState:
+		cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
+		// these are playback checks
+		ticker := time.After(time.Second * 2)
+		if newStepSub != nil {
+			select {
+			case stepMsg := <-newStepSub.Out():
+				m2 := stepMsg.Data().(types.EventDataRoundState)
+				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
+					return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m)
+				}
+			case <-newStepSub.Cancelled():
+				return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was cancelled")
+			case <-ticker:
+				return fmt.Errorf("failed to read off newStepSub.Out()")
+			}
+		}
+	case msgInfo:
+		peerID := m.PeerID
+		if peerID == "" {
+			peerID = "local"
+		}
+		switch msg := m.Msg.(type) {
+		case *ProposalMessage:
+			p := msg.Proposal
+			cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
+				p.BlockID.PartSetHeader, "pol", p.POLRound, "peer", peerID)
+		case *BlockPartMessage:
+			cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID)
+		case *VoteMessage:
+			v := msg.Vote
+			cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
+				"blockID", v.BlockID, "peer", peerID)
+		}
+
+		cs.handleMsg(m)
+	case timeoutInfo:
+		cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
+		cs.handleTimeout(m, cs.RoundState)
+	default:
+		return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
+	}
+	return nil
+}
+
+// Replay only those messages since the last block.  `timeoutRoutine` should
+// run concurrently to read off tickChan.
+func (cs *State) catchupReplay(csHeight int64) error {
+
+	// Set replayMode to true so we don't log signing errors.
+	cs.replayMode = true
+	defer func() { cs.replayMode = false }()
+
+	// Ensure that #ENDHEIGHT for this height doesn't exist.
+	// NOTE: This is just a sanity check. As far as we know things work fine
+	// without it, and Handshake could reuse State if it weren't for
+	// this check (since we can crash after writing #ENDHEIGHT).
+	//
+	// Ignore data corruption errors since this is a sanity check.
+	gr, found, err := cs.wal.SearchForEndHeight(csHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
+	if err != nil {
+		return err
+	}
+	if gr != nil {
+		if err := gr.Close(); err != nil {
+			return err
+		}
+	}
+	if found {
+		return fmt.Errorf("wal should not contain #ENDHEIGHT %d", csHeight)
+	}
+
+	// Search for last height marker.
+	//
+	// Ignore data corruption errors in previous heights because we only care about last height
+	if csHeight < cs.state.InitialHeight {
+		return fmt.Errorf("cannot replay height %v, below initial height %v", csHeight, cs.state.InitialHeight)
+	}
+	endHeight := csHeight - 1
+	if csHeight == cs.state.InitialHeight {
+		endHeight = 0
+	}
+	gr, found, err = cs.wal.SearchForEndHeight(endHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
+	if err == io.EOF {
+		cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", endHeight)
+	} else if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, endHeight)
+	}
+	defer gr.Close()
+
+	cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
+
+	var msg *TimedWALMessage
+	dec := WALDecoder{gr}
+
+LOOP:
+	for {
+		msg, err = dec.Decode()
+		switch {
+		case err == io.EOF:
+			break LOOP
+		case IsDataCorruptionError(err):
+			cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
+			return err
+		case err != nil:
+			return err
+		}
+
+		// NOTE: since the priv key is set when the msgs are received
+		// it will attempt to eg double sign but we can just ignore it
+		// since the votes will be replayed and we'll get to the next step
+		if err := cs.readReplayMessage(msg, nil); err != nil {
+			return err
+		}
+	}
+	cs.Logger.Info("Replay: Done")
+	return nil
+}
+
+//--------------------------------------------------------------------------------
+
+// Parses marker lines of the form:
+// #ENDHEIGHT: 12345
+/*
+func makeHeightSearchFunc(height int64) auto.SearchFunc {
+	return func(line string) (int, error) {
+		line = strings.TrimRight(line, "\n")
+		parts := strings.Split(line, " ")
+		if len(parts) != 2 {
+			return -1, errors.New("line did not have 2 parts")
+		}
+		i, err := strconv.Atoi(parts[1])
+		if err != nil {
+			return -1, errors.New("failed to parse INFO: " + err.Error())
+		}
+		if height < i {
+			return 1, nil
+		} else if height == i {
+			return 0, nil
+		} else {
+			return -1, nil
+		}
+	}
+}*/
+
+//---------------------------------------------------
+// 2. Recover from failure while applying the block.
+// (by handshaking with the app to figure out where
+// we were last, and using the WAL to recover there.)
+//---------------------------------------------------
+
+type Handshaker struct {
+	stateStore   sm.Store
+	initialState sm.State
+	store        sm.BlockStore
+	eventBus     types.BlockEventPublisher
+	genDoc       *types.GenesisDoc
+	logger       log.Logger
+
+	nBlocks int // number of blocks applied to the state
+}
+
+func NewHandshaker(stateStore sm.Store, state sm.State,
+	store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
+
+	return &Handshaker{
+		stateStore:   stateStore,
+		initialState: state,
+		store:        store,
+		eventBus:     types.NopEventBus{},
+		genDoc:       genDoc,
+		logger:       log.NewNopLogger(),
+		nBlocks:      0,
+	}
+}
+
+func (h *Handshaker) SetLogger(l log.Logger) {
+	h.logger = l
+}
+
+// SetEventBus - sets the event bus for publishing block related events.
+// If not called, it defaults to types.NopEventBus.
+func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) {
+	h.eventBus = eventBus
+}
+
+// NBlocks returns the number of blocks applied to the state.
+func (h *Handshaker) NBlocks() int {
+	return h.nBlocks
+}
+
+// TODO: retry the handshake/replay if it fails ?
+func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
+
+	// Handshake is done via ABCI Info on the query conn.
+	res, err := proxyApp.Query().InfoSync(proxy.RequestInfo)
+	if err != nil {
+		return fmt.Errorf("error calling Info: %v", err)
+	}
+
+	blockHeight := res.LastBlockHeight
+	if blockHeight < 0 {
+		return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight)
+	}
+	appHash := res.LastBlockAppHash
+
+	h.logger.Info("ABCI Handshake App Info",
+		"height", blockHeight,
+		"hash", appHash,
+		"software-version", res.Version,
+		"protocol-version", res.AppVersion,
+	)
+
+	// Only set the version if there is no existing state.
+	if h.initialState.LastBlockHeight == 0 {
+		h.initialState.Version.Consensus.App = res.AppVersion
+	}
+
+	// Replay blocks up to the latest in the blockstore.
+	_, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
+	if err != nil {
+		return fmt.Errorf("error on replay: %v", err)
+	}
+
+	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
+		"appHeight", blockHeight, "appHash", appHash)
+
+	// TODO: (on restart) replay mempool
+
+	return nil
+}
+
+// ReplayBlocks replays all blocks since appBlockHeight and ensures the result
+// matches the current state.
+// Returns the final AppHash or an error.
+func (h *Handshaker) ReplayBlocks(
+	state sm.State,
+	appHash []byte,
+	appBlockHeight int64,
+	proxyApp proxy.AppConns,
+) ([]byte, error) {
+	storeBlockBase := h.store.Base()
+	storeBlockHeight := h.store.Height()
+	stateBlockHeight := state.LastBlockHeight
+	h.logger.Info(
+		"ABCI Replay Blocks",
+		"appHeight",
+		appBlockHeight,
+		"storeHeight",
+		storeBlockHeight,
+		"stateHeight",
+		stateBlockHeight)
+
+	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain.
+	if appBlockHeight == 0 {
+		validators := make([]*types.Validator, len(h.genDoc.Validators))
+		for i, val := range h.genDoc.Validators {
+			validators[i] = types.NewValidator(val.PubKey, val.Power)
+		}
+		validatorSet := types.NewValidatorSet(validators)
+		nextVals := types.TM2PB.ValidatorUpdates(validatorSet)
+		csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams)
+		req := abci.RequestInitChain{
+			Time:            h.genDoc.GenesisTime,
+			ChainId:         h.genDoc.ChainID,
+			InitialHeight:   h.genDoc.InitialHeight,
+			ConsensusParams: csParams,
+			Validators:      nextVals,
+			AppStateBytes:   h.genDoc.AppState,
+		}
+		res, err := proxyApp.Consensus().InitChainSync(req)
+		if err != nil {
+			return nil, err
+		}
+
+		appHash = res.AppHash
+
+		if stateBlockHeight == 0 { // we only update state when we are in initial state
+			// If the app did not return an app hash, we keep the one set from the genesis doc in
+			// the state. We don't set appHash since we don't want the genesis doc app hash
+			// recorded in the genesis block. We should probably just remove GenesisDoc.AppHash.
+			if len(res.AppHash) > 0 {
+				state.AppHash = res.AppHash
+			}
+			// If the app returned validators or consensus params, update the state.
+			if len(res.Validators) > 0 {
+				vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
+				if err != nil {
+					return nil, err
+				}
+				state.Validators = types.NewValidatorSet(vals)
+				state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1)
+			} else if len(h.genDoc.Validators) == 0 {
+				// If validator set is not set in genesis and still empty after InitChain, exit.
+				return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")
+			}
+
+			if res.ConsensusParams != nil {
+				state.ConsensusParams = types.UpdateConsensusParams(state.ConsensusParams, res.ConsensusParams)
+				state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion
+			}
+			// We update the last results hash with the empty hash, to conform with RFC-6962.
+			state.LastResultsHash = merkle.HashFromByteSlices(nil)
+			if err := h.stateStore.Save(state); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// First handle edge cases and constraints on the storeBlockHeight and storeBlockBase.
+	switch {
+	case storeBlockHeight == 0:
+		assertAppHashEqualsOneFromState(appHash, state)
+		return appHash, nil
+
+	case appBlockHeight == 0 && state.InitialHeight < storeBlockBase:
+		// the app has no state, and the block store is truncated above the initial height
+		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
+
+	case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1:
+		// the app is too far behind truncated store (can be 1 behind since we replay the next)
+		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
+
+	case storeBlockHeight < appBlockHeight:
+		// the app should never be ahead of the store (but this is under app's control)
+		return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight}
+
+	case storeBlockHeight < stateBlockHeight:
+		// the state should never be ahead of the store (this is under tendermint's control)
+		panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
+
+	case storeBlockHeight > stateBlockHeight+1:
+		// store should be at most one ahead of the state (this is under tendermint's control)
+		panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
+	}
+
+	var err error
+	// Now either store is equal to state, or one ahead.
+	// For each, consider all cases of where the app could be, given app <= store
+	if storeBlockHeight == stateBlockHeight {
+		// Tendermint ran Commit and saved the state.
+		// Either the app is asking for replay, or we're all synced up.
+		if appBlockHeight < storeBlockHeight {
+			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We're good!
+			assertAppHashEqualsOneFromState(appHash, state)
+			return appHash, nil
+		}
+
+	} else if storeBlockHeight == stateBlockHeight+1 {
+		// We saved the block in the store but haven't updated the state,
+		// so we'll need to replay a block using the WAL.
+		switch {
+		case appBlockHeight < stateBlockHeight:
+			// the app is further behind than it should be, so replay blocks
+			// but leave the last block to go through the WAL
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true)
+
+		case appBlockHeight == stateBlockHeight:
+			// We haven't run Commit (both the state and app are one block behind),
+			// so replayBlock with the real app.
+			// NOTE: We could instead use the cs.WAL on cs.Start,
+			// but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT
+			h.logger.Info("Replay last block using real app")
+			state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+			return state.AppHash, err
+
+		case appBlockHeight == storeBlockHeight:
+			// We ran Commit, but didn't save the state, so replayBlock with mock app.
+			abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight)
+			if err != nil {
+				return nil, err
+			}
+			mockApp := newMockProxyApp(appHash, abciResponses)
+			h.logger.Info("Replay last block using mock app")
+			state, err = h.replayBlock(state, storeBlockHeight, mockApp)
+			return state.AppHash, err
+		}
+
+	}
+
+	panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d",
+		appBlockHeight, storeBlockHeight, stateBlockHeight))
+}
+
+func (h *Handshaker) replayBlocks(
+	state sm.State,
+	proxyApp proxy.AppConns,
+	appBlockHeight,
+	storeBlockHeight int64,
+	mutateState bool) ([]byte, error) {
+	// App is further behind than it should be, so we need to replay blocks.
+	// We replay all blocks from appBlockHeight+1.
+	//
+	// Note that we don't have an old version of the state,
+	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
+	// This also means we won't be saving validator sets if they change during this period.
+	// TODO: Load the historical information to fix this and just use state.ApplyBlock
+	//
+	// If mutateState == true, the final block is replayed with h.replayBlock()
+
+	var appHash []byte
+	var err error
+	finalBlock := storeBlockHeight
+	if mutateState {
+		finalBlock--
+	}
+	firstBlock := appBlockHeight + 1
+	if firstBlock == 1 {
+		firstBlock = state.InitialHeight
+	}
+	for i := firstBlock; i <= finalBlock; i++ {
+		h.logger.Info("Applying block", "height", i)
+		block := h.store.LoadBlock(i)
+		// Extra check to ensure the app was not changed in a way it shouldn't have.
+		if len(appHash) > 0 {
+			assertAppHashEqualsOneFromBlock(appHash, block)
+		}
+
+		appHash, err = sm.ExecCommitBlock(proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight)
+		if err != nil {
+			return nil, err
+		}
+
+		h.nBlocks++
+	}
+
+	if mutateState {
+		// sync the final block
+		state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+		if err != nil {
+			return nil, err
+		}
+		appHash = state.AppHash
+	}
+
+	assertAppHashEqualsOneFromState(appHash, state)
+	return appHash, nil
+}
+
+// ApplyBlock on the proxyApp with the last block.
+func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) {
+	block := h.store.LoadBlock(height)
+	meta := h.store.LoadBlockMeta(height)
+
+	// Use stubs for both mempool and evidence pool since no transactions nor
+	// evidence are needed here - block already exists.
+	blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{})
+	blockExec.SetEventBus(h.eventBus)
+
+	var err error
+	state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block)
+	if err != nil {
+		return sm.State{}, err
+	}
+
+	h.nBlocks++
+
+	return state, nil
+}
+
+func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) {
+	if !bytes.Equal(appHash, block.AppHash) {
+		panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X.
+
+Block: %v
+`,
+			appHash, block.AppHash, block))
+	}
+}
+
+func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) {
+	if !bytes.Equal(appHash, state.AppHash) {
+		panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got
+%X, expected %X.
+
+State: %v
+
+Did you reset Tendermint without resetting your application's data?`,
+			appHash, state.AppHash, state))
+	}
+}

internal/consensus/replay_file.go

@@ -0,0 +1,338 @@
+package consensus
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	dbm "github.com/tendermint/tm-db"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// event bus subscriber
+	subscriber = "replay-file"
+)
+
+//--------------------------------------------------------
+// replay messages interactively or all at once
+
+// replay the wal file
+func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) {
+	consensusState := newConsensusStateForReplay(config, csConfig)
+
+	if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil {
+		tmos.Exit(fmt.Sprintf("Error during consensus replay: %v", err))
+	}
+}
+
+// Replay msgs in file or start the console
+func (cs *State) ReplayFile(file string, console bool) error {
+
+	if cs.IsRunning() {
+		return errors.New("cs is already running, cannot replay")
+	}
+	if cs.wal != nil {
+		return errors.New("cs wal is open, cannot replay")
+	}
+
+	cs.startForReplay()
+
+	// ensure all new step events are regenerated as expected
+
+	ctx := context.Background()
+	newStepSub, err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+	if err != nil {
+		return fmt.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)
+	}
+	defer func() {
+		if err := cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep); err != nil {
+			cs.Logger.Error("Error unsubscribing to event bus", "err", err)
+		}
+	}()
+
+	// just open the file for reading, no need to use wal
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0600)
+	if err != nil {
+		return err
+	}
+
+	pb := newPlayback(file, fp, cs, cs.state.Copy())
+	defer pb.fp.Close()
+
+	var nextN int // apply N msgs in a row
+	var msg *TimedWALMessage
+	for {
+		if nextN == 0 && console {
+			nextN = pb.replayConsoleLoop()
+		}
+
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
+			return err
+		}
+
+		if nextN > 0 {
+			nextN--
+		}
+		pb.count++
+	}
+}
+
+//------------------------------------------------
+// playback manager
+
+type playback struct {
+	cs *State
+
+	fp    *os.File
+	dec   *WALDecoder
+	count int // how many lines/msgs into the file are we
+
+	// replays can be reset to beginning
+	fileName     string   // so we can close/reopen the file
+	genesisState sm.State // so the replay session knows where to restart from
+}
+
+func newPlayback(fileName string, fp *os.File, cs *State, genState sm.State) *playback {
+	return &playback{
+		cs:           cs,
+		fp:           fp,
+		fileName:     fileName,
+		genesisState: genState,
+		dec:          NewWALDecoder(fp),
+	}
+}
+
+// go back count steps by resetting the state and running (pb.count - count) steps
+func (pb *playback) replayReset(count int, newStepSub types.Subscription) error {
+	if err := pb.cs.Stop(); err != nil {
+		return err
+	}
+	pb.cs.Wait()
+
+	newCS := NewState(pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec,
+		pb.cs.blockStore, pb.cs.txNotifier, pb.cs.evpool)
+	newCS.SetEventBus(pb.cs.eventBus)
+	newCS.startForReplay()
+
+	if err := pb.fp.Close(); err != nil {
+		return err
+	}
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0600)
+	if err != nil {
+		return err
+	}
+	pb.fp = fp
+	pb.dec = NewWALDecoder(fp)
+	count = pb.count - count
+	fmt.Printf("Reseting from %d to %d\n", pb.count, count)
+	pb.count = 0
+	pb.cs = newCS
+	var msg *TimedWALMessage
+	for i := 0; i < count; i++ {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
+			return err
+		}
+		pb.count++
+	}
+	return nil
+}
+
+func (cs *State) startForReplay() {
+	cs.Logger.Error("Replay commands are disabled until someone updates them and writes tests")
+	/* TODO:!
+	// since we replay tocks we just ignore ticks
+		go func() {
+			for {
+				select {
+				case <-cs.tickChan:
+				case <-cs.Quit:
+					return
+				}
+			}
+		}()*/
+}
+
+// console function for parsing input and running commands
+func (pb *playback) replayConsoleLoop() int {
+	for {
+		fmt.Printf("> ")
+		bufReader := bufio.NewReader(os.Stdin)
+		line, more, err := bufReader.ReadLine()
+		if more {
+			tmos.Exit("input is too long")
+		} else if err != nil {
+			tmos.Exit(err.Error())
+		}
+
+		tokens := strings.Split(string(line), " ")
+		if len(tokens) == 0 {
+			continue
+		}
+
+		switch tokens[0] {
+		case "next":
+			// "next" -> replay next message
+			// "next N" -> replay next N messages
+
+			if len(tokens) == 1 {
+				return 0
+			}
+			i, err := strconv.Atoi(tokens[1])
+			if err != nil {
+				fmt.Println("next takes an integer argument")
+			} else {
+				return i
+			}
+
+		case "back":
+			// "back" -> go back one message
+			// "back N" -> go back N messages
+
+			// NOTE: "back" is not supported in the state machine design,
+			// so we restart and replay up to
+
+			ctx := context.Background()
+			// ensure all new step events are regenerated as expected
+
+			newStepSub, err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+			if err != nil {
+				tmos.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep))
+			}
+			defer func() {
+				if err := pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep); err != nil {
+					pb.cs.Logger.Error("Error unsubscribing from eventBus", "err", err)
+				}
+			}()
+
+			if len(tokens) == 1 {
+				if err := pb.replayReset(1, newStepSub); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("back takes an integer argument")
+				} else if i > pb.count {
+					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
+				} else if err := pb.replayReset(i, newStepSub); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
+			}
+
+		case "rs":
+			// "rs" -> print entire round state
+			// "rs short" -> print height/round/step
+			// "rs <field>" -> print another field of the round state
+
+			rs := pb.cs.RoundState
+			if len(tokens) == 1 {
+				fmt.Println(rs)
+			} else {
+				switch tokens[1] {
+				case "short":
+					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
+				case "validators":
+					fmt.Println(rs.Validators)
+				case "proposal":
+					fmt.Println(rs.Proposal)
+				case "proposal_block":
+					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
+				case "locked_round":
+					fmt.Println(rs.LockedRound)
+				case "locked_block":
+					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
+				case "votes":
+					fmt.Println(rs.Votes.StringIndented("  "))
+
+				default:
+					fmt.Println("Unknown option", tokens[1])
+				}
+			}
+		case "n":
+			fmt.Println(pb.count)
+		}
+	}
+}
+
+//--------------------------------------------------------------------------------
+
+// convenience for replay mode
+func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *State {
+	dbType := dbm.BackendType(config.DBBackend)
+	// Get BlockStore
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	// Get State
+	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	stateStore := sm.NewStore(stateDB)
+	gdoc, err := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	state, err := sm.MakeGenesisState(gdoc)
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+
+	// Create proxyAppConn connection (consensus, mempool, query)
+	clientCreator := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
+	proxyApp := proxy.NewAppConns(clientCreator)
+	err = proxyApp.Start()
+	if err != nil {
+		tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))
+	}
+
+	eventBus := types.NewEventBus()
+	if err := eventBus.Start(); err != nil {
+		tmos.Exit(fmt.Sprintf("Failed to start event bus: %v", err))
+	}
+
+	handshaker := NewHandshaker(stateStore, state, blockStore, gdoc)
+	handshaker.SetEventBus(eventBus)
+	err = handshaker.Handshake(proxyApp)
+	if err != nil {
+		tmos.Exit(fmt.Sprintf("Error on handshake: %v", err))
+	}
+
+	mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+
+	consensusState := NewState(csConfig, state.Copy(), blockExec,
+		blockStore, mempool, evpool)
+
+	consensusState.SetEventBus(eventBus)
+	return consensusState
+}

internal/consensus/replay_stubs.go

@@ -0,0 +1,96 @@
+package consensus
+
+import (
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/clist"
+	mempl "github.com/tendermint/tendermint/mempool"
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	"github.com/tendermint/tendermint/proxy"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+
+type emptyMempool struct{}
+
+var _ mempl.Mempool = emptyMempool{}
+
+func (emptyMempool) Lock()            {}
+func (emptyMempool) Unlock()          {}
+func (emptyMempool) Size() int        { return 0 }
+func (emptyMempool) SizeBytes() int64 { return 0 }
+func (emptyMempool) CheckTx(_ types.Tx, _ func(*abci.Response), _ mempl.TxInfo) error {
+	return nil
+}
+
+func (txmp emptyMempool) RemoveTxByKey(txKey types.TxKey) error {
+	return nil
+}
+
+func (emptyMempool) ReapMaxBytesMaxGas(_, _ int64) types.Txs { return types.Txs{} }
+func (emptyMempool) ReapMaxTxs(n int) types.Txs              { return types.Txs{} }
+func (emptyMempool) Update(
+	_ int64,
+	_ types.Txs,
+	_ []*abci.ResponseDeliverTx,
+	_ mempl.PreCheckFunc,
+	_ mempl.PostCheckFunc,
+) error {
+	return nil
+}
+func (emptyMempool) Flush()                        {}
+func (emptyMempool) FlushAppConn() error           { return nil }
+func (emptyMempool) TxsAvailable() <-chan struct{} { return make(chan struct{}) }
+func (emptyMempool) EnableTxsAvailable()           {}
+func (emptyMempool) TxsBytes() int64               { return 0 }
+
+func (emptyMempool) TxsFront() *clist.CElement    { return nil }
+func (emptyMempool) TxsWaitChan() <-chan struct{} { return nil }
+
+func (emptyMempool) InitWAL() error { return nil }
+func (emptyMempool) CloseWAL()      {}
+
+//-----------------------------------------------------------------------------
+// mockProxyApp uses ABCIResponses to give the right results.
+//
+// Useful because we don't want to call Commit() twice for the same block on
+// the real app.
+
+func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy.AppConnConsensus {
+	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{
+		appHash:       appHash,
+		abciResponses: abciResponses,
+	})
+	cli, _ := clientCreator.NewABCIClient()
+	err := cli.Start()
+	if err != nil {
+		panic(err)
+	}
+	return proxy.NewAppConnConsensus(cli)
+}
+
+type mockProxyApp struct {
+	abci.BaseApplication
+
+	appHash       []byte
+	txCount       int
+	abciResponses *tmstate.ABCIResponses
+}
+
+func (mock *mockProxyApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	r := mock.abciResponses.DeliverTxs[mock.txCount]
+	mock.txCount++
+	if r == nil {
+		return abci.ResponseDeliverTx{}
+	}
+	return *r
+}
+
+func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
+	mock.txCount = 0
+	return *mock.abciResponses.EndBlock
+}
+
+func (mock *mockProxyApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{Data: mock.appHash}
+}

internal/consensus/ticker.go

@@ -0,0 +1,134 @@
+package consensus
+
+import (
+	"time"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/service"
+)
+
+var (
+	tickTockBufferSize = 10
+)
+
+// TimeoutTicker is a timer that schedules timeouts
+// conditional on the height/round/step in the timeoutInfo.
+// The timeoutInfo.Duration may be non-positive.
+type TimeoutTicker interface {
+	Start() error
+	Stop() error
+	Chan() <-chan timeoutInfo       // on which to receive a timeout
+	ScheduleTimeout(ti timeoutInfo) // reset the timer
+
+	SetLogger(log.Logger)
+}
+
+// timeoutTicker wraps time.Timer,
+// scheduling timeouts only for greater height/round/step
+// than what it's already seen.
+// Timeouts are scheduled along the tickChan,
+// and fired on the tockChan.
+type timeoutTicker struct {
+	service.BaseService
+
+	timer    *time.Timer
+	tickChan chan timeoutInfo // for scheduling timeouts
+	tockChan chan timeoutInfo // for notifying about them
+}
+
+// NewTimeoutTicker returns a new TimeoutTicker.
+func NewTimeoutTicker() TimeoutTicker {
+	tt := &timeoutTicker{
+		timer:    time.NewTimer(0),
+		tickChan: make(chan timeoutInfo, tickTockBufferSize),
+		tockChan: make(chan timeoutInfo, tickTockBufferSize),
+	}
+	tt.BaseService = *service.NewBaseService(nil, "TimeoutTicker", tt)
+	tt.stopTimer() // don't want to fire until the first scheduled timeout
+	return tt
+}
+
+// OnStart implements service.Service. It starts the timeout routine.
+func (t *timeoutTicker) OnStart() error {
+
+	go t.timeoutRoutine()
+
+	return nil
+}
+
+// OnStop implements service.Service. It stops the timeout routine.
+func (t *timeoutTicker) OnStop() {
+	t.BaseService.OnStop()
+	t.stopTimer()
+}
+
+// Chan returns a channel on which timeouts are sent.
+func (t *timeoutTicker) Chan() <-chan timeoutInfo {
+	return t.tockChan
+}
+
+// ScheduleTimeout schedules a new timeout by sending on the internal tickChan.
+// The timeoutRoutine is always available to read from tickChan, so this won't block.
+// The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
+func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) {
+	t.tickChan <- ti
+}
+
+//-------------------------------------------------------------
+
+// stop the timer and drain if necessary
+func (t *timeoutTicker) stopTimer() {
+	// Stop() returns false if it was already fired or was stopped
+	if !t.timer.Stop() {
+		select {
+		case <-t.timer.C:
+		default:
+			t.Logger.Debug("Timer already stopped")
+		}
+	}
+}
+
+// send on tickChan to start a new timer.
+// timers are interupted and replaced by new ticks from later steps
+// timeouts of 0 on the tickChan will be immediately relayed to the tockChan
+func (t *timeoutTicker) timeoutRoutine() {
+	t.Logger.Debug("Starting timeout routine")
+	var ti timeoutInfo
+	for {
+		select {
+		case newti := <-t.tickChan:
+			t.Logger.Debug("Received tick", "old_ti", ti, "new_ti", newti)
+
+			// ignore tickers for old height/round/step
+			if newti.Height < ti.Height {
+				continue
+			} else if newti.Height == ti.Height {
+				if newti.Round < ti.Round {
+					continue
+				} else if newti.Round == ti.Round {
+					if ti.Step > 0 && newti.Step <= ti.Step {
+						continue
+					}
+				}
+			}
+
+			// stop the last timer
+			t.stopTimer()
+
+			// update timeoutInfo and reset timer
+			// NOTE time.Timer allows duration to be non-positive
+			ti = newti
+			t.timer.Reset(ti.Duration)
+			t.Logger.Debug("Scheduled timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+		case <-t.timer.C:
+			t.Logger.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+			// go routine here guarantees timeoutRoutine doesn't block.
+			// Determinism comes from playback in the receiveRoutine.
+			// We can eliminate it by merging the timeoutRoutine into receiveRoutine
+			//  and managing the timeouts ourselves with a millisecond ticker
+			go func(toi timeoutInfo) { t.tockChan <- toi }(ti)
+		case <-t.Quit():
+			return
+		}
+	}
+}

internal/consensus/types/height_vote_set.go

@@ -0,0 +1,267 @@
+package types
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	"github.com/tendermint/tendermint/p2p"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+type RoundVoteSet struct {
+	Prevotes   *types.VoteSet
+	Precommits *types.VoteSet
+}
+
+var (
+	ErrGotVoteFromUnwantedRound = errors.New(
+		"peer has sent a vote that does not match our round for more than one round",
+	)
+)
+
+/*
+Keeps track of all VoteSets from round 0 to round 'round'.
+
+Also keeps track of up to one RoundVoteSet greater than
+'round' from each peer, to facilitate catchup syncing of commits.
+
+A commit is +2/3 precommits for a block at a round,
+but which round is not known in advance, so when a peer
+provides a precommit for a round greater than mtx.round,
+we create a new entry in roundVoteSets but also remember the
+peer to prevent abuse.
+We let each peer provide us with up to 2 unexpected "catchup" rounds.
+One for their LastCommit round, and another for the official commit round.
+*/
+type HeightVoteSet struct {
+	chainID string
+	height  int64
+	valSet  *types.ValidatorSet
+
+	mtx               sync.Mutex
+	round             int32                  // max tracked round
+	roundVoteSets     map[int32]RoundVoteSet // keys: [0...round]
+	peerCatchupRounds map[p2p.ID][]int32     // keys: peer.ID; values: at most 2 rounds
+}
+
+func NewHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
+	hvs := &HeightVoteSet{
+		chainID: chainID,
+	}
+	hvs.Reset(height, valSet)
+	return hvs
+}
+
+func (hvs *HeightVoteSet) Reset(height int64, valSet *types.ValidatorSet) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+
+	hvs.height = height
+	hvs.valSet = valSet
+	hvs.roundVoteSets = make(map[int32]RoundVoteSet)
+	hvs.peerCatchupRounds = make(map[p2p.ID][]int32)
+
+	hvs.addRound(0)
+	hvs.round = 0
+}
+
+func (hvs *HeightVoteSet) Height() int64 {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.height
+}
+
+func (hvs *HeightVoteSet) Round() int32 {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.round
+}
+
+// Create more RoundVoteSets up to round.
+func (hvs *HeightVoteSet) SetRound(round int32) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	newRound := tmmath.SafeSubInt32(hvs.round, 1)
+	if hvs.round != 0 && (round < newRound) {
+		panic("SetRound() must increment hvs.round")
+	}
+	for r := newRound; r <= round; r++ {
+		if _, ok := hvs.roundVoteSets[r]; ok {
+			continue // Already exists because peerCatchupRounds.
+		}
+		hvs.addRound(r)
+	}
+	hvs.round = round
+}
+
+func (hvs *HeightVoteSet) addRound(round int32) {
+	if _, ok := hvs.roundVoteSets[round]; ok {
+		panic("addRound() for an existing round")
+	}
+	// log.Debug("addRound(round)", "round", round)
+	prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrevoteType, hvs.valSet)
+	precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrecommitType, hvs.valSet)
+	hvs.roundVoteSets[round] = RoundVoteSet{
+		Prevotes:   prevotes,
+		Precommits: precommits,
+	}
+}
+
+// Duplicate votes return added=false, err=nil.
+// By convention, peerID is "" if origin is self.
+func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerID p2p.ID) (added bool, err error) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	if !types.IsVoteTypeValid(vote.Type) {
+		return
+	}
+	voteSet := hvs.getVoteSet(vote.Round, vote.Type)
+	if voteSet == nil {
+		if rndz := hvs.peerCatchupRounds[peerID]; len(rndz) < 2 {
+			hvs.addRound(vote.Round)
+			voteSet = hvs.getVoteSet(vote.Round, vote.Type)
+			hvs.peerCatchupRounds[peerID] = append(rndz, vote.Round)
+		} else {
+			// punish peer
+			err = ErrGotVoteFromUnwantedRound
+			return
+		}
+	}
+	added, err = voteSet.AddVote(vote)
+	return
+}
+
+func (hvs *HeightVoteSet) Prevotes(round int32) *types.VoteSet {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.getVoteSet(round, tmproto.PrevoteType)
+}
+
+func (hvs *HeightVoteSet) Precommits(round int32) *types.VoteSet {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.getVoteSet(round, tmproto.PrecommitType)
+}
+
+// Last round and blockID that has +2/3 prevotes for a particular block or nil.
+// Returns -1 if no such round exists.
+func (hvs *HeightVoteSet) POLInfo() (polRound int32, polBlockID types.BlockID) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	for r := hvs.round; r >= 0; r-- {
+		rvs := hvs.getVoteSet(r, tmproto.PrevoteType)
+		polBlockID, ok := rvs.TwoThirdsMajority()
+		if ok {
+			return r, polBlockID
+		}
+	}
+	return -1, types.BlockID{}
+}
+
+func (hvs *HeightVoteSet) getVoteSet(round int32, voteType tmproto.SignedMsgType) *types.VoteSet {
+	rvs, ok := hvs.roundVoteSets[round]
+	if !ok {
+		return nil
+	}
+	switch voteType {
+	case tmproto.PrevoteType:
+		return rvs.Prevotes
+	case tmproto.PrecommitType:
+		return rvs.Precommits
+	default:
+		panic(fmt.Sprintf("Unexpected vote type %X", voteType))
+	}
+}
+
+// If a peer claims that it has 2/3 majority for given blockKey, call this.
+// NOTE: if there are too many peers, or too much peer churn,
+// this can cause memory issues.
+// TODO: implement ability to remove peers too
+func (hvs *HeightVoteSet) SetPeerMaj23(
+	round int32,
+	voteType tmproto.SignedMsgType,
+	peerID p2p.ID,
+	blockID types.BlockID) error {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	if !types.IsVoteTypeValid(voteType) {
+		return fmt.Errorf("setPeerMaj23: Invalid vote type %X", voteType)
+	}
+	voteSet := hvs.getVoteSet(round, voteType)
+	if voteSet == nil {
+		return nil // something we don't know about yet
+	}
+	return voteSet.SetPeerMaj23(types.P2PID(peerID), blockID)
+}
+
+//---------------------------------------------------------
+// string and json
+
+func (hvs *HeightVoteSet) String() string {
+	return hvs.StringIndented("")
+}
+
+func (hvs *HeightVoteSet) StringIndented(indent string) string {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	vsStrings := make([]string, 0, (len(hvs.roundVoteSets)+1)*2)
+	// rounds 0 ~ hvs.round inclusive
+	for round := int32(0); round <= hvs.round; round++ {
+		voteSetString := hvs.roundVoteSets[round].Prevotes.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+		voteSetString = hvs.roundVoteSets[round].Precommits.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+	}
+	// all other peer catchup rounds
+	for round, roundVoteSet := range hvs.roundVoteSets {
+		if round <= hvs.round {
+			continue
+		}
+		voteSetString := roundVoteSet.Prevotes.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+		voteSetString = roundVoteSet.Precommits.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+	}
+	return fmt.Sprintf(`HeightVoteSet{H:%v R:0~%v
+%s  %v
+%s}`,
+		hvs.height, hvs.round,
+		indent, strings.Join(vsStrings, "\n"+indent+"  "),
+		indent)
+}
+
+func (hvs *HeightVoteSet) MarshalJSON() ([]byte, error) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return tmjson.Marshal(hvs.toAllRoundVotes())
+}
+
+func (hvs *HeightVoteSet) toAllRoundVotes() []roundVotes {
+	totalRounds := hvs.round + 1
+	allVotes := make([]roundVotes, totalRounds)
+	// rounds 0 ~ hvs.round inclusive
+	for round := int32(0); round < totalRounds; round++ {
+		allVotes[round] = roundVotes{
+			Round:              round,
+			Prevotes:           hvs.roundVoteSets[round].Prevotes.VoteStrings(),
+			PrevotesBitArray:   hvs.roundVoteSets[round].Prevotes.BitArrayString(),
+			Precommits:         hvs.roundVoteSets[round].Precommits.VoteStrings(),
+			PrecommitsBitArray: hvs.roundVoteSets[round].Precommits.BitArrayString(),
+		}
+	}
+	// TODO: all other peer catchup rounds
+	return allVotes
+}
+
+type roundVotes struct {
+	Round              int32    `json:"round"`
+	Prevotes           []string `json:"prevotes"`
+	PrevotesBitArray   string   `json:"prevotes_bit_array"`
+	Precommits         []string `json:"precommits"`
+	PrecommitsBitArray string   `json:"precommits_bit_array"`
+}

internal/consensus/types/height_vote_set_test.go

@@ -0,0 +1,87 @@
+package types
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+var config *cfg.Config // NOTE: must be reset for each _test.go file
+
+func TestMain(m *testing.M) {
+	config = cfg.ResetTestRoot("consensus_height_vote_set_test")
+	code := m.Run()
+	os.RemoveAll(config.RootDir)
+	os.Exit(code)
+}
+
+func TestPeerCatchupRounds(t *testing.T) {
+	valSet, privVals := types.RandValidatorSet(10, 1)
+
+	hvs := NewHeightVoteSet(config.ChainID(), 1, valSet)
+
+	vote999_0 := makeVoteHR(t, 1, 0, 999, privVals)
+	added, err := hvs.AddVote(vote999_0, "peer1")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from peer", added, err)
+	}
+
+	vote1000_0 := makeVoteHR(t, 1, 0, 1000, privVals)
+	added, err = hvs.AddVote(vote1000_0, "peer1")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from peer", added, err)
+	}
+
+	vote1001_0 := makeVoteHR(t, 1, 0, 1001, privVals)
+	added, err = hvs.AddVote(vote1001_0, "peer1")
+	if err != ErrGotVoteFromUnwantedRound {
+		t.Errorf("expected GotVoteFromUnwantedRoundError, but got %v", err)
+	}
+	if added {
+		t.Error("Expected to *not* add vote from peer, too many catchup rounds.")
+	}
+
+	added, err = hvs.AddVote(vote1001_0, "peer2")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from another peer")
+	}
+
+}
+
+func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []types.PrivValidator) *types.Vote {
+	privVal := privVals[valIndex]
+	pubKey, err := privVal.GetPubKey()
+	if err != nil {
+		panic(err)
+	}
+
+	randBytes := tmrand.Bytes(tmhash.Size)
+
+	vote := &types.Vote{
+		ValidatorAddress: pubKey.Address(),
+		ValidatorIndex:   valIndex,
+		Height:           height,
+		Round:            round,
+		Timestamp:        tmtime.Now(),
+		Type:             tmproto.PrecommitType,
+		BlockID:          types.BlockID{Hash: randBytes, PartSetHeader: types.PartSetHeader{}},
+	}
+	chainID := config.ChainID()
+
+	v := vote.ToProto()
+	err = privVal.SignVote(chainID, v)
+	if err != nil {
+		panic(fmt.Sprintf("Error signing vote: %v", err))
+	}
+
+	vote.Signature = v.Signature
+
+	return vote
+}

internal/consensus/types/peer_round_state.go

@@ -0,0 +1,68 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bits"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+
+// PeerRoundState contains the known state of a peer.
+// NOTE: Read-only when returned by PeerState.GetRoundState().
+type PeerRoundState struct {
+	Height int64         `json:"height"` // Height peer is at
+	Round  int32         `json:"round"`  // Round peer is at, -1 if unknown.
+	Step   RoundStepType `json:"step"`   // Step peer is at
+
+	// Estimated start of round 0 at this height
+	StartTime time.Time `json:"start_time"`
+
+	// True if peer has proposal for this round
+	Proposal                   bool                `json:"proposal"`
+	ProposalBlockPartSetHeader types.PartSetHeader `json:"proposal_block_part_set_header"`
+	ProposalBlockParts         *bits.BitArray      `json:"proposal_block_parts"`
+	// Proposal's POL round. -1 if none.
+	ProposalPOLRound int32 `json:"proposal_pol_round"`
+
+	// nil until ProposalPOLMessage received.
+	ProposalPOL     *bits.BitArray `json:"proposal_pol"`
+	Prevotes        *bits.BitArray `json:"prevotes"`          // All votes peer has for this round
+	Precommits      *bits.BitArray `json:"precommits"`        // All precommits peer has for this round
+	LastCommitRound int32          `json:"last_commit_round"` // Round of commit for last height. -1 if none.
+	LastCommit      *bits.BitArray `json:"last_commit"`       // All commit precommits of commit for last height.
+
+	// Round that we have commit for. Not necessarily unique. -1 if none.
+	CatchupCommitRound int32 `json:"catchup_commit_round"`
+
+	// All commit precommits peer has for this height & CatchupCommitRound
+	CatchupCommit *bits.BitArray `json:"catchup_commit"`
+}
+
+// String returns a string representation of the PeerRoundState
+func (prs PeerRoundState) String() string {
+	return prs.StringIndented("")
+}
+
+// StringIndented returns a string representation of the PeerRoundState
+func (prs PeerRoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`PeerRoundState{
+%s  %v/%v/%v @%v
+%s  Proposal %v -> %v
+%s  POL      %v (round %v)
+%s  Prevotes   %v
+%s  Precommits %v
+%s  LastCommit %v (round %v)
+%s  Catchup    %v (round %v)
+%s}`,
+		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
+		indent, prs.ProposalBlockPartSetHeader, prs.ProposalBlockParts,
+		indent, prs.ProposalPOL, prs.ProposalPOLRound,
+		indent, prs.Prevotes,
+		indent, prs.Precommits,
+		indent, prs.LastCommit, prs.LastCommitRound,
+		indent, prs.CatchupCommit, prs.CatchupCommitRound,
+		indent)
+}

internal/consensus/types/round_state.go

@@ -0,0 +1,215 @@
+package types
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+// RoundStepType enum type
+
+// RoundStepType enumerates the state of the consensus state machine
+type RoundStepType uint8 // These must be numeric, ordered.
+
+// RoundStepType
+const (
+	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
+	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
+	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
+	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
+	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
+	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
+	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
+	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
+	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
+
+	// NOTE: Update IsValid method if you change this!
+)
+
+// IsValid returns true if the step is valid, false if unknown/undefined.
+func (rs RoundStepType) IsValid() bool {
+	return uint8(rs) >= 0x01 && uint8(rs) <= 0x08
+}
+
+// String returns a string
+func (rs RoundStepType) String() string {
+	switch rs {
+	case RoundStepNewHeight:
+		return "RoundStepNewHeight"
+	case RoundStepNewRound:
+		return "RoundStepNewRound"
+	case RoundStepPropose:
+		return "RoundStepPropose"
+	case RoundStepPrevote:
+		return "RoundStepPrevote"
+	case RoundStepPrevoteWait:
+		return "RoundStepPrevoteWait"
+	case RoundStepPrecommit:
+		return "RoundStepPrecommit"
+	case RoundStepPrecommitWait:
+		return "RoundStepPrecommitWait"
+	case RoundStepCommit:
+		return "RoundStepCommit"
+	default:
+		return "RoundStepUnknown" // Cannot panic.
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+// RoundState defines the internal consensus state.
+// NOTE: Not thread safe. Should only be manipulated by functions downstream
+// of the cs.receiveRoutine
+type RoundState struct {
+	Height    int64         `json:"height"` // Height we are working on
+	Round     int32         `json:"round"`
+	Step      RoundStepType `json:"step"`
+	StartTime time.Time     `json:"start_time"`
+
+	// Subjective time when +2/3 precommits for Block at Round were found
+	CommitTime         time.Time           `json:"commit_time"`
+	Validators         *types.ValidatorSet `json:"validators"`
+	Proposal           *types.Proposal     `json:"proposal"`
+	ProposalBlock      *types.Block        `json:"proposal_block"`
+	ProposalBlockParts *types.PartSet      `json:"proposal_block_parts"`
+	LockedRound        int32               `json:"locked_round"`
+	LockedBlock        *types.Block        `json:"locked_block"`
+	LockedBlockParts   *types.PartSet      `json:"locked_block_parts"`
+
+	// Last known round with POL for non-nil valid block.
+	ValidRound int32        `json:"valid_round"`
+	ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above.
+
+	// Last known block parts of POL mentioned above.
+	ValidBlockParts           *types.PartSet      `json:"valid_block_parts"`
+	Votes                     *HeightVoteSet      `json:"votes"`
+	CommitRound               int32               `json:"commit_round"` //
+	LastCommit                *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
+	LastValidators            *types.ValidatorSet `json:"last_validators"`
+	TriggeredTimeoutPrecommit bool                `json:"triggered_timeout_precommit"`
+}
+
+// Compressed version of the RoundState for use in RPC
+type RoundStateSimple struct {
+	HeightRoundStep   string              `json:"height/round/step"`
+	StartTime         time.Time           `json:"start_time"`
+	ProposalBlockHash bytes.HexBytes      `json:"proposal_block_hash"`
+	LockedBlockHash   bytes.HexBytes      `json:"locked_block_hash"`
+	ValidBlockHash    bytes.HexBytes      `json:"valid_block_hash"`
+	Votes             json.RawMessage     `json:"height_vote_set"`
+	Proposer          types.ValidatorInfo `json:"proposer"`
+}
+
+// Compress the RoundState to RoundStateSimple
+func (rs *RoundState) RoundStateSimple() RoundStateSimple {
+	votesJSON, err := rs.Votes.MarshalJSON()
+	if err != nil {
+		panic(err)
+	}
+
+	addr := rs.Validators.GetProposer().Address
+	idx, _ := rs.Validators.GetByAddress(addr)
+
+	return RoundStateSimple{
+		HeightRoundStep:   fmt.Sprintf("%d/%d/%d", rs.Height, rs.Round, rs.Step),
+		StartTime:         rs.StartTime,
+		ProposalBlockHash: rs.ProposalBlock.Hash(),
+		LockedBlockHash:   rs.LockedBlock.Hash(),
+		ValidBlockHash:    rs.ValidBlock.Hash(),
+		Votes:             votesJSON,
+		Proposer: types.ValidatorInfo{
+			Address: addr,
+			Index:   idx,
+		},
+	}
+}
+
+// NewRoundEvent returns the RoundState with proposer information as an event.
+func (rs *RoundState) NewRoundEvent() types.EventDataNewRound {
+	addr := rs.Validators.GetProposer().Address
+	idx, _ := rs.Validators.GetByAddress(addr)
+
+	return types.EventDataNewRound{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
+		Proposer: types.ValidatorInfo{
+			Address: addr,
+			Index:   idx,
+		},
+	}
+}
+
+// CompleteProposalEvent returns information about a proposed block as an event.
+func (rs *RoundState) CompleteProposalEvent() types.EventDataCompleteProposal {
+	// We must construct BlockID from ProposalBlock and ProposalBlockParts
+	// cs.Proposal is not guaranteed to be set when this function is called
+	blockID := types.BlockID{
+		Hash:          rs.ProposalBlock.Hash(),
+		PartSetHeader: rs.ProposalBlockParts.Header(),
+	}
+
+	return types.EventDataCompleteProposal{
+		Height:  rs.Height,
+		Round:   rs.Round,
+		Step:    rs.Step.String(),
+		BlockID: blockID,
+	}
+}
+
+// RoundStateEvent returns the H/R/S of the RoundState as an event.
+func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
+	return types.EventDataRoundState{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
+	}
+}
+
+// String returns a string
+func (rs *RoundState) String() string {
+	return rs.StringIndented("")
+}
+
+// StringIndented returns a string
+func (rs *RoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`RoundState{
+%s  H:%v R:%v S:%v
+%s  StartTime:     %v
+%s  CommitTime:    %v
+%s  Validators:    %v
+%s  Proposal:      %v
+%s  ProposalBlock: %v %v
+%s  LockedRound:   %v
+%s  LockedBlock:   %v %v
+%s  ValidRound:   %v
+%s  ValidBlock:   %v %v
+%s  Votes:         %v
+%s  LastCommit:    %v
+%s  LastValidators:%v
+%s}`,
+		indent, rs.Height, rs.Round, rs.Step,
+		indent, rs.StartTime,
+		indent, rs.CommitTime,
+		indent, rs.Validators.StringIndented(indent+"  "),
+		indent, rs.Proposal,
+		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
+		indent, rs.LockedRound,
+		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
+		indent, rs.ValidRound,
+		indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(),
+		indent, rs.Votes.StringIndented(indent+"  "),
+		indent, rs.LastCommit.StringShort(),
+		indent, rs.LastValidators.StringIndented(indent+"  "),
+		indent)
+}
+
+// StringShort returns a string
+func (rs *RoundState) StringShort() string {
+	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
+		rs.Height, rs.Round, rs.Step, rs.StartTime)
+}

internal/consensus/wal.go

@@ -0,0 +1,433 @@
+package consensus
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+
+	auto "github.com/tendermint/tendermint/libs/autofile"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/libs/service"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	// time.Time + max consensus msg size
+	maxMsgSizeBytes = maxMsgSize + 24
+
+	// how often the WAL should be sync'd during period sync'ing
+	walDefaultFlushInterval = 2 * time.Second
+)
+
+//--------------------------------------------------------
+// types and functions for savings consensus messages
+
+// TimedWALMessage wraps WALMessage and adds Time for debugging purposes.
+type TimedWALMessage struct {
+	Time time.Time  `json:"time"`
+	Msg  WALMessage `json:"msg"`
+}
+
+// EndHeightMessage marks the end of the given height inside WAL.
+// @internal used by scripts/wal2json util.
+type EndHeightMessage struct {
+	Height int64 `json:"height"`
+}
+
+type WALMessage interface{}
+
+func init() {
+	tmjson.RegisterType(msgInfo{}, "tendermint/wal/MsgInfo")
+	tmjson.RegisterType(timeoutInfo{}, "tendermint/wal/TimeoutInfo")
+	tmjson.RegisterType(EndHeightMessage{}, "tendermint/wal/EndHeightMessage")
+}
+
+//--------------------------------------------------------
+// Simple write-ahead logger
+
+// WAL is an interface for any write-ahead logger.
+type WAL interface {
+	Write(WALMessage) error
+	WriteSync(WALMessage) error
+	FlushAndSync() error
+
+	SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error)
+
+	// service methods
+	Start() error
+	Stop() error
+	Wait()
+}
+
+// Write ahead logger writes msgs to disk before they are processed.
+// Can be used for crash-recovery and deterministic replay.
+// TODO: currently the wal is overwritten during replay catchup, give it a mode
+// so it's either reading or appending - must read to end to start appending
+// again.
+type BaseWAL struct {
+	service.BaseService
+
+	group *auto.Group
+
+	enc *WALEncoder
+
+	flushTicker   *time.Ticker
+	flushInterval time.Duration
+}
+
+var _ WAL = &BaseWAL{}
+
+// NewWAL returns a new write-ahead logger based on `baseWAL`, which implements
+// WAL. It's flushed and synced to disk every 2s and once when stopped.
+func NewWAL(walFile string, groupOptions ...func(*auto.Group)) (*BaseWAL, error) {
+	err := tmos.EnsureDir(filepath.Dir(walFile), 0700)
+	if err != nil {
+		return nil, fmt.Errorf("failed to ensure WAL directory is in place: %w", err)
+	}
+
+	group, err := auto.OpenGroup(walFile, groupOptions...)
+	if err != nil {
+		return nil, err
+	}
+	wal := &BaseWAL{
+		group:         group,
+		enc:           NewWALEncoder(group),
+		flushInterval: walDefaultFlushInterval,
+	}
+	wal.BaseService = *service.NewBaseService(nil, "baseWAL", wal)
+	return wal, nil
+}
+
+// SetFlushInterval allows us to override the periodic flush interval for the WAL.
+func (wal *BaseWAL) SetFlushInterval(i time.Duration) {
+	wal.flushInterval = i
+}
+
+func (wal *BaseWAL) Group() *auto.Group {
+	return wal.group
+}
+
+func (wal *BaseWAL) SetLogger(l log.Logger) {
+	wal.BaseService.Logger = l
+	wal.group.SetLogger(l)
+}
+
+func (wal *BaseWAL) OnStart() error {
+	size, err := wal.group.Head.Size()
+	if err != nil {
+		return err
+	} else if size == 0 {
+		if err := wal.WriteSync(EndHeightMessage{0}); err != nil {
+			return err
+		}
+	}
+	err = wal.group.Start()
+	if err != nil {
+		return err
+	}
+	wal.flushTicker = time.NewTicker(wal.flushInterval)
+	go wal.processFlushTicks()
+	return nil
+}
+
+func (wal *BaseWAL) processFlushTicks() {
+	for {
+		select {
+		case <-wal.flushTicker.C:
+			if err := wal.FlushAndSync(); err != nil {
+				wal.Logger.Error("Periodic WAL flush failed", "err", err)
+			}
+		case <-wal.Quit():
+			return
+		}
+	}
+}
+
+// FlushAndSync flushes and fsync's the underlying group's data to disk.
+// See auto#FlushAndSync
+func (wal *BaseWAL) FlushAndSync() error {
+	return wal.group.FlushAndSync()
+}
+
+// Stop the underlying autofile group.
+// Use Wait() to ensure it's finished shutting down
+// before cleaning up files.
+func (wal *BaseWAL) OnStop() {
+	wal.flushTicker.Stop()
+	if err := wal.FlushAndSync(); err != nil {
+		wal.Logger.Error("error on flush data to disk", "error", err)
+	}
+	if err := wal.group.Stop(); err != nil {
+		wal.Logger.Error("error trying to stop wal", "error", err)
+	}
+	wal.group.Close()
+}
+
+// Wait for the underlying autofile group to finish shutting down
+// so it's safe to cleanup files.
+func (wal *BaseWAL) Wait() {
+	wal.group.Wait()
+}
+
+// Write is called in newStep and for each receive on the
+// peerMsgQueue and the timeoutTicker.
+// NOTE: does not call fsync()
+func (wal *BaseWAL) Write(msg WALMessage) error {
+	if wal == nil {
+		return nil
+	}
+
+	if err := wal.enc.Encode(&TimedWALMessage{tmtime.Now(), msg}); err != nil {
+		wal.Logger.Error("Error writing msg to consensus wal. WARNING: recover may not be possible for the current height",
+			"err", err, "msg", msg)
+		return err
+	}
+
+	return nil
+}
+
+// WriteSync is called when we receive a msg from ourselves
+// so that we write to disk before sending signed messages.
+// NOTE: calls fsync()
+func (wal *BaseWAL) WriteSync(msg WALMessage) error {
+	if wal == nil {
+		return nil
+	}
+
+	if err := wal.Write(msg); err != nil {
+		return err
+	}
+
+	if err := wal.FlushAndSync(); err != nil {
+		wal.Logger.Error(`WriteSync failed to flush consensus wal.
+		WARNING: may result in creating alternative proposals / votes for the current height iff the node restarted`,
+			"err", err)
+		return err
+	}
+
+	return nil
+}
+
+// WALSearchOptions are optional arguments to SearchForEndHeight.
+type WALSearchOptions struct {
+	// IgnoreDataCorruptionErrors set to true will result in skipping data corruption errors.
+	IgnoreDataCorruptionErrors bool
+}
+
+// SearchForEndHeight searches for the EndHeightMessage with the given height
+// and returns an auto.GroupReader, whenever it was found or not and an error.
+// Group reader will be nil if found equals false.
+//
+// CONTRACT: caller must close group reader.
+func (wal *BaseWAL) SearchForEndHeight(
+	height int64,
+	options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	var (
+		msg *TimedWALMessage
+		gr  *auto.GroupReader
+	)
+	lastHeightFound := int64(-1)
+
+	// NOTE: starting from the last file in the group because we're usually
+	// searching for the last height. See replay.go
+	min, max := wal.group.MinIndex(), wal.group.MaxIndex()
+	wal.Logger.Info("Searching for height", "height", height, "min", min, "max", max)
+	for index := max; index >= min; index-- {
+		gr, err = wal.group.NewReader(index)
+		if err != nil {
+			return nil, false, err
+		}
+
+		dec := NewWALDecoder(gr)
+		for {
+			msg, err = dec.Decode()
+			if err == io.EOF {
+				// OPTIMISATION: no need to look for height in older files if we've seen h < height
+				if lastHeightFound > 0 && lastHeightFound < height {
+					gr.Close()
+					return nil, false, nil
+				}
+				// check next file
+				break
+			}
+			if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) {
+				wal.Logger.Error("Corrupted entry. Skipping...", "err", err)
+				// do nothing
+				continue
+			} else if err != nil {
+				gr.Close()
+				return nil, false, err
+			}
+
+			if m, ok := msg.Msg.(EndHeightMessage); ok {
+				lastHeightFound = m.Height
+				if m.Height == height { // found
+					wal.Logger.Info("Found", "height", height, "index", index)
+					return gr, true, nil
+				}
+			}
+		}
+		gr.Close()
+	}
+
+	return nil, false, nil
+}
+
+// A WALEncoder writes custom-encoded WAL messages to an output stream.
+//
+// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value
+type WALEncoder struct {
+	wr io.Writer
+}
+
+// NewWALEncoder returns a new encoder that writes to wr.
+func NewWALEncoder(wr io.Writer) *WALEncoder {
+	return &WALEncoder{wr}
+}
+
+// Encode writes the custom encoding of v to the stream. It returns an error if
+// the encoded size of v is greater than 1MB. Any error encountered
+// during the write is also returned.
+func (enc *WALEncoder) Encode(v *TimedWALMessage) error {
+	pbMsg, err := WALToProto(v.Msg)
+	if err != nil {
+		return err
+	}
+	pv := tmcons.TimedWALMessage{
+		Time: v.Time,
+		Msg:  pbMsg,
+	}
+
+	data, err := proto.Marshal(&pv)
+	if err != nil {
+		panic(fmt.Errorf("encode timed wall message failure: %w", err))
+	}
+
+	crc := crc32.Checksum(data, crc32c)
+	length := uint32(len(data))
+	if length > maxMsgSizeBytes {
+		return fmt.Errorf("msg is too big: %d bytes, max: %d bytes", length, maxMsgSizeBytes)
+	}
+	totalLength := 8 + int(length)
+
+	msg := make([]byte, totalLength)
+	binary.BigEndian.PutUint32(msg[0:4], crc)
+	binary.BigEndian.PutUint32(msg[4:8], length)
+	copy(msg[8:], data)
+
+	_, err = enc.wr.Write(msg)
+	return err
+}
+
+// IsDataCorruptionError returns true if data has been corrupted inside WAL.
+func IsDataCorruptionError(err error) bool {
+	_, ok := err.(DataCorruptionError)
+	return ok
+}
+
+// DataCorruptionError is an error that occures if data on disk was corrupted.
+type DataCorruptionError struct {
+	cause error
+}
+
+func (e DataCorruptionError) Error() string {
+	return fmt.Sprintf("DataCorruptionError[%v]", e.cause)
+}
+
+func (e DataCorruptionError) Cause() error {
+	return e.cause
+}
+
+// A WALDecoder reads and decodes custom-encoded WAL messages from an input
+// stream. See WALEncoder for the format used.
+//
+// It will also compare the checksums and make sure data size is equal to the
+// length from the header. If that is not the case, error will be returned.
+type WALDecoder struct {
+	rd io.Reader
+}
+
+// NewWALDecoder returns a new decoder that reads from rd.
+func NewWALDecoder(rd io.Reader) *WALDecoder {
+	return &WALDecoder{rd}
+}
+
+// Decode reads the next custom-encoded value from its reader and returns it.
+func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
+	b := make([]byte, 4)
+
+	_, err := dec.rd.Read(b)
+	if errors.Is(err, io.EOF) {
+		return nil, err
+	}
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read checksum: %v", err)}
+	}
+	crc := binary.BigEndian.Uint32(b)
+
+	b = make([]byte, 4)
+	_, err = dec.rd.Read(b)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read length: %v", err)}
+	}
+	length := binary.BigEndian.Uint32(b)
+
+	if length > maxMsgSizeBytes {
+		return nil, DataCorruptionError{fmt.Errorf(
+			"length %d exceeded maximum possible value of %d bytes",
+			length,
+			maxMsgSizeBytes)}
+	}
+
+	data := make([]byte, length)
+	n, err := dec.rd.Read(data)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read data: %v (read: %d, wanted: %d)", err, n, length)}
+	}
+
+	// check checksum before decoding data
+	actualCRC := crc32.Checksum(data, crc32c)
+	if actualCRC != crc {
+		return nil, DataCorruptionError{fmt.Errorf("checksums do not match: read: %v, actual: %v", crc, actualCRC)}
+	}
+
+	var res = new(tmcons.TimedWALMessage)
+	err = proto.Unmarshal(data, res)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to decode data: %v", err)}
+	}
+
+	walMsg, err := WALFromProto(res.Msg)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to convert from proto: %w", err)}
+	}
+	tMsgWal := &TimedWALMessage{
+		Time: res.Time,
+		Msg:  walMsg,
+	}
+
+	return tMsgWal, err
+}
+
+type nilWAL struct{}
+
+var _ WAL = nilWAL{}
+
+func (nilWAL) Write(m WALMessage) error     { return nil }
+func (nilWAL) WriteSync(m WALMessage) error { return nil }
+func (nilWAL) FlushAndSync() error          { return nil }
+func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	return nil, false, nil
+}
+func (nilWAL) Start() error { return nil }
+func (nilWAL) Stop() error  { return nil }
+func (nilWAL) Wait()        {}

internal/consensus/wal_fuzz.go

@@ -0,0 +1,32 @@
+//go:build gofuzz
+// +build gofuzz
+
+package consensus
+
+import (
+	"bytes"
+	"io"
+)
+
+func Fuzz(data []byte) int {
+	dec := NewWALDecoder(bytes.NewReader(data))
+	for {
+		msg, err := dec.Decode()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			if msg != nil {
+				panic("msg != nil on error")
+			}
+			return 0
+		}
+		var w bytes.Buffer
+		enc := NewWALEncoder(&w)
+		err = enc.Encode(msg)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return 1
+}

internal/consensus/wal_generator.go

@@ -0,0 +1,226 @@
+package consensus
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"path/filepath"
+	"testing"
+	"time"
+
+	db "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/privval"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+// WALGenerateNBlocks generates a consensus WAL. It does this by spinning up a
+// stripped down version of node (proxy app, event bus, consensus state) with a
+// persistent kvstore application and special consensus wal instance
+// (byteBufferWAL) and waits until numBlocks are created.
+// If the node fails to produce given numBlocks, it returns an error.
+func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
+	config := getConfig(t)
+
+	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), "wal_generator"))
+
+	logger := log.TestingLogger().With("wal_generator", "wal_generator")
+	logger.Info("generating WAL (last height msg excluded)", "numBlocks", numBlocks)
+
+	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
+	// NOTE: we can't import node package because of circular dependency.
+	// NOTE: we don't do handshake so need to set state.Version.Consensus.App directly.
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
+	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		return fmt.Errorf("failed to read genesis file: %w", err)
+	}
+	blockStoreDB := db.NewMemDB()
+	stateDB := blockStoreDB
+	stateStore := sm.NewStore(stateDB)
+	state, err := sm.MakeGenesisState(genDoc)
+	if err != nil {
+		return fmt.Errorf("failed to make genesis state: %w", err)
+	}
+	state.Version.Consensus.App = kvstore.ProtocolVersion
+	if err = stateStore.Save(state); err != nil {
+		t.Error(err)
+	}
+
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app))
+	proxyApp.SetLogger(logger.With("module", "proxy"))
+	if err := proxyApp.Start(); err != nil {
+		return fmt.Errorf("failed to start proxy app connections: %w", err)
+	}
+	t.Cleanup(func() {
+		if err := proxyApp.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(logger.With("module", "events"))
+	if err := eventBus.Start(); err != nil {
+		return fmt.Errorf("failed to start event bus: %w", err)
+	}
+	t.Cleanup(func() {
+		if err := eventBus.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+	mempool := emptyMempool{}
+	evpool := sm.EmptyEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+	consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
+	consensusState.SetLogger(logger)
+	consensusState.SetEventBus(eventBus)
+	if privValidator != nil {
+		consensusState.SetPrivValidator(privValidator)
+	}
+	// END OF COPY PASTE
+
+	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
+	numBlocksWritten := make(chan struct{})
+	wal := newByteBufferWAL(logger, NewWALEncoder(wr), int64(numBlocks), numBlocksWritten)
+	// see wal.go#103
+	if err := wal.Write(EndHeightMessage{0}); err != nil {
+		t.Error(err)
+	}
+
+	consensusState.wal = wal
+
+	if err := consensusState.Start(); err != nil {
+		return fmt.Errorf("failed to start consensus state: %w", err)
+	}
+
+	select {
+	case <-numBlocksWritten:
+		if err := consensusState.Stop(); err != nil {
+			t.Error(err)
+		}
+		return nil
+	case <-time.After(1 * time.Minute):
+		if err := consensusState.Stop(); err != nil {
+			t.Error(err)
+		}
+		return fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks)
+	}
+}
+
+// WALWithNBlocks returns a WAL content with numBlocks.
+func WALWithNBlocks(t *testing.T, numBlocks int) (data []byte, err error) {
+	var b bytes.Buffer
+	wr := bufio.NewWriter(&b)
+
+	if err := WALGenerateNBlocks(t, wr, numBlocks); err != nil {
+		return []byte{}, err
+	}
+
+	wr.Flush()
+	return b.Bytes(), nil
+}
+
+func randPort() int {
+	// returns between base and base + spread
+	base, spread := 20000, 20000
+	return base + tmrand.Intn(spread)
+}
+
+func makeAddrs() (string, string, string) {
+	start := randPort()
+	return fmt.Sprintf("tcp://127.0.0.1:%d", start),
+		fmt.Sprintf("tcp://127.0.0.1:%d", start+1),
+		fmt.Sprintf("tcp://127.0.0.1:%d", start+2)
+}
+
+// getConfig returns a config for test cases
+func getConfig(t *testing.T) *cfg.Config {
+	c := cfg.ResetTestRoot(t.Name())
+
+	// and we use random ports to run in parallel
+	tm, rpc, grpc := makeAddrs()
+	c.P2P.ListenAddress = tm
+	c.RPC.ListenAddress = rpc
+	c.RPC.GRPCListenAddress = grpc
+	return c
+}
+
+// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
+// when the heightToStop is reached. Client will be notified via
+// signalWhenStopsTo channel.
+type byteBufferWAL struct {
+	enc               *WALEncoder
+	stopped           bool
+	heightToStop      int64
+	signalWhenStopsTo chan<- struct{}
+
+	logger log.Logger
+}
+
+// needed for determinism
+var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+
+func newByteBufferWAL(logger log.Logger, enc *WALEncoder, nBlocks int64, signalStop chan<- struct{}) *byteBufferWAL {
+	return &byteBufferWAL{
+		enc:               enc,
+		heightToStop:      nBlocks,
+		signalWhenStopsTo: signalStop,
+		logger:            logger,
+	}
+}
+
+// Save writes message to the internal buffer except when heightToStop is
+// reached, in which case it will signal the caller via signalWhenStopsTo and
+// skip writing.
+func (w *byteBufferWAL) Write(m WALMessage) error {
+	if w.stopped {
+		w.logger.Debug("WAL already stopped. Not writing message", "msg", m)
+		return nil
+	}
+
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		w.logger.Debug("WAL write end height message", "height", endMsg.Height, "stopHeight", w.heightToStop)
+		if endMsg.Height == w.heightToStop {
+			w.logger.Debug("Stopping WAL at height", "height", endMsg.Height)
+			w.signalWhenStopsTo <- struct{}{}
+			w.stopped = true
+			return nil
+		}
+	}
+
+	w.logger.Debug("WAL Write Message", "msg", m)
+	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
+	if err != nil {
+		panic(fmt.Sprintf("failed to encode the msg %v", m))
+	}
+
+	return nil
+}
+
+func (w *byteBufferWAL) WriteSync(m WALMessage) error {
+	return w.Write(m)
+}
+
+func (w *byteBufferWAL) FlushAndSync() error { return nil }
+
+func (w *byteBufferWAL) SearchForEndHeight(
+	height int64,
+	options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	return nil, false, nil
+}
+
+func (w *byteBufferWAL) Start() error { return nil }
+func (w *byteBufferWAL) Stop() error  { return nil }
+func (w *byteBufferWAL) Wait()        {}

internal/consensus/wal_test.go

@@ -0,0 +1,286 @@
+package consensus
+
+import (
+	"bytes"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	// "sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/consensus/types"
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/libs/autofile"
+	"github.com/tendermint/tendermint/libs/log"
+	tmtypes "github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	walTestFlushInterval = time.Duration(100) * time.Millisecond
+)
+
+func TestWALTruncate(t *testing.T) {
+	walDir, err := ioutil.TempDir("", "wal")
+	require.NoError(t, err)
+	defer os.RemoveAll(walDir)
+
+	walFile := filepath.Join(walDir, "wal")
+
+	// this magic number 4K can truncate the content when RotateFile.
+	// defaultHeadSizeLimit(10M) is hard to simulate.
+	// this magic number 1 * time.Millisecond make RotateFile check frequently.
+	// defaultGroupCheckDuration(5s) is hard to simulate.
+	wal, err := NewWAL(walFile,
+		autofile.GroupHeadSizeLimit(4096),
+		autofile.GroupCheckDuration(1*time.Millisecond),
+	)
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
+	err = wal.Start()
+	require.NoError(t, err)
+	defer func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	}()
+
+	// 60 block's size nearly 70K, greater than group's headBuf size(4096 * 10),
+	// when headBuf is full, truncate content will Flush to the file. at this
+	// time, RotateFile is called, truncate content exist in each file.
+	err = WALGenerateNBlocks(t, wal.Group(), 60)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Millisecond) // wait groupCheckDuration, make sure RotateFile run
+
+	if err := wal.FlushAndSync(); err != nil {
+		t.Error(err)
+	}
+
+	h := int64(50)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	defer gr.Close()
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, "wrong height")
+}
+
+func TestWALEncoderDecoder(t *testing.T) {
+	now := tmtime.Now()
+	msgs := []TimedWALMessage{
+		{Time: now, Msg: EndHeightMessage{0}},
+		{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
+		{Time: now, Msg: tmtypes.EventDataRoundState{Height: 1, Round: 1, Step: ""}},
+	}
+
+	b := new(bytes.Buffer)
+
+	for _, msg := range msgs {
+		msg := msg
+
+		b.Reset()
+
+		enc := NewWALEncoder(b)
+		err := enc.Encode(&msg)
+		require.NoError(t, err)
+
+		dec := NewWALDecoder(b)
+		decoded, err := dec.Decode()
+		require.NoError(t, err)
+		assert.Equal(t, msg.Time.UTC(), decoded.Time)
+		assert.Equal(t, msg.Msg, decoded.Msg)
+	}
+}
+
+func TestWALWrite(t *testing.T) {
+	walDir, err := ioutil.TempDir("", "wal")
+	require.NoError(t, err)
+	defer os.RemoveAll(walDir)
+	walFile := filepath.Join(walDir, "wal")
+
+	wal, err := NewWAL(walFile)
+	require.NoError(t, err)
+	err = wal.Start()
+	require.NoError(t, err)
+	defer func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	}()
+
+	// 1) Write returns an error if msg is too big
+	msg := &BlockPartMessage{
+		Height: 1,
+		Round:  1,
+		Part: &tmtypes.Part{
+			Index: 1,
+			Bytes: make([]byte, 1),
+			Proof: merkle.Proof{
+				Total:    1,
+				Index:    1,
+				LeafHash: make([]byte, maxMsgSizeBytes-30),
+			},
+		},
+	}
+
+	err = wal.Write(msgInfo{
+		Msg: msg,
+	})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "msg is too big")
+	}
+}
+
+func TestWALSearchForEndHeight(t *testing.T) {
+	walBody, err := WALWithNBlocks(t, 6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := tempWALWithData(walBody)
+
+	wal, err := NewWAL(walFile)
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
+
+	h := int64(3)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	defer gr.Close()
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, "wrong height")
+}
+
+func TestWALPeriodicSync(t *testing.T) {
+	walDir, err := ioutil.TempDir("", "wal")
+	require.NoError(t, err)
+	defer os.RemoveAll(walDir)
+
+	walFile := filepath.Join(walDir, "wal")
+	wal, err := NewWAL(walFile, autofile.GroupCheckDuration(1*time.Millisecond))
+	require.NoError(t, err)
+
+	wal.SetFlushInterval(walTestFlushInterval)
+	wal.SetLogger(log.TestingLogger())
+
+	// Generate some data
+	err = WALGenerateNBlocks(t, wal.Group(), 5)
+	require.NoError(t, err)
+
+	// We should have data in the buffer now
+	assert.NotZero(t, wal.Group().Buffered())
+
+	require.NoError(t, wal.Start())
+	defer func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		wal.Wait()
+	}()
+
+	time.Sleep(walTestFlushInterval + (10 * time.Millisecond))
+
+	// The data should have been flushed by the periodic sync
+	assert.Zero(t, wal.Group().Buffered())
+
+	h := int64(4)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	if gr != nil {
+		gr.Close()
+	}
+}
+
+/*
+var initOnce sync.Once
+
+func registerInterfacesOnce() {
+	initOnce.Do(func() {
+		var _ = wire.RegisterInterface(
+			struct{ WALMessage }{},
+			wire.ConcreteType{[]byte{}, 0x10},
+		)
+	})
+}
+*/
+
+func nBytes(n int) []byte {
+	buf := make([]byte, n)
+	n, _ = rand.Read(buf)
+	return buf[:n]
+}
+
+func benchmarkWalDecode(b *testing.B, n int) {
+	// registerInterfacesOnce()
+
+	buf := new(bytes.Buffer)
+	enc := NewWALEncoder(buf)
+
+	data := nBytes(n)
+	if err := enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second).UTC()}); err != nil {
+		b.Error(err)
+	}
+
+	encoded := buf.Bytes()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		buf.Write(encoded)
+		dec := NewWALDecoder(buf)
+		if _, err := dec.Decode(); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ReportAllocs()
+}
+
+func BenchmarkWalDecode512B(b *testing.B) {
+	benchmarkWalDecode(b, 512)
+}
+
+func BenchmarkWalDecode10KB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024)
+}
+func BenchmarkWalDecode100KB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024)
+}
+func BenchmarkWalDecode1MB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024)
+}
+func BenchmarkWalDecode10MB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024*1024)
+}
+func BenchmarkWalDecode100MB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024*1024)
+}
+func BenchmarkWalDecode1GB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024*1024)
+}

internal/rpc/client/event_test.go

@@ -0,0 +1,181 @@
+package client_test
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+var waitForEventTimeout = 8 * time.Second
+
+// MakeTxKV returns a text transaction, allong with expected key, value pair
+func MakeTxKV() ([]byte, []byte, []byte) {
+	k := []byte(tmrand.Str(8))
+	v := []byte(tmrand.Str(8))
+	return k, v, append(k, append([]byte("="), v...)...)
+}
+
+func TestHeaderEvents(t *testing.T) {
+	for i, c := range GetClients() {
+		i, c := i, c
+		t.Run(reflect.TypeOf(c).String(), func(t *testing.T) {
+			// start for this test it if it wasn't already running
+			if !c.IsRunning() {
+				// if so, then we start it, listen, and stop it.
+				err := c.Start()
+				require.Nil(t, err, "%d: %+v", i, err)
+				t.Cleanup(func() {
+					if err := c.Stop(); err != nil {
+						t.Error(err)
+					}
+				})
+			}
+
+			evtTyp := types.EventNewBlockHeader
+			evt, err := client.WaitForOneEvent(c, evtTyp, waitForEventTimeout)
+			require.Nil(t, err, "%d: %+v", i, err)
+			_, ok := evt.(types.EventDataNewBlockHeader)
+			require.True(t, ok, "%d: %#v", i, evt)
+			// TODO: more checks...
+		})
+	}
+}
+
+// subscribe to new blocks and make sure height increments by 1
+func TestBlockEvents(t *testing.T) {
+	for _, c := range GetClients() {
+		c := c
+		t.Run(reflect.TypeOf(c).String(), func(t *testing.T) {
+
+			// start for this test it if it wasn't already running
+			if !c.IsRunning() {
+				// if so, then we start it, listen, and stop it.
+				err := c.Start()
+				require.Nil(t, err)
+				t.Cleanup(func() {
+					if err := c.Stop(); err != nil {
+						t.Error(err)
+					}
+				})
+			}
+
+			const subscriber = "TestBlockEvents"
+
+			eventCh, err := c.Subscribe(context.Background(), subscriber, types.QueryForEvent(types.EventNewBlock).String())
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				if err := c.UnsubscribeAll(context.Background(), subscriber); err != nil {
+					t.Error(err)
+				}
+			})
+
+			var firstBlockHeight int64
+			for i := int64(0); i < 3; i++ {
+				event := <-eventCh
+				blockEvent, ok := event.Data.(types.EventDataNewBlock)
+				require.True(t, ok)
+
+				block := blockEvent.Block
+
+				if firstBlockHeight == 0 {
+					firstBlockHeight = block.Header.Height
+				}
+
+				require.Equal(t, firstBlockHeight+i, block.Header.Height)
+			}
+		})
+	}
+}
+
+func TestTxEventsSentWithBroadcastTxAsync(t *testing.T) { testTxEventsSent(t, "async") }
+func TestTxEventsSentWithBroadcastTxSync(t *testing.T)  { testTxEventsSent(t, "sync") }
+
+func testTxEventsSent(t *testing.T, broadcastMethod string) {
+	for _, c := range GetClients() {
+		c := c
+		t.Run(reflect.TypeOf(c).String(), func(t *testing.T) {
+
+			// start for this test it if it wasn't already running
+			if !c.IsRunning() {
+				// if so, then we start it, listen, and stop it.
+				err := c.Start()
+				require.Nil(t, err)
+				t.Cleanup(func() {
+					if err := c.Stop(); err != nil {
+						t.Error(err)
+					}
+				})
+			}
+
+			// make the tx
+			_, _, tx := MakeTxKV()
+
+			// send
+			go func() {
+				var (
+					txres *ctypes.ResultBroadcastTx
+					err   error
+					ctx   = context.Background()
+				)
+				switch broadcastMethod {
+				case "async":
+					txres, err = c.BroadcastTxAsync(ctx, tx)
+				case "sync":
+					txres, err = c.BroadcastTxSync(ctx, tx)
+				default:
+					panic(fmt.Sprintf("Unknown broadcastMethod %s", broadcastMethod))
+				}
+				if assert.NoError(t, err) {
+					assert.Equal(t, txres.Code, abci.CodeTypeOK)
+				}
+			}()
+
+			// and wait for confirmation
+			evt, err := client.WaitForOneEvent(c, types.EventTx, waitForEventTimeout)
+			require.Nil(t, err)
+
+			// and make sure it has the proper info
+			txe, ok := evt.(types.EventDataTx)
+			require.True(t, ok)
+
+			// make sure this is the proper tx
+			require.EqualValues(t, tx, txe.Tx)
+			require.True(t, txe.Result.IsOK())
+		})
+	}
+}
+
+// Test HTTPClient resubscribes upon disconnect && subscription error.
+// Test Local client resubscribes upon subscription error.
+func TestClientsResubscribe(t *testing.T) {
+	// TODO(melekes)
+}
+
+func TestHTTPReturnsErrorIfClientIsNotRunning(t *testing.T) {
+	c := getHTTPClient()
+
+	// on Subscribe
+	_, err := c.Subscribe(context.Background(), "TestHeaderEvents",
+		types.QueryForEvent(types.EventNewBlockHeader).String())
+	assert.Error(t, err)
+
+	// on Unsubscribe
+	err = c.Unsubscribe(context.Background(), "TestHeaderEvents",
+		types.QueryForEvent(types.EventNewBlockHeader).String())
+	assert.Error(t, err)
+
+	// on UnsubscribeAll
+	err = c.UnsubscribeAll(context.Background(), "TestHeaderEvents")
+	assert.Error(t, err)
+}

internal/rpc/client/evidence_test.go

@@ -0,0 +1,164 @@
+package client_test
+
+import (
+	"bytes"
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/ed25519"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	"github.com/tendermint/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/rpc/client"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
+)
+
+// For some reason the empty node used in tests has a time of
+// 2018-10-10 08:20:13.695936996 +0000 UTC
+// this is because the test genesis time is set here
+// so in order to validate evidence we need evidence to be the same time
+var defaultTestTime = time.Date(2018, 10, 10, 8, 20, 13, 695936996, time.UTC)
+
+func newEvidence(t *testing.T, val *privval.FilePV,
+	vote *types.Vote, vote2 *types.Vote,
+	chainID string) *types.DuplicateVoteEvidence {
+
+	var err error
+
+	v := vote.ToProto()
+	v2 := vote2.ToProto()
+
+	vote.Signature, err = val.Key.PrivKey.Sign(types.VoteSignBytes(chainID, v))
+	require.NoError(t, err)
+
+	vote2.Signature, err = val.Key.PrivKey.Sign(types.VoteSignBytes(chainID, v2))
+	require.NoError(t, err)
+
+	validator := types.NewValidator(val.Key.PubKey, 10)
+	valSet := types.NewValidatorSet([]*types.Validator{validator})
+
+	return types.NewDuplicateVoteEvidence(vote, vote2, defaultTestTime, valSet)
+}
+
+func makeEvidences(
+	t *testing.T,
+	val *privval.FilePV,
+	chainID string,
+) (correct *types.DuplicateVoteEvidence, fakes []*types.DuplicateVoteEvidence) {
+	vote := types.Vote{
+		ValidatorAddress: val.Key.Address,
+		ValidatorIndex:   0,
+		Height:           1,
+		Round:            0,
+		Type:             tmproto.PrevoteType,
+		Timestamp:        defaultTestTime,
+		BlockID: types.BlockID{
+			Hash: tmhash.Sum(tmrand.Bytes(tmhash.Size)),
+			PartSetHeader: types.PartSetHeader{
+				Total: 1000,
+				Hash:  tmhash.Sum([]byte("partset")),
+			},
+		},
+	}
+
+	vote2 := vote
+	vote2.BlockID.Hash = tmhash.Sum([]byte("blockhash2"))
+	correct = newEvidence(t, val, &vote, &vote2, chainID)
+
+	fakes = make([]*types.DuplicateVoteEvidence, 0)
+
+	// different address
+	{
+		v := vote2
+		v.ValidatorAddress = []byte("some_address")
+		fakes = append(fakes, newEvidence(t, val, &vote, &v, chainID))
+	}
+
+	// different height
+	{
+		v := vote2
+		v.Height = vote.Height + 1
+		fakes = append(fakes, newEvidence(t, val, &vote, &v, chainID))
+	}
+
+	// different round
+	{
+		v := vote2
+		v.Round = vote.Round + 1
+		fakes = append(fakes, newEvidence(t, val, &vote, &v, chainID))
+	}
+
+	// different type
+	{
+		v := vote2
+		v.Type = tmproto.PrecommitType
+		fakes = append(fakes, newEvidence(t, val, &vote, &v, chainID))
+	}
+
+	// exactly same vote
+	{
+		v := vote
+		fakes = append(fakes, newEvidence(t, val, &vote, &v, chainID))
+	}
+
+	return correct, fakes
+}
+
+func TestBroadcastEvidence_DuplicateVoteEvidence(t *testing.T) {
+	var (
+		config  = rpctest.GetConfig()
+		chainID = config.ChainID()
+		pv      = privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile())
+	)
+
+	for i, c := range GetClients() {
+		correct, fakes := makeEvidences(t, pv, chainID)
+		t.Logf("client %d", i)
+
+		result, err := c.BroadcastEvidence(context.Background(), correct)
+		require.NoError(t, err, "BroadcastEvidence(%s) failed", correct)
+		assert.Equal(t, correct.Hash(), result.Hash, "expected result hash to match evidence hash")
+
+		status, err := c.Status(context.Background())
+		require.NoError(t, err)
+		err = client.WaitForHeight(c, status.SyncInfo.LatestBlockHeight+2, nil)
+		require.NoError(t, err)
+
+		ed25519pub := pv.Key.PubKey.(ed25519.PubKey)
+		rawpub := ed25519pub.Bytes()
+		result2, err := c.ABCIQuery(context.Background(), "/val", rawpub)
+		require.NoError(t, err)
+		qres := result2.Response
+		require.True(t, qres.IsOK())
+
+		var v abci.ValidatorUpdate
+		err = abci.ReadMessage(bytes.NewReader(qres.Value), &v)
+		require.NoError(t, err, "Error reading query result, value %v", qres.Value)
+
+		pk, err := cryptoenc.PubKeyFromProto(v.PubKey)
+		require.NoError(t, err)
+
+		require.EqualValues(t, rawpub, pk, "Stored PubKey not equal with expected, value %v", string(qres.Value))
+		require.Equal(t, int64(9), v.Power, "Stored Power not equal with expected, value %v", string(qres.Value))
+
+		for _, fake := range fakes {
+			_, err := c.BroadcastEvidence(context.Background(), fake)
+			require.Error(t, err, "BroadcastEvidence(%s) succeeded, but the evidence was fake", fake)
+		}
+	}
+}
+
+func TestBroadcastEmptyEvidence(t *testing.T) {
+	for _, c := range GetClients() {
+		_, err := c.BroadcastEvidence(context.Background(), nil)
+		assert.Error(t, err)
+	}
+}

internal/rpc/client/examples_test.go

@@ -0,0 +1,137 @@
+package client_test
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+func ExampleHTTP_simple() {
+	// Start a tendermint node (and kvstore) in the background to test against
+	app := kvstore.NewApplication()
+	node := rpctest.StartTendermint(app, rpctest.SuppressStdout, rpctest.RecreateConfig)
+	defer rpctest.StopTendermint(node)
+
+	// Create our RPC client
+	rpcAddr := rpctest.GetConfig().RPC.ListenAddress
+	c, err := rpchttp.New(rpcAddr, "/websocket")
+	if err != nil {
+		log.Fatal(err) //nolint:gocritic
+	}
+
+	// Create a transaction
+	k := []byte("name")
+	v := []byte("satoshi")
+	tx := append(k, append([]byte("="), v...)...)
+
+	// Broadcast the transaction and wait for it to commit (rather use
+	// c.BroadcastTxSync though in production).
+	bres, err := c.BroadcastTxCommit(context.Background(), tx)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if bres.CheckTx.IsErr() || bres.DeliverTx.IsErr() {
+		log.Fatal("BroadcastTxCommit transaction failed")
+	}
+
+	// Now try to fetch the value for the key
+	qres, err := c.ABCIQuery(context.Background(), "/key", k)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if qres.Response.IsErr() {
+		log.Fatal("ABCIQuery failed")
+	}
+	if !bytes.Equal(qres.Response.Key, k) {
+		log.Fatal("returned key does not match queried key")
+	}
+	if !bytes.Equal(qres.Response.Value, v) {
+		log.Fatal("returned value does not match sent value")
+	}
+
+	fmt.Println("Sent tx     :", string(tx))
+	fmt.Println("Queried for :", string(qres.Response.Key))
+	fmt.Println("Got value   :", string(qres.Response.Value))
+
+	// Output:
+	// Sent tx     : name=satoshi
+	// Queried for : name
+	// Got value   : satoshi
+}
+
+func ExampleHTTP_batching() {
+	// Start a tendermint node (and kvstore) in the background to test against
+	app := kvstore.NewApplication()
+	node := rpctest.StartTendermint(app, rpctest.SuppressStdout, rpctest.RecreateConfig)
+
+	// Create our RPC client
+	rpcAddr := rpctest.GetConfig().RPC.ListenAddress
+	c, err := rpchttp.New(rpcAddr, "/websocket")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer rpctest.StopTendermint(node)
+
+	// Create our two transactions
+	k1 := []byte("firstName")
+	v1 := []byte("satoshi")
+	tx1 := append(k1, append([]byte("="), v1...)...)
+
+	k2 := []byte("lastName")
+	v2 := []byte("nakamoto")
+	tx2 := append(k2, append([]byte("="), v2...)...)
+
+	txs := [][]byte{tx1, tx2}
+
+	// Create a new batch
+	batch := c.NewBatch()
+
+	// Queue up our transactions
+	for _, tx := range txs {
+		// Broadcast the transaction and wait for it to commit (rather use
+		// c.BroadcastTxSync though in production).
+		if _, err := batch.BroadcastTxCommit(context.Background(), tx); err != nil {
+			log.Fatal(err) //nolint:gocritic
+		}
+	}
+
+	// Send the batch of 2 transactions
+	if _, err := batch.Send(context.Background()); err != nil {
+		log.Fatal(err)
+	}
+
+	// Now let's query for the original results as a batch
+	keys := [][]byte{k1, k2}
+	for _, key := range keys {
+		if _, err := batch.ABCIQuery(context.Background(), "/key", key); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	// Send the 2 queries and keep the results
+	results, err := batch.Send(context.Background())
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Each result in the returned list is the deserialized result of each
+	// respective ABCIQuery response
+	for _, result := range results {
+		qr, ok := result.(*ctypes.ResultABCIQuery)
+		if !ok {
+			log.Fatal("invalid result type from ABCIQuery request")
+		}
+		fmt.Println(string(qr.Response.Key), "=", string(qr.Response.Value))
+	}
+
+	// Output:
+	// firstName = satoshi
+	// lastName = nakamoto
+}

internal/rpc/client/helpers.go

@@ -0,0 +1,83 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/types"
+)
+
+// Waiter is informed of current height, decided whether to quit early
+type Waiter func(delta int64) (abort error)
+
+// DefaultWaitStrategy is the standard backoff algorithm,
+// but you can plug in another one
+func DefaultWaitStrategy(delta int64) (abort error) {
+	if delta > 10 {
+		return fmt.Errorf("waiting for %d blocks... aborting", delta)
+	} else if delta > 0 {
+		// estimate of wait time....
+		// wait half a second for the next block (in progress)
+		// plus one second for every full block
+		delay := time.Duration(delta-1)*time.Second + 500*time.Millisecond
+		time.Sleep(delay)
+	}
+	return nil
+}
+
+// Wait for height will poll status at reasonable intervals until
+// the block at the given height is available.
+//
+// If waiter is nil, we use DefaultWaitStrategy, but you can also
+// provide your own implementation
+func WaitForHeight(c StatusClient, h int64, waiter Waiter) error {
+	if waiter == nil {
+		waiter = DefaultWaitStrategy
+	}
+	delta := int64(1)
+	for delta > 0 {
+		s, err := c.Status(context.Background())
+		if err != nil {
+			return err
+		}
+		delta = h - s.SyncInfo.LatestBlockHeight
+		// wait for the time, or abort early
+		if err := waiter(delta); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// WaitForOneEvent subscribes to a websocket event for the given
+// event time and returns upon receiving it one time, or
+// when the timeout duration has expired.
+//
+// This handles subscribing and unsubscribing under the hood
+func WaitForOneEvent(c EventsClient, evtTyp string, timeout time.Duration) (types.TMEventData, error) {
+	const subscriber = "helpers"
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	// register for the next event of this type
+	eventCh, err := c.Subscribe(ctx, subscriber, types.QueryForEvent(evtTyp).String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to subscribe: %w", err)
+	}
+	// make sure to unregister after the test is over
+	defer func() {
+		if deferErr := c.UnsubscribeAll(ctx, subscriber); deferErr != nil {
+			panic(err)
+		}
+	}()
+
+	select {
+	case event := <-eventCh:
+		return event.Data, nil
+	case <-ctx.Done():
+		return nil, errors.New("timed out waiting for event")
+	}
+}

internal/rpc/client/helpers_test.go

@@ -0,0 +1,77 @@
+package client_test
+
+import (
+	"errors"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/client/mock"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+func TestWaitForHeight(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// test with error result - immediate failure
+	m := &mock.StatusMock{
+		Call: mock.Call{
+			Error: errors.New("bye"),
+		},
+	}
+	r := mock.NewStatusRecorder(m)
+
+	// connection failure always leads to error
+	err := client.WaitForHeight(r, 8, nil)
+	require.NotNil(err)
+	require.Equal("bye", err.Error())
+	// we called status once to check
+	require.Equal(1, len(r.Calls))
+
+	// now set current block height to 10
+	m.Call = mock.Call{
+		Response: &ctypes.ResultStatus{SyncInfo: ctypes.SyncInfo{LatestBlockHeight: 10}},
+	}
+
+	// we will not wait for more than 10 blocks
+	err = client.WaitForHeight(r, 40, nil)
+	require.NotNil(err)
+	require.True(strings.Contains(err.Error(), "aborting"))
+	// we called status once more to check
+	require.Equal(2, len(r.Calls))
+
+	// waiting for the past returns immediately
+	err = client.WaitForHeight(r, 5, nil)
+	require.Nil(err)
+	// we called status once more to check
+	require.Equal(3, len(r.Calls))
+
+	// since we can't update in a background goroutine (test --race)
+	// we use the callback to update the status height
+	myWaiter := func(delta int64) error {
+		// update the height for the next call
+		m.Call.Response = &ctypes.ResultStatus{SyncInfo: ctypes.SyncInfo{LatestBlockHeight: 15}}
+		return client.DefaultWaitStrategy(delta)
+	}
+
+	// we wait for a few blocks
+	err = client.WaitForHeight(r, 12, myWaiter)
+	require.Nil(err)
+	// we called status once to check
+	require.Equal(5, len(r.Calls))
+
+	pre := r.Calls[3]
+	require.Nil(pre.Error)
+	prer, ok := pre.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.Equal(int64(10), prer.SyncInfo.LatestBlockHeight)
+
+	post := r.Calls[4]
+	require.Nil(post.Error)
+	postr, ok := post.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.Equal(int64(15), postr.SyncInfo.LatestBlockHeight)
+}

internal/rpc/client/http/http.go

@@ -0,0 +1,764 @@
+package http
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/libs/service"
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	jsonrpcclient "github.com/tendermint/tendermint/rpc/jsonrpc/client"
+	"github.com/tendermint/tendermint/types"
+)
+
+/*
+HTTP is a Client implementation that communicates with a Tendermint node over
+JSON RPC and WebSockets.
+
+This is the main implementation you probably want to use in production code.
+There are other implementations when calling the Tendermint node in-process
+(Local), or when you want to mock out the server for test code (mock).
+
+You can subscribe for any event published by Tendermint using Subscribe method.
+Note delivery is best-effort. If you don't read events fast enough or network is
+slow, Tendermint might cancel the subscription. The client will attempt to
+resubscribe (you don't need to do anything). It will keep trying every second
+indefinitely until successful.
+
+Request batching is available for JSON RPC requests over HTTP, which conforms to
+the JSON RPC specification (https://www.jsonrpc.org/specification#batch). See
+the example for more details.
+
+Example:
+
+		c, err := New("http://192.168.1.10:26657", "/websocket")
+		if err != nil {
+			// handle error
+		}
+
+		// call Start/Stop if you're subscribing to events
+		err = c.Start()
+		if err != nil {
+			// handle error
+		}
+		defer c.Stop()
+
+		res, err := c.Status()
+		if err != nil {
+			// handle error
+		}
+
+		// handle result
+*/
+type HTTP struct {
+	remote string
+	rpc    *jsonrpcclient.Client
+
+	*baseRPCClient
+	*WSEvents
+}
+
+// BatchHTTP provides the same interface as `HTTP`, but allows for batching of
+// requests (as per https://www.jsonrpc.org/specification#batch). Do not
+// instantiate directly - rather use the HTTP.NewBatch() method to create an
+// instance of this struct.
+//
+// Batching of HTTP requests is thread-safe in the sense that multiple
+// goroutines can each create their own batches and send them using the same
+// HTTP client. Multiple goroutines could also enqueue transactions in a single
+// batch, but ordering of transactions in the batch cannot be guaranteed in such
+// an example.
+type BatchHTTP struct {
+	rpcBatch *jsonrpcclient.RequestBatch
+	*baseRPCClient
+}
+
+// rpcClient is an internal interface to which our RPC clients (batch and
+// non-batch) must conform. Acts as an additional code-level sanity check to
+// make sure the implementations stay coherent.
+type rpcClient interface {
+	rpcclient.ABCIClient
+	rpcclient.HistoryClient
+	rpcclient.NetworkClient
+	rpcclient.SignClient
+	rpcclient.StatusClient
+}
+
+// baseRPCClient implements the basic RPC method logic without the actual
+// underlying RPC call functionality, which is provided by `caller`.
+type baseRPCClient struct {
+	caller jsonrpcclient.Caller
+}
+
+var _ rpcClient = (*HTTP)(nil)
+var _ rpcClient = (*BatchHTTP)(nil)
+var _ rpcClient = (*baseRPCClient)(nil)
+
+//-----------------------------------------------------------------------------
+// HTTP
+
+// New takes a remote endpoint in the form <protocol>://<host>:<port> and
+// the websocket path (which always seems to be "/websocket")
+// An error is returned on invalid remote. The function panics when remote is nil.
+func New(remote, wsEndpoint string) (*HTTP, error) {
+	httpClient, err := jsonrpcclient.DefaultHTTPClient(remote)
+	if err != nil {
+		return nil, err
+	}
+	return NewWithClient(remote, wsEndpoint, httpClient)
+}
+
+// Create timeout enabled http client
+func NewWithTimeout(remote, wsEndpoint string, timeout uint) (*HTTP, error) {
+	httpClient, err := jsonrpcclient.DefaultHTTPClient(remote)
+	if err != nil {
+		return nil, err
+	}
+	httpClient.Timeout = time.Duration(timeout) * time.Second
+	return NewWithClient(remote, wsEndpoint, httpClient)
+}
+
+// NewWithClient allows for setting a custom http client (See New).
+// An error is returned on invalid remote. The function panics when remote is nil.
+func NewWithClient(remote, wsEndpoint string, client *http.Client) (*HTTP, error) {
+	if client == nil {
+		panic("nil http.Client provided")
+	}
+
+	rc, err := jsonrpcclient.NewWithHTTPClient(remote, client)
+	if err != nil {
+		return nil, err
+	}
+
+	wsEvents, err := newWSEvents(remote, wsEndpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	httpClient := &HTTP{
+		rpc:           rc,
+		remote:        remote,
+		baseRPCClient: &baseRPCClient{caller: rc},
+		WSEvents:      wsEvents,
+	}
+
+	return httpClient, nil
+}
+
+var _ rpcclient.Client = (*HTTP)(nil)
+
+// SetLogger sets a logger.
+func (c *HTTP) SetLogger(l log.Logger) {
+	c.WSEvents.SetLogger(l)
+}
+
+// Remote returns the remote network address in a string form.
+func (c *HTTP) Remote() string {
+	return c.remote
+}
+
+// NewBatch creates a new batch client for this HTTP client.
+func (c *HTTP) NewBatch() *BatchHTTP {
+	rpcBatch := c.rpc.NewRequestBatch()
+	return &BatchHTTP{
+		rpcBatch: rpcBatch,
+		baseRPCClient: &baseRPCClient{
+			caller: rpcBatch,
+		},
+	}
+}
+
+//-----------------------------------------------------------------------------
+// BatchHTTP
+
+// Send is a convenience function for an HTTP batch that will trigger the
+// compilation of the batched requests and send them off using the client as a
+// single request. On success, this returns a list of the deserialized results
+// from each request in the sent batch.
+func (b *BatchHTTP) Send(ctx context.Context) ([]interface{}, error) {
+	return b.rpcBatch.Send(ctx)
+}
+
+// Clear will empty out this batch of requests and return the number of requests
+// that were cleared out.
+func (b *BatchHTTP) Clear() int {
+	return b.rpcBatch.Clear()
+}
+
+// Count returns the number of enqueued requests waiting to be sent.
+func (b *BatchHTTP) Count() int {
+	return b.rpcBatch.Count()
+}
+
+//-----------------------------------------------------------------------------
+// baseRPCClient
+
+func (c *baseRPCClient) Status(ctx context.Context) (*ctypes.ResultStatus, error) {
+	result := new(ctypes.ResultStatus)
+	_, err := c.caller.Call(ctx, "status", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *baseRPCClient) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	result := new(ctypes.ResultABCIInfo)
+	_, err := c.caller.Call(ctx, "abci_info", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *baseRPCClient) ABCIQuery(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+) (*ctypes.ResultABCIQuery, error) {
+	return c.ABCIQueryWithOptions(ctx, path, data, rpcclient.DefaultABCIQueryOptions)
+}
+
+func (c *baseRPCClient) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts rpcclient.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	result := new(ctypes.ResultABCIQuery)
+	_, err := c.caller.Call(ctx, "abci_query",
+		map[string]interface{}{"path": path, "data": data, "height": opts.Height, "prove": opts.Prove},
+		result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *baseRPCClient) BroadcastTxCommit(
+	ctx context.Context,
+	tx types.Tx,
+) (*ctypes.ResultBroadcastTxCommit, error) {
+	result := new(ctypes.ResultBroadcastTxCommit)
+	_, err := c.caller.Call(ctx, "broadcast_tx_commit", map[string]interface{}{"tx": tx}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) BroadcastTxAsync(
+	ctx context.Context,
+	tx types.Tx,
+) (*ctypes.ResultBroadcastTx, error) {
+	return c.broadcastTX(ctx, "broadcast_tx_async", tx)
+}
+
+func (c *baseRPCClient) BroadcastTxSync(
+	ctx context.Context,
+	tx types.Tx,
+) (*ctypes.ResultBroadcastTx, error) {
+	return c.broadcastTX(ctx, "broadcast_tx_sync", tx)
+}
+
+func (c *baseRPCClient) broadcastTX(
+	ctx context.Context,
+	route string,
+	tx types.Tx,
+) (*ctypes.ResultBroadcastTx, error) {
+	result := new(ctypes.ResultBroadcastTx)
+	_, err := c.caller.Call(ctx, route, map[string]interface{}{"tx": tx}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) UnconfirmedTxs(
+	ctx context.Context,
+	limit *int,
+) (*ctypes.ResultUnconfirmedTxs, error) {
+	result := new(ctypes.ResultUnconfirmedTxs)
+	params := make(map[string]interface{})
+	if limit != nil {
+		params["limit"] = limit
+	}
+	_, err := c.caller.Call(ctx, "unconfirmed_txs", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) NumUnconfirmedTxs(ctx context.Context) (*ctypes.ResultUnconfirmedTxs, error) {
+	result := new(ctypes.ResultUnconfirmedTxs)
+	_, err := c.caller.Call(ctx, "num_unconfirmed_txs", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) CheckTx(ctx context.Context, tx types.Tx) (*ctypes.ResultCheckTx, error) {
+	result := new(ctypes.ResultCheckTx)
+	_, err := c.caller.Call(ctx, "check_tx", map[string]interface{}{"tx": tx}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) NetInfo(ctx context.Context) (*ctypes.ResultNetInfo, error) {
+	result := new(ctypes.ResultNetInfo)
+	_, err := c.caller.Call(ctx, "net_info", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) DumpConsensusState(ctx context.Context) (*ctypes.ResultDumpConsensusState, error) {
+	result := new(ctypes.ResultDumpConsensusState)
+	_, err := c.caller.Call(ctx, "dump_consensus_state", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) ConsensusState(ctx context.Context) (*ctypes.ResultConsensusState, error) {
+	result := new(ctypes.ResultConsensusState)
+	_, err := c.caller.Call(ctx, "consensus_state", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) ConsensusParams(
+	ctx context.Context,
+	height *int64,
+) (*ctypes.ResultConsensusParams, error) {
+	result := new(ctypes.ResultConsensusParams)
+	params := make(map[string]interface{})
+	if height != nil {
+		params["height"] = height
+	}
+	_, err := c.caller.Call(ctx, "consensus_params", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) Health(ctx context.Context) (*ctypes.ResultHealth, error) {
+	result := new(ctypes.ResultHealth)
+	_, err := c.caller.Call(ctx, "health", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) BlockchainInfo(
+	ctx context.Context,
+	minHeight,
+	maxHeight int64,
+) (*ctypes.ResultBlockchainInfo, error) {
+	result := new(ctypes.ResultBlockchainInfo)
+	_, err := c.caller.Call(ctx, "blockchain",
+		map[string]interface{}{"minHeight": minHeight, "maxHeight": maxHeight},
+		result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) Genesis(ctx context.Context) (*ctypes.ResultGenesis, error) {
+	result := new(ctypes.ResultGenesis)
+	_, err := c.caller.Call(ctx, "genesis", map[string]interface{}{}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) GenesisChunked(ctx context.Context, id uint) (*ctypes.ResultGenesisChunk, error) {
+	result := new(ctypes.ResultGenesisChunk)
+	_, err := c.caller.Call(ctx, "genesis_chunked", map[string]interface{}{"chunk": id}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) {
+	result := new(ctypes.ResultBlock)
+	params := make(map[string]interface{})
+	if height != nil {
+		params["height"] = height
+	}
+	_, err := c.caller.Call(ctx, "block", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) BlockByHash(ctx context.Context, hash []byte) (*ctypes.ResultBlock, error) {
+	result := new(ctypes.ResultBlock)
+	params := map[string]interface{}{
+		"hash": hash,
+	}
+	_, err := c.caller.Call(ctx, "block_by_hash", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) BlockResults(
+	ctx context.Context,
+	height *int64,
+) (*ctypes.ResultBlockResults, error) {
+	result := new(ctypes.ResultBlockResults)
+	params := make(map[string]interface{})
+	if height != nil {
+		params["height"] = height
+	}
+	_, err := c.caller.Call(ctx, "block_results", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) Commit(ctx context.Context, height *int64) (*ctypes.ResultCommit, error) {
+	result := new(ctypes.ResultCommit)
+	params := make(map[string]interface{})
+	if height != nil {
+		params["height"] = height
+	}
+	_, err := c.caller.Call(ctx, "commit", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) Tx(ctx context.Context, hash []byte, prove bool) (*ctypes.ResultTx, error) {
+	result := new(ctypes.ResultTx)
+	params := map[string]interface{}{
+		"hash":  hash,
+		"prove": prove,
+	}
+	_, err := c.caller.Call(ctx, "tx", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) TxSearch(
+	ctx context.Context,
+	query string,
+	prove bool,
+	page,
+	perPage *int,
+	orderBy string,
+) (*ctypes.ResultTxSearch, error) {
+
+	result := new(ctypes.ResultTxSearch)
+	params := map[string]interface{}{
+		"query":    query,
+		"prove":    prove,
+		"order_by": orderBy,
+	}
+
+	if page != nil {
+		params["page"] = page
+	}
+	if perPage != nil {
+		params["per_page"] = perPage
+	}
+
+	_, err := c.caller.Call(ctx, "tx_search", params, result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *baseRPCClient) BlockSearch(
+	ctx context.Context,
+	query string,
+	page, perPage *int,
+	orderBy string,
+) (*ctypes.ResultBlockSearch, error) {
+
+	result := new(ctypes.ResultBlockSearch)
+	params := map[string]interface{}{
+		"query":    query,
+		"order_by": orderBy,
+	}
+
+	if page != nil {
+		params["page"] = page
+	}
+	if perPage != nil {
+		params["per_page"] = perPage
+	}
+
+	_, err := c.caller.Call(ctx, "block_search", params, result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func (c *baseRPCClient) Validators(
+	ctx context.Context,
+	height *int64,
+	page,
+	perPage *int,
+) (*ctypes.ResultValidators, error) {
+	result := new(ctypes.ResultValidators)
+	params := make(map[string]interface{})
+	if page != nil {
+		params["page"] = page
+	}
+	if perPage != nil {
+		params["per_page"] = perPage
+	}
+	if height != nil {
+		params["height"] = height
+	}
+	_, err := c.caller.Call(ctx, "validators", params, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+func (c *baseRPCClient) BroadcastEvidence(
+	ctx context.Context,
+	ev types.Evidence,
+) (*ctypes.ResultBroadcastEvidence, error) {
+	result := new(ctypes.ResultBroadcastEvidence)
+	_, err := c.caller.Call(ctx, "broadcast_evidence", map[string]interface{}{"evidence": ev}, result)
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+//-----------------------------------------------------------------------------
+// WSEvents
+
+var errNotRunning = errors.New("client is not running. Use .Start() method to start")
+
+// WSEvents is a wrapper around WSClient, which implements EventsClient.
+type WSEvents struct {
+	service.BaseService
+	remote   string
+	endpoint string
+	ws       *jsonrpcclient.WSClient
+
+	mtx           tmsync.RWMutex
+	subscriptions map[string]chan ctypes.ResultEvent // query -> chan
+}
+
+func newWSEvents(remote, endpoint string) (*WSEvents, error) {
+	w := &WSEvents{
+		endpoint:      endpoint,
+		remote:        remote,
+		subscriptions: make(map[string]chan ctypes.ResultEvent),
+	}
+	w.BaseService = *service.NewBaseService(nil, "WSEvents", w)
+
+	var err error
+	w.ws, err = jsonrpcclient.NewWS(w.remote, w.endpoint, jsonrpcclient.OnReconnect(func() {
+		// resubscribe immediately
+		w.redoSubscriptionsAfter(0 * time.Second)
+	}))
+	if err != nil {
+		return nil, err
+	}
+	w.ws.SetLogger(w.Logger)
+
+	return w, nil
+}
+
+// OnStart implements service.Service by starting WSClient and event loop.
+func (w *WSEvents) OnStart() error {
+	if err := w.ws.Start(); err != nil {
+		return err
+	}
+
+	go w.eventListener()
+
+	return nil
+}
+
+// OnStop implements service.Service by stopping WSClient.
+func (w *WSEvents) OnStop() {
+	if err := w.ws.Stop(); err != nil {
+		w.Logger.Error("Can't stop ws client", "err", err)
+	}
+}
+
+// Subscribe implements EventsClient by using WSClient to subscribe given
+// subscriber to query. By default, returns a channel with cap=1. Error is
+// returned if it fails to subscribe.
+//
+// Channel is never closed to prevent clients from seeing an erroneous event.
+//
+// It returns an error if WSEvents is not running.
+func (w *WSEvents) Subscribe(ctx context.Context, subscriber, query string,
+	outCapacity ...int) (out <-chan ctypes.ResultEvent, err error) {
+
+	if !w.IsRunning() {
+		return nil, errNotRunning
+	}
+
+	if err := w.ws.Subscribe(ctx, query); err != nil {
+		return nil, err
+	}
+
+	outCap := 1
+	if len(outCapacity) > 0 {
+		outCap = outCapacity[0]
+	}
+
+	outc := make(chan ctypes.ResultEvent, outCap)
+	w.mtx.Lock()
+	// subscriber param is ignored because Tendermint will override it with
+	// remote IP anyway.
+	w.subscriptions[query] = outc
+	w.mtx.Unlock()
+
+	return outc, nil
+}
+
+// Unsubscribe implements EventsClient by using WSClient to unsubscribe given
+// subscriber from query.
+//
+// It returns an error if WSEvents is not running.
+func (w *WSEvents) Unsubscribe(ctx context.Context, subscriber, query string) error {
+	if !w.IsRunning() {
+		return errNotRunning
+	}
+
+	if err := w.ws.Unsubscribe(ctx, query); err != nil {
+		return err
+	}
+
+	w.mtx.Lock()
+	_, ok := w.subscriptions[query]
+	if ok {
+		delete(w.subscriptions, query)
+	}
+	w.mtx.Unlock()
+
+	return nil
+}
+
+// UnsubscribeAll implements EventsClient by using WSClient to unsubscribe
+// given subscriber from all the queries.
+//
+// It returns an error if WSEvents is not running.
+func (w *WSEvents) UnsubscribeAll(ctx context.Context, subscriber string) error {
+	if !w.IsRunning() {
+		return errNotRunning
+	}
+
+	if err := w.ws.UnsubscribeAll(ctx); err != nil {
+		return err
+	}
+
+	w.mtx.Lock()
+	w.subscriptions = make(map[string]chan ctypes.ResultEvent)
+	w.mtx.Unlock()
+
+	return nil
+}
+
+// After being reconnected, it is necessary to redo subscription to server
+// otherwise no data will be automatically received.
+func (w *WSEvents) redoSubscriptionsAfter(d time.Duration) {
+	time.Sleep(d)
+
+	w.mtx.RLock()
+	defer w.mtx.RUnlock()
+	for q := range w.subscriptions {
+		err := w.ws.Subscribe(context.Background(), q)
+		if err != nil {
+			w.Logger.Error("Failed to resubscribe", "err", err)
+		}
+	}
+}
+
+func isErrAlreadySubscribed(err error) bool {
+	return strings.Contains(err.Error(), tmpubsub.ErrAlreadySubscribed.Error())
+}
+
+func (w *WSEvents) eventListener() {
+	for {
+		select {
+		case resp, ok := <-w.ws.ResponsesCh:
+			if !ok {
+				return
+			}
+
+			if resp.Error != nil {
+				w.Logger.Error("WS error", "err", resp.Error.Error())
+				// Error can be ErrAlreadySubscribed or max client (subscriptions per
+				// client) reached or Tendermint exited.
+				// We can ignore ErrAlreadySubscribed, but need to retry in other
+				// cases.
+				if !isErrAlreadySubscribed(resp.Error) {
+					// Resubscribe after 1 second to give Tendermint time to restart (if
+					// crashed).
+					w.redoSubscriptionsAfter(1 * time.Second)
+				}
+				continue
+			}
+
+			result := new(ctypes.ResultEvent)
+			err := tmjson.Unmarshal(resp.Result, result)
+			if err != nil {
+				w.Logger.Error("failed to unmarshal response", "err", err)
+				continue
+			}
+
+			w.mtx.RLock()
+			if out, ok := w.subscriptions[result.Query]; ok {
+				if cap(out) == 0 {
+					out <- *result
+				} else {
+					select {
+					case out <- *result:
+					default:
+						w.Logger.Error("wanted to publish ResultEvent, but out channel is full", "result", result, "query", result.Query)
+					}
+				}
+			}
+			w.mtx.RUnlock()
+		case <-w.Quit():
+			return
+		}
+	}
+}

internal/rpc/client/interface.go

@@ -0,0 +1,152 @@
+package client
+
+/*
+The client package provides a general purpose interface (Client) for connecting
+to a tendermint node, as well as higher-level functionality.
+
+The main implementation for production code is client.HTTP, which
+connects via http to the jsonrpc interface of the tendermint node.
+
+For connecting to a node running in the same process (eg. when
+compiling the abci app in the same process), you can use the client.Local
+implementation.
+
+For mocking out server responses during testing to see behavior for
+arbitrary return values, use the mock package.
+
+In addition to the Client interface, which should be used externally
+for maximum flexibility and testability, and two implementations,
+this package also provides helper functions that work on any Client
+implementation.
+*/
+
+import (
+	"context"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/service"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Client wraps most important rpc calls a client would make if you want to
+// listen for events, test if it also implements events.EventSwitch.
+type Client interface {
+	service.Service
+	ABCIClient
+	EventsClient
+	HistoryClient
+	NetworkClient
+	SignClient
+	StatusClient
+	EvidenceClient
+	MempoolClient
+}
+
+// ABCIClient groups together the functionality that principally affects the
+// ABCI app.
+//
+// In many cases this will be all we want, so we can accept an interface which
+// is easier to mock.
+type ABCIClient interface {
+	// Reading from abci app
+	ABCIInfo(context.Context) (*ctypes.ResultABCIInfo, error)
+	ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*ctypes.ResultABCIQuery, error)
+	ABCIQueryWithOptions(ctx context.Context, path string, data bytes.HexBytes,
+		opts ABCIQueryOptions) (*ctypes.ResultABCIQuery, error)
+
+	// Writing to abci app
+	BroadcastTxCommit(context.Context, types.Tx) (*ctypes.ResultBroadcastTxCommit, error)
+	BroadcastTxAsync(context.Context, types.Tx) (*ctypes.ResultBroadcastTx, error)
+	BroadcastTxSync(context.Context, types.Tx) (*ctypes.ResultBroadcastTx, error)
+}
+
+// SignClient groups together the functionality needed to get valid signatures
+// and prove anything about the chain.
+type SignClient interface {
+	Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error)
+	BlockByHash(ctx context.Context, hash []byte) (*ctypes.ResultBlock, error)
+	BlockResults(ctx context.Context, height *int64) (*ctypes.ResultBlockResults, error)
+	Commit(ctx context.Context, height *int64) (*ctypes.ResultCommit, error)
+	Validators(ctx context.Context, height *int64, page, perPage *int) (*ctypes.ResultValidators, error)
+	Tx(ctx context.Context, hash []byte, prove bool) (*ctypes.ResultTx, error)
+
+	// TxSearch defines a method to search for a paginated set of transactions by
+	// DeliverTx event search criteria.
+	TxSearch(
+		ctx context.Context,
+		query string,
+		prove bool,
+		page, perPage *int,
+		orderBy string,
+	) (*ctypes.ResultTxSearch, error)
+
+	// BlockSearch defines a method to search for a paginated set of blocks by
+	// BeginBlock and EndBlock event search criteria.
+	BlockSearch(
+		ctx context.Context,
+		query string,
+		page, perPage *int,
+		orderBy string,
+	) (*ctypes.ResultBlockSearch, error)
+}
+
+// HistoryClient provides access to data from genesis to now in large chunks.
+type HistoryClient interface {
+	Genesis(context.Context) (*ctypes.ResultGenesis, error)
+	GenesisChunked(context.Context, uint) (*ctypes.ResultGenesisChunk, error)
+	BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error)
+}
+
+// StatusClient provides access to general chain info.
+type StatusClient interface {
+	Status(context.Context) (*ctypes.ResultStatus, error)
+}
+
+// NetworkClient is general info about the network state. May not be needed
+// usually.
+type NetworkClient interface {
+	NetInfo(context.Context) (*ctypes.ResultNetInfo, error)
+	DumpConsensusState(context.Context) (*ctypes.ResultDumpConsensusState, error)
+	ConsensusState(context.Context) (*ctypes.ResultConsensusState, error)
+	ConsensusParams(ctx context.Context, height *int64) (*ctypes.ResultConsensusParams, error)
+	Health(context.Context) (*ctypes.ResultHealth, error)
+}
+
+// EventsClient is reactive, you can subscribe to any message, given the proper
+// string. see tendermint/types/events.go
+type EventsClient interface {
+	// Subscribe subscribes given subscriber to query. Returns a channel with
+	// cap=1 onto which events are published. An error is returned if it fails to
+	// subscribe. outCapacity can be used optionally to set capacity for the
+	// channel. Channel is never closed to prevent accidental reads.
+	//
+	// ctx cannot be used to unsubscribe. To unsubscribe, use either Unsubscribe
+	// or UnsubscribeAll.
+	Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan ctypes.ResultEvent, err error)
+	// Unsubscribe unsubscribes given subscriber from query.
+	Unsubscribe(ctx context.Context, subscriber, query string) error
+	// UnsubscribeAll unsubscribes given subscriber from all the queries.
+	UnsubscribeAll(ctx context.Context, subscriber string) error
+}
+
+// MempoolClient shows us data about current mempool state.
+type MempoolClient interface {
+	UnconfirmedTxs(ctx context.Context, limit *int) (*ctypes.ResultUnconfirmedTxs, error)
+	NumUnconfirmedTxs(context.Context) (*ctypes.ResultUnconfirmedTxs, error)
+	CheckTx(context.Context, types.Tx) (*ctypes.ResultCheckTx, error)
+}
+
+// EvidenceClient is used for submitting an evidence of the malicious
+// behaviour.
+type EvidenceClient interface {
+	BroadcastEvidence(context.Context, types.Evidence) (*ctypes.ResultBroadcastEvidence, error)
+}
+
+// RemoteClient is a Client, which can also return the remote network address.
+type RemoteClient interface {
+	Client
+
+	// Remote returns the remote network address in a string form.
+	Remote() string
+}

internal/rpc/client/local/local.go

@@ -0,0 +1,301 @@
+package local
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/log"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
+	nm "github.com/tendermint/tendermint/node"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+/*
+Local is a Client implementation that directly executes the rpc
+functions on a given node, without going through HTTP or GRPC.
+
+This implementation is useful for:
+
+* Running tests against a node in-process without the overhead
+of going through an http server
+* Communication between an ABCI app and Tendermint core when they
+are compiled in process.
+
+For real clients, you probably want to use client.HTTP.  For more
+powerful control during testing, you probably want the "client/mock" package.
+
+You can subscribe for any event published by Tendermint using Subscribe method.
+Note delivery is best-effort. If you don't read events fast enough, Tendermint
+might cancel the subscription. The client will attempt to resubscribe (you
+don't need to do anything). It will keep trying indefinitely with exponential
+backoff (10ms -> 20ms -> 40ms) until successful.
+*/
+type Local struct {
+	*types.EventBus
+	Logger log.Logger
+	ctx    *rpctypes.Context
+}
+
+// NewLocal configures a client that calls the Node directly.
+//
+// Note that given how rpc/core works with package singletons, that
+// you can only have one node per process.  So make sure test cases
+// don't run in parallel, or try to simulate an entire network in
+// one process...
+func New(node *nm.Node) *Local {
+	if err := node.ConfigureRPC(); err != nil {
+		node.Logger.Error("Error configuring RPC", "err", err)
+	}
+	return &Local{
+		EventBus: node.EventBus(),
+		Logger:   log.NewNopLogger(),
+		ctx:      &rpctypes.Context{},
+	}
+}
+
+var _ rpcclient.Client = (*Local)(nil)
+
+// SetLogger allows to set a logger on the client.
+func (c *Local) SetLogger(l log.Logger) {
+	c.Logger = l
+}
+
+func (c *Local) Status(ctx context.Context) (*ctypes.ResultStatus, error) {
+	return core.Status(c.ctx)
+}
+
+func (c *Local) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	return core.ABCIInfo(c.ctx)
+}
+
+func (c *Local) ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*ctypes.ResultABCIQuery, error) {
+	return c.ABCIQueryWithOptions(ctx, path, data, rpcclient.DefaultABCIQueryOptions)
+}
+
+func (c *Local) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts rpcclient.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	return core.ABCIQuery(c.ctx, path, data, opts.Height, opts.Prove)
+}
+
+func (c *Local) BroadcastTxCommit(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return core.BroadcastTxCommit(c.ctx, tx)
+}
+
+func (c *Local) BroadcastTxAsync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxAsync(c.ctx, tx)
+}
+
+func (c *Local) BroadcastTxSync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxSync(c.ctx, tx)
+}
+
+func (c *Local) UnconfirmedTxs(ctx context.Context, limit *int) (*ctypes.ResultUnconfirmedTxs, error) {
+	return core.UnconfirmedTxs(c.ctx, limit)
+}
+
+func (c *Local) NumUnconfirmedTxs(ctx context.Context) (*ctypes.ResultUnconfirmedTxs, error) {
+	return core.NumUnconfirmedTxs(c.ctx)
+}
+
+func (c *Local) CheckTx(ctx context.Context, tx types.Tx) (*ctypes.ResultCheckTx, error) {
+	return core.CheckTx(c.ctx, tx)
+}
+
+func (c *Local) NetInfo(ctx context.Context) (*ctypes.ResultNetInfo, error) {
+	return core.NetInfo(c.ctx)
+}
+
+func (c *Local) DumpConsensusState(ctx context.Context) (*ctypes.ResultDumpConsensusState, error) {
+	return core.DumpConsensusState(c.ctx)
+}
+
+func (c *Local) ConsensusState(ctx context.Context) (*ctypes.ResultConsensusState, error) {
+	return core.ConsensusState(c.ctx)
+}
+
+func (c *Local) ConsensusParams(ctx context.Context, height *int64) (*ctypes.ResultConsensusParams, error) {
+	return core.ConsensusParams(c.ctx, height)
+}
+
+func (c *Local) Health(ctx context.Context) (*ctypes.ResultHealth, error) {
+	return core.Health(c.ctx)
+}
+
+func (c *Local) DialSeeds(ctx context.Context, seeds []string) (*ctypes.ResultDialSeeds, error) {
+	return core.UnsafeDialSeeds(c.ctx, seeds)
+}
+
+func (c *Local) DialPeers(
+	ctx context.Context,
+	peers []string,
+	persistent,
+	unconditional,
+	private bool,
+) (*ctypes.ResultDialPeers, error) {
+	return core.UnsafeDialPeers(c.ctx, peers, persistent, unconditional, private)
+}
+
+func (c *Local) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error) {
+	return core.BlockchainInfo(c.ctx, minHeight, maxHeight)
+}
+
+func (c *Local) Genesis(ctx context.Context) (*ctypes.ResultGenesis, error) {
+	return core.Genesis(c.ctx)
+}
+
+func (c *Local) GenesisChunked(ctx context.Context, id uint) (*ctypes.ResultGenesisChunk, error) {
+	return core.GenesisChunked(c.ctx, id)
+}
+
+func (c *Local) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) {
+	return core.Block(c.ctx, height)
+}
+
+func (c *Local) BlockByHash(ctx context.Context, hash []byte) (*ctypes.ResultBlock, error) {
+	return core.BlockByHash(c.ctx, hash)
+}
+
+func (c *Local) BlockResults(ctx context.Context, height *int64) (*ctypes.ResultBlockResults, error) {
+	return core.BlockResults(c.ctx, height)
+}
+
+func (c *Local) Commit(ctx context.Context, height *int64) (*ctypes.ResultCommit, error) {
+	return core.Commit(c.ctx, height)
+}
+
+func (c *Local) Validators(ctx context.Context, height *int64, page, perPage *int) (*ctypes.ResultValidators, error) {
+	return core.Validators(c.ctx, height, page, perPage)
+}
+
+func (c *Local) Tx(ctx context.Context, hash []byte, prove bool) (*ctypes.ResultTx, error) {
+	return core.Tx(c.ctx, hash, prove)
+}
+
+func (c *Local) TxSearch(
+	_ context.Context,
+	query string,
+	prove bool,
+	page,
+	perPage *int,
+	orderBy string,
+) (*ctypes.ResultTxSearch, error) {
+	return core.TxSearch(c.ctx, query, prove, page, perPage, orderBy)
+}
+
+func (c *Local) BlockSearch(
+	_ context.Context,
+	query string,
+	page, perPage *int,
+	orderBy string,
+) (*ctypes.ResultBlockSearch, error) {
+	return core.BlockSearch(c.ctx, query, page, perPage, orderBy)
+}
+
+func (c *Local) BroadcastEvidence(ctx context.Context, ev types.Evidence) (*ctypes.ResultBroadcastEvidence, error) {
+	return core.BroadcastEvidence(c.ctx, ev)
+}
+
+func (c *Local) Subscribe(
+	ctx context.Context,
+	subscriber,
+	query string,
+	outCapacity ...int) (out <-chan ctypes.ResultEvent, err error) {
+	q, err := tmquery.New(query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse query: %w", err)
+	}
+
+	outCap := 1
+	if len(outCapacity) > 0 {
+		outCap = outCapacity[0]
+	}
+
+	var sub types.Subscription
+	if outCap > 0 {
+		sub, err = c.EventBus.Subscribe(ctx, subscriber, q, outCap)
+	} else {
+		sub, err = c.EventBus.SubscribeUnbuffered(ctx, subscriber, q)
+	}
+	if err != nil {
+		return nil, fmt.Errorf("failed to subscribe: %w", err)
+	}
+
+	outc := make(chan ctypes.ResultEvent, outCap)
+	go c.eventsRoutine(sub, subscriber, q, outc)
+
+	return outc, nil
+}
+
+func (c *Local) eventsRoutine(
+	sub types.Subscription,
+	subscriber string,
+	q tmpubsub.Query,
+	outc chan<- ctypes.ResultEvent) {
+	for {
+		select {
+		case msg := <-sub.Out():
+			result := ctypes.ResultEvent{Query: q.String(), Data: msg.Data(), Events: msg.Events()}
+			if cap(outc) == 0 {
+				outc <- result
+			} else {
+				select {
+				case outc <- result:
+				default:
+					c.Logger.Error("wanted to publish ResultEvent, but out channel is full", "result", result, "query", result.Query)
+				}
+			}
+		case <-sub.Cancelled():
+			if sub.Err() == tmpubsub.ErrUnsubscribed {
+				return
+			}
+
+			c.Logger.Error("subscription was cancelled, resubscribing...", "err", sub.Err(), "query", q.String())
+			sub = c.resubscribe(subscriber, q)
+			if sub == nil { // client was stopped
+				return
+			}
+		case <-c.Quit():
+			return
+		}
+	}
+}
+
+// Try to resubscribe with exponential backoff.
+func (c *Local) resubscribe(subscriber string, q tmpubsub.Query) types.Subscription {
+	attempts := 0
+	for {
+		if !c.IsRunning() {
+			return nil
+		}
+
+		sub, err := c.EventBus.Subscribe(context.Background(), subscriber, q)
+		if err == nil {
+			return sub
+		}
+
+		attempts++
+		time.Sleep((10 << uint(attempts)) * time.Millisecond) // 10ms -> 20ms -> 40ms
+	}
+}
+
+func (c *Local) Unsubscribe(ctx context.Context, subscriber, query string) error {
+	q, err := tmquery.New(query)
+	if err != nil {
+		return fmt.Errorf("failed to parse query: %w", err)
+	}
+	return c.EventBus.Unsubscribe(ctx, subscriber, q)
+}
+
+func (c *Local) UnsubscribeAll(ctx context.Context, subscriber string) error {
+	return c.EventBus.UnsubscribeAll(ctx, subscriber)
+}

internal/rpc/client/main_test.go

@@ -0,0 +1,31 @@
+package client_test
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	nm "github.com/tendermint/tendermint/node"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+var node *nm.Node
+
+func TestMain(m *testing.M) {
+	// start a tendermint node (and kvstore) in the background to test against
+	dir, err := ioutil.TempDir("/tmp", "rpc-client-test")
+	if err != nil {
+		panic(err)
+	}
+
+	app := kvstore.NewPersistentKVStoreApplication(dir)
+	node = rpctest.StartTendermint(app)
+
+	code := m.Run()
+
+	// and shut down proper at the end
+	rpctest.StopTendermint(node)
+	_ = os.RemoveAll(dir)
+	os.Exit(code)
+}

internal/rpc/client/mock/abci.go

@@ -0,0 +1,241 @@
+package mock
+
+import (
+	"context"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/proxy"
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// ABCIApp will send all abci related request to the named app,
+// so you can test app behavior from a client without needing
+// an entire tendermint node
+type ABCIApp struct {
+	App abci.Application
+}
+
+var (
+	_ client.ABCIClient = ABCIApp{}
+	_ client.ABCIClient = ABCIMock{}
+	_ client.ABCIClient = (*ABCIRecorder)(nil)
+)
+
+func (a ABCIApp) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	return &ctypes.ResultABCIInfo{Response: a.App.Info(proxy.RequestInfo)}, nil
+}
+
+func (a ABCIApp) ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*ctypes.ResultABCIQuery, error) {
+	return a.ABCIQueryWithOptions(ctx, path, data, client.DefaultABCIQueryOptions)
+}
+
+func (a ABCIApp) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts client.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	q := a.App.Query(abci.RequestQuery{
+		Data:   data,
+		Path:   path,
+		Height: opts.Height,
+		Prove:  opts.Prove,
+	})
+	return &ctypes.ResultABCIQuery{Response: q}, nil
+}
+
+// NOTE: Caller should call a.App.Commit() separately,
+// this function does not actually wait for a commit.
+// TODO: Make it wait for a commit and set res.Height appropriately.
+func (a ABCIApp) BroadcastTxCommit(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res := ctypes.ResultBroadcastTxCommit{}
+	res.CheckTx = a.App.CheckTx(abci.RequestCheckTx{Tx: tx})
+	if res.CheckTx.IsErr() {
+		return &res, nil
+	}
+	res.DeliverTx = a.App.DeliverTx(abci.RequestDeliverTx{Tx: tx})
+	res.Height = -1 // TODO
+	return &res, nil
+}
+
+func (a ABCIApp) BroadcastTxAsync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	c := a.App.CheckTx(abci.RequestCheckTx{Tx: tx})
+	// and this gets written in a background thread...
+	if !c.IsErr() {
+		go func() { a.App.DeliverTx(abci.RequestDeliverTx{Tx: tx}) }()
+	}
+	return &ctypes.ResultBroadcastTx{
+		Code:      c.Code,
+		Data:      c.Data,
+		Log:       c.Log,
+		Codespace: c.Codespace,
+		Hash:      tx.Hash(),
+	}, nil
+}
+
+func (a ABCIApp) BroadcastTxSync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	c := a.App.CheckTx(abci.RequestCheckTx{Tx: tx})
+	// and this gets written in a background thread...
+	if !c.IsErr() {
+		go func() { a.App.DeliverTx(abci.RequestDeliverTx{Tx: tx}) }()
+	}
+	return &ctypes.ResultBroadcastTx{
+		Code:      c.Code,
+		Data:      c.Data,
+		Log:       c.Log,
+		Codespace: c.Codespace,
+		Hash:      tx.Hash(),
+	}, nil
+}
+
+// ABCIMock will send all abci related request to the named app,
+// so you can test app behavior from a client without needing
+// an entire tendermint node
+type ABCIMock struct {
+	Info            Call
+	Query           Call
+	BroadcastCommit Call
+	Broadcast       Call
+}
+
+func (m ABCIMock) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	res, err := m.Info.GetResponse(nil)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultABCIInfo{Response: res.(abci.ResponseInfo)}, nil
+}
+
+func (m ABCIMock) ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*ctypes.ResultABCIQuery, error) {
+	return m.ABCIQueryWithOptions(ctx, path, data, client.DefaultABCIQueryOptions)
+}
+
+func (m ABCIMock) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts client.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	res, err := m.Query.GetResponse(QueryArgs{path, data, opts.Height, opts.Prove})
+	if err != nil {
+		return nil, err
+	}
+	resQuery := res.(abci.ResponseQuery)
+	return &ctypes.ResultABCIQuery{Response: resQuery}, nil
+}
+
+func (m ABCIMock) BroadcastTxCommit(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res, err := m.BroadcastCommit.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTxCommit), nil
+}
+
+func (m ABCIMock) BroadcastTxAsync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := m.Broadcast.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTx), nil
+}
+
+func (m ABCIMock) BroadcastTxSync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := m.Broadcast.GetResponse(tx)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultBroadcastTx), nil
+}
+
+// ABCIRecorder can wrap another type (ABCIApp, ABCIMock, or Client)
+// and record all ABCI related calls.
+type ABCIRecorder struct {
+	Client client.ABCIClient
+	Calls  []Call
+}
+
+func NewABCIRecorder(client client.ABCIClient) *ABCIRecorder {
+	return &ABCIRecorder{
+		Client: client,
+		Calls:  []Call{},
+	}
+}
+
+type QueryArgs struct {
+	Path   string
+	Data   bytes.HexBytes
+	Height int64
+	Prove  bool
+}
+
+func (r *ABCIRecorder) addCall(call Call) {
+	r.Calls = append(r.Calls, call)
+}
+
+func (r *ABCIRecorder) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	res, err := r.Client.ABCIInfo(ctx)
+	r.addCall(Call{
+		Name:     "abci_info",
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) ABCIQuery(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+) (*ctypes.ResultABCIQuery, error) {
+	return r.ABCIQueryWithOptions(ctx, path, data, client.DefaultABCIQueryOptions)
+}
+
+func (r *ABCIRecorder) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts client.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	res, err := r.Client.ABCIQueryWithOptions(ctx, path, data, opts)
+	r.addCall(Call{
+		Name:     "abci_query",
+		Args:     QueryArgs{path, data, opts.Height, opts.Prove},
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxCommit(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	res, err := r.Client.BroadcastTxCommit(ctx, tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_commit",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxAsync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := r.Client.BroadcastTxAsync(ctx, tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_async",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}
+
+func (r *ABCIRecorder) BroadcastTxSync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	res, err := r.Client.BroadcastTxSync(ctx, tx)
+	r.addCall(Call{
+		Name:     "broadcast_tx_sync",
+		Args:     tx,
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}

internal/rpc/client/mock/abci_test.go

@@ -0,0 +1,199 @@
+package mock_test
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/client/mock"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestABCIMock(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	key, value := []byte("foo"), []byte("bar")
+	height := int64(10)
+	goodTx := types.Tx{0x01, 0xff}
+	badTx := types.Tx{0x12, 0x21}
+
+	m := mock.ABCIMock{
+		Info: mock.Call{Error: errors.New("foobar")},
+		Query: mock.Call{Response: abci.ResponseQuery{
+			Key:    key,
+			Value:  value,
+			Height: height,
+		}},
+		// Broadcast commit depends on call
+		BroadcastCommit: mock.Call{
+			Args: goodTx,
+			Response: &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   abci.ResponseCheckTx{Data: bytes.HexBytes("stand")},
+				DeliverTx: abci.ResponseDeliverTx{Data: bytes.HexBytes("deliver")},
+			},
+			Error: errors.New("bad tx"),
+		},
+		Broadcast: mock.Call{Error: errors.New("must commit")},
+	}
+
+	// now, let's try to make some calls
+	_, err := m.ABCIInfo(context.Background())
+	require.NotNil(err)
+	assert.Equal("foobar", err.Error())
+
+	// query always returns the response
+	_query, err := m.ABCIQueryWithOptions(context.Background(), "/", nil, client.ABCIQueryOptions{Prove: false})
+	query := _query.Response
+	require.Nil(err)
+	require.NotNil(query)
+	assert.EqualValues(key, query.Key)
+	assert.EqualValues(value, query.Value)
+	assert.Equal(height, query.Height)
+
+	// non-commit calls always return errors
+	_, err = m.BroadcastTxSync(context.Background(), goodTx)
+	require.NotNil(err)
+	assert.Equal("must commit", err.Error())
+	_, err = m.BroadcastTxAsync(context.Background(), goodTx)
+	require.NotNil(err)
+	assert.Equal("must commit", err.Error())
+
+	// commit depends on the input
+	_, err = m.BroadcastTxCommit(context.Background(), badTx)
+	require.NotNil(err)
+	assert.Equal("bad tx", err.Error())
+	bres, err := m.BroadcastTxCommit(context.Background(), goodTx)
+	require.Nil(err, "%+v", err)
+	assert.EqualValues(0, bres.CheckTx.Code)
+	assert.EqualValues("stand", bres.CheckTx.Data)
+	assert.EqualValues("deliver", bres.DeliverTx.Data)
+}
+
+func TestABCIRecorder(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	// This mock returns errors on everything but Query
+	m := mock.ABCIMock{
+		Info: mock.Call{Response: abci.ResponseInfo{
+			Data:    "data",
+			Version: "v0.9.9",
+		}},
+		Query:           mock.Call{Error: errors.New("query")},
+		Broadcast:       mock.Call{Error: errors.New("broadcast")},
+		BroadcastCommit: mock.Call{Error: errors.New("broadcast_commit")},
+	}
+	r := mock.NewABCIRecorder(m)
+
+	require.Equal(0, len(r.Calls))
+
+	_, err := r.ABCIInfo(context.Background())
+	assert.Nil(err, "expected no err on info")
+
+	_, err = r.ABCIQueryWithOptions(
+		context.Background(),
+		"path",
+		bytes.HexBytes("data"),
+		client.ABCIQueryOptions{Prove: false},
+	)
+	assert.NotNil(err, "expected error on query")
+	require.Equal(2, len(r.Calls))
+
+	info := r.Calls[0]
+	assert.Equal("abci_info", info.Name)
+	assert.Nil(info.Error)
+	assert.Nil(info.Args)
+	require.NotNil(info.Response)
+	ir, ok := info.Response.(*ctypes.ResultABCIInfo)
+	require.True(ok)
+	assert.Equal("data", ir.Response.Data)
+	assert.Equal("v0.9.9", ir.Response.Version)
+
+	query := r.Calls[1]
+	assert.Equal("abci_query", query.Name)
+	assert.Nil(query.Response)
+	require.NotNil(query.Error)
+	assert.Equal("query", query.Error.Error())
+	require.NotNil(query.Args)
+	qa, ok := query.Args.(mock.QueryArgs)
+	require.True(ok)
+	assert.Equal("path", qa.Path)
+	assert.EqualValues("data", qa.Data)
+	assert.False(qa.Prove)
+
+	// now add some broadcasts (should all err)
+	txs := []types.Tx{{1}, {2}, {3}}
+	_, err = r.BroadcastTxCommit(context.Background(), txs[0])
+	assert.NotNil(err, "expected err on broadcast")
+	_, err = r.BroadcastTxSync(context.Background(), txs[1])
+	assert.NotNil(err, "expected err on broadcast")
+	_, err = r.BroadcastTxAsync(context.Background(), txs[2])
+	assert.NotNil(err, "expected err on broadcast")
+
+	require.Equal(5, len(r.Calls))
+
+	bc := r.Calls[2]
+	assert.Equal("broadcast_tx_commit", bc.Name)
+	assert.Nil(bc.Response)
+	require.NotNil(bc.Error)
+	assert.EqualValues(bc.Args, txs[0])
+
+	bs := r.Calls[3]
+	assert.Equal("broadcast_tx_sync", bs.Name)
+	assert.Nil(bs.Response)
+	require.NotNil(bs.Error)
+	assert.EqualValues(bs.Args, txs[1])
+
+	ba := r.Calls[4]
+	assert.Equal("broadcast_tx_async", ba.Name)
+	assert.Nil(ba.Response)
+	require.NotNil(ba.Error)
+	assert.EqualValues(ba.Args, txs[2])
+}
+
+func TestABCIApp(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	app := kvstore.NewApplication()
+	m := mock.ABCIApp{app}
+
+	// get some info
+	info, err := m.ABCIInfo(context.Background())
+	require.Nil(err)
+	assert.Equal(`{"size":0}`, info.Response.GetData())
+
+	// add a key
+	key, value := "foo", "bar"
+	tx := fmt.Sprintf("%s=%s", key, value)
+	res, err := m.BroadcastTxCommit(context.Background(), types.Tx(tx))
+	require.Nil(err)
+	assert.True(res.CheckTx.IsOK())
+	require.NotNil(res.DeliverTx)
+	assert.True(res.DeliverTx.IsOK())
+
+	// commit
+	// TODO: This may not be necessary in the future
+	if res.Height == -1 {
+		m.App.Commit()
+	}
+
+	// check the key
+	_qres, err := m.ABCIQueryWithOptions(
+		context.Background(),
+		"/key",
+		bytes.HexBytes(key),
+		client.ABCIQueryOptions{Prove: true},
+	)
+	qres := _qres.Response
+	require.Nil(err)
+	assert.EqualValues(value, qres.Value)
+
+	// XXX Check proof
+}

internal/rpc/client/mock/client.go

@@ -0,0 +1,179 @@
+package mock
+
+/*
+package mock returns a Client implementation that
+accepts various (mock) implementations of the various methods.
+
+This implementation is useful for using in tests, when you don't
+need a real server, but want a high-level of control about
+the server response you want to mock (eg. error handling),
+or if you just want to record the calls to verify in your tests.
+
+For real clients, you probably want the "http" package.  If you
+want to directly call a tendermint node in process, you can use the
+"local" package.
+*/
+
+import (
+	"context"
+	"reflect"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/service"
+	"github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/rpc/core"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Client wraps arbitrary implementations of the various interfaces.
+//
+// We provide a few choices to mock out each one in this package.
+// Nothing hidden here, so no New function, just construct it from
+// some parts, and swap them out them during the tests.
+type Client struct {
+	client.ABCIClient
+	client.SignClient
+	client.HistoryClient
+	client.StatusClient
+	client.EventsClient
+	client.EvidenceClient
+	client.MempoolClient
+	service.Service
+}
+
+var _ client.Client = Client{}
+
+// Call is used by recorders to save a call and response.
+// It can also be used to configure mock responses.
+//
+type Call struct {
+	Name     string
+	Args     interface{}
+	Response interface{}
+	Error    error
+}
+
+// GetResponse will generate the apporiate response for us, when
+// using the Call struct to configure a Mock handler.
+//
+// When configuring a response, if only one of Response or Error is
+// set then that will always be returned. If both are set, then
+// we return Response if the Args match the set args, Error otherwise.
+func (c Call) GetResponse(args interface{}) (interface{}, error) {
+	// handle the case with no response
+	if c.Response == nil {
+		if c.Error == nil {
+			panic("Misconfigured call, you must set either Response or Error")
+		}
+		return nil, c.Error
+	}
+	// response without error
+	if c.Error == nil {
+		return c.Response, nil
+	}
+	// have both, we must check args....
+	if reflect.DeepEqual(args, c.Args) {
+		return c.Response, nil
+	}
+	return nil, c.Error
+}
+
+func (c Client) Status(ctx context.Context) (*ctypes.ResultStatus, error) {
+	return core.Status(&rpctypes.Context{})
+}
+
+func (c Client) ABCIInfo(ctx context.Context) (*ctypes.ResultABCIInfo, error) {
+	return core.ABCIInfo(&rpctypes.Context{})
+}
+
+func (c Client) ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*ctypes.ResultABCIQuery, error) {
+	return c.ABCIQueryWithOptions(ctx, path, data, client.DefaultABCIQueryOptions)
+}
+
+func (c Client) ABCIQueryWithOptions(
+	ctx context.Context,
+	path string,
+	data bytes.HexBytes,
+	opts client.ABCIQueryOptions) (*ctypes.ResultABCIQuery, error) {
+	return core.ABCIQuery(&rpctypes.Context{}, path, data, opts.Height, opts.Prove)
+}
+
+func (c Client) BroadcastTxCommit(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	return core.BroadcastTxCommit(&rpctypes.Context{}, tx)
+}
+
+func (c Client) BroadcastTxAsync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxAsync(&rpctypes.Context{}, tx)
+}
+
+func (c Client) BroadcastTxSync(ctx context.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	return core.BroadcastTxSync(&rpctypes.Context{}, tx)
+}
+
+func (c Client) CheckTx(ctx context.Context, tx types.Tx) (*ctypes.ResultCheckTx, error) {
+	return core.CheckTx(&rpctypes.Context{}, tx)
+}
+
+func (c Client) NetInfo(ctx context.Context) (*ctypes.ResultNetInfo, error) {
+	return core.NetInfo(&rpctypes.Context{})
+}
+
+func (c Client) ConsensusState(ctx context.Context) (*ctypes.ResultConsensusState, error) {
+	return core.ConsensusState(&rpctypes.Context{})
+}
+
+func (c Client) DumpConsensusState(ctx context.Context) (*ctypes.ResultDumpConsensusState, error) {
+	return core.DumpConsensusState(&rpctypes.Context{})
+}
+
+func (c Client) ConsensusParams(ctx context.Context, height *int64) (*ctypes.ResultConsensusParams, error) {
+	return core.ConsensusParams(&rpctypes.Context{}, height)
+}
+
+func (c Client) Health(ctx context.Context) (*ctypes.ResultHealth, error) {
+	return core.Health(&rpctypes.Context{})
+}
+
+func (c Client) DialSeeds(ctx context.Context, seeds []string) (*ctypes.ResultDialSeeds, error) {
+	return core.UnsafeDialSeeds(&rpctypes.Context{}, seeds)
+}
+
+func (c Client) DialPeers(
+	ctx context.Context,
+	peers []string,
+	persistent,
+	unconditional,
+	private bool,
+) (*ctypes.ResultDialPeers, error) {
+	return core.UnsafeDialPeers(&rpctypes.Context{}, peers, persistent, unconditional, private)
+}
+
+func (c Client) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error) {
+	return core.BlockchainInfo(&rpctypes.Context{}, minHeight, maxHeight)
+}
+
+func (c Client) Genesis(ctx context.Context) (*ctypes.ResultGenesis, error) {
+	return core.Genesis(&rpctypes.Context{})
+}
+
+func (c Client) Block(ctx context.Context, height *int64) (*ctypes.ResultBlock, error) {
+	return core.Block(&rpctypes.Context{}, height)
+}
+
+func (c Client) BlockByHash(ctx context.Context, hash []byte) (*ctypes.ResultBlock, error) {
+	return core.BlockByHash(&rpctypes.Context{}, hash)
+}
+
+func (c Client) Commit(ctx context.Context, height *int64) (*ctypes.ResultCommit, error) {
+	return core.Commit(&rpctypes.Context{}, height)
+}
+
+func (c Client) Validators(ctx context.Context, height *int64, page, perPage *int) (*ctypes.ResultValidators, error) {
+	return core.Validators(&rpctypes.Context{}, height, page, perPage)
+}
+
+func (c Client) BroadcastEvidence(ctx context.Context, ev types.Evidence) (*ctypes.ResultBroadcastEvidence, error) {
+	return core.BroadcastEvidence(&rpctypes.Context{}, ev)
+}

internal/rpc/client/mock/status.go

@@ -0,0 +1,54 @@
+package mock
+
+import (
+	"context"
+
+	"github.com/tendermint/tendermint/rpc/client"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+// StatusMock returns the result specified by the Call
+type StatusMock struct {
+	Call
+}
+
+var (
+	_ client.StatusClient = (*StatusMock)(nil)
+	_ client.StatusClient = (*StatusRecorder)(nil)
+)
+
+func (m *StatusMock) Status(ctx context.Context) (*ctypes.ResultStatus, error) {
+	res, err := m.GetResponse(nil)
+	if err != nil {
+		return nil, err
+	}
+	return res.(*ctypes.ResultStatus), nil
+}
+
+// StatusRecorder can wrap another type (StatusMock, full client)
+// and record the status calls
+type StatusRecorder struct {
+	Client client.StatusClient
+	Calls  []Call
+}
+
+func NewStatusRecorder(client client.StatusClient) *StatusRecorder {
+	return &StatusRecorder{
+		Client: client,
+		Calls:  []Call{},
+	}
+}
+
+func (r *StatusRecorder) addCall(call Call) {
+	r.Calls = append(r.Calls, call)
+}
+
+func (r *StatusRecorder) Status(ctx context.Context) (*ctypes.ResultStatus, error) {
+	res, err := r.Client.Status(ctx)
+	r.addCall(Call{
+		Name:     "status",
+		Response: res,
+		Error:    err,
+	})
+	return res, err
+}

internal/rpc/client/mock/status_test.go

@@ -0,0 +1,49 @@
+package mock_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/rpc/client/mock"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+)
+
+func TestStatus(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+
+	m := &mock.StatusMock{
+		Call: mock.Call{
+			Response: &ctypes.ResultStatus{
+				SyncInfo: ctypes.SyncInfo{
+					LatestBlockHash:   bytes.HexBytes("block"),
+					LatestAppHash:     bytes.HexBytes("app"),
+					LatestBlockHeight: 10,
+				},
+			}},
+	}
+
+	r := mock.NewStatusRecorder(m)
+	require.Equal(0, len(r.Calls))
+
+	// make sure response works proper
+	status, err := r.Status(context.Background())
+	require.Nil(err, "%+v", err)
+	assert.EqualValues("block", status.SyncInfo.LatestBlockHash)
+	assert.EqualValues(10, status.SyncInfo.LatestBlockHeight)
+
+	// make sure recorder works properly
+	require.Equal(1, len(r.Calls))
+	rs := r.Calls[0]
+	assert.Equal("status", rs.Name)
+	assert.Nil(rs.Args)
+	assert.Nil(rs.Error)
+	require.NotNil(rs.Response)
+	st, ok := rs.Response.(*ctypes.ResultStatus)
+	require.True(ok)
+	assert.EqualValues("block", st.SyncInfo.LatestBlockHash)
+	assert.EqualValues(10, st.SyncInfo.LatestBlockHeight)
+}

internal/rpc/client/mocks/client.go

@@ -0,0 +1,826 @@
+// Code generated by mockery 2.7.4. DO NOT EDIT.
+
+package mocks
+
+import (
+	bytes "github.com/tendermint/tendermint/libs/bytes"
+	client "github.com/tendermint/tendermint/rpc/client"
+
+	context "context"
+
+	coretypes "github.com/tendermint/tendermint/rpc/core/types"
+
+	log "github.com/tendermint/tendermint/libs/log"
+
+	mock "github.com/stretchr/testify/mock"
+
+	types "github.com/tendermint/tendermint/types"
+)
+
+// Client is an autogenerated mock type for the Client type
+type Client struct {
+	mock.Mock
+}
+
+// ABCIInfo provides a mock function with given fields: _a0
+func (_m *Client) ABCIInfo(_a0 context.Context) (*coretypes.ResultABCIInfo, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultABCIInfo
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultABCIInfo); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultABCIInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ABCIQuery provides a mock function with given fields: ctx, path, data
+func (_m *Client) ABCIQuery(ctx context.Context, path string, data bytes.HexBytes) (*coretypes.ResultABCIQuery, error) {
+	ret := _m.Called(ctx, path, data)
+
+	var r0 *coretypes.ResultABCIQuery
+	if rf, ok := ret.Get(0).(func(context.Context, string, bytes.HexBytes) *coretypes.ResultABCIQuery); ok {
+		r0 = rf(ctx, path, data)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultABCIQuery)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, string, bytes.HexBytes) error); ok {
+		r1 = rf(ctx, path, data)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ABCIQueryWithOptions provides a mock function with given fields: ctx, path, data, opts
+func (_m *Client) ABCIQueryWithOptions(ctx context.Context, path string, data bytes.HexBytes, opts client.ABCIQueryOptions) (*coretypes.ResultABCIQuery, error) {
+	ret := _m.Called(ctx, path, data, opts)
+
+	var r0 *coretypes.ResultABCIQuery
+	if rf, ok := ret.Get(0).(func(context.Context, string, bytes.HexBytes, client.ABCIQueryOptions) *coretypes.ResultABCIQuery); ok {
+		r0 = rf(ctx, path, data, opts)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultABCIQuery)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, string, bytes.HexBytes, client.ABCIQueryOptions) error); ok {
+		r1 = rf(ctx, path, data, opts)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Block provides a mock function with given fields: ctx, height
+func (_m *Client) Block(ctx context.Context, height *int64) (*coretypes.ResultBlock, error) {
+	ret := _m.Called(ctx, height)
+
+	var r0 *coretypes.ResultBlock
+	if rf, ok := ret.Get(0).(func(context.Context, *int64) *coretypes.ResultBlock); ok {
+		r0 = rf(ctx, height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBlock)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int64) error); ok {
+		r1 = rf(ctx, height)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BlockByHash provides a mock function with given fields: ctx, hash
+func (_m *Client) BlockByHash(ctx context.Context, hash []byte) (*coretypes.ResultBlock, error) {
+	ret := _m.Called(ctx, hash)
+
+	var r0 *coretypes.ResultBlock
+	if rf, ok := ret.Get(0).(func(context.Context, []byte) *coretypes.ResultBlock); ok {
+		r0 = rf(ctx, hash)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBlock)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, []byte) error); ok {
+		r1 = rf(ctx, hash)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BlockResults provides a mock function with given fields: ctx, height
+func (_m *Client) BlockResults(ctx context.Context, height *int64) (*coretypes.ResultBlockResults, error) {
+	ret := _m.Called(ctx, height)
+
+	var r0 *coretypes.ResultBlockResults
+	if rf, ok := ret.Get(0).(func(context.Context, *int64) *coretypes.ResultBlockResults); ok {
+		r0 = rf(ctx, height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBlockResults)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int64) error); ok {
+		r1 = rf(ctx, height)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BlockSearch provides a mock function with given fields: ctx, query, page, perPage, orderBy
+func (_m *Client) BlockSearch(ctx context.Context, query string, page *int, perPage *int, orderBy string) (*coretypes.ResultBlockSearch, error) {
+	ret := _m.Called(ctx, query, page, perPage, orderBy)
+
+	var r0 *coretypes.ResultBlockSearch
+	if rf, ok := ret.Get(0).(func(context.Context, string, *int, *int, string) *coretypes.ResultBlockSearch); ok {
+		r0 = rf(ctx, query, page, perPage, orderBy)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBlockSearch)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, string, *int, *int, string) error); ok {
+		r1 = rf(ctx, query, page, perPage, orderBy)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BlockchainInfo provides a mock function with given fields: ctx, minHeight, maxHeight
+func (_m *Client) BlockchainInfo(ctx context.Context, minHeight int64, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) {
+	ret := _m.Called(ctx, minHeight, maxHeight)
+
+	var r0 *coretypes.ResultBlockchainInfo
+	if rf, ok := ret.Get(0).(func(context.Context, int64, int64) *coretypes.ResultBlockchainInfo); ok {
+		r0 = rf(ctx, minHeight, maxHeight)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBlockchainInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, int64, int64) error); ok {
+		r1 = rf(ctx, minHeight, maxHeight)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BroadcastEvidence provides a mock function with given fields: _a0, _a1
+func (_m *Client) BroadcastEvidence(_a0 context.Context, _a1 types.Evidence) (*coretypes.ResultBroadcastEvidence, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultBroadcastEvidence
+	if rf, ok := ret.Get(0).(func(context.Context, types.Evidence) *coretypes.ResultBroadcastEvidence); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBroadcastEvidence)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.Evidence) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BroadcastTxAsync provides a mock function with given fields: _a0, _a1
+func (_m *Client) BroadcastTxAsync(_a0 context.Context, _a1 types.Tx) (*coretypes.ResultBroadcastTx, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultBroadcastTx
+	if rf, ok := ret.Get(0).(func(context.Context, types.Tx) *coretypes.ResultBroadcastTx); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBroadcastTx)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.Tx) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BroadcastTxCommit provides a mock function with given fields: _a0, _a1
+func (_m *Client) BroadcastTxCommit(_a0 context.Context, _a1 types.Tx) (*coretypes.ResultBroadcastTxCommit, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultBroadcastTxCommit
+	if rf, ok := ret.Get(0).(func(context.Context, types.Tx) *coretypes.ResultBroadcastTxCommit); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBroadcastTxCommit)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.Tx) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// BroadcastTxSync provides a mock function with given fields: _a0, _a1
+func (_m *Client) BroadcastTxSync(_a0 context.Context, _a1 types.Tx) (*coretypes.ResultBroadcastTx, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultBroadcastTx
+	if rf, ok := ret.Get(0).(func(context.Context, types.Tx) *coretypes.ResultBroadcastTx); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultBroadcastTx)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.Tx) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// CheckTx provides a mock function with given fields: _a0, _a1
+func (_m *Client) CheckTx(_a0 context.Context, _a1 types.Tx) (*coretypes.ResultCheckTx, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultCheckTx
+	if rf, ok := ret.Get(0).(func(context.Context, types.Tx) *coretypes.ResultCheckTx); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultCheckTx)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, types.Tx) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Commit provides a mock function with given fields: ctx, height
+func (_m *Client) Commit(ctx context.Context, height *int64) (*coretypes.ResultCommit, error) {
+	ret := _m.Called(ctx, height)
+
+	var r0 *coretypes.ResultCommit
+	if rf, ok := ret.Get(0).(func(context.Context, *int64) *coretypes.ResultCommit); ok {
+		r0 = rf(ctx, height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultCommit)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int64) error); ok {
+		r1 = rf(ctx, height)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ConsensusParams provides a mock function with given fields: ctx, height
+func (_m *Client) ConsensusParams(ctx context.Context, height *int64) (*coretypes.ResultConsensusParams, error) {
+	ret := _m.Called(ctx, height)
+
+	var r0 *coretypes.ResultConsensusParams
+	if rf, ok := ret.Get(0).(func(context.Context, *int64) *coretypes.ResultConsensusParams); ok {
+		r0 = rf(ctx, height)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultConsensusParams)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int64) error); ok {
+		r1 = rf(ctx, height)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// ConsensusState provides a mock function with given fields: _a0
+func (_m *Client) ConsensusState(_a0 context.Context) (*coretypes.ResultConsensusState, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultConsensusState
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultConsensusState); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultConsensusState)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// DumpConsensusState provides a mock function with given fields: _a0
+func (_m *Client) DumpConsensusState(_a0 context.Context) (*coretypes.ResultDumpConsensusState, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultDumpConsensusState
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultDumpConsensusState); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultDumpConsensusState)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Genesis provides a mock function with given fields: _a0
+func (_m *Client) Genesis(_a0 context.Context) (*coretypes.ResultGenesis, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultGenesis
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultGenesis); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultGenesis)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// GenesisChunked provides a mock function with given fields: _a0, _a1
+func (_m *Client) GenesisChunked(_a0 context.Context, _a1 uint) (*coretypes.ResultGenesisChunk, error) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *coretypes.ResultGenesisChunk
+	if rf, ok := ret.Get(0).(func(context.Context, uint) *coretypes.ResultGenesisChunk); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultGenesisChunk)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, uint) error); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Health provides a mock function with given fields: _a0
+func (_m *Client) Health(_a0 context.Context) (*coretypes.ResultHealth, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultHealth
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultHealth); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultHealth)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// IsRunning provides a mock function with given fields:
+func (_m *Client) IsRunning() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// NetInfo provides a mock function with given fields: _a0
+func (_m *Client) NetInfo(_a0 context.Context) (*coretypes.ResultNetInfo, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultNetInfo
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultNetInfo); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultNetInfo)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// NumUnconfirmedTxs provides a mock function with given fields: _a0
+func (_m *Client) NumUnconfirmedTxs(_a0 context.Context) (*coretypes.ResultUnconfirmedTxs, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultUnconfirmedTxs
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultUnconfirmedTxs); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultUnconfirmedTxs)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// OnReset provides a mock function with given fields:
+func (_m *Client) OnReset() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// OnStart provides a mock function with given fields:
+func (_m *Client) OnStart() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// OnStop provides a mock function with given fields:
+func (_m *Client) OnStop() {
+	_m.Called()
+}
+
+// Quit provides a mock function with given fields:
+func (_m *Client) Quit() <-chan struct{} {
+	ret := _m.Called()
+
+	var r0 <-chan struct{}
+	if rf, ok := ret.Get(0).(func() <-chan struct{}); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan struct{})
+		}
+	}
+
+	return r0
+}
+
+// Reset provides a mock function with given fields:
+func (_m *Client) Reset() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// SetLogger provides a mock function with given fields: _a0
+func (_m *Client) SetLogger(_a0 log.Logger) {
+	_m.Called(_a0)
+}
+
+// Start provides a mock function with given fields:
+func (_m *Client) Start() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Status provides a mock function with given fields: _a0
+func (_m *Client) Status(_a0 context.Context) (*coretypes.ResultStatus, error) {
+	ret := _m.Called(_a0)
+
+	var r0 *coretypes.ResultStatus
+	if rf, ok := ret.Get(0).(func(context.Context) *coretypes.ResultStatus); ok {
+		r0 = rf(_a0)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultStatus)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Stop provides a mock function with given fields:
+func (_m *Client) Stop() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// String provides a mock function with given fields:
+func (_m *Client) String() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Subscribe provides a mock function with given fields: ctx, subscriber, query, outCapacity
+func (_m *Client) Subscribe(ctx context.Context, subscriber string, query string, outCapacity ...int) (<-chan coretypes.ResultEvent, error) {
+	_va := make([]interface{}, len(outCapacity))
+	for _i := range outCapacity {
+		_va[_i] = outCapacity[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, ctx, subscriber, query)
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 <-chan coretypes.ResultEvent
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, ...int) <-chan coretypes.ResultEvent); ok {
+		r0 = rf(ctx, subscriber, query, outCapacity...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(<-chan coretypes.ResultEvent)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, ...int) error); ok {
+		r1 = rf(ctx, subscriber, query, outCapacity...)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Tx provides a mock function with given fields: ctx, hash, prove
+func (_m *Client) Tx(ctx context.Context, hash []byte, prove bool) (*coretypes.ResultTx, error) {
+	ret := _m.Called(ctx, hash, prove)
+
+	var r0 *coretypes.ResultTx
+	if rf, ok := ret.Get(0).(func(context.Context, []byte, bool) *coretypes.ResultTx); ok {
+		r0 = rf(ctx, hash, prove)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultTx)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, []byte, bool) error); ok {
+		r1 = rf(ctx, hash, prove)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// TxSearch provides a mock function with given fields: ctx, query, prove, page, perPage, orderBy
+func (_m *Client) TxSearch(ctx context.Context, query string, prove bool, page *int, perPage *int, orderBy string) (*coretypes.ResultTxSearch, error) {
+	ret := _m.Called(ctx, query, prove, page, perPage, orderBy)
+
+	var r0 *coretypes.ResultTxSearch
+	if rf, ok := ret.Get(0).(func(context.Context, string, bool, *int, *int, string) *coretypes.ResultTxSearch); ok {
+		r0 = rf(ctx, query, prove, page, perPage, orderBy)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultTxSearch)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, string, bool, *int, *int, string) error); ok {
+		r1 = rf(ctx, query, prove, page, perPage, orderBy)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// UnconfirmedTxs provides a mock function with given fields: ctx, limit
+func (_m *Client) UnconfirmedTxs(ctx context.Context, limit *int) (*coretypes.ResultUnconfirmedTxs, error) {
+	ret := _m.Called(ctx, limit)
+
+	var r0 *coretypes.ResultUnconfirmedTxs
+	if rf, ok := ret.Get(0).(func(context.Context, *int) *coretypes.ResultUnconfirmedTxs); ok {
+		r0 = rf(ctx, limit)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultUnconfirmedTxs)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int) error); ok {
+		r1 = rf(ctx, limit)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Unsubscribe provides a mock function with given fields: ctx, subscriber, query
+func (_m *Client) Unsubscribe(ctx context.Context, subscriber string, query string) error {
+	ret := _m.Called(ctx, subscriber, query)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok {
+		r0 = rf(ctx, subscriber, query)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// UnsubscribeAll provides a mock function with given fields: ctx, subscriber
+func (_m *Client) UnsubscribeAll(ctx context.Context, subscriber string) error {
+	ret := _m.Called(ctx, subscriber)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(context.Context, string) error); ok {
+		r0 = rf(ctx, subscriber)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Validators provides a mock function with given fields: ctx, height, page, perPage
+func (_m *Client) Validators(ctx context.Context, height *int64, page *int, perPage *int) (*coretypes.ResultValidators, error) {
+	ret := _m.Called(ctx, height, page, perPage)
+
+	var r0 *coretypes.ResultValidators
+	if rf, ok := ret.Get(0).(func(context.Context, *int64, *int, *int) *coretypes.ResultValidators); ok {
+		r0 = rf(ctx, height, page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*coretypes.ResultValidators)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(context.Context, *int64, *int, *int) error); ok {
+		r1 = rf(ctx, height, page, perPage)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

internal/rpc/client/rpc_test.go

@@ -0,0 +1,729 @@
+package client_test
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"math"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	mempl "github.com/tendermint/tendermint/mempool"
+	"github.com/tendermint/tendermint/rpc/client"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
+	rpclocal "github.com/tendermint/tendermint/rpc/client/local"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpcclient "github.com/tendermint/tendermint/rpc/jsonrpc/client"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+	"github.com/tendermint/tendermint/types"
+)
+
+var (
+	ctx = context.Background()
+)
+
+func getHTTPClient() *rpchttp.HTTP {
+	rpcAddr := rpctest.GetConfig().RPC.ListenAddress
+	c, err := rpchttp.New(rpcAddr, "/websocket")
+	if err != nil {
+		panic(err)
+	}
+	c.SetLogger(log.TestingLogger())
+	return c
+}
+
+func getHTTPClientWithTimeout(timeout uint) *rpchttp.HTTP {
+	rpcAddr := rpctest.GetConfig().RPC.ListenAddress
+	c, err := rpchttp.NewWithTimeout(rpcAddr, "/websocket", timeout)
+	if err != nil {
+		panic(err)
+	}
+	c.SetLogger(log.TestingLogger())
+	return c
+}
+
+func getLocalClient() *rpclocal.Local {
+	return rpclocal.New(node)
+}
+
+// GetClients returns a slice of clients for table-driven tests
+func GetClients() []client.Client {
+	return []client.Client{
+		getHTTPClient(),
+		getLocalClient(),
+	}
+}
+
+func TestNilCustomHTTPClient(t *testing.T) {
+	require.Panics(t, func() {
+		_, _ = rpchttp.NewWithClient("http://example.com", "/websocket", nil)
+	})
+	require.Panics(t, func() {
+		_, _ = rpcclient.NewWithHTTPClient("http://example.com", nil)
+	})
+}
+
+func TestCustomHTTPClient(t *testing.T) {
+	remote := rpctest.GetConfig().RPC.ListenAddress
+	c, err := rpchttp.NewWithClient(remote, "/websocket", http.DefaultClient)
+	require.Nil(t, err)
+	status, err := c.Status(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, status)
+}
+
+func TestCorsEnabled(t *testing.T) {
+	origin := rpctest.GetConfig().RPC.CORSAllowedOrigins[0]
+	remote := strings.ReplaceAll(rpctest.GetConfig().RPC.ListenAddress, "tcp", "http")
+
+	req, err := http.NewRequest("GET", remote, nil)
+	require.Nil(t, err, "%+v", err)
+	req.Header.Set("Origin", origin)
+	c := &http.Client{}
+	resp, err := c.Do(req)
+	require.Nil(t, err, "%+v", err)
+	defer resp.Body.Close()
+
+	assert.Equal(t, resp.Header.Get("Access-Control-Allow-Origin"), origin)
+}
+
+// Make sure status is correct (we connect properly)
+func TestStatus(t *testing.T) {
+	for i, c := range GetClients() {
+		moniker := rpctest.GetConfig().Moniker
+		status, err := c.Status(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.Equal(t, moniker, status.NodeInfo.Moniker)
+	}
+}
+
+// Make sure info is correct (we connect properly)
+func TestInfo(t *testing.T) {
+	for i, c := range GetClients() {
+		// status, err := c.Status()
+		// require.Nil(t, err, "%+v", err)
+		info, err := c.ABCIInfo(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		// TODO: this is not correct - fix merkleeyes!
+		// assert.EqualValues(t, status.SyncInfo.LatestBlockHeight, info.Response.LastBlockHeight)
+		assert.True(t, strings.Contains(info.Response.Data, "size"))
+	}
+}
+
+func TestNetInfo(t *testing.T) {
+	for i, c := range GetClients() {
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		netinfo, err := nc.NetInfo(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.True(t, netinfo.Listening)
+		assert.Equal(t, 0, len(netinfo.Peers))
+	}
+}
+
+func TestDumpConsensusState(t *testing.T) {
+	for i, c := range GetClients() {
+		// FIXME: fix server so it doesn't panic on invalid input
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		cons, err := nc.DumpConsensusState(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.NotEmpty(t, cons.RoundState)
+		assert.Empty(t, cons.Peers)
+	}
+}
+
+func TestConsensusState(t *testing.T) {
+	for i, c := range GetClients() {
+		// FIXME: fix server so it doesn't panic on invalid input
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		cons, err := nc.ConsensusState(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		assert.NotEmpty(t, cons.RoundState)
+	}
+}
+
+func TestHealth(t *testing.T) {
+	for i, c := range GetClients() {
+		nc, ok := c.(client.NetworkClient)
+		require.True(t, ok, "%d", i)
+		_, err := nc.Health(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+	}
+}
+
+func TestGenesisAndValidators(t *testing.T) {
+	for i, c := range GetClients() {
+
+		// make sure this is the right genesis file
+		gen, err := c.Genesis(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+		// get the genesis validator
+		require.Equal(t, 1, len(gen.Genesis.Validators))
+		gval := gen.Genesis.Validators[0]
+
+		// get the current validators
+		h := int64(1)
+		vals, err := c.Validators(context.Background(), &h, nil, nil)
+		require.Nil(t, err, "%d: %+v", i, err)
+		require.Equal(t, 1, len(vals.Validators))
+		require.Equal(t, 1, vals.Count)
+		require.Equal(t, 1, vals.Total)
+		val := vals.Validators[0]
+
+		// make sure the current set is also the genesis set
+		assert.Equal(t, gval.Power, val.VotingPower)
+		assert.Equal(t, gval.PubKey, val.PubKey)
+	}
+}
+
+func TestGenesisChunked(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	for _, c := range GetClients() {
+		first, err := c.GenesisChunked(ctx, 0)
+		require.NoError(t, err)
+
+		decoded := make([]string, 0, first.TotalChunks)
+		for i := 0; i < first.TotalChunks; i++ {
+			chunk, err := c.GenesisChunked(ctx, uint(i))
+			require.NoError(t, err)
+			data, err := base64.StdEncoding.DecodeString(chunk.Data)
+			require.NoError(t, err)
+			decoded = append(decoded, string(data))
+
+		}
+		doc := []byte(strings.Join(decoded, ""))
+
+		var out types.GenesisDoc
+		require.NoError(t, tmjson.Unmarshal(doc, &out),
+			"first: %+v, doc: %s", first, string(doc))
+	}
+}
+
+func TestABCIQuery(t *testing.T) {
+	for i, c := range GetClients() {
+		// write something
+		k, v, tx := MakeTxKV()
+		bres, err := c.BroadcastTxCommit(context.Background(), tx)
+		require.Nil(t, err, "%d: %+v", i, err)
+		apph := bres.Height + 1 // this is where the tx will be applied to the state
+
+		// wait before querying
+		err = client.WaitForHeight(c, apph, nil)
+		require.NoError(t, err)
+		res, err := c.ABCIQuery(context.Background(), "/key", k)
+		qres := res.Response
+		if assert.Nil(t, err) && assert.True(t, qres.IsOK()) {
+			assert.EqualValues(t, v, qres.Value)
+		}
+	}
+}
+
+// Make some app checks
+func TestAppCalls(t *testing.T) {
+	assert, require := assert.New(t), require.New(t)
+	for i, c := range GetClients() {
+
+		// get an offset of height to avoid racing and guessing
+		s, err := c.Status(context.Background())
+		require.NoError(err)
+		// sh is start height or status height
+		sh := s.SyncInfo.LatestBlockHeight
+
+		// look for the future
+		h := sh + 20
+		_, err = c.Block(context.Background(), &h)
+		require.Error(err) // no block yet
+
+		// write something
+		k, v, tx := MakeTxKV()
+		bres, err := c.BroadcastTxCommit(context.Background(), tx)
+		require.NoError(err)
+		require.True(bres.DeliverTx.IsOK())
+		txh := bres.Height
+		apph := txh + 1 // this is where the tx will be applied to the state
+
+		// wait before querying
+		err = client.WaitForHeight(c, apph, nil)
+		require.NoError(err)
+
+		_qres, err := c.ABCIQueryWithOptions(context.Background(), "/key", k, client.ABCIQueryOptions{Prove: false})
+		require.NoError(err)
+		qres := _qres.Response
+		if assert.True(qres.IsOK()) {
+			assert.Equal(k, qres.Key)
+			assert.EqualValues(v, qres.Value)
+		}
+
+		// make sure we can lookup the tx with proof
+		ptx, err := c.Tx(context.Background(), bres.Hash, true)
+		require.NoError(err)
+		assert.EqualValues(txh, ptx.Height)
+		assert.EqualValues(tx, ptx.Tx)
+
+		// and we can even check the block is added
+		block, err := c.Block(context.Background(), &apph)
+		require.NoError(err)
+		appHash := block.Block.Header.AppHash
+		assert.True(len(appHash) > 0)
+		assert.EqualValues(apph, block.Block.Header.Height)
+
+		blockByHash, err := c.BlockByHash(context.Background(), block.BlockID.Hash)
+		require.NoError(err)
+		require.Equal(block, blockByHash)
+
+		// now check the results
+		blockResults, err := c.BlockResults(context.Background(), &txh)
+		require.Nil(err, "%d: %+v", i, err)
+		assert.Equal(txh, blockResults.Height)
+		if assert.Equal(1, len(blockResults.TxsResults)) {
+			// check success code
+			assert.EqualValues(0, blockResults.TxsResults[0].Code)
+		}
+
+		// check blockchain info, now that we know there is info
+		info, err := c.BlockchainInfo(context.Background(), apph, apph)
+		require.NoError(err)
+		assert.True(info.LastHeight >= apph)
+		if assert.Equal(1, len(info.BlockMetas)) {
+			lastMeta := info.BlockMetas[0]
+			assert.EqualValues(apph, lastMeta.Header.Height)
+			blockData := block.Block
+			assert.Equal(blockData.Header.AppHash, lastMeta.Header.AppHash)
+			assert.Equal(block.BlockID, lastMeta.BlockID)
+		}
+
+		// and get the corresponding commit with the same apphash
+		commit, err := c.Commit(context.Background(), &apph)
+		require.NoError(err)
+		cappHash := commit.Header.AppHash
+		assert.Equal(appHash, cappHash)
+		assert.NotNil(commit.Commit)
+
+		// compare the commits (note Commit(2) has commit from Block(3))
+		h = apph - 1
+		commit2, err := c.Commit(context.Background(), &h)
+		require.NoError(err)
+		assert.Equal(block.Block.LastCommitHash, commit2.Commit.Hash())
+
+		// and we got a proof that works!
+		_pres, err := c.ABCIQueryWithOptions(context.Background(), "/key", k, client.ABCIQueryOptions{Prove: true})
+		require.NoError(err)
+		pres := _pres.Response
+		assert.True(pres.IsOK())
+
+		// XXX Test proof
+	}
+}
+
+func TestBroadcastTxSync(t *testing.T) {
+	require := require.New(t)
+
+	// TODO (melekes): use mempool which is set on RPC rather than getting it from node
+	mempool := node.Mempool()
+	initMempoolSize := mempool.Size()
+
+	for i, c := range GetClients() {
+		_, _, tx := MakeTxKV()
+		bres, err := c.BroadcastTxSync(context.Background(), tx)
+		require.Nil(err, "%d: %+v", i, err)
+		require.Equal(bres.Code, abci.CodeTypeOK) // FIXME
+
+		require.Equal(initMempoolSize+1, mempool.Size())
+
+		txs := mempool.ReapMaxTxs(len(tx))
+		require.EqualValues(tx, txs[0])
+		mempool.Flush()
+	}
+}
+
+func TestBroadcastTxCommit(t *testing.T) {
+	require := require.New(t)
+
+	mempool := node.Mempool()
+	for i, c := range GetClients() {
+		_, _, tx := MakeTxKV()
+		bres, err := c.BroadcastTxCommit(context.Background(), tx)
+		require.Nil(err, "%d: %+v", i, err)
+		require.True(bres.CheckTx.IsOK())
+		require.True(bres.DeliverTx.IsOK())
+
+		require.Equal(0, mempool.Size())
+	}
+}
+
+func TestUnconfirmedTxs(t *testing.T) {
+	_, _, tx := MakeTxKV()
+
+	ch := make(chan *abci.Response, 1)
+	mempool := node.Mempool()
+	err := mempool.CheckTx(tx, func(resp *abci.Response) { ch <- resp }, mempl.TxInfo{})
+	require.NoError(t, err)
+
+	// wait for tx to arrive in mempoool.
+	select {
+	case <-ch:
+	case <-time.After(5 * time.Second):
+		t.Error("Timed out waiting for CheckTx callback")
+	}
+
+	for _, c := range GetClients() {
+		mc := c.(client.MempoolClient)
+		limit := 1
+		res, err := mc.UnconfirmedTxs(context.Background(), &limit)
+		require.NoError(t, err)
+
+		assert.Equal(t, 1, res.Count)
+		assert.Equal(t, 1, res.Total)
+		assert.Equal(t, mempool.SizeBytes(), res.TotalBytes)
+		assert.Exactly(t, types.Txs{tx}, types.Txs(res.Txs))
+	}
+
+	mempool.Flush()
+}
+
+func TestNumUnconfirmedTxs(t *testing.T) {
+	_, _, tx := MakeTxKV()
+
+	ch := make(chan *abci.Response, 1)
+	mempool := node.Mempool()
+	err := mempool.CheckTx(tx, func(resp *abci.Response) { ch <- resp }, mempl.TxInfo{})
+	require.NoError(t, err)
+
+	// wait for tx to arrive in mempoool.
+	select {
+	case <-ch:
+	case <-time.After(5 * time.Second):
+		t.Error("Timed out waiting for CheckTx callback")
+	}
+
+	mempoolSize := mempool.Size()
+	for i, c := range GetClients() {
+		mc, ok := c.(client.MempoolClient)
+		require.True(t, ok, "%d", i)
+		res, err := mc.NumUnconfirmedTxs(context.Background())
+		require.Nil(t, err, "%d: %+v", i, err)
+
+		assert.Equal(t, mempoolSize, res.Count)
+		assert.Equal(t, mempoolSize, res.Total)
+		assert.Equal(t, mempool.SizeBytes(), res.TotalBytes)
+	}
+
+	mempool.Flush()
+}
+
+func TestCheckTx(t *testing.T) {
+	mempool := node.Mempool()
+
+	for _, c := range GetClients() {
+		_, _, tx := MakeTxKV()
+
+		res, err := c.CheckTx(context.Background(), tx)
+		require.NoError(t, err)
+		assert.Equal(t, abci.CodeTypeOK, res.Code)
+
+		assert.Equal(t, 0, mempool.Size(), "mempool must be empty")
+	}
+}
+
+func TestTx(t *testing.T) {
+	// first we broadcast a tx
+	c := getHTTPClient()
+	_, _, tx := MakeTxKV()
+	bres, err := c.BroadcastTxCommit(context.Background(), tx)
+	require.Nil(t, err, "%+v", err)
+
+	txHeight := bres.Height
+	txHash := bres.Hash
+
+	anotherTxHash := types.Tx("a different tx").Hash()
+
+	cases := []struct {
+		valid bool
+		prove bool
+		hash  []byte
+	}{
+		// only valid if correct hash provided
+		{true, false, txHash},
+		{true, true, txHash},
+		{false, false, anotherTxHash},
+		{false, true, anotherTxHash},
+		{false, false, nil},
+		{false, true, nil},
+	}
+
+	for i, c := range GetClients() {
+		for j, tc := range cases {
+			t.Logf("client %d, case %d", i, j)
+
+			// now we query for the tx.
+			// since there's only one tx, we know index=0.
+			ptx, err := c.Tx(context.Background(), tc.hash, tc.prove)
+
+			if !tc.valid {
+				require.NotNil(t, err)
+			} else {
+				require.Nil(t, err, "%+v", err)
+				assert.EqualValues(t, txHeight, ptx.Height)
+				assert.EqualValues(t, tx, ptx.Tx)
+				assert.Zero(t, ptx.Index)
+				assert.True(t, ptx.TxResult.IsOK())
+				assert.EqualValues(t, txHash, ptx.Hash)
+
+				// time to verify the proof
+				proof := ptx.Proof
+				if tc.prove && assert.EqualValues(t, tx, proof.Data) {
+					assert.NoError(t, proof.Proof.Verify(proof.RootHash, txHash))
+				}
+			}
+		}
+	}
+}
+
+func TestTxSearchWithTimeout(t *testing.T) {
+	// Get a client with a time-out of 10 secs.
+	timeoutClient := getHTTPClientWithTimeout(10)
+
+	_, _, tx := MakeTxKV()
+	_, err := timeoutClient.BroadcastTxCommit(context.Background(), tx)
+	require.NoError(t, err)
+
+	// query using a compositeKey (see kvstore application)
+	result, err := timeoutClient.TxSearch(context.Background(), "app.creator='Cosmoshi Netowoko'", false, nil, nil, "asc")
+	require.Nil(t, err)
+	require.Greater(t, len(result.Txs), 0, "expected a lot of transactions")
+}
+
+func TestTxSearch(t *testing.T) {
+	c := getHTTPClient()
+
+	// first we broadcast a few txs
+	for i := 0; i < 10; i++ {
+		_, _, tx := MakeTxKV()
+		_, err := c.BroadcastTxCommit(context.Background(), tx)
+		require.NoError(t, err)
+	}
+
+	// since we're not using an isolated test server, we'll have lingering transactions
+	// from other tests as well
+	result, err := c.TxSearch(context.Background(), "tx.height >= 0", true, nil, nil, "asc")
+	require.NoError(t, err)
+	txCount := len(result.Txs)
+
+	// pick out the last tx to have something to search for in tests
+	find := result.Txs[len(result.Txs)-1]
+	anotherTxHash := types.Tx("a different tx").Hash()
+
+	for i, c := range GetClients() {
+		t.Logf("client %d", i)
+
+		// now we query for the tx.
+		result, err := c.TxSearch(context.Background(), fmt.Sprintf("tx.hash='%v'", find.Hash), true, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Len(t, result.Txs, 1)
+		require.Equal(t, find.Hash, result.Txs[0].Hash)
+
+		ptx := result.Txs[0]
+		assert.EqualValues(t, find.Height, ptx.Height)
+		assert.EqualValues(t, find.Tx, ptx.Tx)
+		assert.Zero(t, ptx.Index)
+		assert.True(t, ptx.TxResult.IsOK())
+		assert.EqualValues(t, find.Hash, ptx.Hash)
+
+		// time to verify the proof
+		if assert.EqualValues(t, find.Tx, ptx.Proof.Data) {
+			assert.NoError(t, ptx.Proof.Proof.Verify(ptx.Proof.RootHash, find.Hash))
+		}
+
+		// query by height
+		result, err = c.TxSearch(context.Background(), fmt.Sprintf("tx.height=%d", find.Height), true, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Len(t, result.Txs, 1)
+
+		// query for non existing tx
+		result, err = c.TxSearch(context.Background(), fmt.Sprintf("tx.hash='%X'", anotherTxHash), false, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Len(t, result.Txs, 0)
+
+		// query using a compositeKey (see kvstore application)
+		result, err = c.TxSearch(context.Background(), "app.creator='Cosmoshi Netowoko'", false, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Greater(t, len(result.Txs), 0, "expected a lot of transactions")
+
+		// query using an index key
+		result, err = c.TxSearch(context.Background(), "app.index_key='index is working'", false, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Greater(t, len(result.Txs), 0, "expected a lot of transactions")
+
+		// query using an noindex key
+		result, err = c.TxSearch(context.Background(), "app.noindex_key='index is working'", false, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Equal(t, len(result.Txs), 0, "expected a lot of transactions")
+
+		// query using a compositeKey (see kvstore application) and height
+		result, err = c.TxSearch(context.Background(),
+			"app.creator='Cosmoshi Netowoko' AND tx.height<10000", true, nil, nil, "asc")
+		require.Nil(t, err)
+		require.Greater(t, len(result.Txs), 0, "expected a lot of transactions")
+
+		// query a non existing tx with page 1 and txsPerPage 1
+		perPage := 1
+		result, err = c.TxSearch(context.Background(), "app.creator='Cosmoshi Neetowoko'", true, nil, &perPage, "asc")
+		require.Nil(t, err)
+		require.Len(t, result.Txs, 0)
+
+		// check sorting
+		result, err = c.TxSearch(context.Background(), "tx.height >= 1", false, nil, nil, "asc")
+		require.Nil(t, err)
+		for k := 0; k < len(result.Txs)-1; k++ {
+			require.LessOrEqual(t, result.Txs[k].Height, result.Txs[k+1].Height)
+			require.LessOrEqual(t, result.Txs[k].Index, result.Txs[k+1].Index)
+		}
+
+		result, err = c.TxSearch(context.Background(), "tx.height >= 1", false, nil, nil, "desc")
+		require.Nil(t, err)
+		for k := 0; k < len(result.Txs)-1; k++ {
+			require.GreaterOrEqual(t, result.Txs[k].Height, result.Txs[k+1].Height)
+			require.GreaterOrEqual(t, result.Txs[k].Index, result.Txs[k+1].Index)
+		}
+		// check pagination
+		perPage = 3
+		var (
+			seen      = map[int64]bool{}
+			maxHeight int64
+			pages     = int(math.Ceil(float64(txCount) / float64(perPage)))
+		)
+
+		for page := 1; page <= pages; page++ {
+			page := page
+			result, err := c.TxSearch(context.Background(), "tx.height >= 1", false, &page, &perPage, "asc")
+			require.NoError(t, err)
+			if page < pages {
+				require.Len(t, result.Txs, perPage)
+			} else {
+				require.LessOrEqual(t, len(result.Txs), perPage)
+			}
+			require.Equal(t, txCount, result.TotalCount)
+			for _, tx := range result.Txs {
+				require.False(t, seen[tx.Height],
+					"Found duplicate height %v in page %v", tx.Height, page)
+				require.Greater(t, tx.Height, maxHeight,
+					"Found decreasing height %v (max seen %v) in page %v", tx.Height, maxHeight, page)
+				seen[tx.Height] = true
+				maxHeight = tx.Height
+			}
+		}
+		require.Len(t, seen, txCount)
+	}
+}
+
+func TestBatchedJSONRPCCalls(t *testing.T) {
+	c := getHTTPClient()
+	testBatchedJSONRPCCalls(t, c)
+}
+
+func testBatchedJSONRPCCalls(t *testing.T, c *rpchttp.HTTP) {
+	k1, v1, tx1 := MakeTxKV()
+	k2, v2, tx2 := MakeTxKV()
+
+	batch := c.NewBatch()
+	r1, err := batch.BroadcastTxCommit(context.Background(), tx1)
+	require.NoError(t, err)
+	r2, err := batch.BroadcastTxCommit(context.Background(), tx2)
+	require.NoError(t, err)
+	require.Equal(t, 2, batch.Count())
+	bresults, err := batch.Send(ctx)
+	require.NoError(t, err)
+	require.Len(t, bresults, 2)
+	require.Equal(t, 0, batch.Count())
+
+	bresult1, ok := bresults[0].(*ctypes.ResultBroadcastTxCommit)
+	require.True(t, ok)
+	require.Equal(t, *bresult1, *r1)
+	bresult2, ok := bresults[1].(*ctypes.ResultBroadcastTxCommit)
+	require.True(t, ok)
+	require.Equal(t, *bresult2, *r2)
+	apph := tmmath.MaxInt64(bresult1.Height, bresult2.Height) + 1
+
+	err = client.WaitForHeight(c, apph, nil)
+	require.NoError(t, err)
+
+	q1, err := batch.ABCIQuery(context.Background(), "/key", k1)
+	require.NoError(t, err)
+	q2, err := batch.ABCIQuery(context.Background(), "/key", k2)
+	require.NoError(t, err)
+	require.Equal(t, 2, batch.Count())
+	qresults, err := batch.Send(ctx)
+	require.NoError(t, err)
+	require.Len(t, qresults, 2)
+	require.Equal(t, 0, batch.Count())
+
+	qresult1, ok := qresults[0].(*ctypes.ResultABCIQuery)
+	require.True(t, ok)
+	require.Equal(t, *qresult1, *q1)
+	qresult2, ok := qresults[1].(*ctypes.ResultABCIQuery)
+	require.True(t, ok)
+	require.Equal(t, *qresult2, *q2)
+
+	require.Equal(t, qresult1.Response.Key, k1)
+	require.Equal(t, qresult2.Response.Key, k2)
+	require.Equal(t, qresult1.Response.Value, v1)
+	require.Equal(t, qresult2.Response.Value, v2)
+}
+
+func TestBatchedJSONRPCCallsCancellation(t *testing.T) {
+	c := getHTTPClient()
+	_, _, tx1 := MakeTxKV()
+	_, _, tx2 := MakeTxKV()
+
+	batch := c.NewBatch()
+	_, err := batch.BroadcastTxCommit(context.Background(), tx1)
+	require.NoError(t, err)
+	_, err = batch.BroadcastTxCommit(context.Background(), tx2)
+	require.NoError(t, err)
+	// we should have 2 requests waiting
+	require.Equal(t, 2, batch.Count())
+	// we want to make sure we cleared 2 pending requests
+	require.Equal(t, 2, batch.Clear())
+	// now there should be no batched requests
+	require.Equal(t, 0, batch.Count())
+}
+
+func TestSendingEmptyRequestBatch(t *testing.T) {
+	c := getHTTPClient()
+	batch := c.NewBatch()
+	_, err := batch.Send(ctx)
+	require.Error(t, err, "sending an empty batch of JSON RPC requests should result in an error")
+}
+
+func TestClearingEmptyRequestBatch(t *testing.T) {
+	c := getHTTPClient()
+	batch := c.NewBatch()
+	require.Zero(t, batch.Clear(), "clearing an empty batch of JSON RPC requests should result in a 0 result")
+}
+
+func TestConcurrentJSONRPCBatching(t *testing.T) {
+	var wg sync.WaitGroup
+	c := getHTTPClient()
+	for i := 0; i < 50; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			testBatchedJSONRPCCalls(t, c)
+		}()
+	}
+	wg.Wait()
+}

internal/rpc/client/types.go

@@ -0,0 +1,11 @@
+package client
+
+// ABCIQueryOptions can be used to provide options for ABCIQuery call other
+// than the DefaultABCIQueryOptions.
+type ABCIQueryOptions struct {
+	Height int64
+	Prove  bool
+}
+
+// DefaultABCIQueryOptions are latest height (0) and prove false.
+var DefaultABCIQueryOptions = ABCIQueryOptions{Height: 0, Prove: false}

internal/rpc/core/CONTRIBUTING.md

@@ -0,0 +1,4 @@
+# Openapi docs
+
+Do not forget to update ../openapi/openapi.yaml if making changes to any
+endpoint.

internal/rpc/core/README.md

@@ -0,0 +1,18 @@
+# Tendermint RPC
+
+## Pagination
+
+Requests that return multiple items will be paginated to 30 items by default.
+You can specify further pages with the ?page parameter. You can also set a
+custom page size up to 100 with the ?per_page parameter.
+
+## Subscribing to events
+
+The user can subscribe to events emitted by Tendermint, using `/subscribe`. If
+the maximum number of clients is reached or the client has too many
+subscriptions, an error will be returned. The subscription timeout is 5 sec.
+Each subscription has a buffer to accommodate short bursts of events or some
+slowness in clients. If the buffer gets full, the subscription will be canceled
+("client is not pulling messages fast enough"). If Tendermint exits, all
+subscriptions are canceled ("Tendermint exited"). The user can unsubscribe
+using either `/unsubscribe` or `/unsubscribe_all`.

internal/rpc/core/abci.go

@@ -0,0 +1,42 @@
+package core
+
+import (
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/proxy"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+// ABCIQuery queries the application for some information.
+// More: https://docs.tendermint.com/master/rpc/#/ABCI/abci_query
+func ABCIQuery(
+	ctx *rpctypes.Context,
+	path string,
+	data bytes.HexBytes,
+	height int64,
+	prove bool,
+) (*ctypes.ResultABCIQuery, error) {
+	resQuery, err := env.ProxyAppQuery.QuerySync(abci.RequestQuery{
+		Path:   path,
+		Data:   data,
+		Height: height,
+		Prove:  prove,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &ctypes.ResultABCIQuery{Response: *resQuery}, nil
+}
+
+// ABCIInfo gets some info about the application.
+// More: https://docs.tendermint.com/master/rpc/#/ABCI/abci_info
+func ABCIInfo(ctx *rpctypes.Context) (*ctypes.ResultABCIInfo, error) {
+	resInfo, err := env.ProxyAppQuery.InfoSync(proxy.RequestInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ctypes.ResultABCIInfo{Response: *resInfo}, nil
+}

internal/rpc/core/blocks.go

@@ -0,0 +1,225 @@
+package core
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	blockidxnull "github.com/tendermint/tendermint/state/indexer/block/null"
+	"github.com/tendermint/tendermint/types"
+)
+
+// BlockchainInfo gets block headers for minHeight <= height <= maxHeight.
+// Block headers are returned in descending order (highest first).
+// More: https://docs.tendermint.com/master/rpc/#/Info/blockchain
+func BlockchainInfo(ctx *rpctypes.Context, minHeight, maxHeight int64) (*ctypes.ResultBlockchainInfo, error) {
+	// maximum 20 block metas
+	const limit int64 = 20
+	var err error
+	minHeight, maxHeight, err = filterMinMax(
+		env.BlockStore.Base(),
+		env.BlockStore.Height(),
+		minHeight,
+		maxHeight,
+		limit)
+	if err != nil {
+		return nil, err
+	}
+	env.Logger.Debug("BlockchainInfoHandler", "maxHeight", maxHeight, "minHeight", minHeight)
+
+	blockMetas := []*types.BlockMeta{}
+	for height := maxHeight; height >= minHeight; height-- {
+		blockMeta := env.BlockStore.LoadBlockMeta(height)
+		blockMetas = append(blockMetas, blockMeta)
+	}
+
+	return &ctypes.ResultBlockchainInfo{
+		LastHeight: env.BlockStore.Height(),
+		BlockMetas: blockMetas}, nil
+}
+
+// error if either min or max are negative or min > max
+// if 0, use blockstore base for min, latest block height for max
+// enforce limit.
+func filterMinMax(base, height, min, max, limit int64) (int64, int64, error) {
+	// filter negatives
+	if min < 0 || max < 0 {
+		return min, max, fmt.Errorf("heights must be non-negative")
+	}
+
+	// adjust for default values
+	if min == 0 {
+		min = 1
+	}
+	if max == 0 {
+		max = height
+	}
+
+	// limit max to the height
+	max = tmmath.MinInt64(height, max)
+
+	// limit min to the base
+	min = tmmath.MaxInt64(base, min)
+
+	// limit min to within `limit` of max
+	// so the total number of blocks returned will be `limit`
+	min = tmmath.MaxInt64(min, max-limit+1)
+
+	if min > max {
+		return min, max, fmt.Errorf("min height %d can't be greater than max height %d", min, max)
+	}
+	return min, max, nil
+}
+
+// Block gets block at a given height.
+// If no height is provided, it will fetch the latest block.
+// More: https://docs.tendermint.com/master/rpc/#/Info/block
+func Block(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlock, error) {
+	height, err := getHeight(env.BlockStore.Height(), heightPtr)
+	if err != nil {
+		return nil, err
+	}
+
+	block := env.BlockStore.LoadBlock(height)
+	blockMeta := env.BlockStore.LoadBlockMeta(height)
+	if blockMeta == nil {
+		return &ctypes.ResultBlock{BlockID: types.BlockID{}, Block: block}, nil
+	}
+	return &ctypes.ResultBlock{BlockID: blockMeta.BlockID, Block: block}, nil
+}
+
+// BlockByHash gets block by hash.
+// More: https://docs.tendermint.com/master/rpc/#/Info/block_by_hash
+func BlockByHash(ctx *rpctypes.Context, hash []byte) (*ctypes.ResultBlock, error) {
+	block := env.BlockStore.LoadBlockByHash(hash)
+	if block == nil {
+		return &ctypes.ResultBlock{BlockID: types.BlockID{}, Block: nil}, nil
+	}
+	// If block is not nil, then blockMeta can't be nil.
+	blockMeta := env.BlockStore.LoadBlockMeta(block.Height)
+	return &ctypes.ResultBlock{BlockID: blockMeta.BlockID, Block: block}, nil
+}
+
+// Commit gets block commit at a given height.
+// If no height is provided, it will fetch the commit for the latest block.
+// More: https://docs.tendermint.com/master/rpc/#/Info/commit
+func Commit(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultCommit, error) {
+	height, err := getHeight(env.BlockStore.Height(), heightPtr)
+	if err != nil {
+		return nil, err
+	}
+
+	blockMeta := env.BlockStore.LoadBlockMeta(height)
+	if blockMeta == nil {
+		return nil, nil
+	}
+	header := blockMeta.Header
+
+	// If the next block has not been committed yet,
+	// use a non-canonical commit
+	if height == env.BlockStore.Height() {
+		commit := env.BlockStore.LoadSeenCommit(height)
+		return ctypes.NewResultCommit(&header, commit, false), nil
+	}
+
+	// Return the canonical commit (comes from the block at height+1)
+	commit := env.BlockStore.LoadBlockCommit(height)
+	return ctypes.NewResultCommit(&header, commit, true), nil
+}
+
+// BlockResults gets ABCIResults at a given height.
+// If no height is provided, it will fetch results for the latest block.
+//
+// Results are for the height of the block containing the txs.
+// Thus response.results.deliver_tx[5] is the results of executing
+// getBlock(h).Txs[5]
+// More: https://docs.tendermint.com/master/rpc/#/Info/block_results
+func BlockResults(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultBlockResults, error) {
+	height, err := getHeight(env.BlockStore.Height(), heightPtr)
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := env.StateStore.LoadABCIResponses(height)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ctypes.ResultBlockResults{
+		Height:                height,
+		TxsResults:            results.DeliverTxs,
+		BeginBlockEvents:      results.BeginBlock.Events,
+		EndBlockEvents:        results.EndBlock.Events,
+		ValidatorUpdates:      results.EndBlock.ValidatorUpdates,
+		ConsensusParamUpdates: results.EndBlock.ConsensusParamUpdates,
+	}, nil
+}
+
+// BlockSearch searches for a paginated set of blocks matching BeginBlock and
+// EndBlock event search criteria.
+func BlockSearch(
+	ctx *rpctypes.Context,
+	query string,
+	pagePtr, perPagePtr *int,
+	orderBy string,
+) (*ctypes.ResultBlockSearch, error) {
+
+	// skip if block indexing is disabled
+	if _, ok := env.BlockIndexer.(*blockidxnull.BlockerIndexer); ok {
+		return nil, errors.New("block indexing is disabled")
+	}
+
+	q, err := tmquery.New(query)
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := env.BlockIndexer.Search(ctx.Context(), q)
+	if err != nil {
+		return nil, err
+	}
+
+	// sort results (must be done before pagination)
+	switch orderBy {
+	case "desc", "":
+		sort.Slice(results, func(i, j int) bool { return results[i] > results[j] })
+
+	case "asc":
+		sort.Slice(results, func(i, j int) bool { return results[i] < results[j] })
+
+	default:
+		return nil, errors.New("expected order_by to be either `asc` or `desc` or empty")
+	}
+
+	// paginate results
+	totalCount := len(results)
+	perPage := validatePerPage(perPagePtr)
+
+	page, err := validatePage(pagePtr, perPage, totalCount)
+	if err != nil {
+		return nil, err
+	}
+
+	skipCount := validateSkipCount(page, perPage)
+	pageSize := tmmath.MinInt(perPage, totalCount-skipCount)
+
+	apiResults := make([]*ctypes.ResultBlock, 0, pageSize)
+	for i := skipCount; i < skipCount+pageSize; i++ {
+		block := env.BlockStore.LoadBlock(results[i])
+		if block != nil {
+			blockMeta := env.BlockStore.LoadBlockMeta(block.Height)
+			if blockMeta != nil {
+				apiResults = append(apiResults, &ctypes.ResultBlock{
+					Block:   block,
+					BlockID: blockMeta.BlockID,
+				})
+			}
+		}
+	}
+
+	return &ctypes.ResultBlockSearch{Blocks: apiResults, TotalCount: totalCount}, nil
+}

internal/rpc/core/blocks_test.go

@@ -0,0 +1,134 @@
+package core
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestBlockchainInfo(t *testing.T) {
+	cases := []struct {
+		min, max     int64
+		base, height int64
+		limit        int64
+		resultLength int64
+		wantErr      bool
+	}{
+
+		// min > max
+		{0, 0, 0, 0, 10, 0, true},  // min set to 1
+		{0, 1, 0, 0, 10, 0, true},  // max set to height (0)
+		{0, 0, 0, 1, 10, 1, false}, // max set to height (1)
+		{2, 0, 0, 1, 10, 0, true},  // max set to height (1)
+		{2, 1, 0, 5, 10, 0, true},
+
+		// negative
+		{1, 10, 0, 14, 10, 10, false}, // control
+		{-1, 10, 0, 14, 10, 0, true},
+		{1, -10, 0, 14, 10, 0, true},
+		{-9223372036854775808, -9223372036854775788, 0, 100, 20, 0, true},
+
+		// check base
+		{1, 1, 1, 1, 1, 1, false},
+		{2, 5, 3, 5, 5, 3, false},
+
+		// check limit and height
+		{1, 1, 0, 1, 10, 1, false},
+		{1, 1, 0, 5, 10, 1, false},
+		{2, 2, 0, 5, 10, 1, false},
+		{1, 2, 0, 5, 10, 2, false},
+		{1, 5, 0, 1, 10, 1, false},
+		{1, 5, 0, 10, 10, 5, false},
+		{1, 15, 0, 10, 10, 10, false},
+		{1, 15, 0, 15, 10, 10, false},
+		{1, 15, 0, 15, 20, 15, false},
+		{1, 20, 0, 15, 20, 15, false},
+		{1, 20, 0, 20, 20, 20, false},
+	}
+
+	for i, c := range cases {
+		caseString := fmt.Sprintf("test %d failed", i)
+		min, max, err := filterMinMax(c.base, c.height, c.min, c.max, c.limit)
+		if c.wantErr {
+			require.Error(t, err, caseString)
+		} else {
+			require.NoError(t, err, caseString)
+			require.Equal(t, 1+max-min, c.resultLength, caseString)
+		}
+	}
+}
+
+func TestBlockResults(t *testing.T) {
+	results := &tmstate.ABCIResponses{
+		DeliverTxs: []*abci.ResponseDeliverTx{
+			{Code: 0, Data: []byte{0x01}, Log: "ok"},
+			{Code: 0, Data: []byte{0x02}, Log: "ok"},
+			{Code: 1, Log: "not ok"},
+		},
+		EndBlock:   &abci.ResponseEndBlock{},
+		BeginBlock: &abci.ResponseBeginBlock{},
+	}
+
+	env = &Environment{}
+	env.StateStore = sm.NewStore(dbm.NewMemDB())
+	err := env.StateStore.SaveABCIResponses(100, results)
+	require.NoError(t, err)
+	env.BlockStore = mockBlockStore{height: 100}
+
+	testCases := []struct {
+		height  int64
+		wantErr bool
+		wantRes *ctypes.ResultBlockResults
+	}{
+		{-1, true, nil},
+		{0, true, nil},
+		{101, true, nil},
+		{100, false, &ctypes.ResultBlockResults{
+			Height:                100,
+			TxsResults:            results.DeliverTxs,
+			BeginBlockEvents:      results.BeginBlock.Events,
+			EndBlockEvents:        results.EndBlock.Events,
+			ValidatorUpdates:      results.EndBlock.ValidatorUpdates,
+			ConsensusParamUpdates: results.EndBlock.ConsensusParamUpdates,
+		}},
+	}
+
+	for _, tc := range testCases {
+		res, err := BlockResults(&rpctypes.Context{}, &tc.height)
+		if tc.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+			assert.Equal(t, tc.wantRes, res)
+		}
+	}
+}
+
+type mockBlockStore struct {
+	height int64
+}
+
+func (mockBlockStore) Base() int64                                       { return 1 }
+func (store mockBlockStore) Height() int64                               { return store.height }
+func (store mockBlockStore) Size() int64                                 { return store.height }
+func (mockBlockStore) LoadBaseMeta() *types.BlockMeta                    { return nil }
+func (mockBlockStore) LoadBlockMeta(height int64) *types.BlockMeta       { return nil }
+func (mockBlockStore) LoadBlock(height int64) *types.Block               { return nil }
+func (mockBlockStore) LoadBlockByHash(hash []byte) *types.Block          { return nil }
+func (mockBlockStore) LoadBlockPart(height int64, index int) *types.Part { return nil }
+func (mockBlockStore) LoadBlockCommit(height int64) *types.Commit        { return nil }
+func (mockBlockStore) LoadSeenCommit(height int64) *types.Commit         { return nil }
+func (mockBlockStore) PruneBlocks(height int64) (uint64, error)          { return 0, nil }
+func (mockBlockStore) SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) {
+}

internal/rpc/core/consensus.go

@@ -0,0 +1,108 @@
+package core
+
+import (
+	cm "github.com/tendermint/tendermint/consensus"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Validators gets the validator set at the given block height.
+//
+// If no height is provided, it will fetch the latest validator set. Note the
+// validators are sorted by their voting power - this is the canonical order
+// for the validators in the set as used in computing their Merkle root.
+//
+// More: https://docs.tendermint.com/master/rpc/#/Info/validators
+func Validators(ctx *rpctypes.Context, heightPtr *int64, pagePtr, perPagePtr *int) (*ctypes.ResultValidators, error) {
+	// The latest validator that we know is the NextValidator of the last block.
+	height, err := getHeight(latestUncommittedHeight(), heightPtr)
+	if err != nil {
+		return nil, err
+	}
+
+	validators, err := env.StateStore.LoadValidators(height)
+	if err != nil {
+		return nil, err
+	}
+
+	totalCount := len(validators.Validators)
+	perPage := validatePerPage(perPagePtr)
+	page, err := validatePage(pagePtr, perPage, totalCount)
+	if err != nil {
+		return nil, err
+	}
+
+	skipCount := validateSkipCount(page, perPage)
+
+	v := validators.Validators[skipCount : skipCount+tmmath.MinInt(perPage, totalCount-skipCount)]
+
+	return &ctypes.ResultValidators{
+		BlockHeight: height,
+		Validators:  v,
+		Count:       len(v),
+		Total:       totalCount}, nil
+}
+
+// DumpConsensusState dumps consensus state.
+// UNSTABLE
+// More: https://docs.tendermint.com/master/rpc/#/Info/dump_consensus_state
+func DumpConsensusState(ctx *rpctypes.Context) (*ctypes.ResultDumpConsensusState, error) {
+	// Get Peer consensus states.
+	peers := env.P2PPeers.Peers().List()
+	peerStates := make([]ctypes.PeerStateInfo, len(peers))
+	for i, peer := range peers {
+		peerState, ok := peer.Get(types.PeerStateKey).(*cm.PeerState)
+		if !ok { // peer does not have a state yet
+			continue
+		}
+		peerStateJSON, err := peerState.ToJSON()
+		if err != nil {
+			return nil, err
+		}
+		peerStates[i] = ctypes.PeerStateInfo{
+			// Peer basic info.
+			NodeAddress: peer.SocketAddr().String(),
+			// Peer consensus state.
+			PeerState: peerStateJSON,
+		}
+	}
+	// Get self round state.
+	roundState, err := env.ConsensusState.GetRoundStateJSON()
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultDumpConsensusState{
+		RoundState: roundState,
+		Peers:      peerStates}, nil
+}
+
+// ConsensusState returns a concise summary of the consensus state.
+// UNSTABLE
+// More: https://docs.tendermint.com/master/rpc/#/Info/consensus_state
+func ConsensusState(ctx *rpctypes.Context) (*ctypes.ResultConsensusState, error) {
+	// Get self round state.
+	bz, err := env.ConsensusState.GetRoundStateSimpleJSON()
+	return &ctypes.ResultConsensusState{RoundState: bz}, err
+}
+
+// ConsensusParams gets the consensus parameters at the given block height.
+// If no height is provided, it will fetch the latest consensus params.
+// More: https://docs.tendermint.com/master/rpc/#/Info/consensus_params
+func ConsensusParams(ctx *rpctypes.Context, heightPtr *int64) (*ctypes.ResultConsensusParams, error) {
+	// The latest consensus params that we know is the consensus params after the
+	// last block.
+	height, err := getHeight(latestUncommittedHeight(), heightPtr)
+	if err != nil {
+		return nil, err
+	}
+
+	consensusParams, err := env.StateStore.LoadConsensusParams(height)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultConsensusParams{
+		BlockHeight:     height,
+		ConsensusParams: consensusParams}, nil
+}

internal/rpc/core/dev.go

@@ -0,0 +1,12 @@
+package core
+
+import (
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+// UnsafeFlushMempool removes all transactions from the mempool.
+func UnsafeFlushMempool(ctx *rpctypes.Context) (*ctypes.ResultUnsafeFlushMempool, error) {
+	env.Mempool.Flush()
+	return &ctypes.ResultUnsafeFlushMempool{}, nil
+}

internal/rpc/core/doc.go

@@ -0,0 +1,45 @@
+/*
+Package core defines the Tendermint RPC endpoints.
+
+Tendermint ships with its own JSONRPC library -
+https://github.com/tendermint/tendermint/tree/master/rpc/jsonrpc.
+
+## Get the list
+
+An HTTP Get request to the root RPC endpoint shows a list of available endpoints.
+
+```bash
+curl 'localhost:26657'
+```
+
+> Response:
+
+```plain
+Available endpoints:
+/abci_info
+/dump_consensus_state
+/genesis
+/net_info
+/num_unconfirmed_txs
+/status
+/health
+/unconfirmed_txs
+/unsafe_flush_mempool
+/validators
+
+Endpoints that require arguments:
+/abci_query?path=_&data=_&prove=_
+/block?height=_
+/blockchain?minHeight=_&maxHeight=_
+/broadcast_tx_async?tx=_
+/broadcast_tx_commit?tx=_
+/broadcast_tx_sync?tx=_
+/commit?height=_
+/dial_seeds?seeds=_
+/dial_persistent_peers?persistent_peers=_
+/subscribe?event=_
+/tx?hash=_&prove=_
+/unsubscribe?event=_
+```
+*/
+package core

internal/rpc/core/doc_template.txt

@@ -0,0 +1,8 @@
+{{with .PDoc}}
+{{comment_md .Doc}}
+{{example_html $ ""}}
+
+{{range .Funcs}}{{$name_html := html .Name}}## [{{$name_html}}]({{posLink_url $ .Decl}})
+{{comment_md .Doc}}{{end}}
+{{end}}
+---

internal/rpc/core/env.go

@@ -0,0 +1,207 @@
+package core
+
+import (
+	"encoding/base64"
+	"fmt"
+	"time"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/consensus"
+	"github.com/tendermint/tendermint/crypto"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	mempl "github.com/tendermint/tendermint/mempool"
+	"github.com/tendermint/tendermint/p2p"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/state/indexer"
+	"github.com/tendermint/tendermint/state/txindex"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// see README
+	defaultPerPage = 30
+	maxPerPage     = 100
+
+	// SubscribeTimeout is the maximum time we wait to subscribe for an event.
+	// must be less than the server's write timeout (see rpcserver.DefaultConfig)
+	SubscribeTimeout = 5 * time.Second
+
+	// genesisChunkSize is the maximum size, in bytes, of each
+	// chunk in the genesis structure for the chunked API
+	genesisChunkSize = 16 * 1024 * 1024 // 16
+)
+
+var (
+	// set by Node
+	env *Environment
+)
+
+// SetEnvironment sets up the given Environment.
+// It will race if multiple Node call SetEnvironment.
+func SetEnvironment(e *Environment) {
+	env = e
+}
+
+//----------------------------------------------
+// These interfaces are used by RPC and must be thread safe
+
+type Consensus interface {
+	GetState() sm.State
+	GetValidators() (int64, []*types.Validator)
+	GetLastHeight() int64
+	GetRoundStateJSON() ([]byte, error)
+	GetRoundStateSimpleJSON() ([]byte, error)
+}
+
+type transport interface {
+	Listeners() []string
+	IsListening() bool
+	NodeInfo() p2p.NodeInfo
+}
+
+type peers interface {
+	AddPersistentPeers([]string) error
+	AddUnconditionalPeerIDs([]string) error
+	AddPrivatePeerIDs([]string) error
+	DialPeersAsync([]string) error
+	Peers() p2p.IPeerSet
+}
+
+//----------------------------------------------
+// Environment contains objects and interfaces used by the RPC. It is expected
+// to be setup once during startup.
+type Environment struct {
+	// external, thread safe interfaces
+	ProxyAppQuery   proxy.AppConnQuery
+	ProxyAppMempool proxy.AppConnMempool
+
+	// interfaces defined in types and above
+	StateStore     sm.Store
+	BlockStore     sm.BlockStore
+	EvidencePool   sm.EvidencePool
+	ConsensusState Consensus
+	P2PPeers       peers
+	P2PTransport   transport
+
+	// objects
+	PubKey           crypto.PubKey
+	GenDoc           *types.GenesisDoc // cache the genesis structure
+	TxIndexer        txindex.TxIndexer
+	BlockIndexer     indexer.BlockIndexer
+	ConsensusReactor *consensus.Reactor
+	EventBus         *types.EventBus // thread safe
+	Mempool          mempl.Mempool
+
+	Logger log.Logger
+
+	Config cfg.RPCConfig
+
+	// cache of chunked genesis data.
+	genChunks []string
+}
+
+//----------------------------------------------
+
+func validatePage(pagePtr *int, perPage, totalCount int) (int, error) {
+	if perPage < 1 {
+		panic(fmt.Sprintf("zero or negative perPage: %d", perPage))
+	}
+
+	if pagePtr == nil { // no page parameter
+		return 1, nil
+	}
+
+	pages := ((totalCount - 1) / perPage) + 1
+	if pages == 0 {
+		pages = 1 // one page (even if it's empty)
+	}
+	page := *pagePtr
+	if page <= 0 || page > pages {
+		return 1, fmt.Errorf("page should be within [1, %d] range, given %d", pages, page)
+	}
+
+	return page, nil
+}
+
+func validatePerPage(perPagePtr *int) int {
+	if perPagePtr == nil { // no per_page parameter
+		return defaultPerPage
+	}
+
+	perPage := *perPagePtr
+	if perPage < 1 {
+		return defaultPerPage
+	} else if perPage > maxPerPage {
+		return maxPerPage
+	}
+	return perPage
+}
+
+// InitGenesisChunks configures the environment and should be called on service
+// startup.
+func InitGenesisChunks() error {
+	if env.genChunks != nil {
+		return nil
+	}
+
+	if env.GenDoc == nil {
+		return nil
+	}
+
+	data, err := tmjson.Marshal(env.GenDoc)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < len(data); i += genesisChunkSize {
+		end := i + genesisChunkSize
+
+		if end > len(data) {
+			end = len(data)
+		}
+
+		env.genChunks = append(env.genChunks, base64.StdEncoding.EncodeToString(data[i:end]))
+	}
+
+	return nil
+}
+
+func validateSkipCount(page, perPage int) int {
+	skipCount := (page - 1) * perPage
+	if skipCount < 0 {
+		return 0
+	}
+
+	return skipCount
+}
+
+// latestHeight can be either latest committed or uncommitted (+1) height.
+func getHeight(latestHeight int64, heightPtr *int64) (int64, error) {
+	if heightPtr != nil {
+		height := *heightPtr
+		if height <= 0 {
+			return 0, fmt.Errorf("height must be greater than 0, but got %d", height)
+		}
+		if height > latestHeight {
+			return 0, fmt.Errorf("height %d must be less than or equal to the current blockchain height %d",
+				height, latestHeight)
+		}
+		base := env.BlockStore.Base()
+		if height < base {
+			return 0, fmt.Errorf("height %d is not available, lowest height is %d",
+				height, base)
+		}
+		return height, nil
+	}
+	return latestHeight, nil
+}
+
+func latestUncommittedHeight() int64 {
+	nodeIsSyncing := env.ConsensusReactor.WaitSync()
+	if nodeIsSyncing {
+		return env.BlockStore.Height()
+	}
+	return env.BlockStore.Height() + 1
+}

internal/rpc/core/env_test.go

@@ -0,0 +1,82 @@
+package core
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestPaginationPage(t *testing.T) {
+	cases := []struct {
+		totalCount int
+		perPage    int
+		page       int
+		newPage    int
+		expErr     bool
+	}{
+		{0, 10, 1, 1, false},
+
+		{0, 10, 0, 1, false},
+		{0, 10, 1, 1, false},
+		{0, 10, 2, 0, true},
+
+		{5, 10, -1, 0, true},
+		{5, 10, 0, 1, false},
+		{5, 10, 1, 1, false},
+		{5, 10, 2, 0, true},
+		{5, 10, 2, 0, true},
+
+		{5, 5, 1, 1, false},
+		{5, 5, 2, 0, true},
+		{5, 5, 3, 0, true},
+
+		{5, 3, 2, 2, false},
+		{5, 3, 3, 0, true},
+
+		{5, 2, 2, 2, false},
+		{5, 2, 3, 3, false},
+		{5, 2, 4, 0, true},
+	}
+
+	for _, c := range cases {
+		p, err := validatePage(&c.page, c.perPage, c.totalCount)
+		if c.expErr {
+			assert.Error(t, err)
+			continue
+		}
+
+		assert.Equal(t, c.newPage, p, fmt.Sprintf("%v", c))
+	}
+
+	// nil case
+	p, err := validatePage(nil, 1, 1)
+	if assert.NoError(t, err) {
+		assert.Equal(t, 1, p)
+	}
+}
+
+func TestPaginationPerPage(t *testing.T) {
+	cases := []struct {
+		totalCount int
+		perPage    int
+		newPerPage int
+	}{
+		{5, 0, defaultPerPage},
+		{5, 1, 1},
+		{5, 2, 2},
+		{5, defaultPerPage, defaultPerPage},
+		{5, maxPerPage - 1, maxPerPage - 1},
+		{5, maxPerPage, maxPerPage},
+		{5, maxPerPage + 1, maxPerPage},
+	}
+
+	for _, c := range cases {
+		p := validatePerPage(&c.perPage)
+		assert.Equal(t, c.newPerPage, p, fmt.Sprintf("%v", c))
+	}
+
+	// nil case
+	p := validatePerPage(nil)
+	assert.Equal(t, defaultPerPage, p)
+}

internal/rpc/core/events.go

@@ -0,0 +1,130 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+const (
+	// maxQueryLength is the maximum length of a query string that will be
+	// accepted. This is just a safety check to avoid outlandish queries.
+	maxQueryLength = 512
+)
+
+// Subscribe for events via WebSocket.
+// More: https://docs.tendermint.com/master/rpc/#/Websocket/subscribe
+func Subscribe(ctx *rpctypes.Context, query string) (*ctypes.ResultSubscribe, error) {
+	addr := ctx.RemoteAddr()
+
+	if env.EventBus.NumClients() >= env.Config.MaxSubscriptionClients {
+		return nil, fmt.Errorf("max_subscription_clients %d reached", env.Config.MaxSubscriptionClients)
+	} else if env.EventBus.NumClientSubscriptions(addr) >= env.Config.MaxSubscriptionsPerClient {
+		return nil, fmt.Errorf("max_subscriptions_per_client %d reached", env.Config.MaxSubscriptionsPerClient)
+	} else if len(query) > maxQueryLength {
+		return nil, errors.New("maximum query length exceeded")
+	}
+
+	env.Logger.Info("Subscribe to query", "remote", addr, "query", query)
+
+	q, err := tmquery.New(query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse query: %w", err)
+	}
+
+	subCtx, cancel := context.WithTimeout(ctx.Context(), SubscribeTimeout)
+	defer cancel()
+
+	sub, err := env.EventBus.Subscribe(subCtx, addr, q, env.Config.SubscriptionBufferSize)
+	if err != nil {
+		return nil, err
+	}
+
+	closeIfSlow := env.Config.CloseOnSlowClient
+
+	// Capture the current ID, since it can change in the future.
+	subscriptionID := ctx.JSONReq.ID
+	go func() {
+		for {
+			select {
+			case msg := <-sub.Out():
+				var (
+					resultEvent = &ctypes.ResultEvent{Query: query, Data: msg.Data(), Events: msg.Events()}
+					resp        = rpctypes.NewRPCSuccessResponse(subscriptionID, resultEvent)
+				)
+				writeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+				defer cancel()
+				if err := ctx.WSConn.WriteRPCResponse(writeCtx, resp); err != nil {
+					env.Logger.Info("Can't write response (slow client)",
+						"to", addr, "subscriptionID", subscriptionID, "err", err)
+
+					if closeIfSlow {
+						var (
+							err  = errors.New("subscription was cancelled (reason: slow client)")
+							resp = rpctypes.RPCServerError(subscriptionID, err)
+						)
+						if !ctx.WSConn.TryWriteRPCResponse(resp) {
+							env.Logger.Info("Can't write response (slow client)",
+								"to", addr, "subscriptionID", subscriptionID, "err", err)
+						}
+						return
+					}
+				}
+			case <-sub.Cancelled():
+				if sub.Err() != tmpubsub.ErrUnsubscribed {
+					var reason string
+					if sub.Err() == nil {
+						reason = "Tendermint exited"
+					} else {
+						reason = sub.Err().Error()
+					}
+					var (
+						err  = fmt.Errorf("subscription was cancelled (reason: %s)", reason)
+						resp = rpctypes.RPCServerError(subscriptionID, err)
+					)
+					if !ctx.WSConn.TryWriteRPCResponse(resp) {
+						env.Logger.Info("Can't write response (slow client)",
+							"to", addr, "subscriptionID", subscriptionID, "err", err)
+					}
+				}
+				return
+			}
+		}
+	}()
+
+	return &ctypes.ResultSubscribe{}, nil
+}
+
+// Unsubscribe from events via WebSocket.
+// More: https://docs.tendermint.com/master/rpc/#/Websocket/unsubscribe
+func Unsubscribe(ctx *rpctypes.Context, query string) (*ctypes.ResultUnsubscribe, error) {
+	addr := ctx.RemoteAddr()
+	env.Logger.Info("Unsubscribe from query", "remote", addr, "query", query)
+	q, err := tmquery.New(query)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse query: %w", err)
+	}
+	err = env.EventBus.Unsubscribe(context.Background(), addr, q)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultUnsubscribe{}, nil
+}
+
+// UnsubscribeAll from all events via WebSocket.
+// More: https://docs.tendermint.com/master/rpc/#/Websocket/unsubscribe_all
+func UnsubscribeAll(ctx *rpctypes.Context) (*ctypes.ResultUnsubscribe, error) {
+	addr := ctx.RemoteAddr()
+	env.Logger.Info("Unsubscribe from all", "remote", addr)
+	err := env.EventBus.UnsubscribeAll(context.Background(), addr)
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultUnsubscribe{}, nil
+}

internal/rpc/core/evidence.go

@@ -0,0 +1,27 @@
+package core
+
+import (
+	"errors"
+	"fmt"
+
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// BroadcastEvidence broadcasts evidence of the misbehavior.
+// More: https://docs.tendermint.com/master/rpc/#/Info/broadcast_evidence
+func BroadcastEvidence(ctx *rpctypes.Context, ev types.Evidence) (*ctypes.ResultBroadcastEvidence, error) {
+	if ev == nil {
+		return nil, errors.New("no evidence was provided")
+	}
+
+	if err := ev.ValidateBasic(); err != nil {
+		return nil, fmt.Errorf("evidence.ValidateBasic failed: %w", err)
+	}
+
+	if err := env.EvidencePool.AddEvidence(ev); err != nil {
+		return nil, fmt.Errorf("failed to add evidence: %w", err)
+	}
+	return &ctypes.ResultBroadcastEvidence{Hash: ev.Hash()}, nil
+}

internal/rpc/core/health.go

@@ -0,0 +1,13 @@
+package core
+
+import (
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+// Health gets node health. Returns empty result (200 OK) on success, no
+// response - in case of an error.
+// More: https://docs.tendermint.com/master/rpc/#/Info/health
+func Health(ctx *rpctypes.Context) (*ctypes.ResultHealth, error) {
+	return &ctypes.ResultHealth{}, nil
+}

internal/rpc/core/mempool.go

@@ -0,0 +1,183 @@
+package core
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	mempl "github.com/tendermint/tendermint/mempool"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+// NOTE: tx should be signed, but this is only checked at the app level (not by Tendermint!)
+
+// BroadcastTxAsync returns right away, with no response. Does not wait for
+// CheckTx nor DeliverTx results.
+// More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_async
+func BroadcastTxAsync(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	err := env.Mempool.CheckTx(tx, nil, mempl.TxInfo{})
+
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultBroadcastTx{Hash: tx.Hash()}, nil
+}
+
+// BroadcastTxSync returns with the response from CheckTx. Does not wait for
+// DeliverTx result.
+// More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_sync
+func BroadcastTxSync(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadcastTx, error) {
+	resCh := make(chan *abci.Response, 1)
+	err := env.Mempool.CheckTx(tx, func(res *abci.Response) {
+		select {
+		case <-ctx.Context().Done():
+		case resCh <- res:
+		}
+
+	}, mempl.TxInfo{})
+	if err != nil {
+		return nil, err
+	}
+
+	select {
+	case <-ctx.Context().Done():
+		return nil, fmt.Errorf("broadcast confirmation not received: %w", ctx.Context().Err())
+	case res := <-resCh:
+		r := res.GetCheckTx()
+		return &ctypes.ResultBroadcastTx{
+			Code:      r.Code,
+			Data:      r.Data,
+			Log:       r.Log,
+			Codespace: r.Codespace,
+			Hash:      tx.Hash(),
+		}, nil
+	}
+}
+
+// BroadcastTxCommit returns with the responses from CheckTx and DeliverTx.
+// More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_commit
+func BroadcastTxCommit(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) {
+	subscriber := ctx.RemoteAddr()
+
+	if env.EventBus.NumClients() >= env.Config.MaxSubscriptionClients {
+		return nil, fmt.Errorf("max_subscription_clients %d reached", env.Config.MaxSubscriptionClients)
+	} else if env.EventBus.NumClientSubscriptions(subscriber) >= env.Config.MaxSubscriptionsPerClient {
+		return nil, fmt.Errorf("max_subscriptions_per_client %d reached", env.Config.MaxSubscriptionsPerClient)
+	}
+
+	// Subscribe to tx being committed in block.
+	subCtx, cancel := context.WithTimeout(ctx.Context(), SubscribeTimeout)
+	defer cancel()
+	q := types.EventQueryTxFor(tx)
+	deliverTxSub, err := env.EventBus.Subscribe(subCtx, subscriber, q)
+	if err != nil {
+		err = fmt.Errorf("failed to subscribe to tx: %w", err)
+		env.Logger.Error("Error on broadcast_tx_commit", "err", err)
+		return nil, err
+	}
+	defer func() {
+		if err := env.EventBus.Unsubscribe(context.Background(), subscriber, q); err != nil {
+			env.Logger.Error("Error unsubscribing from eventBus", "err", err)
+		}
+	}()
+
+	// Broadcast tx and wait for CheckTx result
+	checkTxResCh := make(chan *abci.Response, 1)
+	err = env.Mempool.CheckTx(tx, func(res *abci.Response) {
+		select {
+		case <-ctx.Context().Done():
+		case checkTxResCh <- res:
+		}
+	}, mempl.TxInfo{})
+	if err != nil {
+		env.Logger.Error("Error on broadcastTxCommit", "err", err)
+		return nil, fmt.Errorf("error on broadcastTxCommit: %v", err)
+	}
+	select {
+	case <-ctx.Context().Done():
+		return nil, fmt.Errorf("broadcast confirmation not received: %w", ctx.Context().Err())
+	case checkTxResMsg := <-checkTxResCh:
+		checkTxRes := checkTxResMsg.GetCheckTx()
+		if checkTxRes.Code != abci.CodeTypeOK {
+			return &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   *checkTxRes,
+				DeliverTx: abci.ResponseDeliverTx{},
+				Hash:      tx.Hash(),
+			}, nil
+		}
+
+		// Wait for the tx to be included in a block or timeout.
+		select {
+		case msg := <-deliverTxSub.Out(): // The tx was included in a block.
+			deliverTxRes := msg.Data().(types.EventDataTx)
+			return &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   *checkTxRes,
+				DeliverTx: deliverTxRes.Result,
+				Hash:      tx.Hash(),
+				Height:    deliverTxRes.Height,
+			}, nil
+		case <-deliverTxSub.Cancelled():
+			var reason string
+			if deliverTxSub.Err() == nil {
+				reason = "Tendermint exited"
+			} else {
+				reason = deliverTxSub.Err().Error()
+			}
+			err = fmt.Errorf("deliverTxSub was cancelled (reason: %s)", reason)
+			env.Logger.Error("Error on broadcastTxCommit", "err", err)
+			return &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   *checkTxRes,
+				DeliverTx: abci.ResponseDeliverTx{},
+				Hash:      tx.Hash(),
+			}, err
+		case <-time.After(env.Config.TimeoutBroadcastTxCommit):
+			err = errors.New("timed out waiting for tx to be included in a block")
+			env.Logger.Error("Error on broadcastTxCommit", "err", err)
+			return &ctypes.ResultBroadcastTxCommit{
+				CheckTx:   *checkTxRes,
+				DeliverTx: abci.ResponseDeliverTx{},
+				Hash:      tx.Hash(),
+			}, err
+		}
+	}
+}
+
+// UnconfirmedTxs gets unconfirmed transactions (maximum ?limit entries)
+// including their number.
+// More: https://docs.tendermint.com/master/rpc/#/Info/unconfirmed_txs
+func UnconfirmedTxs(ctx *rpctypes.Context, limitPtr *int) (*ctypes.ResultUnconfirmedTxs, error) {
+	// reuse per_page validator
+	limit := validatePerPage(limitPtr)
+
+	txs := env.Mempool.ReapMaxTxs(limit)
+	return &ctypes.ResultUnconfirmedTxs{
+		Count:      len(txs),
+		Total:      env.Mempool.Size(),
+		TotalBytes: env.Mempool.SizeBytes(),
+		Txs:        txs}, nil
+}
+
+// NumUnconfirmedTxs gets number of unconfirmed transactions.
+// More: https://docs.tendermint.com/master/rpc/#/Info/num_unconfirmed_txs
+func NumUnconfirmedTxs(ctx *rpctypes.Context) (*ctypes.ResultUnconfirmedTxs, error) {
+	return &ctypes.ResultUnconfirmedTxs{
+		Count:      env.Mempool.Size(),
+		Total:      env.Mempool.Size(),
+		TotalBytes: env.Mempool.SizeBytes()}, nil
+}
+
+// CheckTx checks the transaction without executing it. The transaction won't
+// be added to the mempool either.
+// More: https://docs.tendermint.com/master/rpc/#/Tx/check_tx
+func CheckTx(ctx *rpctypes.Context, tx types.Tx) (*ctypes.ResultCheckTx, error) {
+	res, err := env.ProxyAppMempool.CheckTxSync(abci.RequestCheckTx{Tx: tx})
+	if err != nil {
+		return nil, err
+	}
+	return &ctypes.ResultCheckTx{ResponseCheckTx: *res}, nil
+}

internal/rpc/core/net.go

@@ -0,0 +1,139 @@
+package core
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/tendermint/tendermint/p2p"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+// NetInfo returns network info.
+// More: https://docs.tendermint.com/master/rpc/#/Info/net_info
+func NetInfo(ctx *rpctypes.Context) (*ctypes.ResultNetInfo, error) {
+	peersList := env.P2PPeers.Peers().List()
+	peers := make([]ctypes.Peer, 0, len(peersList))
+	for _, peer := range peersList {
+		nodeInfo, ok := peer.NodeInfo().(p2p.DefaultNodeInfo)
+		if !ok {
+			return nil, fmt.Errorf("peer.NodeInfo() is not DefaultNodeInfo")
+		}
+		peers = append(peers, ctypes.Peer{
+			NodeInfo:         nodeInfo,
+			IsOutbound:       peer.IsOutbound(),
+			ConnectionStatus: peer.Status(),
+			RemoteIP:         peer.RemoteIP().String(),
+		})
+	}
+	// TODO: Should we include PersistentPeers and Seeds in here?
+	// PRO: useful info
+	// CON: privacy
+	return &ctypes.ResultNetInfo{
+		Listening: env.P2PTransport.IsListening(),
+		Listeners: env.P2PTransport.Listeners(),
+		NPeers:    len(peers),
+		Peers:     peers,
+	}, nil
+}
+
+// UnsafeDialSeeds dials the given seeds (comma-separated id@IP:PORT).
+func UnsafeDialSeeds(ctx *rpctypes.Context, seeds []string) (*ctypes.ResultDialSeeds, error) {
+	if len(seeds) == 0 {
+		return &ctypes.ResultDialSeeds{}, errors.New("no seeds provided")
+	}
+	env.Logger.Info("DialSeeds", "seeds", seeds)
+	if err := env.P2PPeers.DialPeersAsync(seeds); err != nil {
+		return &ctypes.ResultDialSeeds{}, err
+	}
+	return &ctypes.ResultDialSeeds{Log: "Dialing seeds in progress. See /net_info for details"}, nil
+}
+
+// UnsafeDialPeers dials the given peers (comma-separated id@IP:PORT),
+// optionally making them persistent.
+func UnsafeDialPeers(ctx *rpctypes.Context, peers []string, persistent, unconditional, private bool) (
+	*ctypes.ResultDialPeers, error) {
+	if len(peers) == 0 {
+		return &ctypes.ResultDialPeers{}, errors.New("no peers provided")
+	}
+
+	ids, err := getIDs(peers)
+	if err != nil {
+		return &ctypes.ResultDialPeers{}, err
+	}
+
+	env.Logger.Info("DialPeers", "peers", peers, "persistent",
+		persistent, "unconditional", unconditional, "private", private)
+
+	if persistent {
+		if err := env.P2PPeers.AddPersistentPeers(peers); err != nil {
+			return &ctypes.ResultDialPeers{}, err
+		}
+	}
+
+	if private {
+		if err := env.P2PPeers.AddPrivatePeerIDs(ids); err != nil {
+			return &ctypes.ResultDialPeers{}, err
+		}
+	}
+
+	if unconditional {
+		if err := env.P2PPeers.AddUnconditionalPeerIDs(ids); err != nil {
+			return &ctypes.ResultDialPeers{}, err
+		}
+	}
+
+	if err := env.P2PPeers.DialPeersAsync(peers); err != nil {
+		return &ctypes.ResultDialPeers{}, err
+	}
+
+	return &ctypes.ResultDialPeers{Log: "Dialing peers in progress. See /net_info for details"}, nil
+}
+
+// Genesis returns genesis file.
+// More: https://docs.tendermint.com/master/rpc/#/Info/genesis
+func Genesis(ctx *rpctypes.Context) (*ctypes.ResultGenesis, error) {
+	if len(env.genChunks) > 1 {
+		return nil, errors.New("genesis response is large, please use the genesis_chunked API instead")
+	}
+
+	return &ctypes.ResultGenesis{Genesis: env.GenDoc}, nil
+}
+
+func GenesisChunked(ctx *rpctypes.Context, chunk uint) (*ctypes.ResultGenesisChunk, error) {
+	if env.genChunks == nil {
+		return nil, fmt.Errorf("service configuration error, genesis chunks are not initialized")
+	}
+
+	if len(env.genChunks) == 0 {
+		return nil, fmt.Errorf("service configuration error, there are no chunks")
+	}
+
+	id := int(chunk)
+
+	if id > len(env.genChunks)-1 {
+		return nil, fmt.Errorf("there are %d chunks, %d is invalid", len(env.genChunks)-1, id)
+	}
+
+	return &ctypes.ResultGenesisChunk{
+		TotalChunks: len(env.genChunks),
+		ChunkNumber: id,
+		Data:        env.genChunks[id],
+	}, nil
+}
+
+func getIDs(peers []string) ([]string, error) {
+	ids := make([]string, 0, len(peers))
+
+	for _, peer := range peers {
+
+		spl := strings.Split(peer, "@")
+		if len(spl) != 2 {
+			return nil, p2p.ErrNetAddressNoID{Addr: peer}
+		}
+		ids = append(ids, spl[0])
+
+	}
+	return ids, nil
+}

internal/rpc/core/net_test.go

@@ -0,0 +1,87 @@
+package core
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/p2p"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+func TestUnsafeDialSeeds(t *testing.T) {
+	sw := p2p.MakeSwitch(cfg.DefaultP2PConfig(), 1, "testing", "123.123.123",
+		func(n int, sw *p2p.Switch) *p2p.Switch { return sw })
+	err := sw.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := sw.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	env.Logger = log.TestingLogger()
+	env.P2PPeers = sw
+
+	testCases := []struct {
+		seeds []string
+		isErr bool
+	}{
+		{[]string{}, true},
+		{[]string{"d51fb70907db1c6c2d5237e78379b25cf1a37ab4@127.0.0.1:41198"}, false},
+		{[]string{"127.0.0.1:41198"}, true},
+	}
+
+	for _, tc := range testCases {
+		res, err := UnsafeDialSeeds(&rpctypes.Context{}, tc.seeds)
+		if tc.isErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+			assert.NotNil(t, res)
+		}
+	}
+}
+
+func TestUnsafeDialPeers(t *testing.T) {
+	sw := p2p.MakeSwitch(cfg.DefaultP2PConfig(), 1, "testing", "123.123.123",
+		func(n int, sw *p2p.Switch) *p2p.Switch { return sw })
+	sw.SetAddrBook(&p2p.AddrBookMock{
+		Addrs:        make(map[string]struct{}),
+		OurAddrs:     make(map[string]struct{}),
+		PrivateAddrs: make(map[string]struct{}),
+	})
+	err := sw.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := sw.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	env.Logger = log.TestingLogger()
+	env.P2PPeers = sw
+
+	testCases := []struct {
+		peers                               []string
+		persistence, unconditional, private bool
+		isErr                               bool
+	}{
+		{[]string{}, false, false, false, true},
+		{[]string{"d51fb70907db1c6c2d5237e78379b25cf1a37ab4@127.0.0.1:41198"}, true, true, true, false},
+		{[]string{"127.0.0.1:41198"}, true, true, false, true},
+	}
+
+	for _, tc := range testCases {
+		res, err := UnsafeDialPeers(&rpctypes.Context{}, tc.peers, tc.persistence, tc.unconditional, tc.private)
+		if tc.isErr {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+			assert.NotNil(t, res)
+		}
+	}
+}

internal/rpc/core/routes.go

@@ -0,0 +1,57 @@
+package core
+
+import (
+	rpc "github.com/tendermint/tendermint/rpc/jsonrpc/server"
+)
+
+// TODO: better system than "unsafe" prefix
+
+// Routes is a map of available routes.
+var Routes = map[string]*rpc.RPCFunc{
+	// subscribe/unsubscribe are reserved for websocket events.
+	"subscribe":       rpc.NewWSRPCFunc(Subscribe, "query"),
+	"unsubscribe":     rpc.NewWSRPCFunc(Unsubscribe, "query"),
+	"unsubscribe_all": rpc.NewWSRPCFunc(UnsubscribeAll, ""),
+
+	// info API
+	"health":               rpc.NewRPCFunc(Health, ""),
+	"status":               rpc.NewRPCFunc(Status, ""),
+	"net_info":             rpc.NewRPCFunc(NetInfo, ""),
+	"blockchain":           rpc.NewRPCFunc(BlockchainInfo, "minHeight,maxHeight"),
+	"genesis":              rpc.NewRPCFunc(Genesis, ""),
+	"genesis_chunked":      rpc.NewRPCFunc(GenesisChunked, "chunk"),
+	"block":                rpc.NewRPCFunc(Block, "height"),
+	"block_by_hash":        rpc.NewRPCFunc(BlockByHash, "hash"),
+	"block_results":        rpc.NewRPCFunc(BlockResults, "height"),
+	"commit":               rpc.NewRPCFunc(Commit, "height"),
+	"check_tx":             rpc.NewRPCFunc(CheckTx, "tx"),
+	"tx":                   rpc.NewRPCFunc(Tx, "hash,prove"),
+	"tx_search":            rpc.NewRPCFunc(TxSearch, "query,prove,page,per_page,order_by"),
+	"block_search":         rpc.NewRPCFunc(BlockSearch, "query,page,per_page,order_by"),
+	"validators":           rpc.NewRPCFunc(Validators, "height,page,per_page"),
+	"dump_consensus_state": rpc.NewRPCFunc(DumpConsensusState, ""),
+	"consensus_state":      rpc.NewRPCFunc(ConsensusState, ""),
+	"consensus_params":     rpc.NewRPCFunc(ConsensusParams, "height"),
+	"unconfirmed_txs":      rpc.NewRPCFunc(UnconfirmedTxs, "limit"),
+	"num_unconfirmed_txs":  rpc.NewRPCFunc(NumUnconfirmedTxs, ""),
+
+	// tx broadcast API
+	"broadcast_tx_commit": rpc.NewRPCFunc(BroadcastTxCommit, "tx"),
+	"broadcast_tx_sync":   rpc.NewRPCFunc(BroadcastTxSync, "tx"),
+	"broadcast_tx_async":  rpc.NewRPCFunc(BroadcastTxAsync, "tx"),
+
+	// abci API
+	"abci_query": rpc.NewRPCFunc(ABCIQuery, "path,data,height,prove"),
+	"abci_info":  rpc.NewRPCFunc(ABCIInfo, ""),
+
+	// evidence API
+	"broadcast_evidence": rpc.NewRPCFunc(BroadcastEvidence, "evidence"),
+}
+
+// AddUnsafeRoutes adds unsafe routes.
+func AddUnsafeRoutes() {
+	// control API
+	Routes["dial_seeds"] = rpc.NewRPCFunc(UnsafeDialSeeds, "seeds")
+	Routes["dial_peers"] = rpc.NewRPCFunc(UnsafeDialPeers, "peers,persistent,unconditional,private")
+	Routes["unsafe_flush_mempool"] = rpc.NewRPCFunc(UnsafeFlushMempool, "")
+}

internal/rpc/core/status.go

@@ -0,0 +1,85 @@
+package core
+
+import (
+	"time"
+
+	tmbytes "github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/p2p"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Status returns Tendermint status including node info, pubkey, latest block
+// hash, app hash, block height and time.
+// More: https://docs.tendermint.com/master/rpc/#/Info/status
+func Status(ctx *rpctypes.Context) (*ctypes.ResultStatus, error) {
+	var (
+		earliestBlockHeight   int64
+		earliestBlockHash     tmbytes.HexBytes
+		earliestAppHash       tmbytes.HexBytes
+		earliestBlockTimeNano int64
+	)
+
+	if earliestBlockMeta := env.BlockStore.LoadBaseMeta(); earliestBlockMeta != nil {
+		earliestBlockHeight = earliestBlockMeta.Header.Height
+		earliestAppHash = earliestBlockMeta.Header.AppHash
+		earliestBlockHash = earliestBlockMeta.BlockID.Hash
+		earliestBlockTimeNano = earliestBlockMeta.Header.Time.UnixNano()
+	}
+
+	var (
+		latestBlockHash     tmbytes.HexBytes
+		latestAppHash       tmbytes.HexBytes
+		latestBlockTimeNano int64
+
+		latestHeight = env.BlockStore.Height()
+	)
+
+	if latestHeight != 0 {
+		if latestBlockMeta := env.BlockStore.LoadBlockMeta(latestHeight); latestBlockMeta != nil {
+			latestBlockHash = latestBlockMeta.BlockID.Hash
+			latestAppHash = latestBlockMeta.Header.AppHash
+			latestBlockTimeNano = latestBlockMeta.Header.Time.UnixNano()
+		}
+	}
+
+	// Return the very last voting power, not the voting power of this validator
+	// during the last block.
+	var votingPower int64
+	if val := validatorAtHeight(latestUncommittedHeight()); val != nil {
+		votingPower = val.VotingPower
+	}
+
+	result := &ctypes.ResultStatus{
+		NodeInfo: env.P2PTransport.NodeInfo().(p2p.DefaultNodeInfo),
+		SyncInfo: ctypes.SyncInfo{
+			LatestBlockHash:     latestBlockHash,
+			LatestAppHash:       latestAppHash,
+			LatestBlockHeight:   latestHeight,
+			LatestBlockTime:     time.Unix(0, latestBlockTimeNano),
+			EarliestBlockHash:   earliestBlockHash,
+			EarliestAppHash:     earliestAppHash,
+			EarliestBlockHeight: earliestBlockHeight,
+			EarliestBlockTime:   time.Unix(0, earliestBlockTimeNano),
+			CatchingUp:          env.ConsensusReactor.WaitSync(),
+		},
+		ValidatorInfo: ctypes.ValidatorInfo{
+			Address:     env.PubKey.Address(),
+			PubKey:      env.PubKey,
+			VotingPower: votingPower,
+		},
+	}
+
+	return result, nil
+}
+
+func validatorAtHeight(h int64) *types.Validator {
+	vals, err := env.StateStore.LoadValidators(h)
+	if err != nil {
+		return nil
+	}
+	privValAddress := env.PubKey.Address()
+	_, val := vals.GetByAddress(privValAddress)
+	return val
+}

internal/rpc/core/tx.go

@@ -0,0 +1,135 @@
+package core
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	tmquery "github.com/tendermint/tendermint/libs/pubsub/query"
+	ctypes "github.com/tendermint/tendermint/rpc/core/types"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+	"github.com/tendermint/tendermint/state/txindex/null"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Tx allows you to query the transaction results. `nil` could mean the
+// transaction is in the mempool, invalidated, or was not sent in the first
+// place.
+// More: https://docs.tendermint.com/master/rpc/#/Info/tx
+func Tx(ctx *rpctypes.Context, hash []byte, prove bool) (*ctypes.ResultTx, error) {
+	// if index is disabled, return error
+	if _, ok := env.TxIndexer.(*null.TxIndex); ok {
+		return nil, fmt.Errorf("transaction indexing is disabled")
+	}
+
+	r, err := env.TxIndexer.Get(hash)
+	if err != nil {
+		return nil, err
+	}
+
+	if r == nil {
+		return nil, fmt.Errorf("tx (%X) not found", hash)
+	}
+
+	height := r.Height
+	index := r.Index
+
+	var proof types.TxProof
+	if prove {
+		block := env.BlockStore.LoadBlock(height)
+		proof = block.Data.Txs.Proof(int(index)) // XXX: overflow on 32-bit machines
+	}
+
+	return &ctypes.ResultTx{
+		Hash:     hash,
+		Height:   height,
+		Index:    index,
+		TxResult: r.Result,
+		Tx:       r.Tx,
+		Proof:    proof,
+	}, nil
+}
+
+// TxSearch allows you to query for multiple transactions results. It returns a
+// list of transactions (maximum ?per_page entries) and the total count.
+// More: https://docs.tendermint.com/master/rpc/#/Info/tx_search
+func TxSearch(
+	ctx *rpctypes.Context,
+	query string,
+	prove bool,
+	pagePtr, perPagePtr *int,
+	orderBy string,
+) (*ctypes.ResultTxSearch, error) {
+
+	// if index is disabled, return error
+	if _, ok := env.TxIndexer.(*null.TxIndex); ok {
+		return nil, errors.New("transaction indexing is disabled")
+	} else if len(query) > maxQueryLength {
+		return nil, errors.New("maximum query length exceeded")
+	}
+
+	q, err := tmquery.New(query)
+	if err != nil {
+		return nil, err
+	}
+
+	results, err := env.TxIndexer.Search(ctx.Context(), q)
+	if err != nil {
+		return nil, err
+	}
+
+	// sort results (must be done before pagination)
+	switch orderBy {
+	case "desc":
+		sort.Slice(results, func(i, j int) bool {
+			if results[i].Height == results[j].Height {
+				return results[i].Index > results[j].Index
+			}
+			return results[i].Height > results[j].Height
+		})
+	case "asc", "":
+		sort.Slice(results, func(i, j int) bool {
+			if results[i].Height == results[j].Height {
+				return results[i].Index < results[j].Index
+			}
+			return results[i].Height < results[j].Height
+		})
+	default:
+		return nil, errors.New("expected order_by to be either `asc` or `desc` or empty")
+	}
+
+	// paginate results
+	totalCount := len(results)
+	perPage := validatePerPage(perPagePtr)
+
+	page, err := validatePage(pagePtr, perPage, totalCount)
+	if err != nil {
+		return nil, err
+	}
+
+	skipCount := validateSkipCount(page, perPage)
+	pageSize := tmmath.MinInt(perPage, totalCount-skipCount)
+
+	apiResults := make([]*ctypes.ResultTx, 0, pageSize)
+	for i := skipCount; i < skipCount+pageSize; i++ {
+		r := results[i]
+
+		var proof types.TxProof
+		if prove {
+			block := env.BlockStore.LoadBlock(r.Height)
+			proof = block.Data.Txs.Proof(int(r.Index)) // XXX: overflow on 32-bit machines
+		}
+
+		apiResults = append(apiResults, &ctypes.ResultTx{
+			Hash:     types.Tx(r.Tx).Hash(),
+			Height:   r.Height,
+			Index:    r.Index,
+			TxResult: r.Result,
+			Tx:       r.Tx,
+			Proof:    proof,
+		})
+	}
+
+	return &ctypes.ResultTxSearch{Txs: apiResults, TotalCount: totalCount}, nil
+}

internal/rpc/core/types/responses.go

@@ -0,0 +1,251 @@
+package coretypes
+
+import (
+	"encoding/json"
+	"time"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/p2p"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// List of blocks
+type ResultBlockchainInfo struct {
+	LastHeight int64              `json:"last_height"`
+	BlockMetas []*types.BlockMeta `json:"block_metas"`
+}
+
+// Genesis file
+type ResultGenesis struct {
+	Genesis *types.GenesisDoc `json:"genesis"`
+}
+
+// ResultGenesisChunk is the output format for the chunked/paginated
+// interface. These chunks are produced by converting the genesis
+// document to JSON and then splitting the resulting payload into
+// 16 megabyte blocks and then base64 encoding each block.
+type ResultGenesisChunk struct {
+	ChunkNumber int    `json:"chunk"`
+	TotalChunks int    `json:"total"`
+	Data        string `json:"data"`
+}
+
+// Single block (with meta)
+type ResultBlock struct {
+	BlockID types.BlockID `json:"block_id"`
+	Block   *types.Block  `json:"block"`
+}
+
+// Commit and Header
+type ResultCommit struct {
+	types.SignedHeader `json:"signed_header"`
+	CanonicalCommit    bool `json:"canonical"`
+}
+
+// ABCI results from a block
+type ResultBlockResults struct {
+	Height                int64                     `json:"height"`
+	TxsResults            []*abci.ResponseDeliverTx `json:"txs_results"`
+	BeginBlockEvents      []abci.Event              `json:"begin_block_events"`
+	EndBlockEvents        []abci.Event              `json:"end_block_events"`
+	ValidatorUpdates      []abci.ValidatorUpdate    `json:"validator_updates"`
+	ConsensusParamUpdates *abci.ConsensusParams     `json:"consensus_param_updates"`
+}
+
+// NewResultCommit is a helper to initialize the ResultCommit with
+// the embedded struct
+func NewResultCommit(header *types.Header, commit *types.Commit,
+	canonical bool) *ResultCommit {
+
+	return &ResultCommit{
+		SignedHeader: types.SignedHeader{
+			Header: header,
+			Commit: commit,
+		},
+		CanonicalCommit: canonical,
+	}
+}
+
+// Info about the node's syncing state
+type SyncInfo struct {
+	LatestBlockHash   bytes.HexBytes `json:"latest_block_hash"`
+	LatestAppHash     bytes.HexBytes `json:"latest_app_hash"`
+	LatestBlockHeight int64          `json:"latest_block_height"`
+	LatestBlockTime   time.Time      `json:"latest_block_time"`
+
+	EarliestBlockHash   bytes.HexBytes `json:"earliest_block_hash"`
+	EarliestAppHash     bytes.HexBytes `json:"earliest_app_hash"`
+	EarliestBlockHeight int64          `json:"earliest_block_height"`
+	EarliestBlockTime   time.Time      `json:"earliest_block_time"`
+
+	CatchingUp bool `json:"catching_up"`
+}
+
+// Info about the node's validator
+type ValidatorInfo struct {
+	Address     bytes.HexBytes `json:"address"`
+	PubKey      crypto.PubKey  `json:"pub_key"`
+	VotingPower int64          `json:"voting_power"`
+}
+
+// Node Status
+type ResultStatus struct {
+	NodeInfo      p2p.DefaultNodeInfo `json:"node_info"`
+	SyncInfo      SyncInfo            `json:"sync_info"`
+	ValidatorInfo ValidatorInfo       `json:"validator_info"`
+}
+
+// Is TxIndexing enabled
+func (s *ResultStatus) TxIndexEnabled() bool {
+	if s == nil {
+		return false
+	}
+	return s.NodeInfo.Other.TxIndex == "on"
+}
+
+// Info about peer connections
+type ResultNetInfo struct {
+	Listening bool     `json:"listening"`
+	Listeners []string `json:"listeners"`
+	NPeers    int      `json:"n_peers"`
+	Peers     []Peer   `json:"peers"`
+}
+
+// Log from dialing seeds
+type ResultDialSeeds struct {
+	Log string `json:"log"`
+}
+
+// Log from dialing peers
+type ResultDialPeers struct {
+	Log string `json:"log"`
+}
+
+// A peer
+type Peer struct {
+	NodeInfo         p2p.DefaultNodeInfo  `json:"node_info"`
+	IsOutbound       bool                 `json:"is_outbound"`
+	ConnectionStatus p2p.ConnectionStatus `json:"connection_status"`
+	RemoteIP         string               `json:"remote_ip"`
+}
+
+// Validators for a height.
+type ResultValidators struct {
+	BlockHeight int64              `json:"block_height"`
+	Validators  []*types.Validator `json:"validators"`
+	// Count of actual validators in this result
+	Count int `json:"count"`
+	// Total number of validators
+	Total int `json:"total"`
+}
+
+// ConsensusParams for given height
+type ResultConsensusParams struct {
+	BlockHeight     int64                   `json:"block_height"`
+	ConsensusParams tmproto.ConsensusParams `json:"consensus_params"`
+}
+
+// Info about the consensus state.
+// UNSTABLE
+type ResultDumpConsensusState struct {
+	RoundState json.RawMessage `json:"round_state"`
+	Peers      []PeerStateInfo `json:"peers"`
+}
+
+// UNSTABLE
+type PeerStateInfo struct {
+	NodeAddress string          `json:"node_address"`
+	PeerState   json.RawMessage `json:"peer_state"`
+}
+
+// UNSTABLE
+type ResultConsensusState struct {
+	RoundState json.RawMessage `json:"round_state"`
+}
+
+// CheckTx result
+type ResultBroadcastTx struct {
+	Code      uint32         `json:"code"`
+	Data      bytes.HexBytes `json:"data"`
+	Log       string         `json:"log"`
+	Codespace string         `json:"codespace"`
+
+	Hash bytes.HexBytes `json:"hash"`
+}
+
+// CheckTx and DeliverTx results
+type ResultBroadcastTxCommit struct {
+	CheckTx   abci.ResponseCheckTx   `json:"check_tx"`
+	DeliverTx abci.ResponseDeliverTx `json:"deliver_tx"`
+	Hash      bytes.HexBytes         `json:"hash"`
+	Height    int64                  `json:"height"`
+}
+
+// ResultCheckTx wraps abci.ResponseCheckTx.
+type ResultCheckTx struct {
+	abci.ResponseCheckTx
+}
+
+// Result of querying for a tx
+type ResultTx struct {
+	Hash     bytes.HexBytes         `json:"hash"`
+	Height   int64                  `json:"height"`
+	Index    uint32                 `json:"index"`
+	TxResult abci.ResponseDeliverTx `json:"tx_result"`
+	Tx       types.Tx               `json:"tx"`
+	Proof    types.TxProof          `json:"proof,omitempty"`
+}
+
+// Result of searching for txs
+type ResultTxSearch struct {
+	Txs        []*ResultTx `json:"txs"`
+	TotalCount int         `json:"total_count"`
+}
+
+// ResultBlockSearch defines the RPC response type for a block search by events.
+type ResultBlockSearch struct {
+	Blocks     []*ResultBlock `json:"blocks"`
+	TotalCount int            `json:"total_count"`
+}
+
+// List of mempool txs
+type ResultUnconfirmedTxs struct {
+	Count      int        `json:"n_txs"`
+	Total      int        `json:"total"`
+	TotalBytes int64      `json:"total_bytes"`
+	Txs        []types.Tx `json:"txs"`
+}
+
+// Info abci msg
+type ResultABCIInfo struct {
+	Response abci.ResponseInfo `json:"response"`
+}
+
+// Query abci msg
+type ResultABCIQuery struct {
+	Response abci.ResponseQuery `json:"response"`
+}
+
+// Result of broadcasting evidence
+type ResultBroadcastEvidence struct {
+	Hash []byte `json:"hash"`
+}
+
+// empty results
+type (
+	ResultUnsafeFlushMempool struct{}
+	ResultUnsafeProfile      struct{}
+	ResultSubscribe          struct{}
+	ResultUnsubscribe        struct{}
+	ResultHealth             struct{}
+)
+
+// Event data from a subscription
+type ResultEvent struct {
+	Query  string              `json:"query"`
+	Data   types.TMEventData   `json:"data"`
+	Events map[string][]string `json:"events"`
+}

internal/rpc/core/types/responses_test.go

@@ -0,0 +1,35 @@
+package coretypes
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/tendermint/tendermint/p2p"
+)
+
+func TestStatusIndexer(t *testing.T) {
+	var status *ResultStatus
+	assert.False(t, status.TxIndexEnabled())
+
+	status = &ResultStatus{}
+	assert.False(t, status.TxIndexEnabled())
+
+	status.NodeInfo = p2p.DefaultNodeInfo{}
+	assert.False(t, status.TxIndexEnabled())
+
+	cases := []struct {
+		expected bool
+		other    p2p.DefaultNodeInfoOther
+	}{
+		{false, p2p.DefaultNodeInfoOther{}},
+		{false, p2p.DefaultNodeInfoOther{TxIndex: "aa"}},
+		{false, p2p.DefaultNodeInfoOther{TxIndex: "off"}},
+		{true, p2p.DefaultNodeInfoOther{TxIndex: "on"}},
+	}
+
+	for _, tc := range cases {
+		status.NodeInfo.Other = tc.other
+		assert.Equal(t, tc.expected, status.TxIndexEnabled())
+	}
+}

internal/rpc/grpc/api.go

@@ -0,0 +1,39 @@
+package coregrpc
+
+import (
+	"context"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	core "github.com/tendermint/tendermint/rpc/core"
+	rpctypes "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+type broadcastAPI struct {
+}
+
+func (bapi *broadcastAPI) Ping(ctx context.Context, req *RequestPing) (*ResponsePing, error) {
+	// kvstore so we can check if the server is up
+	return &ResponsePing{}, nil
+}
+
+func (bapi *broadcastAPI) BroadcastTx(ctx context.Context, req *RequestBroadcastTx) (*ResponseBroadcastTx, error) {
+	// NOTE: there's no way to get client's remote address
+	// see https://stackoverflow.com/questions/33684570/session-and-remote-ip-address-in-grpc-go
+	res, err := core.BroadcastTxCommit(&rpctypes.Context{}, req.Tx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ResponseBroadcastTx{
+		CheckTx: &abci.ResponseCheckTx{
+			Code: res.CheckTx.Code,
+			Data: res.CheckTx.Data,
+			Log:  res.CheckTx.Log,
+		},
+		DeliverTx: &abci.ResponseDeliverTx{
+			Code: res.DeliverTx.Code,
+			Data: res.DeliverTx.Data,
+			Log:  res.DeliverTx.Log,
+		},
+	}, nil
+}

internal/rpc/grpc/client_server.go

@@ -0,0 +1,39 @@
+package coregrpc
+
+import (
+	"net"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	tmnet "github.com/tendermint/tendermint/libs/net"
+)
+
+// Config is an gRPC server configuration.
+type Config struct {
+	MaxOpenConnections int
+}
+
+// StartGRPCServer starts a new gRPC BroadcastAPIServer using the given
+// net.Listener.
+// NOTE: This function blocks - you may want to call it in a go-routine.
+func StartGRPCServer(ln net.Listener) error {
+	grpcServer := grpc.NewServer()
+	RegisterBroadcastAPIServer(grpcServer, &broadcastAPI{})
+	return grpcServer.Serve(ln)
+}
+
+// StartGRPCClient dials the gRPC server using protoAddr and returns a new
+// BroadcastAPIClient.
+func StartGRPCClient(protoAddr string) BroadcastAPIClient {
+	//nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option.
+	conn, err := grpc.Dial(protoAddr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc))
+	if err != nil {
+		panic(err)
+	}
+	return NewBroadcastAPIClient(conn)
+}
+
+func dialerFunc(ctx context.Context, addr string) (net.Conn, error) {
+	return tmnet.Connect(addr)
+}

internal/rpc/grpc/grpc_test.go

@@ -0,0 +1,35 @@
+package coregrpc_test
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	core_grpc "github.com/tendermint/tendermint/rpc/grpc"
+	rpctest "github.com/tendermint/tendermint/rpc/test"
+)
+
+func TestMain(m *testing.M) {
+	// start a tendermint node in the background to test against
+	app := kvstore.NewApplication()
+	node := rpctest.StartTendermint(app)
+
+	code := m.Run()
+
+	// and shut down proper at the end
+	rpctest.StopTendermint(node)
+	os.Exit(code)
+}
+
+func TestBroadcastTx(t *testing.T) {
+	res, err := rpctest.GetGRPCClient().BroadcastTx(
+		context.Background(),
+		&core_grpc.RequestBroadcastTx{Tx: []byte("this is a tx")},
+	)
+	require.NoError(t, err)
+	require.EqualValues(t, 0, res.CheckTx.Code)
+	require.EqualValues(t, 0, res.DeliverTx.Code)
+}

internal/rpc/grpc/types.pb.go

@@ -0,0 +1,924 @@
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: tendermint/rpc/grpc/types.proto
+
+package coregrpc
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	types "github.com/tendermint/tendermint/abci/types"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	io "io"
+	math "math"
+	math_bits "math/bits"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package
+
+type RequestPing struct {
+}
+
+func (m *RequestPing) Reset()         { *m = RequestPing{} }
+func (m *RequestPing) String() string { return proto.CompactTextString(m) }
+func (*RequestPing) ProtoMessage()    {}
+func (*RequestPing) Descriptor() ([]byte, []int) {
+	return fileDescriptor_0ffff5682c662b95, []int{0}
+}
+func (m *RequestPing) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *RequestPing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_RequestPing.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *RequestPing) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_RequestPing.Merge(m, src)
+}
+func (m *RequestPing) XXX_Size() int {
+	return m.Size()
+}
+func (m *RequestPing) XXX_DiscardUnknown() {
+	xxx_messageInfo_RequestPing.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RequestPing proto.InternalMessageInfo
+
+type RequestBroadcastTx struct {
+	Tx []byte `protobuf:"bytes,1,opt,name=tx,proto3" json:"tx,omitempty"`
+}
+
+func (m *RequestBroadcastTx) Reset()         { *m = RequestBroadcastTx{} }
+func (m *RequestBroadcastTx) String() string { return proto.CompactTextString(m) }
+func (*RequestBroadcastTx) ProtoMessage()    {}
+func (*RequestBroadcastTx) Descriptor() ([]byte, []int) {
+	return fileDescriptor_0ffff5682c662b95, []int{1}
+}
+func (m *RequestBroadcastTx) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *RequestBroadcastTx) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_RequestBroadcastTx.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *RequestBroadcastTx) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_RequestBroadcastTx.Merge(m, src)
+}
+func (m *RequestBroadcastTx) XXX_Size() int {
+	return m.Size()
+}
+func (m *RequestBroadcastTx) XXX_DiscardUnknown() {
+	xxx_messageInfo_RequestBroadcastTx.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_RequestBroadcastTx proto.InternalMessageInfo
+
+func (m *RequestBroadcastTx) GetTx() []byte {
+	if m != nil {
+		return m.Tx
+	}
+	return nil
+}
+
+type ResponsePing struct {
+}
+
+func (m *ResponsePing) Reset()         { *m = ResponsePing{} }
+func (m *ResponsePing) String() string { return proto.CompactTextString(m) }
+func (*ResponsePing) ProtoMessage()    {}
+func (*ResponsePing) Descriptor() ([]byte, []int) {
+	return fileDescriptor_0ffff5682c662b95, []int{2}
+}
+func (m *ResponsePing) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ResponsePing) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ResponsePing.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ResponsePing) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ResponsePing.Merge(m, src)
+}
+func (m *ResponsePing) XXX_Size() int {
+	return m.Size()
+}
+func (m *ResponsePing) XXX_DiscardUnknown() {
+	xxx_messageInfo_ResponsePing.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ResponsePing proto.InternalMessageInfo
+
+type ResponseBroadcastTx struct {
+	CheckTx   *types.ResponseCheckTx   `protobuf:"bytes,1,opt,name=check_tx,json=checkTx,proto3" json:"check_tx,omitempty"`
+	DeliverTx *types.ResponseDeliverTx `protobuf:"bytes,2,opt,name=deliver_tx,json=deliverTx,proto3" json:"deliver_tx,omitempty"`
+}
+
+func (m *ResponseBroadcastTx) Reset()         { *m = ResponseBroadcastTx{} }
+func (m *ResponseBroadcastTx) String() string { return proto.CompactTextString(m) }
+func (*ResponseBroadcastTx) ProtoMessage()    {}
+func (*ResponseBroadcastTx) Descriptor() ([]byte, []int) {
+	return fileDescriptor_0ffff5682c662b95, []int{3}
+}
+func (m *ResponseBroadcastTx) XXX_Unmarshal(b []byte) error {
+	return m.Unmarshal(b)
+}
+func (m *ResponseBroadcastTx) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	if deterministic {
+		return xxx_messageInfo_ResponseBroadcastTx.Marshal(b, m, deterministic)
+	} else {
+		b = b[:cap(b)]
+		n, err := m.MarshalToSizedBuffer(b)
+		if err != nil {
+			return nil, err
+		}
+		return b[:n], nil
+	}
+}
+func (m *ResponseBroadcastTx) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ResponseBroadcastTx.Merge(m, src)
+}
+func (m *ResponseBroadcastTx) XXX_Size() int {
+	return m.Size()
+}
+func (m *ResponseBroadcastTx) XXX_DiscardUnknown() {
+	xxx_messageInfo_ResponseBroadcastTx.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ResponseBroadcastTx proto.InternalMessageInfo
+
+func (m *ResponseBroadcastTx) GetCheckTx() *types.ResponseCheckTx {
+	if m != nil {
+		return m.CheckTx
+	}
+	return nil
+}
+
+func (m *ResponseBroadcastTx) GetDeliverTx() *types.ResponseDeliverTx {
+	if m != nil {
+		return m.DeliverTx
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*RequestPing)(nil), "tendermint.rpc.grpc.RequestPing")
+	proto.RegisterType((*RequestBroadcastTx)(nil), "tendermint.rpc.grpc.RequestBroadcastTx")
+	proto.RegisterType((*ResponsePing)(nil), "tendermint.rpc.grpc.ResponsePing")
+	proto.RegisterType((*ResponseBroadcastTx)(nil), "tendermint.rpc.grpc.ResponseBroadcastTx")
+}
+
+func init() { proto.RegisterFile("tendermint/rpc/grpc/types.proto", fileDescriptor_0ffff5682c662b95) }
+
+var fileDescriptor_0ffff5682c662b95 = []byte{
+	// 316 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x2f, 0x49, 0xcd, 0x4b,
+	0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x2f, 0x2a, 0x48, 0xd6, 0x4f, 0x07, 0x11, 0x25, 0x95,
+	0x05, 0xa9, 0xc5, 0x7a, 0x05, 0x45, 0xf9, 0x25, 0xf9, 0x42, 0xc2, 0x08, 0x05, 0x7a, 0x45, 0x05,
+	0xc9, 0x7a, 0x20, 0x05, 0x52, 0xd2, 0x48, 0xba, 0x12, 0x93, 0x92, 0x33, 0x91, 0x75, 0x28, 0xf1,
+	0x72, 0x71, 0x07, 0xa5, 0x16, 0x96, 0xa6, 0x16, 0x97, 0x04, 0x64, 0xe6, 0xa5, 0x2b, 0xa9, 0x70,
+	0x09, 0x41, 0xb9, 0x4e, 0x45, 0xf9, 0x89, 0x29, 0xc9, 0x89, 0xc5, 0x25, 0x21, 0x15, 0x42, 0x7c,
+	0x5c, 0x4c, 0x25, 0x15, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x3c, 0x41, 0x4c, 0x25, 0x15, 0x4a, 0x7c,
+	0x5c, 0x3c, 0x41, 0xa9, 0xc5, 0x05, 0xf9, 0x79, 0xc5, 0xa9, 0x60, 0x5d, 0x53, 0x19, 0xb9, 0x84,
+	0x61, 0x02, 0xc8, 0xfa, 0xac, 0xb9, 0x38, 0x92, 0x33, 0x52, 0x93, 0xb3, 0xe3, 0xa1, 0xba, 0xb9,
+	0x8d, 0x14, 0xf4, 0x90, 0x5c, 0x08, 0x72, 0x8c, 0x1e, 0x4c, 0x9f, 0x33, 0x48, 0x61, 0x48, 0x45,
+	0x10, 0x7b, 0x32, 0x84, 0x21, 0xe4, 0xc8, 0xc5, 0x95, 0x92, 0x9a, 0x93, 0x59, 0x96, 0x5a, 0x04,
+	0xd2, 0xce, 0x04, 0xd6, 0xae, 0x84, 0x53, 0xbb, 0x0b, 0x44, 0x69, 0x48, 0x45, 0x10, 0x67, 0x0a,
+	0x8c, 0x69, 0xb4, 0x97, 0x91, 0x8b, 0x07, 0xee, 0x1e, 0xc7, 0x00, 0x4f, 0x21, 0x6f, 0x2e, 0x16,
+	0x90, 0x83, 0x85, 0x50, 0x9c, 0x01, 0x0b, 0x28, 0x3d, 0xa4, 0x80, 0x90, 0x52, 0xc4, 0xa1, 0x02,
+	0xe1, 0x6b, 0xa1, 0x04, 0x2e, 0x6e, 0x64, 0xcf, 0xaa, 0xe3, 0x33, 0x13, 0x49, 0xa1, 0x94, 0x06,
+	0x5e, 0xa3, 0x91, 0x54, 0x3a, 0xf9, 0x9c, 0x78, 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, 0xe3, 0x83,
+	0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x5c, 0x78, 0x2c, 0xc7, 0x70, 0xe3, 0xb1, 0x1c, 0x43,
+	0x94, 0x51, 0x7a, 0x66, 0x49, 0x46, 0x69, 0x92, 0x5e, 0x72, 0x7e, 0xae, 0x3e, 0x52, 0xf4, 0x62,
+	0x49, 0x1f, 0xd6, 0xc9, 0xf9, 0x45, 0xa9, 0x20, 0x46, 0x12, 0x1b, 0x38, 0xc6, 0x8d, 0x01, 0x01,
+	0x00, 0x00, 0xff, 0xff, 0xf6, 0x4b, 0x02, 0xd8, 0x46, 0x02, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// BroadcastAPIClient is the client API for BroadcastAPI service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type BroadcastAPIClient interface {
+	Ping(ctx context.Context, in *RequestPing, opts ...grpc.CallOption) (*ResponsePing, error)
+	BroadcastTx(ctx context.Context, in *RequestBroadcastTx, opts ...grpc.CallOption) (*ResponseBroadcastTx, error)
+}
+
+type broadcastAPIClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewBroadcastAPIClient(cc *grpc.ClientConn) BroadcastAPIClient {
+	return &broadcastAPIClient{cc}
+}
+
+func (c *broadcastAPIClient) Ping(ctx context.Context, in *RequestPing, opts ...grpc.CallOption) (*ResponsePing, error) {
+	out := new(ResponsePing)
+	err := c.cc.Invoke(ctx, "/tendermint.rpc.grpc.BroadcastAPI/Ping", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *broadcastAPIClient) BroadcastTx(ctx context.Context, in *RequestBroadcastTx, opts ...grpc.CallOption) (*ResponseBroadcastTx, error) {
+	out := new(ResponseBroadcastTx)
+	err := c.cc.Invoke(ctx, "/tendermint.rpc.grpc.BroadcastAPI/BroadcastTx", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// BroadcastAPIServer is the server API for BroadcastAPI service.
+type BroadcastAPIServer interface {
+	Ping(context.Context, *RequestPing) (*ResponsePing, error)
+	BroadcastTx(context.Context, *RequestBroadcastTx) (*ResponseBroadcastTx, error)
+}
+
+// UnimplementedBroadcastAPIServer can be embedded to have forward compatible implementations.
+type UnimplementedBroadcastAPIServer struct {
+}
+
+func (*UnimplementedBroadcastAPIServer) Ping(ctx context.Context, req *RequestPing) (*ResponsePing, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented")
+}
+func (*UnimplementedBroadcastAPIServer) BroadcastTx(ctx context.Context, req *RequestBroadcastTx) (*ResponseBroadcastTx, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method BroadcastTx not implemented")
+}
+
+func RegisterBroadcastAPIServer(s *grpc.Server, srv BroadcastAPIServer) {
+	s.RegisterService(&_BroadcastAPI_serviceDesc, srv)
+}
+
+func _BroadcastAPI_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RequestPing)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BroadcastAPIServer).Ping(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/tendermint.rpc.grpc.BroadcastAPI/Ping",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BroadcastAPIServer).Ping(ctx, req.(*RequestPing))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BroadcastAPI_BroadcastTx_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(RequestBroadcastTx)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BroadcastAPIServer).BroadcastTx(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/tendermint.rpc.grpc.BroadcastAPI/BroadcastTx",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BroadcastAPIServer).BroadcastTx(ctx, req.(*RequestBroadcastTx))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _BroadcastAPI_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "tendermint.rpc.grpc.BroadcastAPI",
+	HandlerType: (*BroadcastAPIServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Ping",
+			Handler:    _BroadcastAPI_Ping_Handler,
+		},
+		{
+			MethodName: "BroadcastTx",
+			Handler:    _BroadcastAPI_BroadcastTx_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "tendermint/rpc/grpc/types.proto",
+}
+
+func (m *RequestPing) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *RequestPing) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *RequestPing) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	return len(dAtA) - i, nil
+}
+
+func (m *RequestBroadcastTx) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *RequestBroadcastTx) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *RequestBroadcastTx) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if len(m.Tx) > 0 {
+		i -= len(m.Tx)
+		copy(dAtA[i:], m.Tx)
+		i = encodeVarintTypes(dAtA, i, uint64(len(m.Tx)))
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func (m *ResponsePing) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ResponsePing) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ResponsePing) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	return len(dAtA) - i, nil
+}
+
+func (m *ResponseBroadcastTx) Marshal() (dAtA []byte, err error) {
+	size := m.Size()
+	dAtA = make([]byte, size)
+	n, err := m.MarshalToSizedBuffer(dAtA[:size])
+	if err != nil {
+		return nil, err
+	}
+	return dAtA[:n], nil
+}
+
+func (m *ResponseBroadcastTx) MarshalTo(dAtA []byte) (int, error) {
+	size := m.Size()
+	return m.MarshalToSizedBuffer(dAtA[:size])
+}
+
+func (m *ResponseBroadcastTx) MarshalToSizedBuffer(dAtA []byte) (int, error) {
+	i := len(dAtA)
+	_ = i
+	var l int
+	_ = l
+	if m.DeliverTx != nil {
+		{
+			size, err := m.DeliverTx.MarshalToSizedBuffer(dAtA[:i])
+			if err != nil {
+				return 0, err
+			}
+			i -= size
+			i = encodeVarintTypes(dAtA, i, uint64(size))
+		}
+		i--
+		dAtA[i] = 0x12
+	}
+	if m.CheckTx != nil {
+		{
+			size, err := m.CheckTx.MarshalToSizedBuffer(dAtA[:i])
+			if err != nil {
+				return 0, err
+			}
+			i -= size
+			i = encodeVarintTypes(dAtA, i, uint64(size))
+		}
+		i--
+		dAtA[i] = 0xa
+	}
+	return len(dAtA) - i, nil
+}
+
+func encodeVarintTypes(dAtA []byte, offset int, v uint64) int {
+	offset -= sovTypes(v)
+	base := offset
+	for v >= 1<<7 {
+		dAtA[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	dAtA[offset] = uint8(v)
+	return base
+}
+func (m *RequestPing) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	return n
+}
+
+func (m *RequestBroadcastTx) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	l = len(m.Tx)
+	if l > 0 {
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+func (m *ResponsePing) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	return n
+}
+
+func (m *ResponseBroadcastTx) Size() (n int) {
+	if m == nil {
+		return 0
+	}
+	var l int
+	_ = l
+	if m.CheckTx != nil {
+		l = m.CheckTx.Size()
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	if m.DeliverTx != nil {
+		l = m.DeliverTx.Size()
+		n += 1 + l + sovTypes(uint64(l))
+	}
+	return n
+}
+
+func sovTypes(x uint64) (n int) {
+	return (math_bits.Len64(x|1) + 6) / 7
+}
+func sozTypes(x uint64) (n int) {
+	return sovTypes(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *RequestPing) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: RequestPing: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: RequestPing: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *RequestBroadcastTx) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: RequestBroadcastTx: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: RequestBroadcastTx: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Tx", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				byteLen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Tx = append(m.Tx[:0], dAtA[iNdEx:postIndex]...)
+			if m.Tx == nil {
+				m.Tx = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ResponsePing) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ResponsePing: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ResponsePing: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *ResponseBroadcastTx) Unmarshal(dAtA []byte) error {
+	l := len(dAtA)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= uint64(b&0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: ResponseBroadcastTx: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: ResponseBroadcastTx: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field CheckTx", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.CheckTx == nil {
+				m.CheckTx = &types.ResponseCheckTx{}
+			}
+			if err := m.CheckTx.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field DeliverTx", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				msglen |= int(b&0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthTypes
+			}
+			postIndex := iNdEx + msglen
+			if postIndex < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if m.DeliverTx == nil {
+				m.DeliverTx = &types.ResponseDeliverTx{}
+			}
+			if err := m.DeliverTx.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipTypes(dAtA[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if (skippy < 0) || (iNdEx+skippy) < 0 {
+				return ErrInvalidLengthTypes
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipTypes(dAtA []byte) (n int, err error) {
+	l := len(dAtA)
+	iNdEx := 0
+	depth := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowTypes
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := dAtA[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if dAtA[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+		case 1:
+			iNdEx += 8
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowTypes
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if length < 0 {
+				return 0, ErrInvalidLengthTypes
+			}
+			iNdEx += length
+		case 3:
+			depth++
+		case 4:
+			if depth == 0 {
+				return 0, ErrUnexpectedEndOfGroupTypes
+			}
+			depth--
+		case 5:
+			iNdEx += 4
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+		if iNdEx < 0 {
+			return 0, ErrInvalidLengthTypes
+		}
+		if depth == 0 {
+			return iNdEx, nil
+		}
+	}
+	return 0, io.ErrUnexpectedEOF
+}
+
+var (
+	ErrInvalidLengthTypes        = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowTypes          = fmt.Errorf("proto: integer overflow")
+	ErrUnexpectedEndOfGroupTypes = fmt.Errorf("proto: unexpected end of group")
+)

internal/rpc/jsonrpc/client/args_test.go

@@ -0,0 +1,39 @@
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type Tx []byte
+
+type Foo struct {
+	Bar int
+	Baz string
+}
+
+func TestArgToJSON(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+
+	cases := []struct {
+		input    interface{}
+		expected string
+	}{
+		{[]byte("1234"), "0x31323334"},
+		{Tx("654"), "0x363534"},
+		{Foo{7, "hello"}, `{"Bar":"7","Baz":"hello"}`},
+	}
+
+	for i, tc := range cases {
+		args := map[string]interface{}{"data": tc.input}
+		err := argsToJSON(args)
+		require.Nil(err, "%d: %+v", i, err)
+		require.Equal(1, len(args), "%d", i)
+		data, ok := args["data"].(string)
+		require.True(ok, "%d: %#v", i, args["data"])
+		assert.Equal(tc.expected, data, "%d", i)
+	}
+}

internal/rpc/jsonrpc/client/decode.go

@@ -0,0 +1,126 @@
+package client
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	types "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+func unmarshalResponseBytes(
+	responseBytes []byte,
+	expectedID types.JSONRPCIntID,
+	result interface{},
+) (interface{}, error) {
+
+	// Read response.  If rpc/core/types is imported, the result will unmarshal
+	// into the correct type.
+	response := &types.RPCResponse{}
+	if err := json.Unmarshal(responseBytes, response); err != nil {
+		return nil, fmt.Errorf("error unmarshalling: %w", err)
+	}
+
+	if response.Error != nil {
+		return nil, response.Error
+	}
+
+	if err := validateAndVerifyID(response, expectedID); err != nil {
+		return nil, fmt.Errorf("wrong ID: %w", err)
+	}
+
+	// Unmarshal the RawMessage into the result.
+	if err := tmjson.Unmarshal(response.Result, result); err != nil {
+		return nil, fmt.Errorf("error unmarshalling result: %w", err)
+	}
+
+	return result, nil
+}
+
+func unmarshalResponseBytesArray(
+	responseBytes []byte,
+	expectedIDs []types.JSONRPCIntID,
+	results []interface{},
+) ([]interface{}, error) {
+
+	var (
+		responses []types.RPCResponse
+	)
+
+	if err := json.Unmarshal(responseBytes, &responses); err != nil {
+		return nil, fmt.Errorf("error unmarshalling: %w", err)
+	}
+
+	// No response error checking here as there may be a mixture of successful
+	// and unsuccessful responses.
+
+	if len(results) != len(responses) {
+		return nil, fmt.Errorf(
+			"expected %d result objects into which to inject responses, but got %d",
+			len(responses),
+			len(results),
+		)
+	}
+
+	// Intersect IDs from responses with expectedIDs.
+	ids := make([]types.JSONRPCIntID, len(responses))
+	var ok bool
+	for i, resp := range responses {
+		ids[i], ok = resp.ID.(types.JSONRPCIntID)
+		if !ok {
+			return nil, fmt.Errorf("expected JSONRPCIntID, got %T", resp.ID)
+		}
+	}
+	if err := validateResponseIDs(ids, expectedIDs); err != nil {
+		return nil, fmt.Errorf("wrong IDs: %w", err)
+	}
+
+	for i := 0; i < len(responses); i++ {
+		if err := tmjson.Unmarshal(responses[i].Result, results[i]); err != nil {
+			return nil, fmt.Errorf("error unmarshalling #%d result: %w", i, err)
+		}
+	}
+
+	return results, nil
+}
+
+func validateResponseIDs(ids, expectedIDs []types.JSONRPCIntID) error {
+	m := make(map[types.JSONRPCIntID]bool, len(expectedIDs))
+	for _, expectedID := range expectedIDs {
+		m[expectedID] = true
+	}
+
+	for i, id := range ids {
+		if m[id] {
+			delete(m, id)
+		} else {
+			return fmt.Errorf("unsolicited ID #%d: %v", i, id)
+		}
+	}
+
+	return nil
+}
+
+// From the JSON-RPC 2.0 spec:
+// id: It MUST be the same as the value of the id member in the Request Object.
+func validateAndVerifyID(res *types.RPCResponse, expectedID types.JSONRPCIntID) error {
+	if err := validateResponseID(res.ID); err != nil {
+		return err
+	}
+	if expectedID != res.ID.(types.JSONRPCIntID) { // validateResponseID ensured res.ID has the right type
+		return fmt.Errorf("response ID (%d) does not match request ID (%d)", res.ID, expectedID)
+	}
+	return nil
+}
+
+func validateResponseID(id interface{}) error {
+	if id == nil {
+		return errors.New("no ID")
+	}
+	_, ok := id.(types.JSONRPCIntID)
+	if !ok {
+		return fmt.Errorf("expected JSONRPCIntID, but got: %T", id)
+	}
+	return nil
+}

internal/rpc/jsonrpc/client/encode.go

@@ -0,0 +1,46 @@
+package client
+
+import (
+	"fmt"
+	"net/url"
+	"reflect"
+
+	tmjson "github.com/tendermint/tendermint/libs/json"
+)
+
+func argsToURLValues(args map[string]interface{}) (url.Values, error) {
+	values := make(url.Values)
+	if len(args) == 0 {
+		return values, nil
+	}
+
+	err := argsToJSON(args)
+	if err != nil {
+		return nil, err
+	}
+
+	for key, val := range args {
+		values.Set(key, val.(string))
+	}
+
+	return values, nil
+}
+
+func argsToJSON(args map[string]interface{}) error {
+	for k, v := range args {
+		rt := reflect.TypeOf(v)
+		isByteSlice := rt.Kind() == reflect.Slice && rt.Elem().Kind() == reflect.Uint8
+		if isByteSlice {
+			bytes := reflect.ValueOf(v).Bytes()
+			args[k] = fmt.Sprintf("0x%X", bytes)
+			continue
+		}
+
+		data, err := tmjson.Marshal(v)
+		if err != nil {
+			return err
+		}
+		args[k] = string(data)
+	}
+	return nil
+}

internal/rpc/jsonrpc/client/http_json_client.go

@@ -0,0 +1,406 @@
+package client
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+
+	tmsync "github.com/tendermint/tendermint/libs/sync"
+	types "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+const (
+	protoHTTP  = "http"
+	protoHTTPS = "https"
+	protoWSS   = "wss"
+	protoWS    = "ws"
+	protoTCP   = "tcp"
+	protoUNIX  = "unix"
+)
+
+//-------------------------------------------------------------
+
+// Parsed URL structure
+type parsedURL struct {
+	url.URL
+
+	isUnixSocket bool
+}
+
+// Parse URL and set defaults
+func newParsedURL(remoteAddr string) (*parsedURL, error) {
+	u, err := url.Parse(remoteAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	// default to tcp if nothing specified
+	if u.Scheme == "" {
+		u.Scheme = protoTCP
+	}
+
+	pu := &parsedURL{
+		URL:          *u,
+		isUnixSocket: false,
+	}
+
+	if u.Scheme == protoUNIX {
+		pu.isUnixSocket = true
+	}
+
+	return pu, nil
+}
+
+// Change protocol to HTTP for unknown protocols and TCP protocol - useful for RPC connections
+func (u *parsedURL) SetDefaultSchemeHTTP() {
+	// protocol to use for http operations, to support both http and https
+	switch u.Scheme {
+	case protoHTTP, protoHTTPS, protoWS, protoWSS:
+		// known protocols not changed
+	default:
+		// default to http for unknown protocols (ex. tcp)
+		u.Scheme = protoHTTP
+	}
+}
+
+// Get full address without the protocol - useful for Dialer connections
+func (u parsedURL) GetHostWithPath() string {
+	// Remove protocol, userinfo and # fragment, assume opaque is empty
+	return u.Host + u.EscapedPath()
+}
+
+// Get a trimmed address - useful for WS connections
+func (u parsedURL) GetTrimmedHostWithPath() string {
+	// if it's not an unix socket we return the normal URL
+	if !u.isUnixSocket {
+		return u.GetHostWithPath()
+	}
+	// if it's a unix socket we replace the host slashes with a period
+	// this is because otherwise the http.Client would think that the
+	// domain is invalid.
+	return strings.ReplaceAll(u.GetHostWithPath(), "/", ".")
+}
+
+// GetDialAddress returns the endpoint to dial for the parsed URL
+func (u parsedURL) GetDialAddress() string {
+	// if it's not a unix socket we return the host, example: localhost:443
+	if !u.isUnixSocket {
+		return u.Host
+	}
+	// otherwise we return the path of the unix socket, ex /tmp/socket
+	return u.GetHostWithPath()
+}
+
+// Get a trimmed address with protocol - useful as address in RPC connections
+func (u parsedURL) GetTrimmedURL() string {
+	return u.Scheme + "://" + u.GetTrimmedHostWithPath()
+}
+
+//-------------------------------------------------------------
+
+// HTTPClient is a common interface for JSON-RPC HTTP clients.
+type HTTPClient interface {
+	// Call calls the given method with the params and returns a result.
+	Call(ctx context.Context, method string, params map[string]interface{}, result interface{}) (interface{}, error)
+}
+
+// Caller implementers can facilitate calling the JSON-RPC endpoint.
+type Caller interface {
+	Call(ctx context.Context, method string, params map[string]interface{}, result interface{}) (interface{}, error)
+}
+
+//-------------------------------------------------------------
+
+// Client is a JSON-RPC client, which sends POST HTTP requests to the
+// remote server.
+//
+// Client is safe for concurrent use by multiple goroutines.
+type Client struct {
+	address  string
+	username string
+	password string
+
+	client *http.Client
+
+	mtx       tmsync.Mutex
+	nextReqID int
+}
+
+var _ HTTPClient = (*Client)(nil)
+
+// Both Client and RequestBatch can facilitate calls to the JSON
+// RPC endpoint.
+var _ Caller = (*Client)(nil)
+var _ Caller = (*RequestBatch)(nil)
+
+// New returns a Client pointed at the given address.
+// An error is returned on invalid remote. The function panics when remote is nil.
+func New(remote string) (*Client, error) {
+	httpClient, err := DefaultHTTPClient(remote)
+	if err != nil {
+		return nil, err
+	}
+	return NewWithHTTPClient(remote, httpClient)
+}
+
+// NewWithHTTPClient returns a Client pointed at the given
+// address using a custom http client. An error is returned on invalid remote.
+// The function panics when remote is nil.
+func NewWithHTTPClient(remote string, client *http.Client) (*Client, error) {
+	if client == nil {
+		panic("nil http.Client provided")
+	}
+
+	parsedURL, err := newParsedURL(remote)
+	if err != nil {
+		return nil, fmt.Errorf("invalid remote %s: %s", remote, err)
+	}
+
+	parsedURL.SetDefaultSchemeHTTP()
+
+	address := parsedURL.GetTrimmedURL()
+	username := parsedURL.User.Username()
+	password, _ := parsedURL.User.Password()
+
+	rpcClient := &Client{
+		address:  address,
+		username: username,
+		password: password,
+		client:   client,
+	}
+
+	return rpcClient, nil
+}
+
+// Call issues a POST HTTP request. Requests are JSON encoded. Content-Type:
+// application/json.
+func (c *Client) Call(
+	ctx context.Context,
+	method string,
+	params map[string]interface{},
+	result interface{},
+) (interface{}, error) {
+	id := c.nextRequestID()
+
+	request, err := types.MapToRequest(id, method, params)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode params: %w", err)
+	}
+
+	requestBytes, err := json.Marshal(request)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	requestBuf := bytes.NewBuffer(requestBytes)
+	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, c.address, requestBuf)
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
+	}
+
+	httpRequest.Header.Set("Content-Type", "application/json")
+
+	if c.username != "" || c.password != "" {
+		httpRequest.SetBasicAuth(c.username, c.password)
+	}
+
+	httpResponse, err := c.client.Do(httpRequest)
+	if err != nil {
+		return nil, fmt.Errorf("post failed: %w", err)
+	}
+
+	defer httpResponse.Body.Close()
+
+	responseBytes, err := ioutil.ReadAll(httpResponse.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return unmarshalResponseBytes(responseBytes, id, result)
+}
+
+// NewRequestBatch starts a batch of requests for this client.
+func (c *Client) NewRequestBatch() *RequestBatch {
+	return &RequestBatch{
+		requests: make([]*jsonRPCBufferedRequest, 0),
+		client:   c,
+	}
+}
+
+func (c *Client) sendBatch(ctx context.Context, requests []*jsonRPCBufferedRequest) ([]interface{}, error) {
+	reqs := make([]types.RPCRequest, 0, len(requests))
+	results := make([]interface{}, 0, len(requests))
+	for _, req := range requests {
+		reqs = append(reqs, req.request)
+		results = append(results, req.result)
+	}
+
+	// serialize the array of requests into a single JSON object
+	requestBytes, err := json.Marshal(reqs)
+	if err != nil {
+		return nil, fmt.Errorf("json marshal: %w", err)
+	}
+
+	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, c.address, bytes.NewBuffer(requestBytes))
+	if err != nil {
+		return nil, fmt.Errorf("new request: %w", err)
+	}
+
+	httpRequest.Header.Set("Content-Type", "application/json")
+
+	if c.username != "" || c.password != "" {
+		httpRequest.SetBasicAuth(c.username, c.password)
+	}
+
+	httpResponse, err := c.client.Do(httpRequest)
+	if err != nil {
+		return nil, fmt.Errorf("post: %w", err)
+	}
+
+	defer httpResponse.Body.Close()
+
+	responseBytes, err := ioutil.ReadAll(httpResponse.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response body: %w", err)
+	}
+
+	// collect ids to check responses IDs in unmarshalResponseBytesArray
+	ids := make([]types.JSONRPCIntID, len(requests))
+	for i, req := range requests {
+		ids[i] = req.request.ID.(types.JSONRPCIntID)
+	}
+
+	return unmarshalResponseBytesArray(responseBytes, ids, results)
+}
+
+func (c *Client) nextRequestID() types.JSONRPCIntID {
+	c.mtx.Lock()
+	id := c.nextReqID
+	c.nextReqID++
+	c.mtx.Unlock()
+	return types.JSONRPCIntID(id)
+}
+
+//------------------------------------------------------------------------------------
+
+// jsonRPCBufferedRequest encapsulates a single buffered request, as well as its
+// anticipated response structure.
+type jsonRPCBufferedRequest struct {
+	request types.RPCRequest
+	result  interface{} // The result will be deserialized into this object.
+}
+
+// RequestBatch allows us to buffer multiple request/response structures
+// into a single batch request. Note that this batch acts like a FIFO queue, and
+// is thread-safe.
+type RequestBatch struct {
+	client *Client
+
+	mtx      tmsync.Mutex
+	requests []*jsonRPCBufferedRequest
+}
+
+// Count returns the number of enqueued requests waiting to be sent.
+func (b *RequestBatch) Count() int {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+	return len(b.requests)
+}
+
+func (b *RequestBatch) enqueue(req *jsonRPCBufferedRequest) {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+	b.requests = append(b.requests, req)
+}
+
+// Clear empties out the request batch.
+func (b *RequestBatch) Clear() int {
+	b.mtx.Lock()
+	defer b.mtx.Unlock()
+	return b.clear()
+}
+
+func (b *RequestBatch) clear() int {
+	count := len(b.requests)
+	b.requests = make([]*jsonRPCBufferedRequest, 0)
+	return count
+}
+
+// Send will attempt to send the current batch of enqueued requests, and then
+// will clear out the requests once done. On success, this returns the
+// deserialized list of results from each of the enqueued requests.
+func (b *RequestBatch) Send(ctx context.Context) ([]interface{}, error) {
+	b.mtx.Lock()
+	defer func() {
+		b.clear()
+		b.mtx.Unlock()
+	}()
+	return b.client.sendBatch(ctx, b.requests)
+}
+
+// Call enqueues a request to call the given RPC method with the specified
+// parameters, in the same way that the `Client.Call` function would.
+func (b *RequestBatch) Call(
+	_ context.Context,
+	method string,
+	params map[string]interface{},
+	result interface{},
+) (interface{}, error) {
+	id := b.client.nextRequestID()
+	request, err := types.MapToRequest(id, method, params)
+	if err != nil {
+		return nil, err
+	}
+	b.enqueue(&jsonRPCBufferedRequest{request: request, result: result})
+	return result, nil
+}
+
+//-------------------------------------------------------------
+
+func makeHTTPDialer(remoteAddr string) (func(string, string) (net.Conn, error), error) {
+	u, err := newParsedURL(remoteAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	protocol := u.Scheme
+
+	// accept http(s) as an alias for tcp
+	switch protocol {
+	case protoHTTP, protoHTTPS:
+		protocol = protoTCP
+	}
+
+	dialFn := func(proto, addr string) (net.Conn, error) {
+		return net.Dial(protocol, u.GetDialAddress())
+	}
+
+	return dialFn, nil
+}
+
+// DefaultHTTPClient is used to create an http client with some default parameters.
+// We overwrite the http.Client.Dial so we can do http over tcp or unix.
+// remoteAddr should be fully featured (eg. with tcp:// or unix://).
+// An error will be returned in case of invalid remoteAddr.
+func DefaultHTTPClient(remoteAddr string) (*http.Client, error) {
+	dialFn, err := makeHTTPDialer(remoteAddr)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			// Set to true to prevent GZIP-bomb DoS attacks
+			DisableCompression: true,
+			Dial:               dialFn,
+		},
+	}
+
+	return client, nil
+}

internal/rpc/jsonrpc/client/http_json_client_test.go

@@ -0,0 +1,86 @@
+package client
+
+import (
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPClientMakeHTTPDialer(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("Hi!\n"))
+	})
+	ts := httptest.NewServer(handler)
+	defer ts.Close()
+
+	tsTLS := httptest.NewTLSServer(handler)
+	defer tsTLS.Close()
+	// This silences a TLS handshake error, caused by the dialer just immediately
+	// disconnecting, which we can just ignore.
+	tsTLS.Config.ErrorLog = log.New(ioutil.Discard, "", 0)
+
+	for _, testURL := range []string{ts.URL, tsTLS.URL} {
+		u, err := newParsedURL(testURL)
+		require.NoError(t, err)
+		dialFn, err := makeHTTPDialer(testURL)
+		require.Nil(t, err)
+
+		addr, err := dialFn(u.Scheme, u.GetHostWithPath())
+		require.NoError(t, err)
+		require.NotNil(t, addr)
+	}
+}
+
+func Test_parsedURL(t *testing.T) {
+	type test struct {
+		url                  string
+		expectedURL          string
+		expectedHostWithPath string
+		expectedDialAddress  string
+	}
+
+	tests := map[string]test{
+		"unix endpoint": {
+			url:                  "unix:///tmp/test",
+			expectedURL:          "unix://.tmp.test",
+			expectedHostWithPath: "/tmp/test",
+			expectedDialAddress:  "/tmp/test",
+		},
+
+		"http endpoint": {
+			url:                  "https://example.com",
+			expectedURL:          "https://example.com",
+			expectedHostWithPath: "example.com",
+			expectedDialAddress:  "example.com",
+		},
+
+		"http endpoint with port": {
+			url:                  "https://example.com:8080",
+			expectedURL:          "https://example.com:8080",
+			expectedHostWithPath: "example.com:8080",
+			expectedDialAddress:  "example.com:8080",
+		},
+
+		"http path routed endpoint": {
+			url:                  "https://example.com:8080/rpc",
+			expectedURL:          "https://example.com:8080/rpc",
+			expectedHostWithPath: "example.com:8080/rpc",
+			expectedDialAddress:  "example.com:8080",
+		},
+	}
+
+	for name, tt := range tests {
+		tt := tt // suppressing linter
+		t.Run(name, func(t *testing.T) {
+			parsed, err := newParsedURL(tt.url)
+			require.NoError(t, err)
+			require.Equal(t, tt.expectedDialAddress, parsed.GetDialAddress())
+			require.Equal(t, tt.expectedURL, parsed.GetTrimmedURL())
+			require.Equal(t, tt.expectedHostWithPath, parsed.GetHostWithPath())
+		})
+	}
+}

internal/rpc/jsonrpc/client/http_uri_client.go

@@ -0,0 +1,85 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	types "github.com/tendermint/tendermint/rpc/jsonrpc/types"
+)
+
+const (
+	// URIClientRequestID in a request ID used by URIClient
+	URIClientRequestID = types.JSONRPCIntID(-1)
+)
+
+// URIClient is a JSON-RPC client, which sends POST form HTTP requests to the
+// remote server.
+//
+// URIClient is safe for concurrent use by multiple goroutines.
+type URIClient struct {
+	address string
+	client  *http.Client
+}
+
+var _ HTTPClient = (*URIClient)(nil)
+
+// NewURI returns a new client.
+// An error is returned on invalid remote.
+// The function panics when remote is nil.
+func NewURI(remote string) (*URIClient, error) {
+	parsedURL, err := newParsedURL(remote)
+	if err != nil {
+		return nil, err
+	}
+
+	httpClient, err := DefaultHTTPClient(remote)
+	if err != nil {
+		return nil, err
+	}
+
+	parsedURL.SetDefaultSchemeHTTP()
+
+	uriClient := &URIClient{
+		address: parsedURL.GetTrimmedURL(),
+		client:  httpClient,
+	}
+
+	return uriClient, nil
+}
+
+// Call issues a POST form HTTP request.
+func (c *URIClient) Call(ctx context.Context, method string,
+	params map[string]interface{}, result interface{}) (interface{}, error) {
+
+	values, err := argsToURLValues(params)
+	if err != nil {
+		return nil, fmt.Errorf("failed to encode params: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		http.MethodPost,
+		c.address+"/"+method,
+		strings.NewReader(values.Encode()),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("new request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("post: %w", err)
+	}
+	defer resp.Body.Close()
+
+	responseBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read response body: %w", err)
+	}
+
+	return unmarshalResponseBytes(responseBytes, URIClientRequestID, result)
+}