p2p: renames for reactors and routing layer internal moves (#6547)

internal/consensus/README.md

@@ -0,0 +1,3 @@
+# Consensus
+
+See the [consensus spec](https://github.com/tendermint/spec/tree/master/spec/consensus).

internal/consensus/byzantine_test.go

@@ -0,0 +1,535 @@
+package consensus
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/internal/evidence"
+	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	"github.com/tendermint/tendermint/libs/log"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+// Byzantine node sends two different prevotes (nil and blockID) to the same
+// validator.
+func TestByzantinePrevoteEquivocation(t *testing.T) {
+	config := configSetup(t)
+
+	nValidators := 4
+	prevoteHeight := int64(2)
+	testName := "consensus_byzantine_test"
+	tickerFunc := newMockTickerFunc(true)
+	appFunc := newCounter
+
+	genDoc, privVals := factory.RandGenesisDoc(config, nValidators, false, 30)
+	states := make([]*State, nValidators)
+
+	for i := 0; i < nValidators; i++ {
+		func() {
+			logger := consensusLogger().With("test", "byzantine", "validator", i)
+			stateDB := dbm.NewMemDB() // each state needs its own db
+			stateStore := sm.NewStore(stateDB)
+			state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+
+			thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+			defer os.RemoveAll(thisConfig.RootDir)
+
+			ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+			app := appFunc()
+			vals := types.TM2PB.ValidatorUpdates(state.Validators)
+			app.InitChain(abci.RequestInitChain{Validators: vals})
+
+			blockDB := dbm.NewMemDB()
+			blockStore := store.NewBlockStore(blockDB)
+
+			// one for mempool, one for consensus
+			mtx := new(tmsync.RWMutex)
+			proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
+			proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+
+			// Make Mempool
+			mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+			mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
+			if thisConfig.Consensus.WaitForTxs() {
+				mempool.EnableTxsAvailable()
+			}
+
+			// Make a full instance of the evidence pool
+			evidenceDB := dbm.NewMemDB()
+			evpool, err := evidence.NewPool(logger.With("module", "evidence"), evidenceDB, stateStore, blockStore)
+			require.NoError(t, err)
+
+			// Make State
+			blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+			cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+			cs.SetLogger(cs.Logger)
+			// set private validator
+			pv := privVals[i]
+			cs.SetPrivValidator(pv)
+
+			eventBus := types.NewEventBus()
+			eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+			err = eventBus.Start()
+			require.NoError(t, err)
+			cs.SetEventBus(eventBus)
+
+			cs.SetTimeoutTicker(tickerFunc())
+			cs.SetLogger(logger)
+
+			states[i] = cs
+		}()
+	}
+
+	rts := setup(t, nValidators, states, 100) // buffer must be large enough to not deadlock
+
+	var bzNodeID p2p.NodeID
+
+	// Set the first state's reactor as the dedicated byzantine reactor and grab
+	// the NodeID that corresponds to the state so we can reference the reactor.
+	bzNodeState := states[0]
+	for nID, s := range rts.states {
+		if s == bzNodeState {
+			bzNodeID = nID
+			break
+		}
+	}
+
+	bzReactor := rts.reactors[bzNodeID]
+
+	// alter prevote so that the byzantine node double votes when height is 2
+	bzNodeState.doPrevote = func(height int64, round int32) {
+		// allow first height to happen normally so that byzantine validator is no longer proposer
+		if height == prevoteHeight {
+			prevote1, err := bzNodeState.signVote(
+				tmproto.PrevoteType,
+				bzNodeState.ProposalBlock.Hash(),
+				bzNodeState.ProposalBlockParts.Header(),
+			)
+			require.NoError(t, err)
+
+			prevote2, err := bzNodeState.signVote(tmproto.PrevoteType, nil, types.PartSetHeader{})
+			require.NoError(t, err)
+
+			// send two votes to all peers (1st to one half, 2nd to another half)
+			i := 0
+			for _, ps := range bzReactor.peers {
+				if i < len(bzReactor.peers)/2 {
+					bzNodeState.Logger.Info("signed and pushed vote", "vote", prevote1, "peer", ps.peerID)
+					bzReactor.voteCh.Out <- p2p.Envelope{
+						To: ps.peerID,
+						Message: &tmcons.Vote{
+							Vote: prevote1.ToProto(),
+						},
+					}
+				} else {
+					bzNodeState.Logger.Info("signed and pushed vote", "vote", prevote2, "peer", ps.peerID)
+					bzReactor.voteCh.Out <- p2p.Envelope{
+						To: ps.peerID,
+						Message: &tmcons.Vote{
+							Vote: prevote2.ToProto(),
+						},
+					}
+				}
+
+				i++
+			}
+		} else {
+			bzNodeState.Logger.Info("behaving normally")
+			bzNodeState.defaultDoPrevote(height, round)
+		}
+	}
+
+	// Introducing a lazy proposer means that the time of the block committed is
+	// different to the timestamp that the other nodes have. This tests to ensure
+	// that the evidence that finally gets proposed will have a valid timestamp.
+	// lazyProposer := states[1]
+	lazyNodeState := states[1]
+
+	lazyNodeState.decideProposal = func(height int64, round int32) {
+		lazyNodeState.Logger.Info("Lazy Proposer proposing condensed commit")
+		require.NotNil(t, lazyNodeState.privValidator)
+
+		var commit *types.Commit
+		switch {
+		case lazyNodeState.Height == lazyNodeState.state.InitialHeight:
+			// We're creating a proposal for the first block.
+			// The commit is empty, but not nil.
+			commit = types.NewCommit(0, 0, types.BlockID{}, nil)
+		case lazyNodeState.LastCommit.HasTwoThirdsMajority():
+			// Make the commit from LastCommit
+			commit = lazyNodeState.LastCommit.MakeCommit()
+		default: // This shouldn't happen.
+			lazyNodeState.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block")
+			return
+		}
+
+		// omit the last signature in the commit
+		commit.Signatures[len(commit.Signatures)-1] = types.NewCommitSigAbsent()
+
+		if lazyNodeState.privValidatorPubKey == nil {
+			// If this node is a validator & proposer in the current round, it will
+			// miss the opportunity to create a block.
+			lazyNodeState.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet))
+			return
+		}
+		proposerAddr := lazyNodeState.privValidatorPubKey.Address()
+
+		block, blockParts := lazyNodeState.blockExec.CreateProposalBlock(
+			lazyNodeState.Height, lazyNodeState.state, commit, proposerAddr,
+		)
+
+		// Flush the WAL. Otherwise, we may not recompute the same proposal to sign,
+		// and the privValidator will refuse to sign anything.
+		if err := lazyNodeState.wal.FlushAndSync(); err != nil {
+			lazyNodeState.Logger.Error("Error flushing to disk")
+		}
+
+		// Make proposal
+		propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
+		proposal := types.NewProposal(height, round, lazyNodeState.ValidRound, propBlockID)
+		p := proposal.ToProto()
+		if err := lazyNodeState.privValidator.SignProposal(context.Background(), lazyNodeState.state.ChainID, p); err == nil {
+			proposal.Signature = p.Signature
+
+			// send proposal and block parts on internal msg queue
+			lazyNodeState.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""})
+			for i := 0; i < int(blockParts.Total()); i++ {
+				part := blockParts.GetPart(i)
+				lazyNodeState.sendInternalMessage(msgInfo{&BlockPartMessage{lazyNodeState.Height, lazyNodeState.Round, part}, ""})
+			}
+			lazyNodeState.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal)
+			lazyNodeState.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block))
+		} else if !lazyNodeState.replayMode {
+			lazyNodeState.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err)
+		}
+	}
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	// Evidence should be submitted and committed at the third height but
+	// we will check the first six just in case
+	evidenceFromEachValidator := make([]types.Evidence, nValidators)
+
+	wg := new(sync.WaitGroup)
+	i := 0
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		go func(j int, s types.Subscription) {
+			defer wg.Done()
+			for {
+				select {
+				case msg := <-s.Out():
+					require.NotNil(t, msg)
+					block := msg.Data().(types.EventDataNewBlock).Block
+					if len(block.Evidence.Evidence) != 0 {
+						evidenceFromEachValidator[j] = block.Evidence.Evidence[0]
+						return
+					}
+				case <-s.Canceled():
+					require.Fail(t, "subscription failed for %d", j)
+					return
+				}
+			}
+		}(i, sub)
+
+		i++
+	}
+
+	wg.Wait()
+
+	pubkey, err := bzNodeState.privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	for idx, ev := range evidenceFromEachValidator {
+		if assert.NotNil(t, ev, idx) {
+			ev, ok := ev.(*types.DuplicateVoteEvidence)
+			assert.True(t, ok)
+			assert.Equal(t, pubkey.Address(), ev.VoteA.ValidatorAddress)
+			assert.Equal(t, prevoteHeight, ev.Height())
+		}
+	}
+}
+
+// 4 validators. 1 is byzantine. The other three are partitioned into A (1 val) and B (2 vals).
+// byzantine validator sends conflicting proposals into A and B,
+// and prevotes/precommits on both of them.
+// B sees a commit, A doesn't.
+// Heal partition and ensure A sees the commit
+func TestByzantineConflictingProposalsWithPartition(t *testing.T) {
+	// TODO: https://github.com/tendermint/tendermint/issues/6092
+	t.SkipNow()
+
+	// n := 4
+	// logger := consensusLogger().With("test", "byzantine")
+	// app := newCounter
+
+	// states, cleanup := randConsensusState(n, "consensus_byzantine_test", newMockTickerFunc(false), app)
+	// t.Cleanup(cleanup)
+
+	// // give the byzantine validator a normal ticker
+	// ticker := NewTimeoutTicker()
+	// ticker.SetLogger(states[0].Logger)
+	// states[0].SetTimeoutTicker(ticker)
+
+	// p2pLogger := logger.With("module", "p2p")
+
+	// blocksSubs := make([]types.Subscription, n)
+	// reactors := make([]p2p.Reactor, n)
+	// for i := 0; i < n; i++ {
+	// 	// enable txs so we can create different proposals
+	// 	assertMempool(states[i].txNotifier).EnableTxsAvailable()
+
+	// 	eventBus := states[i].eventBus
+	// 	eventBus.SetLogger(logger.With("module", "events", "validator", i))
+
+	// 	var err error
+	// 	blocksSubs[i], err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock)
+	// 	require.NoError(t, err)
+
+	// 	conR := NewReactor(states[i], true) // so we don't start the consensus states
+	// 	conR.SetLogger(logger.With("validator", i))
+	// 	conR.SetEventBus(eventBus)
+
+	// 	var conRI p2p.Reactor = conR
+
+	// 	// make first val byzantine
+	// 	if i == 0 {
+	// 		conRI = NewByzantineReactor(conR)
+	// 	}
+
+	// 	reactors[i] = conRI
+	// 	err = states[i].blockExec.Store().Save(states[i].state) // for save height 1's validators info
+	// 	require.NoError(t, err)
+	// }
+
+	// switches := p2p.MakeConnectedSwitches(config.P2P, N, func(i int, sw *p2p.Switch) *p2p.Switch {
+	// 	sw.SetLogger(p2pLogger.With("validator", i))
+	// 	sw.AddReactor("CONSENSUS", reactors[i])
+	// 	return sw
+	// }, func(sws []*p2p.Switch, i, j int) {
+	// 	// the network starts partitioned with globally active adversary
+	// 	if i != 0 {
+	// 		return
+	// 	}
+	// 	p2p.Connect2Switches(sws, i, j)
+	// })
+
+	// // make first val byzantine
+	// // NOTE: Now, test validators are MockPV, which by default doesn't
+	// // do any safety checks.
+	// states[0].privValidator.(types.MockPV).DisableChecks()
+	// states[0].decideProposal = func(j int32) func(int64, int32) {
+	// 	return func(height int64, round int32) {
+	// 		byzantineDecideProposalFunc(t, height, round, states[j], switches[j])
+	// 	}
+	// }(int32(0))
+	// // We are setting the prevote function to do nothing because the prevoting
+	// // and precommitting are done alongside the proposal.
+	// states[0].doPrevote = func(height int64, round int32) {}
+
+	// defer func() {
+	// 	for _, sw := range switches {
+	// 		err := sw.Stop()
+	// 		require.NoError(t, err)
+	// 	}
+	// }()
+
+	// // start the non-byz state machines.
+	// // note these must be started before the byz
+	// for i := 1; i < n; i++ {
+	// 	cr := reactors[i].(*Reactor)
+	// 	cr.SwitchToConsensus(cr.conS.GetState(), false)
+	// }
+
+	// // start the byzantine state machine
+	// byzR := reactors[0].(*ByzantineReactor)
+	// s := byzR.reactor.conS.GetState()
+	// byzR.reactor.SwitchToConsensus(s, false)
+
+	// // byz proposer sends one block to peers[0]
+	// // and the other block to peers[1] and peers[2].
+	// // note peers and switches order don't match.
+	// peers := switches[0].Peers().List()
+
+	// // partition A
+	// ind0 := getSwitchIndex(switches, peers[0])
+
+	// // partition B
+	// ind1 := getSwitchIndex(switches, peers[1])
+	// ind2 := getSwitchIndex(switches, peers[2])
+	// p2p.Connect2Switches(switches, ind1, ind2)
+
+	// // wait for someone in the big partition (B) to make a block
+	// <-blocksSubs[ind2].Out()
+
+	// t.Log("A block has been committed. Healing partition")
+	// p2p.Connect2Switches(switches, ind0, ind1)
+	// p2p.Connect2Switches(switches, ind0, ind2)
+
+	// // wait till everyone makes the first new block
+	// // (one of them already has)
+	// wg := new(sync.WaitGroup)
+	// for i := 1; i < N-1; i++ {
+	// 	wg.Add(1)
+	// 	go func(j int) {
+	// 		<-blocksSubs[j].Out()
+	// 		wg.Done()
+	// 	}(i)
+	// }
+
+	// done := make(chan struct{})
+	// go func() {
+	// 	wg.Wait()
+	// 	close(done)
+	// }()
+
+	// tick := time.NewTicker(time.Second * 10)
+	// select {
+	// case <-done:
+	// case <-tick.C:
+	// 	for i, reactor := range reactors {
+	// 		t.Log(fmt.Sprintf("Consensus Reactor %v", i))
+	// 		t.Log(fmt.Sprintf("%v", reactor))
+	// 	}
+	// 	t.Fatalf("Timed out waiting for all validators to commit first block")
+	// }
+}
+
+// func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch) {
+// 	// byzantine user should create two proposals and try to split the vote.
+// 	// Avoid sending on internalMsgQueue and running consensus state.
+
+// 	// Create a new proposal block from state/txs from the mempool.
+// 	block1, blockParts1 := cs.createProposalBlock()
+// 	polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()}
+// 	proposal1 := types.NewProposal(height, round, polRound, propBlockID)
+// 	p1 := proposal1.ToProto()
+// 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil {
+// 		t.Error(err)
+// 	}
+
+// 	proposal1.Signature = p1.Signature
+
+// 	// some new transactions come in (this ensures that the proposals are different)
+// 	deliverTxsRange(cs, 0, 1)
+
+// 	// Create a new proposal block from state/txs from the mempool.
+// 	block2, blockParts2 := cs.createProposalBlock()
+// 	polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()}
+// 	proposal2 := types.NewProposal(height, round, polRound, propBlockID)
+// 	p2 := proposal2.ToProto()
+// 	if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil {
+// 		t.Error(err)
+// 	}
+
+// 	proposal2.Signature = p2.Signature
+
+// 	block1Hash := block1.Hash()
+// 	block2Hash := block2.Hash()
+
+// 	// broadcast conflicting proposals/block parts to peers
+// 	peers := sw.Peers().List()
+// 	t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers))
+// 	for i, peer := range peers {
+// 		if i < len(peers)/2 {
+// 			go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1)
+// 		} else {
+// 			go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2)
+// 		}
+// 	}
+// }
+
+// func sendProposalAndParts(
+// 	height int64,
+// 	round int32,
+// 	cs *State,
+// 	peer p2p.Peer,
+// 	proposal *types.Proposal,
+// 	blockHash []byte,
+// 	parts *types.PartSet,
+// ) {
+// 	// proposal
+// 	msg := &ProposalMessage{Proposal: proposal}
+// 	peer.Send(DataChannel, MustEncode(msg))
+
+// 	// parts
+// 	for i := 0; i < int(parts.Total()); i++ {
+// 		part := parts.GetPart(i)
+// 		msg := &BlockPartMessage{
+// 			Height: height, // This tells peer that this part applies to us.
+// 			Round:  round,  // This tells peer that this part applies to us.
+// 			Part:   part,
+// 		}
+// 		peer.Send(DataChannel, MustEncode(msg))
+// 	}
+
+// 	// votes
+// 	cs.mtx.Lock()
+// 	prevote, _ := cs.signVote(tmproto.PrevoteType, blockHash, parts.Header())
+// 	precommit, _ := cs.signVote(tmproto.PrecommitType, blockHash, parts.Header())
+// 	cs.mtx.Unlock()
+
+// 	peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote}))
+// 	peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit}))
+// }
+
+// type ByzantineReactor struct {
+// 	service.Service
+// 	reactor *Reactor
+// }
+
+// func NewByzantineReactor(conR *Reactor) *ByzantineReactor {
+// 	return &ByzantineReactor{
+// 		Service: conR,
+// 		reactor: conR,
+// 	}
+// }
+
+// func (br *ByzantineReactor) SetSwitch(s *p2p.Switch)               { br.reactor.SetSwitch(s) }
+// func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() }
+
+// func (br *ByzantineReactor) AddPeer(peer p2p.Peer) {
+// 	if !br.reactor.IsRunning() {
+// 		return
+// 	}
+
+// 	// Create peerState for peer
+// 	peerState := NewPeerState(peer).SetLogger(br.reactor.Logger)
+// 	peer.Set(types.PeerStateKey, peerState)
+
+// 	// Send our state to peer.
+// 	// If we're syncing, broadcast a RoundStepMessage later upon SwitchToConsensus().
+// 	if !br.reactor.waitSync {
+// 		br.reactor.sendNewRoundStepMessage(peer)
+// 	}
+// }
+
+// func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) {
+// 	br.reactor.RemovePeer(peer, reason)
+// }
+
+// func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) {
+// 	br.reactor.Receive(chID, peer, msgBytes)
+// }
+
+// func (br *ByzantineReactor) InitPeer(peer p2p.Peer) p2p.Peer { return peer }

internal/consensus/common_test.go

@@ -0,0 +1,883 @@
+package consensus
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"path"
+
+	dbm "github.com/tendermint/tm-db"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/example/counter"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
+	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	tmbytes "github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	tmpubsub "github.com/tendermint/tendermint/libs/pubsub"
+	"github.com/tendermint/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	testSubscriber = "test-client"
+
+	// genesis, chain_id, priv_val
+	ensureTimeout = time.Millisecond * 200
+)
+
+// A cleanupFunc cleans up any config / test files created for a particular
+// test.
+type cleanupFunc func()
+
+func configSetup(t *testing.T) *cfg.Config {
+	t.Helper()
+
+	config := ResetConfig("consensus_reactor_test")
+
+	consensusReplayConfig := ResetConfig("consensus_replay_test")
+	configStateTest := ResetConfig("consensus_state_test")
+	configMempoolTest := ResetConfig("consensus_mempool_test")
+	configByzantineTest := ResetConfig("consensus_byzantine_test")
+
+	t.Cleanup(func() {
+		os.RemoveAll(config.RootDir)
+		os.RemoveAll(consensusReplayConfig.RootDir)
+		os.RemoveAll(configStateTest.RootDir)
+		os.RemoveAll(configMempoolTest.RootDir)
+		os.RemoveAll(configByzantineTest.RootDir)
+	})
+	return config
+}
+
+func ensureDir(dir string, mode os.FileMode) {
+	if err := tmos.EnsureDir(dir, mode); err != nil {
+		panic(err)
+	}
+}
+
+func ResetConfig(name string) *cfg.Config {
+	return cfg.ResetTestRoot(name)
+}
+
+//-------------------------------------------------------------------------------
+// validator stub (a kvstore consensus peer we control)
+
+type validatorStub struct {
+	Index  int32 // Validator index. NOTE: we don't assume validator set changes.
+	Height int64
+	Round  int32
+	types.PrivValidator
+	VotingPower int64
+}
+
+const testMinPower int64 = 10
+
+func newValidatorStub(privValidator types.PrivValidator, valIndex int32) *validatorStub {
+	return &validatorStub{
+		Index:         valIndex,
+		PrivValidator: privValidator,
+		VotingPower:   testMinPower,
+	}
+}
+
+func (vs *validatorStub) signVote(
+	config *cfg.Config,
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader) (*types.Vote, error) {
+
+	pubKey, err := vs.PrivValidator.GetPubKey(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("can't get pubkey: %w", err)
+	}
+
+	vote := &types.Vote{
+		ValidatorIndex:   vs.Index,
+		ValidatorAddress: pubKey.Address(),
+		Height:           vs.Height,
+		Round:            vs.Round,
+		Timestamp:        tmtime.Now(),
+		Type:             voteType,
+		BlockID:          types.BlockID{Hash: hash, PartSetHeader: header},
+	}
+	v := vote.ToProto()
+	err = vs.PrivValidator.SignVote(context.Background(), config.ChainID(), v)
+	vote.Signature = v.Signature
+
+	return vote, err
+}
+
+// Sign vote for type/hash/header
+func signVote(
+	vs *validatorStub,
+	config *cfg.Config,
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader) *types.Vote {
+
+	v, err := vs.signVote(config, voteType, hash, header)
+	if err != nil {
+		panic(fmt.Errorf("failed to sign vote: %v", err))
+	}
+	return v
+}
+
+func signVotes(
+	config *cfg.Config,
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader,
+	vss ...*validatorStub) []*types.Vote {
+	votes := make([]*types.Vote, len(vss))
+	for i, vs := range vss {
+		votes[i] = signVote(vs, config, voteType, hash, header)
+	}
+	return votes
+}
+
+func incrementHeight(vss ...*validatorStub) {
+	for _, vs := range vss {
+		vs.Height++
+	}
+}
+
+func incrementRound(vss ...*validatorStub) {
+	for _, vs := range vss {
+		vs.Round++
+	}
+}
+
+type ValidatorStubsByPower []*validatorStub
+
+func (vss ValidatorStubsByPower) Len() int {
+	return len(vss)
+}
+
+func (vss ValidatorStubsByPower) Less(i, j int) bool {
+	vssi, err := vss[i].GetPubKey(context.Background())
+	if err != nil {
+		panic(err)
+	}
+	vssj, err := vss[j].GetPubKey(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	if vss[i].VotingPower == vss[j].VotingPower {
+		return bytes.Compare(vssi.Address(), vssj.Address()) == -1
+	}
+	return vss[i].VotingPower > vss[j].VotingPower
+}
+
+func (vss ValidatorStubsByPower) Swap(i, j int) {
+	it := vss[i]
+	vss[i] = vss[j]
+	vss[i].Index = int32(i)
+	vss[j] = it
+	vss[j].Index = int32(j)
+}
+
+//-------------------------------------------------------------------------------
+// Functions for transitioning the consensus state
+
+func startTestRound(cs *State, height int64, round int32) {
+	cs.enterNewRound(height, round)
+	cs.startRoutines(0)
+}
+
+// Create proposal block from cs1 but sign it with vs.
+func decideProposal(
+	cs1 *State,
+	vs *validatorStub,
+	height int64,
+	round int32,
+) (proposal *types.Proposal, block *types.Block) {
+	cs1.mtx.Lock()
+	block, blockParts := cs1.createProposalBlock()
+	validRound := cs1.ValidRound
+	chainID := cs1.state.ChainID
+	cs1.mtx.Unlock()
+	if block == nil {
+		panic("Failed to createProposalBlock. Did you forget to add commit for previous block?")
+	}
+
+	// Make proposal
+	polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}
+	proposal = types.NewProposal(height, round, polRound, propBlockID)
+	p := proposal.ToProto()
+	if err := vs.SignProposal(context.Background(), chainID, p); err != nil {
+		panic(err)
+	}
+
+	proposal.Signature = p.Signature
+
+	return
+}
+
+func addVotes(to *State, votes ...*types.Vote) {
+	for _, vote := range votes {
+		to.peerMsgQueue <- msgInfo{Msg: &VoteMessage{vote}}
+	}
+}
+
+func signAddVotes(
+	config *cfg.Config,
+	to *State,
+	voteType tmproto.SignedMsgType,
+	hash []byte,
+	header types.PartSetHeader,
+	vss ...*validatorStub,
+) {
+	votes := signVotes(config, voteType, hash, header, vss...)
+	addVotes(to, votes...)
+}
+
+func validatePrevote(t *testing.T, cs *State, round int32, privVal *validatorStub, blockHash []byte) {
+	prevotes := cs.Votes.Prevotes(round)
+	pubKey, err := privVal.GetPubKey(context.Background())
+	require.NoError(t, err)
+	address := pubKey.Address()
+	var vote *types.Vote
+	if vote = prevotes.GetByAddress(address); vote == nil {
+		panic("Failed to find prevote from validator")
+	}
+	if blockHash == nil {
+		if vote.BlockID.Hash != nil {
+			panic(fmt.Sprintf("Expected prevote to be for nil, got %X", vote.BlockID.Hash))
+		}
+	} else {
+		if !bytes.Equal(vote.BlockID.Hash, blockHash) {
+			panic(fmt.Sprintf("Expected prevote to be for %X, got %X", blockHash, vote.BlockID.Hash))
+		}
+	}
+}
+
+func validateLastPrecommit(t *testing.T, cs *State, privVal *validatorStub, blockHash []byte) {
+	votes := cs.LastCommit
+	pv, err := privVal.GetPubKey(context.Background())
+	require.NoError(t, err)
+	address := pv.Address()
+	var vote *types.Vote
+	if vote = votes.GetByAddress(address); vote == nil {
+		panic("Failed to find precommit from validator")
+	}
+	if !bytes.Equal(vote.BlockID.Hash, blockHash) {
+		panic(fmt.Sprintf("Expected precommit to be for %X, got %X", blockHash, vote.BlockID.Hash))
+	}
+}
+
+func validatePrecommit(
+	t *testing.T,
+	cs *State,
+	thisRound,
+	lockRound int32,
+	privVal *validatorStub,
+	votedBlockHash,
+	lockedBlockHash []byte,
+) {
+	precommits := cs.Votes.Precommits(thisRound)
+	pv, err := privVal.GetPubKey(context.Background())
+	require.NoError(t, err)
+	address := pv.Address()
+	var vote *types.Vote
+	if vote = precommits.GetByAddress(address); vote == nil {
+		panic("Failed to find precommit from validator")
+	}
+
+	if votedBlockHash == nil {
+		if vote.BlockID.Hash != nil {
+			panic("Expected precommit to be for nil")
+		}
+	} else {
+		if !bytes.Equal(vote.BlockID.Hash, votedBlockHash) {
+			panic("Expected precommit to be for proposal block")
+		}
+	}
+
+	if lockedBlockHash == nil {
+		if cs.LockedRound != lockRound || cs.LockedBlock != nil {
+			panic(fmt.Sprintf(
+				"Expected to be locked on nil at round %d. Got locked at round %d with block %v",
+				lockRound,
+				cs.LockedRound,
+				cs.LockedBlock))
+		}
+	} else {
+		if cs.LockedRound != lockRound || !bytes.Equal(cs.LockedBlock.Hash(), lockedBlockHash) {
+			panic(fmt.Sprintf(
+				"Expected block to be locked on round %d, got %d. Got locked block %X, expected %X",
+				lockRound,
+				cs.LockedRound,
+				cs.LockedBlock.Hash(),
+				lockedBlockHash))
+		}
+	}
+}
+
+func validatePrevoteAndPrecommit(
+	t *testing.T,
+	cs *State,
+	thisRound,
+	lockRound int32,
+	privVal *validatorStub,
+	votedBlockHash,
+	lockedBlockHash []byte,
+) {
+	// verify the prevote
+	validatePrevote(t, cs, thisRound, privVal, votedBlockHash)
+	// verify precommit
+	cs.mtx.Lock()
+	validatePrecommit(t, cs, thisRound, lockRound, privVal, votedBlockHash, lockedBlockHash)
+	cs.mtx.Unlock()
+}
+
+func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message {
+	votesSub, err := cs.eventBus.SubscribeUnbuffered(context.Background(), testSubscriber, types.EventQueryVote)
+	if err != nil {
+		panic(fmt.Sprintf("failed to subscribe %s to %v", testSubscriber, types.EventQueryVote))
+	}
+	ch := make(chan tmpubsub.Message)
+	go func() {
+		for msg := range votesSub.Out() {
+			vote := msg.Data().(types.EventDataVote)
+			// we only fire for our own votes
+			if bytes.Equal(addr, vote.Vote.ValidatorAddress) {
+				ch <- msg
+			}
+		}
+	}()
+	return ch
+}
+
+//-------------------------------------------------------------------------------
+// consensus states
+
+func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State {
+	config := cfg.ResetTestRoot("consensus_state_test")
+	return newStateWithConfig(config, state, pv, app)
+}
+
+func newStateWithConfig(
+	thisConfig *cfg.Config,
+	state sm.State,
+	pv types.PrivValidator,
+	app abci.Application,
+) *State {
+	blockDB := dbm.NewMemDB()
+	return newStateWithConfigAndBlockStore(thisConfig, state, pv, app, blockDB)
+}
+
+func newStateWithConfigAndBlockStore(
+	thisConfig *cfg.Config,
+	state sm.State,
+	pv types.PrivValidator,
+	app abci.Application,
+	blockDB dbm.DB,
+) *State {
+	// Get BlockStore
+	blockStore := store.NewBlockStore(blockDB)
+
+	// one for mempool, one for consensus
+	mtx := new(tmsync.RWMutex)
+	proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
+	proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+
+	// Make Mempool
+	mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+	mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
+	if thisConfig.Consensus.WaitForTxs() {
+		mempool.EnableTxsAvailable()
+	}
+
+	evpool := sm.EmptyEvidencePool{}
+
+	// Make State
+	stateDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(stateDB)
+	if err := stateStore.Save(state); err != nil { // for save height 1's validators info
+		panic(err)
+	}
+
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+	cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+	cs.SetLogger(log.TestingLogger().With("module", "consensus"))
+	cs.SetPrivValidator(pv)
+
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+	err := eventBus.Start()
+	if err != nil {
+		panic(err)
+	}
+	cs.SetEventBus(eventBus)
+	return cs
+}
+
+func loadPrivValidator(config *cfg.Config) *privval.FilePV {
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	ensureDir(filepath.Dir(privValidatorKeyFile), 0700)
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator, err := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
+	if err != nil {
+		panic(err)
+	}
+	privValidator.Reset()
+	return privValidator
+}
+
+func randState(config *cfg.Config, nValidators int) (*State, []*validatorStub) {
+	// Get State
+	state, privVals := randGenesisState(config, nValidators, false, 10)
+
+	vss := make([]*validatorStub, nValidators)
+
+	cs := newState(state, privVals[0], counter.NewApplication(true))
+
+	for i := 0; i < nValidators; i++ {
+		vss[i] = newValidatorStub(privVals[i], int32(i))
+	}
+	// since cs1 starts at 1
+	incrementHeight(vss[1:]...)
+
+	return cs, vss
+}
+
+//-------------------------------------------------------------------------------
+
+func ensureNoNewEvent(ch <-chan tmpubsub.Message, timeout time.Duration,
+	errorMessage string) {
+	select {
+	case <-time.After(timeout):
+		break
+	case <-ch:
+		panic(errorMessage)
+	}
+}
+
+func ensureNoNewEventOnChannel(ch <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		ch,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving new event on the channel")
+}
+
+func ensureNoNewRoundStep(stepCh <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		stepCh,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving NewRoundStep event")
+}
+
+func ensureNoNewUnlock(unlockCh <-chan tmpubsub.Message) {
+	ensureNoNewEvent(
+		unlockCh,
+		ensureTimeout,
+		"We should be stuck waiting, not receiving Unlock event")
+}
+
+func ensureNoNewTimeout(stepCh <-chan tmpubsub.Message, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
+	ensureNoNewEvent(
+		stepCh,
+		timeoutDuration,
+		"We should be stuck waiting, not receiving NewTimeout event")
+}
+
+func ensureNewEvent(ch <-chan tmpubsub.Message, height int64, round int32, timeout time.Duration, errorMessage string) {
+	select {
+	case <-time.After(timeout):
+		panic(errorMessage)
+	case msg := <-ch:
+		roundStateEvent, ok := msg.Data().(types.EventDataRoundState)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataRoundState, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if roundStateEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, roundStateEvent.Height))
+		}
+		if roundStateEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, roundStateEvent.Round))
+		}
+		// TODO: We could check also for a step at this point!
+	}
+}
+
+func ensureNewRound(roundCh <-chan tmpubsub.Message, height int64, round int32) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewRound event")
+	case msg := <-roundCh:
+		newRoundEvent, ok := msg.Data().(types.EventDataNewRound)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewRound, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if newRoundEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, newRoundEvent.Height))
+		}
+		if newRoundEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, newRoundEvent.Round))
+		}
+	}
+}
+
+func ensureNewTimeout(timeoutCh <-chan tmpubsub.Message, height int64, round int32, timeout int64) {
+	timeoutDuration := time.Duration(timeout*10) * time.Nanosecond
+	ensureNewEvent(timeoutCh, height, round, timeoutDuration,
+		"Timeout expired while waiting for NewTimeout event")
+}
+
+func ensureNewProposal(proposalCh <-chan tmpubsub.Message, height int64, round int32) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+	}
+}
+
+func ensureNewValidBlock(validBlockCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureNewEvent(validBlockCh, height, round, ensureTimeout,
+		"Timeout expired while waiting for NewValidBlock event")
+}
+
+func ensureNewBlock(blockCh <-chan tmpubsub.Message, height int64) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewBlock event")
+	case msg := <-blockCh:
+		blockEvent, ok := msg.Data().(types.EventDataNewBlock)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewBlock, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if blockEvent.Block.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockEvent.Block.Height))
+		}
+	}
+}
+
+func ensureNewBlockHeader(blockCh <-chan tmpubsub.Message, height int64, blockHash tmbytes.HexBytes) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewBlockHeader event")
+	case msg := <-blockCh:
+		blockHeaderEvent, ok := msg.Data().(types.EventDataNewBlockHeader)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataNewBlockHeader, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if blockHeaderEvent.Header.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, blockHeaderEvent.Header.Height))
+		}
+		if !bytes.Equal(blockHeaderEvent.Header.Hash(), blockHash) {
+			panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeaderEvent.Header.Hash()))
+		}
+	}
+}
+
+func ensureNewUnlock(unlockCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureNewEvent(unlockCh, height, round, ensureTimeout,
+		"Timeout expired while waiting for NewUnlock event")
+}
+
+func ensureProposal(proposalCh <-chan tmpubsub.Message, height int64, round int32, propID types.BlockID) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewProposal event")
+	case msg := <-proposalCh:
+		proposalEvent, ok := msg.Data().(types.EventDataCompleteProposal)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataCompleteProposal, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		if proposalEvent.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, proposalEvent.Height))
+		}
+		if proposalEvent.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, proposalEvent.Round))
+		}
+		if !proposalEvent.BlockID.Equals(propID) {
+			panic(fmt.Sprintf("Proposed block does not match expected block (%v != %v)", proposalEvent.BlockID, propID))
+		}
+	}
+}
+
+func ensurePrecommit(voteCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureVote(voteCh, height, round, tmproto.PrecommitType)
+}
+
+func ensurePrevote(voteCh <-chan tmpubsub.Message, height int64, round int32) {
+	ensureVote(voteCh, height, round, tmproto.PrevoteType)
+}
+
+func ensureVote(voteCh <-chan tmpubsub.Message, height int64, round int32,
+	voteType tmproto.SignedMsgType) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for NewVote event")
+	case msg := <-voteCh:
+		voteEvent, ok := msg.Data().(types.EventDataVote)
+		if !ok {
+			panic(fmt.Sprintf("expected a EventDataVote, got %T. Wrong subscription channel?",
+				msg.Data()))
+		}
+		vote := voteEvent.Vote
+		if vote.Height != height {
+			panic(fmt.Sprintf("expected height %v, got %v", height, vote.Height))
+		}
+		if vote.Round != round {
+			panic(fmt.Sprintf("expected round %v, got %v", round, vote.Round))
+		}
+		if vote.Type != voteType {
+			panic(fmt.Sprintf("expected type %v, got %v", voteType, vote.Type))
+		}
+	}
+}
+
+func ensurePrecommitTimeout(ch <-chan tmpubsub.Message) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for the Precommit to Timeout")
+	case <-ch:
+	}
+}
+
+func ensureNewEventOnChannel(ch <-chan tmpubsub.Message) {
+	select {
+	case <-time.After(ensureTimeout):
+		panic("Timeout expired while waiting for new activity on the channel")
+	case <-ch:
+	}
+}
+
+//-------------------------------------------------------------------------------
+// consensus nets
+
+// consensusLogger is a TestingLogger which uses a different
+// color for each validator ("validator" key must exist).
+func consensusLogger() log.Logger {
+	return log.TestingLogger().With("module", "consensus")
+}
+
+func randConsensusState(
+	config *cfg.Config,
+	nValidators int,
+	testName string,
+	tickerFunc func() TimeoutTicker,
+	appFunc func() abci.Application,
+	configOpts ...func(*cfg.Config),
+) ([]*State, cleanupFunc) {
+
+	genDoc, privVals := factory.RandGenesisDoc(config, nValidators, false, 30)
+	css := make([]*State, nValidators)
+	logger := consensusLogger()
+
+	closeFuncs := make([]func() error, 0, nValidators)
+
+	configRootDirs := make([]string, 0, nValidators)
+
+	for i := 0; i < nValidators; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
+
+		for _, opt := range configOpts {
+			opt(thisConfig)
+		}
+
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+
+		app := appFunc()
+
+		if appCloser, ok := app.(io.Closer); ok {
+			closeFuncs = append(closeFuncs, appCloser.Close)
+		}
+
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		css[i] = newStateWithConfigAndBlockStore(thisConfig, state, privVals[i], app, stateDB)
+		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+	}
+
+	return css, func() {
+		for _, closer := range closeFuncs {
+			_ = closer()
+		}
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
+}
+
+// nPeers = nValidators + nNotValidator
+func randConsensusNetWithPeers(
+	config *cfg.Config,
+	nValidators,
+	nPeers int,
+	testName string,
+	tickerFunc func() TimeoutTicker,
+	appFunc func(string) abci.Application,
+) ([]*State, *types.GenesisDoc, *cfg.Config, cleanupFunc) {
+	genDoc, privVals := factory.RandGenesisDoc(config, nValidators, false, testMinPower)
+	css := make([]*State, nPeers)
+	logger := consensusLogger()
+
+	var peer0Config *cfg.Config
+	configRootDirs := make([]string, 0, nPeers)
+	for i := 0; i < nPeers; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+		configRootDirs = append(configRootDirs, thisConfig.RootDir)
+		ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		if i == 0 {
+			peer0Config = thisConfig
+		}
+		var privVal types.PrivValidator
+		if i < nValidators {
+			privVal = privVals[i]
+		} else {
+			tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_")
+			if err != nil {
+				panic(err)
+			}
+			tempStateFile, err := ioutil.TempFile("", "priv_validator_state_")
+			if err != nil {
+				panic(err)
+			}
+
+			privVal, err = privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name(), "")
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		app := appFunc(path.Join(config.DBDir(), fmt.Sprintf("%s_%d", testName, i)))
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		if _, ok := app.(*kvstore.PersistentKVStoreApplication); ok {
+			// simulate handshake, receive app version. If don't do this, replay test will fail
+			state.Version.Consensus.App = kvstore.ProtocolVersion
+		}
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+		// sm.SaveState(stateDB,state)	//height 1's validatorsInfo already saved in LoadStateFromDBOrGenesisDoc above
+
+		css[i] = newStateWithConfig(thisConfig, state, privVal, app)
+		css[i].SetTimeoutTicker(tickerFunc())
+		css[i].SetLogger(logger.With("validator", i, "module", "consensus"))
+	}
+	return css, genDoc, peer0Config, func() {
+		for _, dir := range configRootDirs {
+			os.RemoveAll(dir)
+		}
+	}
+}
+
+func randGenesisState(
+	config *cfg.Config,
+	numValidators int,
+	randPower bool,
+	minPower int64) (sm.State, []types.PrivValidator) {
+
+	genDoc, privValidators := factory.RandGenesisDoc(config, numValidators, randPower, minPower)
+	s0, _ := sm.MakeGenesisState(genDoc)
+	return s0, privValidators
+}
+
+func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker {
+	return func() TimeoutTicker {
+		return &mockTicker{
+			c:        make(chan timeoutInfo, 10),
+			onlyOnce: onlyOnce,
+		}
+	}
+}
+
+// mock ticker only fires on RoundStepNewHeight
+// and only once if onlyOnce=true
+type mockTicker struct {
+	c chan timeoutInfo
+
+	mtx      sync.Mutex
+	onlyOnce bool
+	fired    bool
+}
+
+func (m *mockTicker) Start() error {
+	return nil
+}
+
+func (m *mockTicker) Stop() error {
+	return nil
+}
+
+func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	if m.onlyOnce && m.fired {
+		return
+	}
+	if ti.Step == cstypes.RoundStepNewHeight {
+		m.c <- ti
+		m.fired = true
+	}
+}
+
+func (m *mockTicker) Chan() <-chan timeoutInfo {
+	return m.c
+}
+
+func (*mockTicker) SetLogger(log.Logger) {}
+
+func newCounter() abci.Application {
+	return counter.NewApplication(true)
+}
+
+func newPersistentKVStore() abci.Application {
+	dir, err := ioutil.TempDir("", "persistent-kvstore")
+	if err != nil {
+		panic(err)
+	}
+	return kvstore.NewPersistentKVStoreApplication(dir)
+}
+
+func newPersistentKVStoreWithPath(dbDir string) abci.Application {
+	return kvstore.NewPersistentKVStoreApplication(dbDir)
+}

internal/consensus/invalid_test.go

@@ -0,0 +1,118 @@
+package consensus
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/libs/bytes"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestReactorInvalidPrecommit(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	states, cleanup := randConsensusState(config, n, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	t.Cleanup(cleanup)
+
+	for i := 0; i < 4; i++ {
+		ticker := NewTimeoutTicker()
+		ticker.SetLogger(states[i].Logger)
+		states[i].SetTimeoutTicker(ticker)
+	}
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	// this val sends a random precommit at each height
+	node := rts.network.RandomNode()
+
+	byzState := rts.states[node.NodeID]
+	byzReactor := rts.reactors[node.NodeID]
+
+	// Update the doPrevote function to just send a valid precommit for a random
+	// block and otherwise disable the priv validator.
+	byzState.mtx.Lock()
+	privVal := byzState.privValidator
+	byzState.doPrevote = func(height int64, round int32) {
+		invalidDoPrevoteFunc(t, height, round, byzState, byzReactor, privVal)
+	}
+	byzState.mtx.Unlock()
+
+	// wait for a bunch of blocks
+	//
+	// TODO: Make this tighter by ensuring the halt happens by block 2.
+	var wg sync.WaitGroup
+	for i := 0; i < 10; i++ {
+		for _, sub := range rts.subs {
+			wg.Add(1)
+
+			go func(s types.Subscription) {
+				<-s.Out()
+				wg.Done()
+			}(sub)
+		}
+	}
+
+	wg.Wait()
+}
+
+func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, r *Reactor, pv types.PrivValidator) {
+	// routine to:
+	// - precommit for a random block
+	// - send precommit to all peers
+	// - disable privValidator (so we don't do normal precommits)
+	go func() {
+		cs.mtx.Lock()
+		cs.privValidator = pv
+
+		pubKey, err := cs.privValidator.GetPubKey(context.Background())
+		require.NoError(t, err)
+
+		addr := pubKey.Address()
+		valIndex, _ := cs.Validators.GetByAddress(addr)
+
+		// precommit a random block
+		blockHash := bytes.HexBytes(tmrand.Bytes(32))
+		precommit := &types.Vote{
+			ValidatorAddress: addr,
+			ValidatorIndex:   valIndex,
+			Height:           cs.Height,
+			Round:            cs.Round,
+			Timestamp:        cs.voteTime(),
+			Type:             tmproto.PrecommitType,
+			BlockID: types.BlockID{
+				Hash:          blockHash,
+				PartSetHeader: types.PartSetHeader{Total: 1, Hash: tmrand.Bytes(32)}},
+		}
+
+		p := precommit.ToProto()
+		err = cs.privValidator.SignVote(context.Background(), cs.state.ChainID, p)
+		require.NoError(t, err)
+
+		precommit.Signature = p.Signature
+		cs.privValidator = nil // disable priv val so we don't do normal votes
+		cs.mtx.Unlock()
+
+		for _, ps := range r.peers {
+			cs.Logger.Info("sending bad vote", "block", blockHash, "peer", ps.peerID)
+
+			r.voteCh.Out <- p2p.Envelope{
+				To: ps.peerID,
+				Message: &tmcons.Vote{
+					Vote: precommit.ToProto(),
+				},
+			}
+		}
+	}()
+}

internal/consensus/mempool_test.go

@@ -0,0 +1,271 @@
+package consensus
+
+import (
+	"context"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/abci/example/code"
+	abci "github.com/tendermint/tendermint/abci/types"
+	mempl "github.com/tendermint/tendermint/internal/mempool"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+// for testing
+func assertMempool(txn txNotifier) mempl.Mempool {
+	return txn.(mempl.Mempool)
+}
+
+func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) {
+	baseConfig := configSetup(t)
+
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) })
+
+	config.Consensus.CreateEmptyBlocks = false
+	state, privVals := randGenesisState(baseConfig, 1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+	height, round := cs.Height, cs.Round
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	startTestRound(cs, height, round)
+
+	ensureNewEventOnChannel(newBlockCh) // first block gets committed
+	ensureNoNewEventOnChannel(newBlockCh)
+	deliverTxsRange(cs, 0, 1)
+	ensureNewEventOnChannel(newBlockCh) // commit txs
+	ensureNewEventOnChannel(newBlockCh) // commit updated app hash
+	ensureNoNewEventOnChannel(newBlockCh)
+}
+
+func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) {
+	baseConfig := configSetup(t)
+
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) })
+
+	config.Consensus.CreateEmptyBlocksInterval = ensureTimeout
+	state, privVals := randGenesisState(baseConfig, 1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	startTestRound(cs, cs.Height, cs.Round)
+
+	ensureNewEventOnChannel(newBlockCh)   // first block gets committed
+	ensureNoNewEventOnChannel(newBlockCh) // then we dont make a block ...
+	ensureNewEventOnChannel(newBlockCh)   // until the CreateEmptyBlocksInterval has passed
+}
+
+func TestMempoolProgressInHigherRound(t *testing.T) {
+	baseConfig := configSetup(t)
+
+	config := ResetConfig("consensus_mempool_txs_available_test")
+	t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) })
+
+	config.Consensus.CreateEmptyBlocks = false
+	state, privVals := randGenesisState(baseConfig, 1, false, 10)
+	cs := newStateWithConfig(config, state, privVals[0], NewCounterApplication())
+	assertMempool(cs.txNotifier).EnableTxsAvailable()
+	height, round := cs.Height, cs.Round
+	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound)
+	timeoutCh := subscribe(cs.eventBus, types.EventQueryTimeoutPropose)
+	cs.setProposal = func(proposal *types.Proposal) error {
+		if cs.Height == 2 && cs.Round == 0 {
+			// dont set the proposal in round 0 so we timeout and
+			// go to next round
+			cs.Logger.Info("Ignoring set proposal at height 2, round 0")
+			return nil
+		}
+		return cs.defaultSetProposal(proposal)
+	}
+	startTestRound(cs, height, round)
+
+	ensureNewRound(newRoundCh, height, round) // first round at first height
+	ensureNewEventOnChannel(newBlockCh)       // first block gets committed
+
+	height++ // moving to the next height
+	round = 0
+
+	ensureNewRound(newRoundCh, height, round) // first round at next height
+	deliverTxsRange(cs, 0, 1)                 // we deliver txs, but dont set a proposal so we get the next round
+	ensureNewTimeout(timeoutCh, height, round, cs.config.TimeoutPropose.Nanoseconds())
+
+	round++                                   // moving to the next round
+	ensureNewRound(newRoundCh, height, round) // wait for the next round
+	ensureNewEventOnChannel(newBlockCh)       // now we can commit the block
+}
+
+func deliverTxsRange(cs *State, start, end int) {
+	// Deliver some txs.
+	for i := start; i < end; i++ {
+		txBytes := make([]byte, 8)
+		binary.BigEndian.PutUint64(txBytes, uint64(i))
+		err := assertMempool(cs.txNotifier).CheckTx(context.Background(), txBytes, nil, mempl.TxInfo{})
+		if err != nil {
+			panic(fmt.Sprintf("Error after CheckTx: %v", err))
+		}
+	}
+}
+
+func TestMempoolTxConcurrentWithCommit(t *testing.T) {
+	config := configSetup(t)
+
+	state, privVals := randGenesisState(config, 1, false, 10)
+	blockDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(blockDB)
+	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB)
+	err := stateStore.Save(state)
+	require.NoError(t, err)
+	newBlockHeaderCh := subscribe(cs.eventBus, types.EventQueryNewBlockHeader)
+
+	const numTxs int64 = 3000
+	go deliverTxsRange(cs, 0, int(numTxs))
+
+	startTestRound(cs, cs.Height, cs.Round)
+	for n := int64(0); n < numTxs; {
+		select {
+		case msg := <-newBlockHeaderCh:
+			headerEvent := msg.Data().(types.EventDataNewBlockHeader)
+			n += headerEvent.NumTxs
+		case <-time.After(30 * time.Second):
+			t.Fatal("Timed out waiting 30s to commit blocks with transactions")
+		}
+	}
+}
+
+func TestMempoolRmBadTx(t *testing.T) {
+	config := configSetup(t)
+
+	state, privVals := randGenesisState(config, 1, false, 10)
+	app := NewCounterApplication()
+	blockDB := dbm.NewMemDB()
+	stateStore := sm.NewStore(blockDB)
+	cs := newStateWithConfigAndBlockStore(config, state, privVals[0], app, blockDB)
+	err := stateStore.Save(state)
+	require.NoError(t, err)
+
+	// increment the counter by 1
+	txBytes := make([]byte, 8)
+	binary.BigEndian.PutUint64(txBytes, uint64(0))
+
+	resDeliver := app.DeliverTx(abci.RequestDeliverTx{Tx: txBytes})
+	assert.False(t, resDeliver.IsErr(), fmt.Sprintf("expected no error. got %v", resDeliver))
+
+	resCommit := app.Commit()
+	assert.True(t, len(resCommit.Data) > 0)
+
+	emptyMempoolCh := make(chan struct{})
+	checkTxRespCh := make(chan struct{})
+	go func() {
+		// Try to send the tx through the mempool.
+		// CheckTx should not err, but the app should return a bad abci code
+		// and the tx should get removed from the pool
+		err := assertMempool(cs.txNotifier).CheckTx(context.Background(), txBytes, func(r *abci.Response) {
+			if r.GetCheckTx().Code != code.CodeTypeBadNonce {
+				t.Errorf("expected checktx to return bad nonce, got %v", r)
+				return
+			}
+			checkTxRespCh <- struct{}{}
+		}, mempl.TxInfo{})
+		if err != nil {
+			t.Errorf("error after CheckTx: %v", err)
+			return
+		}
+
+		// check for the tx
+		for {
+			txs := assertMempool(cs.txNotifier).ReapMaxBytesMaxGas(int64(len(txBytes)), -1)
+			if len(txs) == 0 {
+				emptyMempoolCh <- struct{}{}
+				return
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+	}()
+
+	// Wait until the tx returns
+	ticker := time.After(time.Second * 5)
+	select {
+	case <-checkTxRespCh:
+		// success
+	case <-ticker:
+		t.Errorf("timed out waiting for tx to return")
+		return
+	}
+
+	// Wait until the tx is removed
+	ticker = time.After(time.Second * 5)
+	select {
+	case <-emptyMempoolCh:
+		// success
+	case <-ticker:
+		t.Errorf("timed out waiting for tx to be removed")
+		return
+	}
+}
+
+// CounterApplication that maintains a mempool state and resets it upon commit
+type CounterApplication struct {
+	abci.BaseApplication
+
+	txCount        int
+	mempoolTxCount int
+}
+
+func NewCounterApplication() *CounterApplication {
+	return &CounterApplication{}
+}
+
+func (app *CounterApplication) Info(req abci.RequestInfo) abci.ResponseInfo {
+	return abci.ResponseInfo{Data: fmt.Sprintf("txs:%v", app.txCount)}
+}
+
+func (app *CounterApplication) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	txValue := txAsUint64(req.Tx)
+	if txValue != uint64(app.txCount) {
+		return abci.ResponseDeliverTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.txCount, txValue)}
+	}
+	app.txCount++
+	return abci.ResponseDeliverTx{Code: code.CodeTypeOK}
+}
+
+func (app *CounterApplication) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx {
+	txValue := txAsUint64(req.Tx)
+	if txValue != uint64(app.mempoolTxCount) {
+		return abci.ResponseCheckTx{
+			Code: code.CodeTypeBadNonce,
+			Log:  fmt.Sprintf("Invalid nonce. Expected %v, got %v", app.mempoolTxCount, txValue)}
+	}
+	app.mempoolTxCount++
+	return abci.ResponseCheckTx{Code: code.CodeTypeOK}
+}
+
+func txAsUint64(tx []byte) uint64 {
+	tx8 := make([]byte, 8)
+	copy(tx8[len(tx8)-len(tx):], tx)
+	return binary.BigEndian.Uint64(tx8)
+}
+
+func (app *CounterApplication) Commit() abci.ResponseCommit {
+	app.mempoolTxCount = app.txCount
+	if app.txCount == 0 {
+		return abci.ResponseCommit{}
+	}
+	hash := make([]byte, 8)
+	binary.BigEndian.PutUint64(hash, uint64(app.txCount))
+	return abci.ResponseCommit{Data: hash}
+}

internal/consensus/metrics.go

@@ -0,0 +1,220 @@
+package consensus
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/discard"
+
+	prometheus "github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	// MetricsSubsystem is a subsystem shared by all metrics exposed by this
+	// package.
+	MetricsSubsystem = "consensus"
+)
+
+// Metrics contains metrics exposed by this package.
+type Metrics struct {
+	// Height of the chain.
+	Height metrics.Gauge
+
+	// ValidatorLastSignedHeight of a validator.
+	ValidatorLastSignedHeight metrics.Gauge
+
+	// Number of rounds.
+	Rounds metrics.Gauge
+
+	// Number of validators.
+	Validators metrics.Gauge
+	// Total power of all validators.
+	ValidatorsPower metrics.Gauge
+	// Power of a validator.
+	ValidatorPower metrics.Gauge
+	// Amount of blocks missed by a validator.
+	ValidatorMissedBlocks metrics.Gauge
+	// Number of validators who did not sign.
+	MissingValidators metrics.Gauge
+	// Total power of the missing validators.
+	MissingValidatorsPower metrics.Gauge
+	// Number of validators who tried to double sign.
+	ByzantineValidators metrics.Gauge
+	// Total power of the byzantine validators.
+	ByzantineValidatorsPower metrics.Gauge
+
+	// Time between this and the last block.
+	BlockIntervalSeconds metrics.Histogram
+
+	// Number of transactions.
+	NumTxs metrics.Gauge
+	// Size of the block.
+	BlockSizeBytes metrics.Gauge
+	// Total number of transactions.
+	TotalTxs metrics.Gauge
+	// The latest block height.
+	CommittedHeight metrics.Gauge
+	// Whether or not a node is fast syncing. 1 if yes, 0 if no.
+	FastSyncing metrics.Gauge
+	// Whether or not a node is state syncing. 1 if yes, 0 if no.
+	StateSyncing metrics.Gauge
+
+	// Number of blockparts transmitted by peer.
+	BlockParts metrics.Counter
+}
+
+// PrometheusMetrics returns Metrics build using Prometheus client library.
+// Optionally, labels can be provided along with their values ("foo",
+// "fooValue").
+func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics {
+	labels := []string{}
+	for i := 0; i < len(labelsAndValues); i += 2 {
+		labels = append(labels, labelsAndValues[i])
+	}
+	return &Metrics{
+		Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "height",
+			Help:      "Height of the chain.",
+		}, labels).With(labelsAndValues...),
+		Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "rounds",
+			Help:      "Number of rounds.",
+		}, labels).With(labelsAndValues...),
+
+		Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validators",
+			Help:      "Number of validators.",
+		}, labels).With(labelsAndValues...),
+		ValidatorLastSignedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_last_signed_height",
+			Help:      "Last signed height for a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		ValidatorMissedBlocks: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_missed_blocks",
+			Help:      "Total missed blocks for a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validators_power",
+			Help:      "Total power of all validators.",
+		}, labels).With(labelsAndValues...),
+		ValidatorPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "validator_power",
+			Help:      "Power of a validator",
+		}, append(labels, "validator_address")).With(labelsAndValues...),
+		MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "missing_validators",
+			Help:      "Number of validators who did not sign.",
+		}, labels).With(labelsAndValues...),
+		MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "missing_validators_power",
+			Help:      "Total power of the missing validators.",
+		}, labels).With(labelsAndValues...),
+		ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "byzantine_validators",
+			Help:      "Number of validators who tried to double sign.",
+		}, labels).With(labelsAndValues...),
+		ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "byzantine_validators_power",
+			Help:      "Total power of the byzantine validators.",
+		}, labels).With(labelsAndValues...),
+		BlockIntervalSeconds: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_interval_seconds",
+			Help:      "Time between this and the last block.",
+		}, labels).With(labelsAndValues...),
+		NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "num_txs",
+			Help:      "Number of transactions.",
+		}, labels).With(labelsAndValues...),
+		BlockSizeBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_size_bytes",
+			Help:      "Size of the block.",
+		}, labels).With(labelsAndValues...),
+		TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "total_txs",
+			Help:      "Total number of transactions.",
+		}, labels).With(labelsAndValues...),
+		CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "latest_block_height",
+			Help:      "The latest block height.",
+		}, labels).With(labelsAndValues...),
+		FastSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "fast_syncing",
+			Help:      "Whether or not a node is fast syncing. 1 if yes, 0 if no.",
+		}, labels).With(labelsAndValues...),
+		StateSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "state_syncing",
+			Help:      "Whether or not a node is state syncing. 1 if yes, 0 if no.",
+		}, labels).With(labelsAndValues...),
+		BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "block_parts",
+			Help:      "Number of blockparts transmitted by peer.",
+		}, append(labels, "peer_id")).With(labelsAndValues...),
+	}
+}
+
+// NopMetrics returns no-op Metrics.
+func NopMetrics() *Metrics {
+	return &Metrics{
+		Height: discard.NewGauge(),
+
+		ValidatorLastSignedHeight: discard.NewGauge(),
+
+		Rounds: discard.NewGauge(),
+
+		Validators:               discard.NewGauge(),
+		ValidatorsPower:          discard.NewGauge(),
+		ValidatorPower:           discard.NewGauge(),
+		ValidatorMissedBlocks:    discard.NewGauge(),
+		MissingValidators:        discard.NewGauge(),
+		MissingValidatorsPower:   discard.NewGauge(),
+		ByzantineValidators:      discard.NewGauge(),
+		ByzantineValidatorsPower: discard.NewGauge(),
+
+		BlockIntervalSeconds: discard.NewHistogram(),
+
+		NumTxs:          discard.NewGauge(),
+		BlockSizeBytes:  discard.NewGauge(),
+		TotalTxs:        discard.NewGauge(),
+		CommittedHeight: discard.NewGauge(),
+		FastSyncing:     discard.NewGauge(),
+		StateSyncing:    discard.NewGauge(),
+		BlockParts:      discard.NewCounter(),
+	}
+}

internal/consensus/msgs.go

@@ -0,0 +1,685 @@
+package consensus
+
+import (
+	"errors"
+	"fmt"
+
+	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/libs/bits"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+// Message defines an interface that the consensus domain types implement. When
+// a proto message is received on a consensus p2p Channel, it is wrapped and then
+// converted to a Message via MsgFromProto.
+type Message interface {
+	ValidateBasic() error
+}
+
+func init() {
+	tmjson.RegisterType(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage")
+	tmjson.RegisterType(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage")
+	tmjson.RegisterType(&ProposalMessage{}, "tendermint/Proposal")
+	tmjson.RegisterType(&ProposalPOLMessage{}, "tendermint/ProposalPOL")
+	tmjson.RegisterType(&BlockPartMessage{}, "tendermint/BlockPart")
+	tmjson.RegisterType(&VoteMessage{}, "tendermint/Vote")
+	tmjson.RegisterType(&HasVoteMessage{}, "tendermint/HasVote")
+	tmjson.RegisterType(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23")
+	tmjson.RegisterType(&VoteSetBitsMessage{}, "tendermint/VoteSetBits")
+}
+
+// NewRoundStepMessage is sent for every step taken in the ConsensusState.
+// For every height/round/step transition
+type NewRoundStepMessage struct {
+	Height                int64
+	Round                 int32
+	Step                  cstypes.RoundStepType
+	SecondsSinceStartTime int64
+	LastCommitRound       int32
+}
+
+// ValidateBasic performs basic validation.
+func (m *NewRoundStepMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.Round < 0 {
+		return errors.New("negative Round")
+	}
+	if !m.Step.IsValid() {
+		return errors.New("invalid Step")
+	}
+
+	// NOTE: SecondsSinceStartTime may be negative
+
+	// LastCommitRound will be -1 for the initial height, but we don't know what height this is
+	// since it can be specified in genesis. The reactor will have to validate this via
+	// ValidateHeight().
+	if m.LastCommitRound < -1 {
+		return errors.New("invalid LastCommitRound (cannot be < -1)")
+	}
+
+	return nil
+}
+
+// ValidateHeight validates the height given the chain's initial height.
+func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error {
+	if m.Height < initialHeight {
+		return fmt.Errorf("invalid Height %v (lower than initial height %v)",
+			m.Height, initialHeight)
+	}
+	if m.Height == initialHeight && m.LastCommitRound != -1 {
+		return fmt.Errorf("invalid LastCommitRound %v (must be -1 for initial height %v)",
+			m.LastCommitRound, initialHeight)
+	}
+	if m.Height > initialHeight && m.LastCommitRound < 0 {
+		return fmt.Errorf("LastCommitRound can only be negative for initial height %v", // nolint
+			initialHeight)
+	}
+	return nil
+}
+
+// String returns a string representation.
+func (m *NewRoundStepMessage) String() string {
+	return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]",
+		m.Height, m.Round, m.Step, m.LastCommitRound)
+}
+
+// NewValidBlockMessage is sent when a validator observes a valid block B in some round r,
+// i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r.
+// In case the block is also committed, then IsCommit flag is set to true.
+type NewValidBlockMessage struct {
+	Height             int64
+	Round              int32
+	BlockPartSetHeader types.PartSetHeader
+	BlockParts         *bits.BitArray
+	IsCommit           bool
+}
+
+// ValidateBasic performs basic validation.
+func (m *NewValidBlockMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.Round < 0 {
+		return errors.New("negative Round")
+	}
+	if err := m.BlockPartSetHeader.ValidateBasic(); err != nil {
+		return fmt.Errorf("wrong BlockPartSetHeader: %v", err)
+	}
+	if m.BlockParts.Size() == 0 {
+		return errors.New("empty blockParts")
+	}
+	if m.BlockParts.Size() != int(m.BlockPartSetHeader.Total) {
+		return fmt.Errorf("blockParts bit array size %d not equal to BlockPartSetHeader.Total %d",
+			m.BlockParts.Size(),
+			m.BlockPartSetHeader.Total)
+	}
+	if m.BlockParts.Size() > int(types.MaxBlockPartsCount) {
+		return fmt.Errorf("blockParts bit array is too big: %d, max: %d", m.BlockParts.Size(), types.MaxBlockPartsCount)
+	}
+	return nil
+}
+
+// String returns a string representation.
+func (m *NewValidBlockMessage) String() string {
+	return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]",
+		m.Height, m.Round, m.BlockPartSetHeader, m.BlockParts, m.IsCommit)
+}
+
+// ProposalMessage is sent when a new block is proposed.
+type ProposalMessage struct {
+	Proposal *types.Proposal
+}
+
+// ValidateBasic performs basic validation.
+func (m *ProposalMessage) ValidateBasic() error {
+	return m.Proposal.ValidateBasic()
+}
+
+// String returns a string representation.
+func (m *ProposalMessage) String() string {
+	return fmt.Sprintf("[Proposal %v]", m.Proposal)
+}
+
+// ProposalPOLMessage is sent when a previous proposal is re-proposed.
+type ProposalPOLMessage struct {
+	Height           int64
+	ProposalPOLRound int32
+	ProposalPOL      *bits.BitArray
+}
+
+// ValidateBasic performs basic validation.
+func (m *ProposalPOLMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.ProposalPOLRound < 0 {
+		return errors.New("negative ProposalPOLRound")
+	}
+	if m.ProposalPOL.Size() == 0 {
+		return errors.New("empty ProposalPOL bit array")
+	}
+	if m.ProposalPOL.Size() > types.MaxVotesCount {
+		return fmt.Errorf("proposalPOL bit array is too big: %d, max: %d", m.ProposalPOL.Size(), types.MaxVotesCount)
+	}
+	return nil
+}
+
+// String returns a string representation.
+func (m *ProposalPOLMessage) String() string {
+	return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL)
+}
+
+// BlockPartMessage is sent when gossipping a piece of the proposed block.
+type BlockPartMessage struct {
+	Height int64
+	Round  int32
+	Part   *types.Part
+}
+
+// ValidateBasic performs basic validation.
+func (m *BlockPartMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.Round < 0 {
+		return errors.New("negative Round")
+	}
+	if err := m.Part.ValidateBasic(); err != nil {
+		return fmt.Errorf("wrong Part: %v", err)
+	}
+	return nil
+}
+
+// String returns a string representation.
+func (m *BlockPartMessage) String() string {
+	return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part)
+}
+
+// VoteMessage is sent when voting for a proposal (or lack thereof).
+type VoteMessage struct {
+	Vote *types.Vote
+}
+
+// ValidateBasic performs basic validation.
+func (m *VoteMessage) ValidateBasic() error {
+	return m.Vote.ValidateBasic()
+}
+
+// String returns a string representation.
+func (m *VoteMessage) String() string {
+	return fmt.Sprintf("[Vote %v]", m.Vote)
+}
+
+// HasVoteMessage is sent to indicate that a particular vote has been received.
+type HasVoteMessage struct {
+	Height int64
+	Round  int32
+	Type   tmproto.SignedMsgType
+	Index  int32
+}
+
+// ValidateBasic performs basic validation.
+func (m *HasVoteMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.Round < 0 {
+		return errors.New("negative Round")
+	}
+	if !types.IsVoteTypeValid(m.Type) {
+		return errors.New("invalid Type")
+	}
+	if m.Index < 0 {
+		return errors.New("negative Index")
+	}
+	return nil
+}
+
+// String returns a string representation.
+func (m *HasVoteMessage) String() string {
+	return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type)
+}
+
+// VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes.
+type VoteSetMaj23Message struct {
+	Height  int64
+	Round   int32
+	Type    tmproto.SignedMsgType
+	BlockID types.BlockID
+}
+
+// ValidateBasic performs basic validation.
+func (m *VoteSetMaj23Message) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if m.Round < 0 {
+		return errors.New("negative Round")
+	}
+	if !types.IsVoteTypeValid(m.Type) {
+		return errors.New("invalid Type")
+	}
+	if err := m.BlockID.ValidateBasic(); err != nil {
+		return fmt.Errorf("wrong BlockID: %v", err)
+	}
+
+	return nil
+}
+
+// String returns a string representation.
+func (m *VoteSetMaj23Message) String() string {
+	return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID)
+}
+
+// VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the
+// BlockID.
+type VoteSetBitsMessage struct {
+	Height  int64
+	Round   int32
+	Type    tmproto.SignedMsgType
+	BlockID types.BlockID
+	Votes   *bits.BitArray
+}
+
+// ValidateBasic performs basic validation.
+func (m *VoteSetBitsMessage) ValidateBasic() error {
+	if m.Height < 0 {
+		return errors.New("negative Height")
+	}
+	if !types.IsVoteTypeValid(m.Type) {
+		return errors.New("invalid Type")
+	}
+	if err := m.BlockID.ValidateBasic(); err != nil {
+		return fmt.Errorf("wrong BlockID: %v", err)
+	}
+
+	// NOTE: Votes.Size() can be zero if the node does not have any
+	if m.Votes.Size() > types.MaxVotesCount {
+		return fmt.Errorf("votes bit array is too big: %d, max: %d", m.Votes.Size(), types.MaxVotesCount)
+	}
+
+	return nil
+}
+
+// String returns a string representation.
+func (m *VoteSetBitsMessage) String() string {
+	return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes)
+}
+
+// MsgToProto takes a consensus message type and returns the proto defined
+// consensus message.
+//
+// TODO: This needs to be removed, but WALToProto depends on this.
+func MsgToProto(msg Message) (*tmcons.Message, error) {
+	if msg == nil {
+		return nil, errors.New("consensus: message is nil")
+	}
+	var pb tmcons.Message
+
+	switch msg := msg.(type) {
+	case *NewRoundStepMessage:
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_NewRoundStep{
+				NewRoundStep: &tmcons.NewRoundStep{
+					Height:                msg.Height,
+					Round:                 msg.Round,
+					Step:                  uint32(msg.Step),
+					SecondsSinceStartTime: msg.SecondsSinceStartTime,
+					LastCommitRound:       msg.LastCommitRound,
+				},
+			},
+		}
+	case *NewValidBlockMessage:
+		pbPartSetHeader := msg.BlockPartSetHeader.ToProto()
+		pbBits := msg.BlockParts.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_NewValidBlock{
+				NewValidBlock: &tmcons.NewValidBlock{
+					Height:             msg.Height,
+					Round:              msg.Round,
+					BlockPartSetHeader: pbPartSetHeader,
+					BlockParts:         pbBits,
+					IsCommit:           msg.IsCommit,
+				},
+			},
+		}
+	case *ProposalMessage:
+		pbP := msg.Proposal.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_Proposal{
+				Proposal: &tmcons.Proposal{
+					Proposal: *pbP,
+				},
+			},
+		}
+	case *ProposalPOLMessage:
+		pbBits := msg.ProposalPOL.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_ProposalPol{
+				ProposalPol: &tmcons.ProposalPOL{
+					Height:           msg.Height,
+					ProposalPolRound: msg.ProposalPOLRound,
+					ProposalPol:      *pbBits,
+				},
+			},
+		}
+	case *BlockPartMessage:
+		parts, err := msg.Part.ToProto()
+		if err != nil {
+			return nil, fmt.Errorf("msg to proto error: %w", err)
+		}
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_BlockPart{
+				BlockPart: &tmcons.BlockPart{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Part:   *parts,
+				},
+			},
+		}
+	case *VoteMessage:
+		vote := msg.Vote.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_Vote{
+				Vote: &tmcons.Vote{
+					Vote: vote,
+				},
+			},
+		}
+	case *HasVoteMessage:
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_HasVote{
+				HasVote: &tmcons.HasVote{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Type:   msg.Type,
+					Index:  msg.Index,
+				},
+			},
+		}
+	case *VoteSetMaj23Message:
+		bi := msg.BlockID.ToProto()
+		pb = tmcons.Message{
+			Sum: &tmcons.Message_VoteSetMaj23{
+				VoteSetMaj23: &tmcons.VoteSetMaj23{
+					Height:  msg.Height,
+					Round:   msg.Round,
+					Type:    msg.Type,
+					BlockID: bi,
+				},
+			},
+		}
+	case *VoteSetBitsMessage:
+		bi := msg.BlockID.ToProto()
+		bits := msg.Votes.ToProto()
+
+		vsb := &tmcons.Message_VoteSetBits{
+			VoteSetBits: &tmcons.VoteSetBits{
+				Height:  msg.Height,
+				Round:   msg.Round,
+				Type:    msg.Type,
+				BlockID: bi,
+			},
+		}
+
+		if bits != nil {
+			vsb.VoteSetBits.Votes = *bits
+		}
+
+		pb = tmcons.Message{
+			Sum: vsb,
+		}
+
+	default:
+		return nil, fmt.Errorf("consensus: message not recognized: %T", msg)
+	}
+
+	return &pb, nil
+}
+
+// MsgFromProto takes a consensus proto message and returns the native go type.
+func MsgFromProto(msg *tmcons.Message) (Message, error) {
+	if msg == nil {
+		return nil, errors.New("consensus: nil message")
+	}
+	var pb Message
+
+	switch msg := msg.Sum.(type) {
+	case *tmcons.Message_NewRoundStep:
+		rs, err := tmmath.SafeConvertUint8(int64(msg.NewRoundStep.Step))
+		// deny message based on possible overflow
+		if err != nil {
+			return nil, fmt.Errorf("denying message due to possible overflow: %w", err)
+		}
+		pb = &NewRoundStepMessage{
+			Height:                msg.NewRoundStep.Height,
+			Round:                 msg.NewRoundStep.Round,
+			Step:                  cstypes.RoundStepType(rs),
+			SecondsSinceStartTime: msg.NewRoundStep.SecondsSinceStartTime,
+			LastCommitRound:       msg.NewRoundStep.LastCommitRound,
+		}
+	case *tmcons.Message_NewValidBlock:
+		pbPartSetHeader, err := types.PartSetHeaderFromProto(&msg.NewValidBlock.BlockPartSetHeader)
+		if err != nil {
+			return nil, fmt.Errorf("parts header to proto error: %w", err)
+		}
+
+		pbBits := new(bits.BitArray)
+		err = pbBits.FromProto(msg.NewValidBlock.BlockParts)
+		if err != nil {
+			return nil, fmt.Errorf("parts to proto error: %w", err)
+		}
+
+		pb = &NewValidBlockMessage{
+			Height:             msg.NewValidBlock.Height,
+			Round:              msg.NewValidBlock.Round,
+			BlockPartSetHeader: *pbPartSetHeader,
+			BlockParts:         pbBits,
+			IsCommit:           msg.NewValidBlock.IsCommit,
+		}
+	case *tmcons.Message_Proposal:
+		pbP, err := types.ProposalFromProto(&msg.Proposal.Proposal)
+		if err != nil {
+			return nil, fmt.Errorf("proposal msg to proto error: %w", err)
+		}
+
+		pb = &ProposalMessage{
+			Proposal: pbP,
+		}
+	case *tmcons.Message_ProposalPol:
+		pbBits := new(bits.BitArray)
+		err := pbBits.FromProto(&msg.ProposalPol.ProposalPol)
+		if err != nil {
+			return nil, fmt.Errorf("proposal PoL to proto error: %w", err)
+		}
+		pb = &ProposalPOLMessage{
+			Height:           msg.ProposalPol.Height,
+			ProposalPOLRound: msg.ProposalPol.ProposalPolRound,
+			ProposalPOL:      pbBits,
+		}
+	case *tmcons.Message_BlockPart:
+		parts, err := types.PartFromProto(&msg.BlockPart.Part)
+		if err != nil {
+			return nil, fmt.Errorf("blockpart msg to proto error: %w", err)
+		}
+		pb = &BlockPartMessage{
+			Height: msg.BlockPart.Height,
+			Round:  msg.BlockPart.Round,
+			Part:   parts,
+		}
+	case *tmcons.Message_Vote:
+		vote, err := types.VoteFromProto(msg.Vote.Vote)
+		if err != nil {
+			return nil, fmt.Errorf("vote msg to proto error: %w", err)
+		}
+
+		pb = &VoteMessage{
+			Vote: vote,
+		}
+	case *tmcons.Message_HasVote:
+		pb = &HasVoteMessage{
+			Height: msg.HasVote.Height,
+			Round:  msg.HasVote.Round,
+			Type:   msg.HasVote.Type,
+			Index:  msg.HasVote.Index,
+		}
+	case *tmcons.Message_VoteSetMaj23:
+		bi, err := types.BlockIDFromProto(&msg.VoteSetMaj23.BlockID)
+		if err != nil {
+			return nil, fmt.Errorf("voteSetMaj23 msg to proto error: %w", err)
+		}
+		pb = &VoteSetMaj23Message{
+			Height:  msg.VoteSetMaj23.Height,
+			Round:   msg.VoteSetMaj23.Round,
+			Type:    msg.VoteSetMaj23.Type,
+			BlockID: *bi,
+		}
+	case *tmcons.Message_VoteSetBits:
+		bi, err := types.BlockIDFromProto(&msg.VoteSetBits.BlockID)
+		if err != nil {
+			return nil, fmt.Errorf("block ID to proto error: %w", err)
+		}
+		bits := new(bits.BitArray)
+		err = bits.FromProto(&msg.VoteSetBits.Votes)
+		if err != nil {
+			return nil, fmt.Errorf("votes to proto error: %w", err)
+		}
+
+		pb = &VoteSetBitsMessage{
+			Height:  msg.VoteSetBits.Height,
+			Round:   msg.VoteSetBits.Round,
+			Type:    msg.VoteSetBits.Type,
+			BlockID: *bi,
+			Votes:   bits,
+		}
+	default:
+		return nil, fmt.Errorf("consensus: message not recognized: %T", msg)
+	}
+
+	if err := pb.ValidateBasic(); err != nil {
+		return nil, err
+	}
+
+	return pb, nil
+}
+
+// WALToProto takes a WAL message and return a proto walMessage and error.
+func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) {
+	var pb tmcons.WALMessage
+
+	switch msg := msg.(type) {
+	case types.EventDataRoundState:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EventDataRoundState{
+				EventDataRoundState: &tmproto.EventDataRoundState{
+					Height: msg.Height,
+					Round:  msg.Round,
+					Step:   msg.Step,
+				},
+			},
+		}
+	case msgInfo:
+		consMsg, err := MsgToProto(msg.Msg)
+		if err != nil {
+			return nil, err
+		}
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_MsgInfo{
+				MsgInfo: &tmcons.MsgInfo{
+					Msg:    *consMsg,
+					PeerID: string(msg.PeerID),
+				},
+			},
+		}
+
+	case timeoutInfo:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_TimeoutInfo{
+				TimeoutInfo: &tmcons.TimeoutInfo{
+					Duration: msg.Duration,
+					Height:   msg.Height,
+					Round:    msg.Round,
+					Step:     uint32(msg.Step),
+				},
+			},
+		}
+
+	case EndHeightMessage:
+		pb = tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EndHeight{
+				EndHeight: &tmcons.EndHeight{
+					Height: msg.Height,
+				},
+			},
+		}
+
+	default:
+		return nil, fmt.Errorf("to proto: wal message not recognized: %T", msg)
+	}
+
+	return &pb, nil
+}
+
+// WALFromProto takes a proto wal message and return a consensus walMessage and
+// error.
+func WALFromProto(msg *tmcons.WALMessage) (WALMessage, error) {
+	if msg == nil {
+		return nil, errors.New("nil WAL message")
+	}
+
+	var pb WALMessage
+
+	switch msg := msg.Sum.(type) {
+	case *tmcons.WALMessage_EventDataRoundState:
+		pb = types.EventDataRoundState{
+			Height: msg.EventDataRoundState.Height,
+			Round:  msg.EventDataRoundState.Round,
+			Step:   msg.EventDataRoundState.Step,
+		}
+
+	case *tmcons.WALMessage_MsgInfo:
+		walMsg, err := MsgFromProto(&msg.MsgInfo.Msg)
+		if err != nil {
+			return nil, fmt.Errorf("msgInfo from proto error: %w", err)
+		}
+		pb = msgInfo{
+			Msg:    walMsg,
+			PeerID: p2p.NodeID(msg.MsgInfo.PeerID),
+		}
+
+	case *tmcons.WALMessage_TimeoutInfo:
+		tis, err := tmmath.SafeConvertUint8(int64(msg.TimeoutInfo.Step))
+		// deny message based on possible overflow
+		if err != nil {
+			return nil, fmt.Errorf("denying message due to possible overflow: %w", err)
+		}
+
+		pb = timeoutInfo{
+			Duration: msg.TimeoutInfo.Duration,
+			Height:   msg.TimeoutInfo.Height,
+			Round:    msg.TimeoutInfo.Round,
+			Step:     cstypes.RoundStepType(tis),
+		}
+
+		return pb, nil
+
+	case *tmcons.WALMessage_EndHeight:
+		pb := EndHeightMessage{
+			Height: msg.EndHeight.Height,
+		}
+
+		return pb, nil
+
+	default:
+		return nil, fmt.Errorf("from proto: wal message not recognized: %T", msg)
+	}
+
+	return pb, nil
+}

internal/consensus/msgs_test.go

@@ -0,0 +1,733 @@
+package consensus
+
+import (
+	"encoding/hex"
+	"fmt"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	"github.com/tendermint/tendermint/libs/bits"
+	"github.com/tendermint/tendermint/libs/bytes"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestMsgToProto(t *testing.T) {
+	psh := types.PartSetHeader{
+		Total: 1,
+		Hash:  tmrand.Bytes(32),
+	}
+	pbPsh := psh.ToProto()
+	bi := types.BlockID{
+		Hash:          tmrand.Bytes(32),
+		PartSetHeader: psh,
+	}
+	pbBi := bi.ToProto()
+	bits := bits.NewBitArray(1)
+	pbBits := bits.ToProto()
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: tmrand.Bytes(32),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	proposal := types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     1,
+		POLRound:  1,
+		BlockID:   bi,
+		Timestamp: time.Now(),
+		Signature: tmrand.Bytes(20),
+	}
+	pbProposal := proposal.ToProto()
+
+	pv := types.NewMockPV()
+	vote, err := factory.MakeVote(pv, factory.DefaultTestChainID,
+		0, 1, 0, 2, types.BlockID{}, time.Now())
+	require.NoError(t, err)
+	pbVote := vote.ToProto()
+
+	testsCases := []struct {
+		testName string
+		msg      Message
+		want     *tmcons.Message
+		wantErr  bool
+	}{
+		{"successful NewRoundStepMessage", &NewRoundStepMessage{
+			Height:                2,
+			Round:                 1,
+			Step:                  1,
+			SecondsSinceStartTime: 1,
+			LastCommitRound:       2,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_NewRoundStep{
+				NewRoundStep: &tmcons.NewRoundStep{
+					Height:                2,
+					Round:                 1,
+					Step:                  1,
+					SecondsSinceStartTime: 1,
+					LastCommitRound:       2,
+				},
+			},
+		}, false},
+
+		{"successful NewValidBlockMessage", &NewValidBlockMessage{
+			Height:             1,
+			Round:              1,
+			BlockPartSetHeader: psh,
+			BlockParts:         bits,
+			IsCommit:           false,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_NewValidBlock{
+				NewValidBlock: &tmcons.NewValidBlock{
+					Height:             1,
+					Round:              1,
+					BlockPartSetHeader: pbPsh,
+					BlockParts:         pbBits,
+					IsCommit:           false,
+				},
+			},
+		}, false},
+		{"successful BlockPartMessage", &BlockPartMessage{
+			Height: 100,
+			Round:  1,
+			Part:   &parts,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_BlockPart{
+				BlockPart: &tmcons.BlockPart{
+					Height: 100,
+					Round:  1,
+					Part:   *pbParts,
+				},
+			},
+		}, false},
+		{"successful ProposalPOLMessage", &ProposalPOLMessage{
+			Height:           1,
+			ProposalPOLRound: 1,
+			ProposalPOL:      bits,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_ProposalPol{
+				ProposalPol: &tmcons.ProposalPOL{
+					Height:           1,
+					ProposalPolRound: 1,
+					ProposalPol:      *pbBits,
+				},
+			}}, false},
+		{"successful ProposalMessage", &ProposalMessage{
+			Proposal: &proposal,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_Proposal{
+				Proposal: &tmcons.Proposal{
+					Proposal: *pbProposal,
+				},
+			},
+		}, false},
+		{"successful VoteMessage", &VoteMessage{
+			Vote: vote,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_Vote{
+				Vote: &tmcons.Vote{
+					Vote: pbVote,
+				},
+			},
+		}, false},
+		{"successful VoteSetMaj23", &VoteSetMaj23Message{
+			Height:  1,
+			Round:   1,
+			Type:    1,
+			BlockID: bi,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_VoteSetMaj23{
+				VoteSetMaj23: &tmcons.VoteSetMaj23{
+					Height:  1,
+					Round:   1,
+					Type:    1,
+					BlockID: pbBi,
+				},
+			},
+		}, false},
+		{"successful VoteSetBits", &VoteSetBitsMessage{
+			Height:  1,
+			Round:   1,
+			Type:    1,
+			BlockID: bi,
+			Votes:   bits,
+		}, &tmcons.Message{
+			Sum: &tmcons.Message_VoteSetBits{
+				VoteSetBits: &tmcons.VoteSetBits{
+					Height:  1,
+					Round:   1,
+					Type:    1,
+					BlockID: pbBi,
+					Votes:   *pbBits,
+				},
+			},
+		}, false},
+		{"failure", nil, &tmcons.Message{}, true},
+	}
+	for _, tt := range testsCases {
+		tt := tt
+		t.Run(tt.testName, func(t *testing.T) {
+			pb, err := MsgToProto(tt.msg)
+			if tt.wantErr == true {
+				assert.Equal(t, err != nil, tt.wantErr)
+				return
+			}
+			assert.EqualValues(t, tt.want, pb, tt.testName)
+
+			msg, err := MsgFromProto(pb)
+
+			if !tt.wantErr {
+				require.NoError(t, err)
+				bcm := assert.Equal(t, tt.msg, msg, tt.testName)
+				assert.True(t, bcm, tt.testName)
+			} else {
+				require.Error(t, err, tt.testName)
+			}
+		})
+	}
+}
+
+func TestWALMsgProto(t *testing.T) {
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: tmrand.Bytes(32),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	testsCases := []struct {
+		testName string
+		msg      WALMessage
+		want     *tmcons.WALMessage
+		wantErr  bool
+	}{
+		{"successful EventDataRoundState", types.EventDataRoundState{
+			Height: 2,
+			Round:  1,
+			Step:   "ronies",
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EventDataRoundState{
+				EventDataRoundState: &tmproto.EventDataRoundState{
+					Height: 2,
+					Round:  1,
+					Step:   "ronies",
+				},
+			},
+		}, false},
+		{"successful msgInfo", msgInfo{
+			Msg: &BlockPartMessage{
+				Height: 100,
+				Round:  1,
+				Part:   &parts,
+			},
+			PeerID: p2p.NodeID("string"),
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_MsgInfo{
+				MsgInfo: &tmcons.MsgInfo{
+					Msg: tmcons.Message{
+						Sum: &tmcons.Message_BlockPart{
+							BlockPart: &tmcons.BlockPart{
+								Height: 100,
+								Round:  1,
+								Part:   *pbParts,
+							},
+						},
+					},
+					PeerID: "string",
+				},
+			},
+		}, false},
+		{"successful timeoutInfo", timeoutInfo{
+			Duration: time.Duration(100),
+			Height:   1,
+			Round:    1,
+			Step:     1,
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_TimeoutInfo{
+				TimeoutInfo: &tmcons.TimeoutInfo{
+					Duration: time.Duration(100),
+					Height:   1,
+					Round:    1,
+					Step:     1,
+				},
+			},
+		}, false},
+		{"successful EndHeightMessage", EndHeightMessage{
+			Height: 1,
+		}, &tmcons.WALMessage{
+			Sum: &tmcons.WALMessage_EndHeight{
+				EndHeight: &tmcons.EndHeight{
+					Height: 1,
+				},
+			},
+		}, false},
+		{"failure", nil, &tmcons.WALMessage{}, true},
+	}
+	for _, tt := range testsCases {
+		tt := tt
+		t.Run(tt.testName, func(t *testing.T) {
+			pb, err := WALToProto(tt.msg)
+			if tt.wantErr == true {
+				assert.Equal(t, err != nil, tt.wantErr)
+				return
+			}
+			assert.EqualValues(t, tt.want, pb, tt.testName)
+
+			msg, err := WALFromProto(pb)
+
+			if !tt.wantErr {
+				require.NoError(t, err)
+				assert.Equal(t, tt.msg, msg, tt.testName) // need the concrete type as WAL Message is a empty interface
+			} else {
+				require.Error(t, err, tt.testName)
+			}
+		})
+	}
+}
+
+// nolint:lll //ignore line length for tests
+func TestConsMsgsVectors(t *testing.T) {
+	date := time.Date(2018, 8, 30, 12, 0, 0, 0, time.UTC)
+	psh := types.PartSetHeader{
+		Total: 1,
+		Hash:  []byte("add_more_exclamation_marks_code-"),
+	}
+	pbPsh := psh.ToProto()
+
+	bi := types.BlockID{
+		Hash:          []byte("add_more_exclamation_marks_code-"),
+		PartSetHeader: psh,
+	}
+	pbBi := bi.ToProto()
+	bits := bits.NewBitArray(1)
+	pbBits := bits.ToProto()
+
+	parts := types.Part{
+		Index: 1,
+		Bytes: []byte("test"),
+		Proof: merkle.Proof{
+			Total:    1,
+			Index:    1,
+			LeafHash: []byte("add_more_exclamation_marks_code-"),
+			Aunts:    [][]byte{},
+		},
+	}
+	pbParts, err := parts.ToProto()
+	require.NoError(t, err)
+
+	proposal := types.Proposal{
+		Type:      tmproto.ProposalType,
+		Height:    1,
+		Round:     1,
+		POLRound:  1,
+		BlockID:   bi,
+		Timestamp: date,
+		Signature: []byte("add_more_exclamation"),
+	}
+	pbProposal := proposal.ToProto()
+
+	v := &types.Vote{
+		ValidatorAddress: []byte("add_more_exclamation"),
+		ValidatorIndex:   1,
+		Height:           1,
+		Round:            0,
+		Timestamp:        date,
+		Type:             tmproto.PrecommitType,
+		BlockID:          bi,
+	}
+	vpb := v.ToProto()
+
+	testCases := []struct {
+		testName string
+		cMsg     proto.Message
+		expBytes string
+	}{
+		{"NewRoundStep", &tmcons.Message{Sum: &tmcons.Message_NewRoundStep{NewRoundStep: &tmcons.NewRoundStep{
+			Height:                1,
+			Round:                 1,
+			Step:                  1,
+			SecondsSinceStartTime: 1,
+			LastCommitRound:       1,
+		}}}, "0a0a08011001180120012801"},
+		{"NewRoundStep Max", &tmcons.Message{Sum: &tmcons.Message_NewRoundStep{NewRoundStep: &tmcons.NewRoundStep{
+			Height:                math.MaxInt64,
+			Round:                 math.MaxInt32,
+			Step:                  math.MaxUint32,
+			SecondsSinceStartTime: math.MaxInt64,
+			LastCommitRound:       math.MaxInt32,
+		}}}, "0a2608ffffffffffffffff7f10ffffffff0718ffffffff0f20ffffffffffffffff7f28ffffffff07"},
+		{"NewValidBlock", &tmcons.Message{Sum: &tmcons.Message_NewValidBlock{
+			NewValidBlock: &tmcons.NewValidBlock{
+				Height: 1, Round: 1, BlockPartSetHeader: pbPsh, BlockParts: pbBits, IsCommit: false}}},
+			"1231080110011a24080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d22050801120100"},
+		{"Proposal", &tmcons.Message{Sum: &tmcons.Message_Proposal{Proposal: &tmcons.Proposal{Proposal: *pbProposal}}},
+			"1a720a7008201001180120012a480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d320608c0b89fdc053a146164645f6d6f72655f6578636c616d6174696f6e"},
+		{"ProposalPol", &tmcons.Message{Sum: &tmcons.Message_ProposalPol{
+			ProposalPol: &tmcons.ProposalPOL{Height: 1, ProposalPolRound: 1}}},
+			"2206080110011a00"},
+		{"BlockPart", &tmcons.Message{Sum: &tmcons.Message_BlockPart{
+			BlockPart: &tmcons.BlockPart{Height: 1, Round: 1, Part: *pbParts}}},
+			"2a36080110011a3008011204746573741a26080110011a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d"},
+		{"Vote", &tmcons.Message{Sum: &tmcons.Message_Vote{
+			Vote: &tmcons.Vote{Vote: vpb}}},
+			"32700a6e0802100122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d2a0608c0b89fdc0532146164645f6d6f72655f6578636c616d6174696f6e3801"},
+		{"HasVote", &tmcons.Message{Sum: &tmcons.Message_HasVote{
+			HasVote: &tmcons.HasVote{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 1}}},
+			"3a080801100118012001"},
+		{"HasVote", &tmcons.Message{Sum: &tmcons.Message_HasVote{
+			HasVote: &tmcons.HasVote{Height: math.MaxInt64, Round: math.MaxInt32,
+				Type: tmproto.PrevoteType, Index: math.MaxInt32}}},
+			"3a1808ffffffffffffffff7f10ffffffff07180120ffffffff07"},
+		{"VoteSetMaj23", &tmcons.Message{Sum: &tmcons.Message_VoteSetMaj23{
+			VoteSetMaj23: &tmcons.VoteSetMaj23{Height: 1, Round: 1, Type: tmproto.PrevoteType, BlockID: pbBi}}},
+			"425008011001180122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d"},
+		{"VoteSetBits", &tmcons.Message{Sum: &tmcons.Message_VoteSetBits{
+			VoteSetBits: &tmcons.VoteSetBits{Height: 1, Round: 1, Type: tmproto.PrevoteType, BlockID: pbBi, Votes: *pbBits}}},
+			"4a5708011001180122480a206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d1224080112206164645f6d6f72655f6578636c616d6174696f6e5f6d61726b735f636f64652d2a050801120100"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			bz, err := proto.Marshal(tc.cMsg)
+			require.NoError(t, err)
+
+			require.Equal(t, tc.expBytes, hex.EncodeToString(bz))
+		})
+	}
+}
+
+func TestVoteSetMaj23MessageValidateBasic(t *testing.T) {
+	const (
+		validSignedMsgType   tmproto.SignedMsgType = 0x01
+		invalidSignedMsgType tmproto.SignedMsgType = 0x03
+	)
+
+	validBlockID := types.BlockID{}
+	invalidBlockID := types.BlockID{
+		Hash: bytes.HexBytes{},
+		PartSetHeader: types.PartSetHeader{
+			Total: 1,
+			Hash:  []byte{0},
+		},
+	}
+
+	testCases := []struct { // nolint: maligned
+		expectErr      bool
+		messageRound   int32
+		messageHeight  int64
+		testName       string
+		messageType    tmproto.SignedMsgType
+		messageBlockID types.BlockID
+	}{
+		{false, 0, 0, "Valid Message", validSignedMsgType, validBlockID},
+		{true, -1, 0, "Invalid Message", validSignedMsgType, validBlockID},
+		{true, 0, -1, "Invalid Message", validSignedMsgType, validBlockID},
+		{true, 0, 0, "Invalid Message", invalidSignedMsgType, validBlockID},
+		{true, 0, 0, "Invalid Message", validSignedMsgType, invalidBlockID},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := VoteSetMaj23Message{
+				Height:  tc.messageHeight,
+				Round:   tc.messageRound,
+				Type:    tc.messageType,
+				BlockID: tc.messageBlockID,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}
+
+func TestVoteSetBitsMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*VoteSetBitsMessage)
+		expErr     string
+	}{
+		{func(msg *VoteSetBitsMessage) {}, ""},
+		{func(msg *VoteSetBitsMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *VoteSetBitsMessage) { msg.Type = 0x03 }, "invalid Type"},
+		{func(msg *VoteSetBitsMessage) {
+			msg.BlockID = types.BlockID{
+				Hash: bytes.HexBytes{},
+				PartSetHeader: types.PartSetHeader{
+					Total: 1,
+					Hash:  []byte{0},
+				},
+			}
+		}, "wrong BlockID: wrong PartSetHeader: wrong Hash:"},
+		{func(msg *VoteSetBitsMessage) { msg.Votes = bits.NewBitArray(types.MaxVotesCount + 1) },
+			"votes bit array is too big: 10001, max: 10000"},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &VoteSetBitsMessage{
+				Height:  1,
+				Round:   0,
+				Type:    0x01,
+				Votes:   bits.NewBitArray(1),
+				BlockID: types.BlockID{},
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}
+
+func TestNewRoundStepMessageValidateBasic(t *testing.T) {
+	testCases := []struct { // nolint: maligned
+		expectErr              bool
+		messageRound           int32
+		messageLastCommitRound int32
+		messageHeight          int64
+		testName               string
+		messageStep            cstypes.RoundStepType
+	}{
+		{false, 0, 0, 0, "Valid Message", cstypes.RoundStepNewHeight},
+		{true, -1, 0, 0, "Negative round", cstypes.RoundStepNewHeight},
+		{true, 0, 0, -1, "Negative height", cstypes.RoundStepNewHeight},
+		{true, 0, 0, 0, "Invalid Step", cstypes.RoundStepCommit + 1},
+		// The following cases will be handled by ValidateHeight
+		{false, 0, 0, 1, "H == 1 but LCR != -1 ", cstypes.RoundStepNewHeight},
+		{false, 0, -1, 2, "H > 1 but LCR < 0", cstypes.RoundStepNewHeight},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := NewRoundStepMessage{
+				Height:          tc.messageHeight,
+				Round:           tc.messageRound,
+				Step:            tc.messageStep,
+				LastCommitRound: tc.messageLastCommitRound,
+			}
+
+			err := message.ValidateBasic()
+			if tc.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestNewRoundStepMessageValidateHeight(t *testing.T) {
+	initialHeight := int64(10)
+	testCases := []struct { // nolint: maligned
+		expectErr              bool
+		messageLastCommitRound int32
+		messageHeight          int64
+		testName               string
+	}{
+		{false, 0, 11, "Valid Message"},
+		{true, 0, -1, "Negative height"},
+		{true, 0, 0, "Zero height"},
+		{true, 0, 10, "Initial height but LCR != -1 "},
+		{true, -1, 11, "Normal height but LCR < 0"},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := NewRoundStepMessage{
+				Height:          tc.messageHeight,
+				Round:           0,
+				Step:            cstypes.RoundStepNewHeight,
+				LastCommitRound: tc.messageLastCommitRound,
+			}
+
+			err := message.ValidateHeight(initialHeight)
+			if tc.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestNewValidBlockMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*NewValidBlockMessage)
+		expErr     string
+	}{
+		{func(msg *NewValidBlockMessage) {}, ""},
+		{func(msg *NewValidBlockMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *NewValidBlockMessage) { msg.Round = -1 }, "negative Round"},
+		{
+			func(msg *NewValidBlockMessage) { msg.BlockPartSetHeader.Total = 2 },
+			"blockParts bit array size 1 not equal to BlockPartSetHeader.Total 2",
+		},
+		{
+			func(msg *NewValidBlockMessage) {
+				msg.BlockPartSetHeader.Total = 0
+				msg.BlockParts = bits.NewBitArray(0)
+			},
+			"empty blockParts",
+		},
+		{
+			func(msg *NewValidBlockMessage) { msg.BlockParts = bits.NewBitArray(int(types.MaxBlockPartsCount) + 1) },
+			"blockParts bit array size 1602 not equal to BlockPartSetHeader.Total 1",
+		},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &NewValidBlockMessage{
+				Height: 1,
+				Round:  0,
+				BlockPartSetHeader: types.PartSetHeader{
+					Total: 1,
+				},
+				BlockParts: bits.NewBitArray(1),
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}
+
+func TestProposalPOLMessageValidateBasic(t *testing.T) {
+	testCases := []struct {
+		malleateFn func(*ProposalPOLMessage)
+		expErr     string
+	}{
+		{func(msg *ProposalPOLMessage) {}, ""},
+		{func(msg *ProposalPOLMessage) { msg.Height = -1 }, "negative Height"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOLRound = -1 }, "negative ProposalPOLRound"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(0) }, "empty ProposalPOL bit array"},
+		{func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(types.MaxVotesCount + 1) },
+			"proposalPOL bit array is too big: 10001, max: 10000"},
+	}
+
+	for i, tc := range testCases {
+		tc := tc
+		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
+			msg := &ProposalPOLMessage{
+				Height:           1,
+				ProposalPOLRound: 1,
+				ProposalPOL:      bits.NewBitArray(1),
+			}
+
+			tc.malleateFn(msg)
+			err := msg.ValidateBasic()
+			if tc.expErr != "" && assert.Error(t, err) {
+				assert.Contains(t, err.Error(), tc.expErr)
+			}
+		})
+	}
+}
+
+func TestBlockPartMessageValidateBasic(t *testing.T) {
+	testPart := new(types.Part)
+	testPart.Proof.LeafHash = tmhash.Sum([]byte("leaf"))
+	testCases := []struct {
+		testName      string
+		messageHeight int64
+		messageRound  int32
+		messagePart   *types.Part
+		expectErr     bool
+	}{
+		{"Valid Message", 0, 0, testPart, false},
+		{"Invalid Message", -1, 0, testPart, true},
+		{"Invalid Message", 0, -1, testPart, true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := BlockPartMessage{
+				Height: tc.messageHeight,
+				Round:  tc.messageRound,
+				Part:   tc.messagePart,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+
+	message := BlockPartMessage{Height: 0, Round: 0, Part: new(types.Part)}
+	message.Part.Index = 1
+
+	assert.Equal(t, true, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+}
+
+func TestHasVoteMessageValidateBasic(t *testing.T) {
+	const (
+		validSignedMsgType   tmproto.SignedMsgType = 0x01
+		invalidSignedMsgType tmproto.SignedMsgType = 0x03
+	)
+
+	testCases := []struct { // nolint: maligned
+		expectErr     bool
+		messageRound  int32
+		messageIndex  int32
+		messageHeight int64
+		testName      string
+		messageType   tmproto.SignedMsgType
+	}{
+		{false, 0, 0, 0, "Valid Message", validSignedMsgType},
+		{true, -1, 0, 0, "Invalid Message", validSignedMsgType},
+		{true, 0, -1, 0, "Invalid Message", validSignedMsgType},
+		{true, 0, 0, 0, "Invalid Message", invalidSignedMsgType},
+		{true, 0, 0, -1, "Invalid Message", validSignedMsgType},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.testName, func(t *testing.T) {
+			message := HasVoteMessage{
+				Height: tc.messageHeight,
+				Round:  tc.messageRound,
+				Type:   tc.messageType,
+				Index:  tc.messageIndex,
+			}
+
+			assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result")
+		})
+	}
+}

internal/consensus/peer_state.go

@@ -0,0 +1,528 @@
+package consensus
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	cstypes "github.com/tendermint/tendermint/internal/consensus/types"
+	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/libs/bits"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+var (
+	ErrPeerStateHeightRegression = errors.New("peer state height regression")
+	ErrPeerStateInvalidStartTime = errors.New("peer state invalid startTime")
+)
+
+// peerStateStats holds internal statistics for a peer.
+type peerStateStats struct {
+	Votes      int `json:"votes"`
+	BlockParts int `json:"block_parts"`
+}
+
+func (pss peerStateStats) String() string {
+	return fmt.Sprintf("peerStateStats{votes: %d, blockParts: %d}", pss.Votes, pss.BlockParts)
+}
+
+// PeerState contains the known state of a peer, including its connection and
+// threadsafe access to its PeerRoundState.
+// NOTE: THIS GETS DUMPED WITH rpc/core/consensus.go.
+// Be mindful of what you Expose.
+type PeerState struct {
+	peerID p2p.NodeID
+	logger log.Logger
+
+	// NOTE: Modify below using setters, never directly.
+	mtx     tmsync.RWMutex
+	running bool
+	PRS     cstypes.PeerRoundState `json:"round_state"`
+	Stats   *peerStateStats        `json:"stats"`
+
+	broadcastWG sync.WaitGroup
+	closer      *tmsync.Closer
+}
+
+// NewPeerState returns a new PeerState for the given node ID.
+func NewPeerState(logger log.Logger, peerID p2p.NodeID) *PeerState {
+	return &PeerState{
+		peerID: peerID,
+		logger: logger,
+		closer: tmsync.NewCloser(),
+		PRS: cstypes.PeerRoundState{
+			Round:              -1,
+			ProposalPOLRound:   -1,
+			LastCommitRound:    -1,
+			CatchupCommitRound: -1,
+		},
+		Stats: &peerStateStats{},
+	}
+}
+
+// SetRunning sets the running state of the peer.
+func (ps *PeerState) SetRunning(v bool) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	ps.running = v
+}
+
+// IsRunning returns true if a PeerState is considered running where multiple
+// broadcasting goroutines exist for the peer.
+func (ps *PeerState) IsRunning() bool {
+	ps.mtx.RLock()
+	defer ps.mtx.RUnlock()
+
+	return ps.running
+}
+
+// GetRoundState returns a shallow copy of the PeerRoundState. There's no point
+// in mutating it since it won't change PeerState.
+func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	prs := ps.PRS.Copy()
+	return &prs
+}
+
+// ToJSON returns a json of PeerState.
+func (ps *PeerState) ToJSON() ([]byte, error) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	return tmjson.Marshal(ps)
+}
+
+// GetHeight returns an atomic snapshot of the PeerRoundState's height used by
+// the mempool to ensure peers are caught up before broadcasting new txs.
+func (ps *PeerState) GetHeight() int64 {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	return ps.PRS.Height
+}
+
+// SetHasProposal sets the given proposal as known for the peer.
+func (ps *PeerState) SetHasProposal(proposal *types.Proposal) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round {
+		return
+	}
+
+	if ps.PRS.Proposal {
+		return
+	}
+
+	ps.PRS.Proposal = true
+
+	// ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage
+	if ps.PRS.ProposalBlockParts != nil {
+		return
+	}
+
+	ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader
+	ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total))
+	ps.PRS.ProposalPOLRound = proposal.POLRound
+	ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received.
+}
+
+// InitProposalBlockParts initializes the peer's proposal block parts header
+// and bit array.
+func (ps *PeerState) InitProposalBlockParts(partSetHeader types.PartSetHeader) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.ProposalBlockParts != nil {
+		return
+	}
+
+	ps.PRS.ProposalBlockPartSetHeader = partSetHeader
+	ps.PRS.ProposalBlockParts = bits.NewBitArray(int(partSetHeader.Total))
+}
+
+// SetHasProposalBlockPart sets the given block part index as known for the peer.
+func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index int) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.Height != height || ps.PRS.Round != round {
+		return
+	}
+
+	ps.PRS.ProposalBlockParts.SetIndex(index, true)
+}
+
+// PickVoteToSend picks a vote to send to the peer. It will return true if a
+// vote was picked.
+//
+// NOTE: `votes` must be the correct Size() for the Height().
+func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (*types.Vote, bool) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if votes.Size() == 0 {
+		return nil, false
+	}
+
+	var (
+		height    = votes.GetHeight()
+		round     = votes.GetRound()
+		votesType = tmproto.SignedMsgType(votes.Type())
+		size      = votes.Size()
+	)
+
+	// lazily set data using 'votes'
+	if votes.IsCommit() {
+		ps.ensureCatchupCommitRound(height, round, size)
+	}
+
+	ps.ensureVoteBitArrays(height, size)
+
+	psVotes := ps.getVoteBitArray(height, round, votesType)
+	if psVotes == nil {
+		return nil, false // not something worth sending
+	}
+
+	if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok {
+		return votes.GetByIndex(int32(index)), true
+	}
+
+	return nil, false
+}
+
+func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmproto.SignedMsgType) *bits.BitArray {
+	if !types.IsVoteTypeValid(votesType) {
+		return nil
+	}
+
+	if ps.PRS.Height == height {
+		if ps.PRS.Round == round {
+			switch votesType {
+			case tmproto.PrevoteType:
+				return ps.PRS.Prevotes
+
+			case tmproto.PrecommitType:
+				return ps.PRS.Precommits
+			}
+		}
+
+		if ps.PRS.CatchupCommitRound == round {
+			switch votesType {
+			case tmproto.PrevoteType:
+				return nil
+
+			case tmproto.PrecommitType:
+				return ps.PRS.CatchupCommit
+			}
+		}
+
+		if ps.PRS.ProposalPOLRound == round {
+			switch votesType {
+			case tmproto.PrevoteType:
+				return ps.PRS.ProposalPOL
+
+			case tmproto.PrecommitType:
+				return nil
+			}
+		}
+
+		return nil
+	}
+	if ps.PRS.Height == height+1 {
+		if ps.PRS.LastCommitRound == round {
+			switch votesType {
+			case tmproto.PrevoteType:
+				return nil
+
+			case tmproto.PrecommitType:
+				return ps.PRS.LastCommit
+			}
+		}
+
+		return nil
+	}
+
+	return nil
+}
+
+// 'round': A round for which we have a +2/3 commit.
+func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) {
+	if ps.PRS.Height != height {
+		return
+	}
+
+	/*
+		NOTE: This is wrong, 'round' could change.
+		e.g. if orig round is not the same as block LastCommit round.
+		if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round {
+			panic(fmt.Sprintf(
+				"Conflicting CatchupCommitRound. Height: %v,
+				Orig: %v,
+				New: %v",
+				height,
+				ps.CatchupCommitRound,
+				round))
+		}
+	*/
+
+	if ps.PRS.CatchupCommitRound == round {
+		return // Nothing to do!
+	}
+
+	ps.PRS.CatchupCommitRound = round
+	if round == ps.PRS.Round {
+		ps.PRS.CatchupCommit = ps.PRS.Precommits
+	} else {
+		ps.PRS.CatchupCommit = bits.NewBitArray(numValidators)
+	}
+}
+
+// EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking
+// what votes this peer has received.
+// NOTE: It's important to make sure that numValidators actually matches
+// what the node sees as the number of validators for height.
+func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	ps.ensureVoteBitArrays(height, numValidators)
+}
+
+func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) {
+	if ps.PRS.Height == height {
+		if ps.PRS.Prevotes == nil {
+			ps.PRS.Prevotes = bits.NewBitArray(numValidators)
+		}
+		if ps.PRS.Precommits == nil {
+			ps.PRS.Precommits = bits.NewBitArray(numValidators)
+		}
+		if ps.PRS.CatchupCommit == nil {
+			ps.PRS.CatchupCommit = bits.NewBitArray(numValidators)
+		}
+		if ps.PRS.ProposalPOL == nil {
+			ps.PRS.ProposalPOL = bits.NewBitArray(numValidators)
+		}
+	} else if ps.PRS.Height == height+1 {
+		if ps.PRS.LastCommit == nil {
+			ps.PRS.LastCommit = bits.NewBitArray(numValidators)
+		}
+	}
+}
+
+// RecordVote increments internal votes related statistics for this peer.
+// It returns the total number of added votes.
+func (ps *PeerState) RecordVote() int {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	ps.Stats.Votes++
+
+	return ps.Stats.Votes
+}
+
+// VotesSent returns the number of blocks for which peer has been sending us
+// votes.
+func (ps *PeerState) VotesSent() int {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	return ps.Stats.Votes
+}
+
+// RecordBlockPart increments internal block part related statistics for this peer.
+// It returns the total number of added block parts.
+func (ps *PeerState) RecordBlockPart() int {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	ps.Stats.BlockParts++
+	return ps.Stats.BlockParts
+}
+
+// BlockPartsSent returns the number of useful block parts the peer has sent us.
+func (ps *PeerState) BlockPartsSent() int {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	return ps.Stats.BlockParts
+}
+
+// SetHasVote sets the given vote as known by the peer
+func (ps *PeerState) SetHasVote(vote *types.Vote) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex)
+}
+
+func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) {
+	logger := ps.logger.With(
+		"peerH/R", fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round),
+		"H/R", fmt.Sprintf("%d/%d", height, round),
+	)
+
+	logger.Debug("setHasVote", "type", voteType, "index", index)
+
+	// NOTE: some may be nil BitArrays -> no side effects
+	psVotes := ps.getVoteBitArray(height, round, voteType)
+	if psVotes != nil {
+		psVotes.SetIndex(int(index), true)
+	}
+}
+
+// ApplyNewRoundStepMessage updates the peer state for the new round.
+func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	// ignore duplicates or decreases
+	if CompareHRS(msg.Height, msg.Round, msg.Step, ps.PRS.Height, ps.PRS.Round, ps.PRS.Step) <= 0 {
+		return
+	}
+
+	var (
+		psHeight             = ps.PRS.Height
+		psRound              = ps.PRS.Round
+		psCatchupCommitRound = ps.PRS.CatchupCommitRound
+		psCatchupCommit      = ps.PRS.CatchupCommit
+		startTime            = tmtime.Now().Add(-1 * time.Duration(msg.SecondsSinceStartTime) * time.Second)
+	)
+
+	ps.PRS.Height = msg.Height
+	ps.PRS.Round = msg.Round
+	ps.PRS.Step = msg.Step
+	ps.PRS.StartTime = startTime
+
+	if psHeight != msg.Height || psRound != msg.Round {
+		ps.PRS.Proposal = false
+		ps.PRS.ProposalBlockPartSetHeader = types.PartSetHeader{}
+		ps.PRS.ProposalBlockParts = nil
+		ps.PRS.ProposalPOLRound = -1
+		ps.PRS.ProposalPOL = nil
+
+		// we'll update the BitArray capacity later
+		ps.PRS.Prevotes = nil
+		ps.PRS.Precommits = nil
+	}
+
+	if psHeight == msg.Height && psRound != msg.Round && msg.Round == psCatchupCommitRound {
+		// Peer caught up to CatchupCommitRound.
+		// Preserve psCatchupCommit!
+		// NOTE: We prefer to use prs.Precommits if
+		// pr.Round matches pr.CatchupCommitRound.
+		ps.PRS.Precommits = psCatchupCommit
+	}
+
+	if psHeight != msg.Height {
+		// shift Precommits to LastCommit
+		if psHeight+1 == msg.Height && psRound == msg.LastCommitRound {
+			ps.PRS.LastCommitRound = msg.LastCommitRound
+			ps.PRS.LastCommit = ps.PRS.Precommits
+		} else {
+			ps.PRS.LastCommitRound = msg.LastCommitRound
+			ps.PRS.LastCommit = nil
+		}
+
+		// we'll update the BitArray capacity later
+		ps.PRS.CatchupCommitRound = -1
+		ps.PRS.CatchupCommit = nil
+	}
+}
+
+// ApplyNewValidBlockMessage updates the peer state for the new valid block.
+func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.Height != msg.Height {
+		return
+	}
+
+	if ps.PRS.Round != msg.Round && !msg.IsCommit {
+		return
+	}
+
+	ps.PRS.ProposalBlockPartSetHeader = msg.BlockPartSetHeader
+	ps.PRS.ProposalBlockParts = msg.BlockParts
+}
+
+// ApplyProposalPOLMessage updates the peer state for the new proposal POL.
+func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.Height != msg.Height {
+		return
+	}
+	if ps.PRS.ProposalPOLRound != msg.ProposalPOLRound {
+		return
+	}
+
+	// TODO: Merge onto existing ps.PRS.ProposalPOL?
+	// We might have sent some prevotes in the meantime.
+	ps.PRS.ProposalPOL = msg.ProposalPOL
+}
+
+// ApplyHasVoteMessage updates the peer state for the new vote.
+func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	if ps.PRS.Height != msg.Height {
+		return
+	}
+
+	ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index)
+}
+
+// ApplyVoteSetBitsMessage updates the peer state for the bit-array of votes
+// it claims to have for the corresponding BlockID.
+// `ourVotes` is a BitArray of votes we have for msg.BlockID
+// NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height),
+// we conservatively overwrite ps's votes w/ msg.Votes.
+func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *bits.BitArray) {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+
+	votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type)
+	if votes != nil {
+		if ourVotes == nil {
+			votes.Update(msg.Votes)
+		} else {
+			otherVotes := votes.Sub(ourVotes)
+			hasVotes := otherVotes.Or(msg.Votes)
+			votes.Update(hasVotes)
+		}
+	}
+}
+
+// String returns a string representation of the PeerState
+func (ps *PeerState) String() string {
+	return ps.StringIndented("")
+}
+
+// StringIndented returns a string representation of the PeerState
+func (ps *PeerState) StringIndented(indent string) string {
+	ps.mtx.Lock()
+	defer ps.mtx.Unlock()
+	return fmt.Sprintf(`PeerState{
+%s  Key        %v
+%s  RoundState %v
+%s  Stats      %v
+%s}`,
+		indent, ps.peerID,
+		indent, ps.PRS.StringIndented(indent+"  "),
+		indent, ps.Stats,
+		indent,
+	)
+}

internal/consensus/reactor_test.go

@@ -0,0 +1,718 @@
+package consensus
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/fortytw2/leaktest"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	abcicli "github.com/tendermint/tendermint/abci/client"
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	abci "github.com/tendermint/tendermint/abci/types"
+	cfg "github.com/tendermint/tendermint/config"
+	cryptoenc "github.com/tendermint/tendermint/crypto/encoding"
+	tmsync "github.com/tendermint/tendermint/internal/libs/sync"
+	"github.com/tendermint/tendermint/internal/mempool"
+	mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0"
+	"github.com/tendermint/tendermint/internal/p2p"
+	"github.com/tendermint/tendermint/internal/p2p/p2ptest"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	"github.com/tendermint/tendermint/libs/log"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	sm "github.com/tendermint/tendermint/state"
+	statemocks "github.com/tendermint/tendermint/state/mocks"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+var (
+	defaultTestTime = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC)
+)
+
+type reactorTestSuite struct {
+	network             *p2ptest.Network
+	states              map[p2p.NodeID]*State
+	reactors            map[p2p.NodeID]*Reactor
+	subs                map[p2p.NodeID]types.Subscription
+	stateChannels       map[p2p.NodeID]*p2p.Channel
+	dataChannels        map[p2p.NodeID]*p2p.Channel
+	voteChannels        map[p2p.NodeID]*p2p.Channel
+	voteSetBitsChannels map[p2p.NodeID]*p2p.Channel
+}
+
+func chDesc(chID p2p.ChannelID) p2p.ChannelDescriptor {
+	return p2p.ChannelDescriptor{
+		ID: byte(chID),
+	}
+}
+
+func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSuite {
+	t.Helper()
+
+	rts := &reactorTestSuite{
+		network:  p2ptest.MakeNetwork(t, p2ptest.NetworkOptions{NumNodes: numNodes}),
+		states:   make(map[p2p.NodeID]*State),
+		reactors: make(map[p2p.NodeID]*Reactor, numNodes),
+		subs:     make(map[p2p.NodeID]types.Subscription, numNodes),
+	}
+
+	rts.stateChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(StateChannel), new(tmcons.Message), size)
+	rts.dataChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(DataChannel), new(tmcons.Message), size)
+	rts.voteChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteChannel), new(tmcons.Message), size)
+	rts.voteSetBitsChannels = rts.network.MakeChannelsNoCleanup(t, chDesc(VoteSetBitsChannel), new(tmcons.Message), size)
+
+	i := 0
+	for nodeID, node := range rts.network.Nodes {
+		state := states[i]
+
+		reactor := NewReactor(
+			state.Logger.With("node", nodeID),
+			state,
+			rts.stateChannels[nodeID],
+			rts.dataChannels[nodeID],
+			rts.voteChannels[nodeID],
+			rts.voteSetBitsChannels[nodeID],
+			node.MakePeerUpdates(t),
+			true,
+		)
+
+		reactor.SetEventBus(state.eventBus)
+
+		blocksSub, err := state.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, size)
+		require.NoError(t, err)
+
+		rts.states[nodeID] = state
+		rts.subs[nodeID] = blocksSub
+		rts.reactors[nodeID] = reactor
+
+		// simulate handle initChain in handshake
+		if state.state.LastBlockHeight == 0 {
+			require.NoError(t, state.blockExec.Store().Save(state.state))
+		}
+
+		require.NoError(t, reactor.Start())
+		require.True(t, reactor.IsRunning())
+
+		i++
+	}
+
+	require.Len(t, rts.reactors, numNodes)
+
+	// start the in-memory network and connect all peers with each other
+	rts.network.Start(t)
+
+	t.Cleanup(func() {
+		for nodeID, r := range rts.reactors {
+			require.NoError(t, rts.states[nodeID].eventBus.Stop())
+			require.NoError(t, r.Stop())
+			require.False(t, r.IsRunning())
+		}
+
+		leaktest.Check(t)
+	})
+
+	return rts
+}
+
+func validateBlock(block *types.Block, activeVals map[string]struct{}) error {
+	if block.LastCommit.Size() != len(activeVals) {
+		return fmt.Errorf(
+			"commit size doesn't match number of active validators. Got %d, expected %d",
+			block.LastCommit.Size(), len(activeVals),
+		)
+	}
+
+	for _, commitSig := range block.LastCommit.Signatures {
+		if _, ok := activeVals[string(commitSig.ValidatorAddress)]; !ok {
+			return fmt.Errorf("found vote for inactive validator %X", commitSig.ValidatorAddress)
+		}
+	}
+
+	return nil
+}
+
+func waitForAndValidateBlock(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	states []*State,
+	txs ...[]byte,
+) {
+
+	fn := func(j int) {
+		msg := <-blocksSubs[j].Out()
+		newBlock := msg.Data().(types.EventDataNewBlock).Block
+
+		require.NoError(t, validateBlock(newBlock, activeVals))
+
+		for _, tx := range txs {
+			require.NoError(t, assertMempool(states[j].txNotifier).CheckTx(context.Background(), tx, nil, mempool.TxInfo{}))
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(n)
+
+	for i := 0; i < n; i++ {
+		go func(j int) {
+			fn(j)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func waitForAndValidateBlockWithTx(
+	t *testing.T,
+	n int,
+	activeVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	states []*State,
+	txs ...[]byte,
+) {
+
+	fn := func(j int) {
+		ntxs := 0
+	BLOCK_TX_LOOP:
+		for {
+			msg := <-blocksSubs[j].Out()
+			newBlock := msg.Data().(types.EventDataNewBlock).Block
+
+			require.NoError(t, validateBlock(newBlock, activeVals))
+
+			// check that txs match the txs we're waiting for.
+			// note they could be spread over multiple blocks,
+			// but they should be in order.
+			for _, tx := range newBlock.Data.Txs {
+				require.EqualValues(t, txs[ntxs], tx)
+				ntxs++
+			}
+
+			if ntxs == len(txs) {
+				break BLOCK_TX_LOOP
+			}
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(n)
+
+	for i := 0; i < n; i++ {
+		go func(j int) {
+			fn(j)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func waitForBlockWithUpdatedValsAndValidateIt(
+	t *testing.T,
+	n int,
+	updatedVals map[string]struct{},
+	blocksSubs []types.Subscription,
+	css []*State,
+) {
+
+	fn := func(j int) {
+		var newBlock *types.Block
+
+	LOOP:
+		for {
+			msg := <-blocksSubs[j].Out()
+			newBlock = msg.Data().(types.EventDataNewBlock).Block
+			if newBlock.LastCommit.Size() == len(updatedVals) {
+				break LOOP
+			}
+		}
+
+		require.NoError(t, validateBlock(newBlock, updatedVals))
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(n)
+
+	for i := 0; i < n; i++ {
+		go func(j int) {
+			fn(j)
+			wg.Done()
+		}(i)
+	}
+
+	wg.Wait()
+}
+
+func TestReactorBasic(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	states, cleanup := randConsensusState(config, n, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	t.Cleanup(cleanup)
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// wait till everyone makes the first new block
+		go func(s types.Subscription) {
+			<-s.Out()
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+}
+
+func TestReactorWithEvidence(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	testName := "consensus_reactor_test"
+	tickerFunc := newMockTickerFunc(true)
+	appFunc := newCounter
+
+	genDoc, privVals := factory.RandGenesisDoc(config, n, false, 30)
+	states := make([]*State, n)
+	logger := consensusLogger()
+
+	for i := 0; i < n; i++ {
+		stateDB := dbm.NewMemDB() // each state needs its own db
+		stateStore := sm.NewStore(stateDB)
+		state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc)
+		thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i))
+
+		defer os.RemoveAll(thisConfig.RootDir)
+
+		ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal
+		app := appFunc()
+		vals := types.TM2PB.ValidatorUpdates(state.Validators)
+		app.InitChain(abci.RequestInitChain{Validators: vals})
+
+		pv := privVals[i]
+		blockDB := dbm.NewMemDB()
+		blockStore := store.NewBlockStore(blockDB)
+
+		// one for mempool, one for consensus
+		mtx := new(tmsync.RWMutex)
+		proxyAppConnMem := abcicli.NewLocalClient(mtx, app)
+		proxyAppConnCon := abcicli.NewLocalClient(mtx, app)
+
+		mempool := mempoolv0.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0)
+		mempool.SetLogger(log.TestingLogger().With("module", "mempool"))
+		if thisConfig.Consensus.WaitForTxs() {
+			mempool.EnableTxsAvailable()
+		}
+
+		// mock the evidence pool
+		// everyone includes evidence of another double signing
+		vIdx := (i + 1) % n
+
+		ev := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID())
+		evpool := &statemocks.EvidencePool{}
+		evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil)
+		evpool.On("PendingEvidence", mock.AnythingOfType("int64")).Return([]types.Evidence{
+			ev}, int64(len(ev.Bytes())))
+		evpool.On("Update", mock.AnythingOfType("state.State"), mock.AnythingOfType("types.EvidenceList")).Return()
+
+		evpool2 := sm.EmptyEvidencePool{}
+
+		blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
+		cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2)
+		cs.SetLogger(log.TestingLogger().With("module", "consensus"))
+		cs.SetPrivValidator(pv)
+
+		eventBus := types.NewEventBus()
+		eventBus.SetLogger(log.TestingLogger().With("module", "events"))
+		err := eventBus.Start()
+		require.NoError(t, err)
+		cs.SetEventBus(eventBus)
+
+		cs.SetTimeoutTicker(tickerFunc())
+		cs.SetLogger(logger.With("validator", i, "module", "consensus"))
+
+		states[i] = cs
+	}
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// We expect for each validator that is the proposer to propose one piece of
+		// evidence.
+		go func(s types.Subscription) {
+			msg := <-s.Out()
+			block := msg.Data().(types.EventDataNewBlock).Block
+
+			require.Len(t, block.Evidence.Evidence, 1)
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+}
+
+func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	states, cleanup := randConsensusState(
+		config,
+		n,
+		"consensus_reactor_test",
+		newMockTickerFunc(true),
+		newCounter,
+		func(c *cfg.Config) {
+			c.Consensus.CreateEmptyBlocks = false
+		},
+	)
+
+	t.Cleanup(cleanup)
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	// send a tx
+	require.NoError(
+		t,
+		assertMempool(states[3].txNotifier).CheckTx(
+			context.Background(),
+			[]byte{1, 2, 3},
+			nil,
+			mempool.TxInfo{},
+		),
+	)
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// wait till everyone makes the first new block
+		go func(s types.Subscription) {
+			<-s.Out()
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+}
+
+func TestReactorRecordsVotesAndBlockParts(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	states, cleanup := randConsensusState(config, n, "consensus_reactor_test", newMockTickerFunc(true), newCounter)
+	t.Cleanup(cleanup)
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// wait till everyone makes the first new block
+		go func(s types.Subscription) {
+			<-s.Out()
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+
+	// Require at least one node to have sent block parts, but we can't know which
+	// peer sent it.
+	require.Eventually(
+		t,
+		func() bool {
+			for _, reactor := range rts.reactors {
+				for _, ps := range reactor.peers {
+					if ps.BlockPartsSent() > 0 {
+						return true
+					}
+				}
+			}
+
+			return false
+		},
+		time.Second,
+		10*time.Millisecond,
+		"number of block parts sent should've increased",
+	)
+
+	nodeID := rts.network.RandomNode().NodeID
+	reactor := rts.reactors[nodeID]
+	peers := rts.network.Peers(nodeID)
+
+	ps, ok := reactor.GetPeerState(peers[0].NodeID)
+	require.True(t, ok)
+	require.NotNil(t, ps)
+	require.Greater(t, ps.VotesSent(), 0, "number of votes sent should've increased")
+}
+
+func TestReactorVotingPowerChange(t *testing.T) {
+	config := configSetup(t)
+
+	n := 4
+	states, cleanup := randConsensusState(
+		config,
+		n,
+		"consensus_voting_power_changes_test",
+		newMockTickerFunc(true),
+		newPersistentKVStore,
+	)
+
+	t.Cleanup(cleanup)
+
+	rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	// map of active validators
+	activeVals := make(map[string]struct{})
+	for i := 0; i < n; i++ {
+		pubKey, err := states[i].privValidator.GetPubKey(context.Background())
+		require.NoError(t, err)
+
+		addr := pubKey.Address()
+		activeVals[string(addr)] = struct{}{}
+	}
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// wait till everyone makes the first new block
+		go func(s types.Subscription) {
+			<-s.Out()
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+
+	blocksSubs := []types.Subscription{}
+	for _, sub := range rts.subs {
+		blocksSubs = append(blocksSubs, sub)
+	}
+
+	val1PubKey, err := states[0].privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	val1PubKeyABCI, err := cryptoenc.PubKeyToProto(val1PubKey)
+	require.NoError(t, err)
+
+	updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25)
+	previousTotalVotingPower := states[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+
+	require.NotEqualf(
+		t, previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(),
+		"expected voting power to change (before: %d, after: %d)",
+		previousTotalVotingPower,
+		states[0].GetRoundState().LastValidators.TotalVotingPower(),
+	)
+
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2)
+	previousTotalVotingPower = states[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+
+	require.NotEqualf(
+		t, states[0].GetRoundState().LastValidators.TotalVotingPower(), previousTotalVotingPower,
+		"expected voting power to change (before: %d, after: %d)",
+		previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(),
+	)
+
+	updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26)
+	previousTotalVotingPower = states[0].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+	waitForAndValidateBlock(t, n, activeVals, blocksSubs, states)
+
+	require.NotEqualf(
+		t, previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(),
+		"expected voting power to change (before: %d, after: %d)",
+		previousTotalVotingPower,
+		states[0].GetRoundState().LastValidators.TotalVotingPower(),
+	)
+}
+
+func TestReactorValidatorSetChanges(t *testing.T) {
+	config := configSetup(t)
+
+	nPeers := 7
+	nVals := 4
+	states, _, _, cleanup := randConsensusNetWithPeers(
+		config,
+		nVals,
+		nPeers,
+		"consensus_val_set_changes_test",
+		newMockTickerFunc(true),
+		newPersistentKVStoreWithPath,
+	)
+	t.Cleanup(cleanup)
+
+	rts := setup(t, nPeers, states, 100) // buffer must be large enough to not deadlock
+
+	for _, reactor := range rts.reactors {
+		state := reactor.state.GetState()
+		reactor.SwitchToConsensus(state, false)
+	}
+
+	// map of active validators
+	activeVals := make(map[string]struct{})
+	for i := 0; i < nVals; i++ {
+		pubKey, err := states[i].privValidator.GetPubKey(context.Background())
+		require.NoError(t, err)
+
+		activeVals[string(pubKey.Address())] = struct{}{}
+	}
+
+	var wg sync.WaitGroup
+	for _, sub := range rts.subs {
+		wg.Add(1)
+
+		// wait till everyone makes the first new block
+		go func(s types.Subscription) {
+			<-s.Out()
+			wg.Done()
+		}(sub)
+	}
+
+	wg.Wait()
+
+	newValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1)
+	require.NoError(t, err)
+
+	newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower)
+
+	blocksSubs := []types.Subscription{}
+	for _, sub := range rts.subs {
+		blocksSubs = append(blocksSubs, sub)
+	}
+
+	// wait till everyone makes block 2
+	// ensure the commit includes all validators
+	// send newValTx to change vals in block 3
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, newValidatorTx1)
+
+	// wait till everyone makes block 3.
+	// it includes the commit for block 2, which is by the original validator set
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, newValidatorTx1)
+
+	// wait till everyone makes block 4.
+	// it includes the commit for block 3, which is by the original validator set
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states)
+
+	// the commits for block 4 should be with the updated validator set
+	activeVals[string(newValidatorPubKey1.Address())] = struct{}{}
+
+	// wait till everyone makes block 5
+	// it includes the commit for block 4, which should have the updated validator set
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states)
+
+	updateValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1)
+	require.NoError(t, err)
+
+	updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25)
+	previousTotalVotingPower := states[nVals].GetRoundState().LastValidators.TotalVotingPower()
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, updateValidatorTx1)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, updateValidatorTx1)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states)
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states)
+
+	require.NotEqualf(
+		t, states[nVals].GetRoundState().LastValidators.TotalVotingPower(), previousTotalVotingPower,
+		"expected voting power to change (before: %d, after: %d)",
+		previousTotalVotingPower, states[nVals].GetRoundState().LastValidators.TotalVotingPower(),
+	)
+
+	newValidatorPubKey2, err := states[nVals+1].privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2)
+	require.NoError(t, err)
+
+	newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower)
+
+	newValidatorPubKey3, err := states[nVals+2].privValidator.GetPubKey(context.Background())
+	require.NoError(t, err)
+
+	newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3)
+	require.NoError(t, err)
+
+	newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower)
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, newValidatorTx2, newValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states)
+
+	activeVals[string(newValidatorPubKey2.Address())] = struct{}{}
+	activeVals[string(newValidatorPubKey3.Address())] = struct{}{}
+
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states)
+
+	removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0)
+	removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0)
+
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, removeValidatorTx2, removeValidatorTx3)
+	waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states)
+
+	delete(activeVals, string(newValidatorPubKey2.Address()))
+	delete(activeVals, string(newValidatorPubKey3.Address()))
+
+	waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states)
+}

internal/consensus/replay.go

@@ -0,0 +1,548 @@
+package consensus
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"reflect"
+	"time"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/types"
+)
+
+var crc32c = crc32.MakeTable(crc32.Castagnoli)
+
+// Functionality to replay blocks and messages on recovery from a crash.
+// There are two general failure scenarios:
+//
+//  1. failure during consensus
+//  2. failure while applying the block
+//
+// The former is handled by the WAL, the latter by the proxyApp Handshake on
+// restart, which ultimately hands off the work to the WAL.
+
+//-----------------------------------------
+// 1. Recover from failure during consensus
+// (by replaying messages from the WAL)
+//-----------------------------------------
+
+// Unmarshal and apply a single message to the consensus state as if it were
+// received in receiveRoutine.  Lines that start with "#" are ignored.
+// NOTE: receiveRoutine should not be running.
+func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub types.Subscription) error {
+	// Skip meta messages which exist for demarcating boundaries.
+	if _, ok := msg.Msg.(EndHeightMessage); ok {
+		return nil
+	}
+
+	// for logging
+	switch m := msg.Msg.(type) {
+	case types.EventDataRoundState:
+		cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step)
+		// these are playback checks
+		ticker := time.After(time.Second * 2)
+		if newStepSub != nil {
+			select {
+			case stepMsg := <-newStepSub.Out():
+				m2 := stepMsg.Data().(types.EventDataRoundState)
+				if m.Height != m2.Height || m.Round != m2.Round || m.Step != m2.Step {
+					return fmt.Errorf("roundState mismatch. Got %v; Expected %v", m2, m)
+				}
+			case <-newStepSub.Canceled():
+				return fmt.Errorf("failed to read off newStepSub.Out(). newStepSub was canceled")
+			case <-ticker:
+				return fmt.Errorf("failed to read off newStepSub.Out()")
+			}
+		}
+	case msgInfo:
+		peerID := m.PeerID
+		if peerID == "" {
+			peerID = "local"
+		}
+		switch msg := m.Msg.(type) {
+		case *ProposalMessage:
+			p := msg.Proposal
+			cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header",
+				p.BlockID.PartSetHeader, "pol", p.POLRound, "peer", peerID)
+		case *BlockPartMessage:
+			cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID)
+		case *VoteMessage:
+			v := msg.Vote
+			cs.Logger.Info("Replay: Vote", "height", v.Height, "round", v.Round, "type", v.Type,
+				"blockID", v.BlockID, "peer", peerID)
+		}
+
+		cs.handleMsg(m)
+	case timeoutInfo:
+		cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration)
+		cs.handleTimeout(m, cs.RoundState)
+	default:
+		return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg))
+	}
+	return nil
+}
+
+// Replay only those messages since the last block.  `timeoutRoutine` should
+// run concurrently to read off tickChan.
+func (cs *State) catchupReplay(csHeight int64) error {
+
+	// Set replayMode to true so we don't log signing errors.
+	cs.replayMode = true
+	defer func() { cs.replayMode = false }()
+
+	// Ensure that #ENDHEIGHT for this height doesn't exist.
+	// NOTE: This is just a sanity check. As far as we know things work fine
+	// without it, and Handshake could reuse State if it weren't for
+	// this check (since we can crash after writing #ENDHEIGHT).
+	//
+	// Ignore data corruption errors since this is a sanity check.
+	gr, found, err := cs.wal.SearchForEndHeight(csHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
+	if err != nil {
+		return err
+	}
+	if gr != nil {
+		if err := gr.Close(); err != nil {
+			return err
+		}
+	}
+	if found {
+		return fmt.Errorf("wal should not contain #ENDHEIGHT %d", csHeight)
+	}
+
+	// Search for last height marker.
+	//
+	// Ignore data corruption errors in previous heights because we only care about last height
+	if csHeight < cs.state.InitialHeight {
+		return fmt.Errorf("cannot replay height %v, below initial height %v", csHeight, cs.state.InitialHeight)
+	}
+	endHeight := csHeight - 1
+	if csHeight == cs.state.InitialHeight {
+		endHeight = 0
+	}
+	gr, found, err = cs.wal.SearchForEndHeight(endHeight, &WALSearchOptions{IgnoreDataCorruptionErrors: true})
+	if err == io.EOF {
+		cs.Logger.Error("Replay: wal.group.Search returned EOF", "#ENDHEIGHT", endHeight)
+	} else if err != nil {
+		return err
+	}
+	if !found {
+		return fmt.Errorf("cannot replay height %d. WAL does not contain #ENDHEIGHT for %d", csHeight, endHeight)
+	}
+	defer gr.Close()
+
+	cs.Logger.Info("Catchup by replaying consensus messages", "height", csHeight)
+
+	var msg *TimedWALMessage
+	dec := WALDecoder{gr}
+
+LOOP:
+	for {
+		msg, err = dec.Decode()
+		switch {
+		case err == io.EOF:
+			break LOOP
+		case IsDataCorruptionError(err):
+			cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight)
+			return err
+		case err != nil:
+			return err
+		}
+
+		// NOTE: since the priv key is set when the msgs are received
+		// it will attempt to eg double sign but we can just ignore it
+		// since the votes will be replayed and we'll get to the next step
+		if err := cs.readReplayMessage(msg, nil); err != nil {
+			return err
+		}
+	}
+	cs.Logger.Info("Replay: Done")
+	return nil
+}
+
+//--------------------------------------------------------------------------------
+
+// Parses marker lines of the form:
+// #ENDHEIGHT: 12345
+/*
+func makeHeightSearchFunc(height int64) auto.SearchFunc {
+	return func(line string) (int, error) {
+		line = strings.TrimRight(line, "\n")
+		parts := strings.Split(line, " ")
+		if len(parts) != 2 {
+			return -1, errors.New("line did not have 2 parts")
+		}
+		i, err := strconv.Atoi(parts[1])
+		if err != nil {
+			return -1, errors.New("failed to parse INFO: " + err.Error())
+		}
+		if height < i {
+			return 1, nil
+		} else if height == i {
+			return 0, nil
+		} else {
+			return -1, nil
+		}
+	}
+}*/
+
+//---------------------------------------------------
+// 2. Recover from failure while applying the block.
+// (by handshaking with the app to figure out where
+// we were last, and using the WAL to recover there.)
+//---------------------------------------------------
+
+type Handshaker struct {
+	stateStore   sm.Store
+	initialState sm.State
+	store        sm.BlockStore
+	eventBus     types.BlockEventPublisher
+	genDoc       *types.GenesisDoc
+	logger       log.Logger
+
+	nBlocks int // number of blocks applied to the state
+}
+
+func NewHandshaker(stateStore sm.Store, state sm.State,
+	store sm.BlockStore, genDoc *types.GenesisDoc) *Handshaker {
+
+	return &Handshaker{
+		stateStore:   stateStore,
+		initialState: state,
+		store:        store,
+		eventBus:     types.NopEventBus{},
+		genDoc:       genDoc,
+		logger:       log.NewNopLogger(),
+		nBlocks:      0,
+	}
+}
+
+func (h *Handshaker) SetLogger(l log.Logger) {
+	h.logger = l
+}
+
+// SetEventBus - sets the event bus for publishing block related events.
+// If not called, it defaults to types.NopEventBus.
+func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) {
+	h.eventBus = eventBus
+}
+
+// NBlocks returns the number of blocks applied to the state.
+func (h *Handshaker) NBlocks() int {
+	return h.nBlocks
+}
+
+// TODO: retry the handshake/replay if it fails ?
+func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error {
+
+	// Handshake is done via ABCI Info on the query conn.
+	res, err := proxyApp.Query().InfoSync(context.Background(), proxy.RequestInfo)
+	if err != nil {
+		return fmt.Errorf("error calling Info: %v", err)
+	}
+
+	blockHeight := res.LastBlockHeight
+	if blockHeight < 0 {
+		return fmt.Errorf("got a negative last block height (%d) from the app", blockHeight)
+	}
+	appHash := res.LastBlockAppHash
+
+	h.logger.Info("ABCI Handshake App Info",
+		"height", blockHeight,
+		"hash", appHash,
+		"software-version", res.Version,
+		"protocol-version", res.AppVersion,
+	)
+
+	// Only set the version if there is no existing state.
+	if h.initialState.LastBlockHeight == 0 {
+		h.initialState.Version.Consensus.App = res.AppVersion
+	}
+
+	// Replay blocks up to the latest in the blockstore.
+	_, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp)
+	if err != nil {
+		return fmt.Errorf("error on replay: %v", err)
+	}
+
+	h.logger.Info("Completed ABCI Handshake - Tendermint and App are synced",
+		"appHeight", blockHeight, "appHash", appHash)
+
+	// TODO: (on restart) replay mempool
+
+	return nil
+}
+
+// ReplayBlocks replays all blocks since appBlockHeight and ensures the result
+// matches the current state.
+// Returns the final AppHash or an error.
+func (h *Handshaker) ReplayBlocks(
+	state sm.State,
+	appHash []byte,
+	appBlockHeight int64,
+	proxyApp proxy.AppConns,
+) ([]byte, error) {
+	storeBlockBase := h.store.Base()
+	storeBlockHeight := h.store.Height()
+	stateBlockHeight := state.LastBlockHeight
+	h.logger.Info(
+		"ABCI Replay Blocks",
+		"appHeight",
+		appBlockHeight,
+		"storeHeight",
+		storeBlockHeight,
+		"stateHeight",
+		stateBlockHeight)
+
+	// If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain.
+	if appBlockHeight == 0 {
+		validators := make([]*types.Validator, len(h.genDoc.Validators))
+		for i, val := range h.genDoc.Validators {
+			validators[i] = types.NewValidator(val.PubKey, val.Power)
+		}
+		validatorSet := types.NewValidatorSet(validators)
+		nextVals := types.TM2PB.ValidatorUpdates(validatorSet)
+		pbParams := h.genDoc.ConsensusParams.ToProto()
+		req := abci.RequestInitChain{
+			Time:            h.genDoc.GenesisTime,
+			ChainId:         h.genDoc.ChainID,
+			InitialHeight:   h.genDoc.InitialHeight,
+			ConsensusParams: &pbParams,
+			Validators:      nextVals,
+			AppStateBytes:   h.genDoc.AppState,
+		}
+		res, err := proxyApp.Consensus().InitChainSync(context.Background(), req)
+		if err != nil {
+			return nil, err
+		}
+
+		appHash = res.AppHash
+
+		if stateBlockHeight == 0 { // we only update state when we are in initial state
+			// If the app did not return an app hash, we keep the one set from the genesis doc in
+			// the state. We don't set appHash since we don't want the genesis doc app hash
+			// recorded in the genesis block. We should probably just remove GenesisDoc.AppHash.
+			if len(res.AppHash) > 0 {
+				state.AppHash = res.AppHash
+			}
+			// If the app returned validators or consensus params, update the state.
+			if len(res.Validators) > 0 {
+				vals, err := types.PB2TM.ValidatorUpdates(res.Validators)
+				if err != nil {
+					return nil, err
+				}
+				state.Validators = types.NewValidatorSet(vals)
+				state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1)
+			} else if len(h.genDoc.Validators) == 0 {
+				// If validator set is not set in genesis and still empty after InitChain, exit.
+				return nil, fmt.Errorf("validator set is nil in genesis and still empty after InitChain")
+			}
+
+			if res.ConsensusParams != nil {
+				state.ConsensusParams = state.ConsensusParams.UpdateConsensusParams(res.ConsensusParams)
+				state.Version.Consensus.App = state.ConsensusParams.Version.AppVersion
+			}
+			// We update the last results hash with the empty hash, to conform with RFC-6962.
+			state.LastResultsHash = merkle.HashFromByteSlices(nil)
+			if err := h.stateStore.Save(state); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// First handle edge cases and constraints on the storeBlockHeight and storeBlockBase.
+	switch {
+	case storeBlockHeight == 0:
+		assertAppHashEqualsOneFromState(appHash, state)
+		return appHash, nil
+
+	case appBlockHeight == 0 && state.InitialHeight < storeBlockBase:
+		// the app has no state, and the block store is truncated above the initial height
+		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
+
+	case appBlockHeight > 0 && appBlockHeight < storeBlockBase-1:
+		// the app is too far behind truncated store (can be 1 behind since we replay the next)
+		return appHash, sm.ErrAppBlockHeightTooLow{AppHeight: appBlockHeight, StoreBase: storeBlockBase}
+
+	case storeBlockHeight < appBlockHeight:
+		// the app should never be ahead of the store (but this is under app's control)
+		return appHash, sm.ErrAppBlockHeightTooHigh{CoreHeight: storeBlockHeight, AppHeight: appBlockHeight}
+
+	case storeBlockHeight < stateBlockHeight:
+		// the state should never be ahead of the store (this is under tendermint's control)
+		panic(fmt.Sprintf("StateBlockHeight (%d) > StoreBlockHeight (%d)", stateBlockHeight, storeBlockHeight))
+
+	case storeBlockHeight > stateBlockHeight+1:
+		// store should be at most one ahead of the state (this is under tendermint's control)
+		panic(fmt.Sprintf("StoreBlockHeight (%d) > StateBlockHeight + 1 (%d)", storeBlockHeight, stateBlockHeight+1))
+	}
+
+	var err error
+	// Now either store is equal to state, or one ahead.
+	// For each, consider all cases of where the app could be, given app <= store
+	if storeBlockHeight == stateBlockHeight {
+		// Tendermint ran Commit and saved the state.
+		// Either the app is asking for replay, or we're all synced up.
+		if appBlockHeight < storeBlockHeight {
+			// the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store)
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false)
+
+		} else if appBlockHeight == storeBlockHeight {
+			// We're good!
+			assertAppHashEqualsOneFromState(appHash, state)
+			return appHash, nil
+		}
+
+	} else if storeBlockHeight == stateBlockHeight+1 {
+		// We saved the block in the store but haven't updated the state,
+		// so we'll need to replay a block using the WAL.
+		switch {
+		case appBlockHeight < stateBlockHeight:
+			// the app is further behind than it should be, so replay blocks
+			// but leave the last block to go through the WAL
+			return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true)
+
+		case appBlockHeight == stateBlockHeight:
+			// We haven't run Commit (both the state and app are one block behind),
+			// so replayBlock with the real app.
+			// NOTE: We could instead use the cs.WAL on cs.Start,
+			// but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT
+			h.logger.Info("Replay last block using real app")
+			state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+			return state.AppHash, err
+
+		case appBlockHeight == storeBlockHeight:
+			// We ran Commit, but didn't save the state, so replayBlock with mock app.
+			abciResponses, err := h.stateStore.LoadABCIResponses(storeBlockHeight)
+			if err != nil {
+				return nil, err
+			}
+			mockApp := newMockProxyApp(appHash, abciResponses)
+			h.logger.Info("Replay last block using mock app")
+			state, err = h.replayBlock(state, storeBlockHeight, mockApp)
+			return state.AppHash, err
+		}
+
+	}
+
+	panic(fmt.Sprintf("uncovered case! appHeight: %d, storeHeight: %d, stateHeight: %d",
+		appBlockHeight, storeBlockHeight, stateBlockHeight))
+}
+
+func (h *Handshaker) replayBlocks(
+	state sm.State,
+	proxyApp proxy.AppConns,
+	appBlockHeight,
+	storeBlockHeight int64,
+	mutateState bool) ([]byte, error) {
+	// App is further behind than it should be, so we need to replay blocks.
+	// We replay all blocks from appBlockHeight+1.
+	//
+	// Note that we don't have an old version of the state,
+	// so we by-pass state validation/mutation using sm.ExecCommitBlock.
+	// This also means we won't be saving validator sets if they change during this period.
+	// TODO: Load the historical information to fix this and just use state.ApplyBlock
+	//
+	// If mutateState == true, the final block is replayed with h.replayBlock()
+
+	var appHash []byte
+	var err error
+	finalBlock := storeBlockHeight
+	if mutateState {
+		finalBlock--
+	}
+	firstBlock := appBlockHeight + 1
+	if firstBlock == 1 {
+		firstBlock = state.InitialHeight
+	}
+	for i := firstBlock; i <= finalBlock; i++ {
+		h.logger.Info("Applying block", "height", i)
+		block := h.store.LoadBlock(i)
+		// Extra check to ensure the app was not changed in a way it shouldn't have.
+		if len(appHash) > 0 {
+			assertAppHashEqualsOneFromBlock(appHash, block)
+		}
+
+		if i == finalBlock && !mutateState {
+			// We emit events for the index services at the final block due to the sync issue when
+			// the node shutdown during the block committing status.
+			blockExec := sm.NewBlockExecutor(
+				h.stateStore, h.logger, proxyApp.Consensus(), emptyMempool{}, sm.EmptyEvidencePool{})
+			blockExec.SetEventBus(h.eventBus)
+			appHash, err = sm.ExecCommitBlock(
+				blockExec, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			appHash, err = sm.ExecCommitBlock(
+				nil, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		h.nBlocks++
+	}
+
+	if mutateState {
+		// sync the final block
+		state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus())
+		if err != nil {
+			return nil, err
+		}
+		appHash = state.AppHash
+	}
+
+	assertAppHashEqualsOneFromState(appHash, state)
+	return appHash, nil
+}
+
+// ApplyBlock on the proxyApp with the last block.
+func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) {
+	block := h.store.LoadBlock(height)
+	meta := h.store.LoadBlockMeta(height)
+
+	// Use stubs for both mempool and evidence pool since no transactions nor
+	// evidence are needed here - block already exists.
+	blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{})
+	blockExec.SetEventBus(h.eventBus)
+
+	var err error
+	state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block)
+	if err != nil {
+		return sm.State{}, err
+	}
+
+	h.nBlocks++
+
+	return state, nil
+}
+
+func assertAppHashEqualsOneFromBlock(appHash []byte, block *types.Block) {
+	if !bytes.Equal(appHash, block.AppHash) {
+		panic(fmt.Sprintf(`block.AppHash does not match AppHash after replay. Got %X, expected %X.
+
+Block: %v
+`,
+			appHash, block.AppHash, block))
+	}
+}
+
+func assertAppHashEqualsOneFromState(appHash []byte, state sm.State) {
+	if !bytes.Equal(appHash, state.AppHash) {
+		panic(fmt.Sprintf(`state.AppHash does not match AppHash after replay. Got
+%X, expected %X.
+
+State: %v
+
+Did you reset Tendermint without resetting your application's data?`,
+			appHash, state.AppHash, state))
+	}
+}

internal/consensus/replay_file.go

@@ -0,0 +1,338 @@
+package consensus
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+	"strings"
+
+	dbm "github.com/tendermint/tm-db"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+const (
+	// event bus subscriber
+	subscriber = "replay-file"
+)
+
+//--------------------------------------------------------
+// replay messages interactively or all at once
+
+// replay the wal file
+func RunReplayFile(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig, console bool) {
+	consensusState := newConsensusStateForReplay(config, csConfig)
+
+	if err := consensusState.ReplayFile(csConfig.WalFile(), console); err != nil {
+		tmos.Exit(fmt.Sprintf("Error during consensus replay: %v", err))
+	}
+}
+
+// Replay msgs in file or start the console
+func (cs *State) ReplayFile(file string, console bool) error {
+
+	if cs.IsRunning() {
+		return errors.New("cs is already running, cannot replay")
+	}
+	if cs.wal != nil {
+		return errors.New("cs wal is open, cannot replay")
+	}
+
+	cs.startForReplay()
+
+	// ensure all new step events are regenerated as expected
+
+	ctx := context.Background()
+	newStepSub, err := cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+	if err != nil {
+		return fmt.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)
+	}
+	defer func() {
+		if err := cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep); err != nil {
+			cs.Logger.Error("Error unsubscribing to event bus", "err", err)
+		}
+	}()
+
+	// just open the file for reading, no need to use wal
+	fp, err := os.OpenFile(file, os.O_RDONLY, 0600)
+	if err != nil {
+		return err
+	}
+
+	pb := newPlayback(file, fp, cs, cs.state.Copy())
+	defer pb.fp.Close()
+
+	var nextN int // apply N msgs in a row
+	var msg *TimedWALMessage
+	for {
+		if nextN == 0 && console {
+			nextN = pb.replayConsoleLoop()
+		}
+
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
+			return err
+		}
+
+		if nextN > 0 {
+			nextN--
+		}
+		pb.count++
+	}
+}
+
+//------------------------------------------------
+// playback manager
+
+type playback struct {
+	cs *State
+
+	fp    *os.File
+	dec   *WALDecoder
+	count int // how many lines/msgs into the file are we
+
+	// replays can be reset to beginning
+	fileName     string   // so we can close/reopen the file
+	genesisState sm.State // so the replay session knows where to restart from
+}
+
+func newPlayback(fileName string, fp *os.File, cs *State, genState sm.State) *playback {
+	return &playback{
+		cs:           cs,
+		fp:           fp,
+		fileName:     fileName,
+		genesisState: genState,
+		dec:          NewWALDecoder(fp),
+	}
+}
+
+// go back count steps by resetting the state and running (pb.count - count) steps
+func (pb *playback) replayReset(count int, newStepSub types.Subscription) error {
+	if err := pb.cs.Stop(); err != nil {
+		return err
+	}
+	pb.cs.Wait()
+
+	newCS := NewState(pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec,
+		pb.cs.blockStore, pb.cs.txNotifier, pb.cs.evpool)
+	newCS.SetEventBus(pb.cs.eventBus)
+	newCS.startForReplay()
+
+	if err := pb.fp.Close(); err != nil {
+		return err
+	}
+	fp, err := os.OpenFile(pb.fileName, os.O_RDONLY, 0600)
+	if err != nil {
+		return err
+	}
+	pb.fp = fp
+	pb.dec = NewWALDecoder(fp)
+	count = pb.count - count
+	fmt.Printf("Reseting from %d to %d\n", pb.count, count)
+	pb.count = 0
+	pb.cs = newCS
+	var msg *TimedWALMessage
+	for i := 0; i < count; i++ {
+		msg, err = pb.dec.Decode()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
+		if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil {
+			return err
+		}
+		pb.count++
+	}
+	return nil
+}
+
+func (cs *State) startForReplay() {
+	cs.Logger.Error("Replay commands are disabled until someone updates them and writes tests")
+	/* TODO:!
+	// since we replay tocks we just ignore ticks
+		go func() {
+			for {
+				select {
+				case <-cs.tickChan:
+				case <-cs.Quit:
+					return
+				}
+			}
+		}()*/
+}
+
+// console function for parsing input and running commands
+func (pb *playback) replayConsoleLoop() int {
+	for {
+		fmt.Printf("> ")
+		bufReader := bufio.NewReader(os.Stdin)
+		line, more, err := bufReader.ReadLine()
+		if more {
+			tmos.Exit("input is too long")
+		} else if err != nil {
+			tmos.Exit(err.Error())
+		}
+
+		tokens := strings.Split(string(line), " ")
+		if len(tokens) == 0 {
+			continue
+		}
+
+		switch tokens[0] {
+		case "next":
+			// "next" -> replay next message
+			// "next N" -> replay next N messages
+
+			if len(tokens) == 1 {
+				return 0
+			}
+			i, err := strconv.Atoi(tokens[1])
+			if err != nil {
+				fmt.Println("next takes an integer argument")
+			} else {
+				return i
+			}
+
+		case "back":
+			// "back" -> go back one message
+			// "back N" -> go back N messages
+
+			// NOTE: "back" is not supported in the state machine design,
+			// so we restart and replay up to
+
+			ctx := context.Background()
+			// ensure all new step events are regenerated as expected
+
+			newStepSub, err := pb.cs.eventBus.Subscribe(ctx, subscriber, types.EventQueryNewRoundStep)
+			if err != nil {
+				tmos.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep))
+			}
+			defer func() {
+				if err := pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep); err != nil {
+					pb.cs.Logger.Error("Error unsubscribing from eventBus", "err", err)
+				}
+			}()
+
+			if len(tokens) == 1 {
+				if err := pb.replayReset(1, newStepSub); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
+			} else {
+				i, err := strconv.Atoi(tokens[1])
+				if err != nil {
+					fmt.Println("back takes an integer argument")
+				} else if i > pb.count {
+					fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count)
+				} else if err := pb.replayReset(i, newStepSub); err != nil {
+					pb.cs.Logger.Error("Replay reset error", "err", err)
+				}
+			}
+
+		case "rs":
+			// "rs" -> print entire round state
+			// "rs short" -> print height/round/step
+			// "rs <field>" -> print another field of the round state
+
+			rs := pb.cs.RoundState
+			if len(tokens) == 1 {
+				fmt.Println(rs)
+			} else {
+				switch tokens[1] {
+				case "short":
+					fmt.Printf("%v/%v/%v\n", rs.Height, rs.Round, rs.Step)
+				case "validators":
+					fmt.Println(rs.Validators)
+				case "proposal":
+					fmt.Println(rs.Proposal)
+				case "proposal_block":
+					fmt.Printf("%v %v\n", rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort())
+				case "locked_round":
+					fmt.Println(rs.LockedRound)
+				case "locked_block":
+					fmt.Printf("%v %v\n", rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort())
+				case "votes":
+					fmt.Println(rs.Votes.StringIndented("  "))
+
+				default:
+					fmt.Println("Unknown option", tokens[1])
+				}
+			}
+		case "n":
+			fmt.Println(pb.count)
+		}
+	}
+}
+
+//--------------------------------------------------------------------------------
+
+// convenience for replay mode
+func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusConfig) *State {
+	dbType := dbm.BackendType(config.DBBackend)
+	// Get BlockStore
+	blockStoreDB, err := dbm.NewDB("blockstore", dbType, config.DBDir())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	// Get State
+	stateDB, err := dbm.NewDB("state", dbType, config.DBDir())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	stateStore := sm.NewStore(stateDB)
+	gdoc, err := sm.MakeGenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+	state, err := sm.MakeGenesisState(gdoc)
+	if err != nil {
+		tmos.Exit(err.Error())
+	}
+
+	// Create proxyAppConn connection (consensus, mempool, query)
+	clientCreator, _ := proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir())
+	proxyApp := proxy.NewAppConns(clientCreator)
+	err = proxyApp.Start()
+	if err != nil {
+		tmos.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err))
+	}
+
+	eventBus := types.NewEventBus()
+	if err := eventBus.Start(); err != nil {
+		tmos.Exit(fmt.Sprintf("Failed to start event bus: %v", err))
+	}
+
+	handshaker := NewHandshaker(stateStore, state, blockStore, gdoc)
+	handshaker.SetEventBus(eventBus)
+	err = handshaker.Handshake(proxyApp)
+	if err != nil {
+		tmos.Exit(fmt.Sprintf("Error on handshake: %v", err))
+	}
+
+	mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+
+	consensusState := NewState(csConfig, state.Copy(), blockExec,
+		blockStore, mempool, evpool)
+
+	consensusState.SetEventBus(eventBus)
+	return consensusState
+}

internal/consensus/replay_stubs.go

@@ -0,0 +1,92 @@
+package consensus
+
+import (
+	"context"
+
+	abci "github.com/tendermint/tendermint/abci/types"
+	"github.com/tendermint/tendermint/internal/libs/clist"
+	mempl "github.com/tendermint/tendermint/internal/mempool"
+	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
+	"github.com/tendermint/tendermint/proxy"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+
+type emptyMempool struct{}
+
+var _ mempl.Mempool = emptyMempool{}
+
+func (emptyMempool) Lock()     {}
+func (emptyMempool) Unlock()   {}
+func (emptyMempool) Size() int { return 0 }
+func (emptyMempool) CheckTx(_ context.Context, _ types.Tx, _ func(*abci.Response), _ mempl.TxInfo) error {
+	return nil
+}
+func (emptyMempool) ReapMaxBytesMaxGas(_, _ int64) types.Txs { return types.Txs{} }
+func (emptyMempool) ReapMaxTxs(n int) types.Txs              { return types.Txs{} }
+func (emptyMempool) Update(
+	_ int64,
+	_ types.Txs,
+	_ []*abci.ResponseDeliverTx,
+	_ mempl.PreCheckFunc,
+	_ mempl.PostCheckFunc,
+) error {
+	return nil
+}
+func (emptyMempool) Flush()                        {}
+func (emptyMempool) FlushAppConn() error           { return nil }
+func (emptyMempool) TxsAvailable() <-chan struct{} { return make(chan struct{}) }
+func (emptyMempool) EnableTxsAvailable()           {}
+func (emptyMempool) SizeBytes() int64              { return 0 }
+
+func (emptyMempool) TxsFront() *clist.CElement    { return nil }
+func (emptyMempool) TxsWaitChan() <-chan struct{} { return nil }
+
+func (emptyMempool) InitWAL() error { return nil }
+func (emptyMempool) CloseWAL()      {}
+
+//-----------------------------------------------------------------------------
+// mockProxyApp uses ABCIResponses to give the right results.
+//
+// Useful because we don't want to call Commit() twice for the same block on
+// the real app.
+
+func newMockProxyApp(appHash []byte, abciResponses *tmstate.ABCIResponses) proxy.AppConnConsensus {
+	clientCreator := proxy.NewLocalClientCreator(&mockProxyApp{
+		appHash:       appHash,
+		abciResponses: abciResponses,
+	})
+	cli, _ := clientCreator.NewABCIClient()
+	err := cli.Start()
+	if err != nil {
+		panic(err)
+	}
+	return proxy.NewAppConnConsensus(cli)
+}
+
+type mockProxyApp struct {
+	abci.BaseApplication
+
+	appHash       []byte
+	txCount       int
+	abciResponses *tmstate.ABCIResponses
+}
+
+func (mock *mockProxyApp) DeliverTx(req abci.RequestDeliverTx) abci.ResponseDeliverTx {
+	r := mock.abciResponses.DeliverTxs[mock.txCount]
+	mock.txCount++
+	if r == nil {
+		return abci.ResponseDeliverTx{}
+	}
+	return *r
+}
+
+func (mock *mockProxyApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock {
+	mock.txCount = 0
+	return *mock.abciResponses.EndBlock
+}
+
+func (mock *mockProxyApp) Commit() abci.ResponseCommit {
+	return abci.ResponseCommit{Data: mock.appHash}
+}

internal/consensus/ticker.go

@@ -0,0 +1,134 @@
+package consensus
+
+import (
+	"time"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/libs/service"
+)
+
+var (
+	tickTockBufferSize = 10
+)
+
+// TimeoutTicker is a timer that schedules timeouts
+// conditional on the height/round/step in the timeoutInfo.
+// The timeoutInfo.Duration may be non-positive.
+type TimeoutTicker interface {
+	Start() error
+	Stop() error
+	Chan() <-chan timeoutInfo       // on which to receive a timeout
+	ScheduleTimeout(ti timeoutInfo) // reset the timer
+
+	SetLogger(log.Logger)
+}
+
+// timeoutTicker wraps time.Timer,
+// scheduling timeouts only for greater height/round/step
+// than what it's already seen.
+// Timeouts are scheduled along the tickChan,
+// and fired on the tockChan.
+type timeoutTicker struct {
+	service.BaseService
+
+	timer    *time.Timer
+	tickChan chan timeoutInfo // for scheduling timeouts
+	tockChan chan timeoutInfo // for notifying about them
+}
+
+// NewTimeoutTicker returns a new TimeoutTicker.
+func NewTimeoutTicker() TimeoutTicker {
+	tt := &timeoutTicker{
+		timer:    time.NewTimer(0),
+		tickChan: make(chan timeoutInfo, tickTockBufferSize),
+		tockChan: make(chan timeoutInfo, tickTockBufferSize),
+	}
+	tt.BaseService = *service.NewBaseService(nil, "TimeoutTicker", tt)
+	tt.stopTimer() // don't want to fire until the first scheduled timeout
+	return tt
+}
+
+// OnStart implements service.Service. It starts the timeout routine.
+func (t *timeoutTicker) OnStart() error {
+
+	go t.timeoutRoutine()
+
+	return nil
+}
+
+// OnStop implements service.Service. It stops the timeout routine.
+func (t *timeoutTicker) OnStop() {
+	t.BaseService.OnStop()
+	t.stopTimer()
+}
+
+// Chan returns a channel on which timeouts are sent.
+func (t *timeoutTicker) Chan() <-chan timeoutInfo {
+	return t.tockChan
+}
+
+// ScheduleTimeout schedules a new timeout by sending on the internal tickChan.
+// The timeoutRoutine is always available to read from tickChan, so this won't block.
+// The scheduling may fail if the timeoutRoutine has already scheduled a timeout for a later height/round/step.
+func (t *timeoutTicker) ScheduleTimeout(ti timeoutInfo) {
+	t.tickChan <- ti
+}
+
+//-------------------------------------------------------------
+
+// stop the timer and drain if necessary
+func (t *timeoutTicker) stopTimer() {
+	// Stop() returns false if it was already fired or was stopped
+	if !t.timer.Stop() {
+		select {
+		case <-t.timer.C:
+		default:
+			t.Logger.Debug("Timer already stopped")
+		}
+	}
+}
+
+// send on tickChan to start a new timer.
+// timers are interupted and replaced by new ticks from later steps
+// timeouts of 0 on the tickChan will be immediately relayed to the tockChan
+func (t *timeoutTicker) timeoutRoutine() {
+	t.Logger.Debug("Starting timeout routine")
+	var ti timeoutInfo
+	for {
+		select {
+		case newti := <-t.tickChan:
+			t.Logger.Debug("Received tick", "old_ti", ti, "new_ti", newti)
+
+			// ignore tickers for old height/round/step
+			if newti.Height < ti.Height {
+				continue
+			} else if newti.Height == ti.Height {
+				if newti.Round < ti.Round {
+					continue
+				} else if newti.Round == ti.Round {
+					if ti.Step > 0 && newti.Step <= ti.Step {
+						continue
+					}
+				}
+			}
+
+			// stop the last timer
+			t.stopTimer()
+
+			// update timeoutInfo and reset timer
+			// NOTE time.Timer allows duration to be non-positive
+			ti = newti
+			t.timer.Reset(ti.Duration)
+			t.Logger.Debug("Scheduled timeout", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+		case <-t.timer.C:
+			t.Logger.Info("Timed out", "dur", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step)
+			// go routine here guarantees timeoutRoutine doesn't block.
+			// Determinism comes from playback in the receiveRoutine.
+			// We can eliminate it by merging the timeoutRoutine into receiveRoutine
+			//  and managing the timeouts ourselves with a millisecond ticker
+			go func(toi timeoutInfo) { t.tockChan <- toi }(ti)
+		case <-t.Quit():
+			return
+		}
+	}
+}

internal/consensus/types/height_vote_set.go

@@ -0,0 +1,267 @@
+package types
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/tendermint/tendermint/internal/p2p"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	tmmath "github.com/tendermint/tendermint/libs/math"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+)
+
+type RoundVoteSet struct {
+	Prevotes   *types.VoteSet
+	Precommits *types.VoteSet
+}
+
+var (
+	ErrGotVoteFromUnwantedRound = errors.New(
+		"peer has sent a vote that does not match our round for more than one round",
+	)
+)
+
+/*
+Keeps track of all VoteSets from round 0 to round 'round'.
+
+Also keeps track of up to one RoundVoteSet greater than
+'round' from each peer, to facilitate catchup syncing of commits.
+
+A commit is +2/3 precommits for a block at a round,
+but which round is not known in advance, so when a peer
+provides a precommit for a round greater than mtx.round,
+we create a new entry in roundVoteSets but also remember the
+peer to prevent abuse.
+We let each peer provide us with up to 2 unexpected "catchup" rounds.
+One for their LastCommit round, and another for the official commit round.
+*/
+type HeightVoteSet struct {
+	chainID string
+	height  int64
+	valSet  *types.ValidatorSet
+
+	mtx               sync.Mutex
+	round             int32                  // max tracked round
+	roundVoteSets     map[int32]RoundVoteSet // keys: [0...round]
+	peerCatchupRounds map[p2p.NodeID][]int32 // keys: peer.ID; values: at most 2 rounds
+}
+
+func NewHeightVoteSet(chainID string, height int64, valSet *types.ValidatorSet) *HeightVoteSet {
+	hvs := &HeightVoteSet{
+		chainID: chainID,
+	}
+	hvs.Reset(height, valSet)
+	return hvs
+}
+
+func (hvs *HeightVoteSet) Reset(height int64, valSet *types.ValidatorSet) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+
+	hvs.height = height
+	hvs.valSet = valSet
+	hvs.roundVoteSets = make(map[int32]RoundVoteSet)
+	hvs.peerCatchupRounds = make(map[p2p.NodeID][]int32)
+
+	hvs.addRound(0)
+	hvs.round = 0
+}
+
+func (hvs *HeightVoteSet) Height() int64 {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.height
+}
+
+func (hvs *HeightVoteSet) Round() int32 {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.round
+}
+
+// Create more RoundVoteSets up to round.
+func (hvs *HeightVoteSet) SetRound(round int32) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	newRound := tmmath.SafeSubInt32(hvs.round, 1)
+	if hvs.round != 0 && (round < newRound) {
+		panic("SetRound() must increment hvs.round")
+	}
+	for r := newRound; r <= round; r++ {
+		if _, ok := hvs.roundVoteSets[r]; ok {
+			continue // Already exists because peerCatchupRounds.
+		}
+		hvs.addRound(r)
+	}
+	hvs.round = round
+}
+
+func (hvs *HeightVoteSet) addRound(round int32) {
+	if _, ok := hvs.roundVoteSets[round]; ok {
+		panic("addRound() for an existing round")
+	}
+	// log.Debug("addRound(round)", "round", round)
+	prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrevoteType, hvs.valSet)
+	precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, tmproto.PrecommitType, hvs.valSet)
+	hvs.roundVoteSets[round] = RoundVoteSet{
+		Prevotes:   prevotes,
+		Precommits: precommits,
+	}
+}
+
+// Duplicate votes return added=false, err=nil.
+// By convention, peerID is "" if origin is self.
+func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err error) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	if !types.IsVoteTypeValid(vote.Type) {
+		return
+	}
+	voteSet := hvs.getVoteSet(vote.Round, vote.Type)
+	if voteSet == nil {
+		if rndz := hvs.peerCatchupRounds[peerID]; len(rndz) < 2 {
+			hvs.addRound(vote.Round)
+			voteSet = hvs.getVoteSet(vote.Round, vote.Type)
+			hvs.peerCatchupRounds[peerID] = append(rndz, vote.Round)
+		} else {
+			// punish peer
+			err = ErrGotVoteFromUnwantedRound
+			return
+		}
+	}
+	added, err = voteSet.AddVote(vote)
+	return
+}
+
+func (hvs *HeightVoteSet) Prevotes(round int32) *types.VoteSet {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.getVoteSet(round, tmproto.PrevoteType)
+}
+
+func (hvs *HeightVoteSet) Precommits(round int32) *types.VoteSet {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return hvs.getVoteSet(round, tmproto.PrecommitType)
+}
+
+// Last round and blockID that has +2/3 prevotes for a particular block or nil.
+// Returns -1 if no such round exists.
+func (hvs *HeightVoteSet) POLInfo() (polRound int32, polBlockID types.BlockID) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	for r := hvs.round; r >= 0; r-- {
+		rvs := hvs.getVoteSet(r, tmproto.PrevoteType)
+		polBlockID, ok := rvs.TwoThirdsMajority()
+		if ok {
+			return r, polBlockID
+		}
+	}
+	return -1, types.BlockID{}
+}
+
+func (hvs *HeightVoteSet) getVoteSet(round int32, voteType tmproto.SignedMsgType) *types.VoteSet {
+	rvs, ok := hvs.roundVoteSets[round]
+	if !ok {
+		return nil
+	}
+	switch voteType {
+	case tmproto.PrevoteType:
+		return rvs.Prevotes
+	case tmproto.PrecommitType:
+		return rvs.Precommits
+	default:
+		panic(fmt.Sprintf("Unexpected vote type %X", voteType))
+	}
+}
+
+// If a peer claims that it has 2/3 majority for given blockKey, call this.
+// NOTE: if there are too many peers, or too much peer churn,
+// this can cause memory issues.
+// TODO: implement ability to remove peers too
+func (hvs *HeightVoteSet) SetPeerMaj23(
+	round int32,
+	voteType tmproto.SignedMsgType,
+	peerID p2p.NodeID,
+	blockID types.BlockID) error {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	if !types.IsVoteTypeValid(voteType) {
+		return fmt.Errorf("setPeerMaj23: Invalid vote type %X", voteType)
+	}
+	voteSet := hvs.getVoteSet(round, voteType)
+	if voteSet == nil {
+		return nil // something we don't know about yet
+	}
+	return voteSet.SetPeerMaj23(types.P2PID(peerID), blockID)
+}
+
+//---------------------------------------------------------
+// string and json
+
+func (hvs *HeightVoteSet) String() string {
+	return hvs.StringIndented("")
+}
+
+func (hvs *HeightVoteSet) StringIndented(indent string) string {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	vsStrings := make([]string, 0, (len(hvs.roundVoteSets)+1)*2)
+	// rounds 0 ~ hvs.round inclusive
+	for round := int32(0); round <= hvs.round; round++ {
+		voteSetString := hvs.roundVoteSets[round].Prevotes.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+		voteSetString = hvs.roundVoteSets[round].Precommits.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+	}
+	// all other peer catchup rounds
+	for round, roundVoteSet := range hvs.roundVoteSets {
+		if round <= hvs.round {
+			continue
+		}
+		voteSetString := roundVoteSet.Prevotes.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+		voteSetString = roundVoteSet.Precommits.StringShort()
+		vsStrings = append(vsStrings, voteSetString)
+	}
+	return fmt.Sprintf(`HeightVoteSet{H:%v R:0~%v
+%s  %v
+%s}`,
+		hvs.height, hvs.round,
+		indent, strings.Join(vsStrings, "\n"+indent+"  "),
+		indent)
+}
+
+func (hvs *HeightVoteSet) MarshalJSON() ([]byte, error) {
+	hvs.mtx.Lock()
+	defer hvs.mtx.Unlock()
+	return tmjson.Marshal(hvs.toAllRoundVotes())
+}
+
+func (hvs *HeightVoteSet) toAllRoundVotes() []roundVotes {
+	totalRounds := hvs.round + 1
+	allVotes := make([]roundVotes, totalRounds)
+	// rounds 0 ~ hvs.round inclusive
+	for round := int32(0); round < totalRounds; round++ {
+		allVotes[round] = roundVotes{
+			Round:              round,
+			Prevotes:           hvs.roundVoteSets[round].Prevotes.VoteStrings(),
+			PrevotesBitArray:   hvs.roundVoteSets[round].Prevotes.BitArrayString(),
+			Precommits:         hvs.roundVoteSets[round].Precommits.VoteStrings(),
+			PrecommitsBitArray: hvs.roundVoteSets[round].Precommits.BitArrayString(),
+		}
+	}
+	// TODO: all other peer catchup rounds
+	return allVotes
+}
+
+type roundVotes struct {
+	Round              int32    `json:"round"`
+	Prevotes           []string `json:"prevotes"`
+	PrevotesBitArray   string   `json:"prevotes_bit_array"`
+	Precommits         []string `json:"precommits"`
+	PrecommitsBitArray string   `json:"precommits_bit_array"`
+}

internal/consensus/types/height_vote_set_test.go

@@ -0,0 +1,89 @@
+package types
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/internal/test/factory"
+	tmrand "github.com/tendermint/tendermint/libs/rand"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	"github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+var config *cfg.Config // NOTE: must be reset for each _test.go file
+
+func TestMain(m *testing.M) {
+	config = cfg.ResetTestRoot("consensus_height_vote_set_test")
+	code := m.Run()
+	os.RemoveAll(config.RootDir)
+	os.Exit(code)
+}
+
+func TestPeerCatchupRounds(t *testing.T) {
+	valSet, privVals := factory.RandValidatorSet(10, 1)
+
+	hvs := NewHeightVoteSet(config.ChainID(), 1, valSet)
+
+	vote999_0 := makeVoteHR(t, 1, 0, 999, privVals)
+	added, err := hvs.AddVote(vote999_0, "peer1")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from peer", added, err)
+	}
+
+	vote1000_0 := makeVoteHR(t, 1, 0, 1000, privVals)
+	added, err = hvs.AddVote(vote1000_0, "peer1")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from peer", added, err)
+	}
+
+	vote1001_0 := makeVoteHR(t, 1, 0, 1001, privVals)
+	added, err = hvs.AddVote(vote1001_0, "peer1")
+	if err != ErrGotVoteFromUnwantedRound {
+		t.Errorf("expected GotVoteFromUnwantedRoundError, but got %v", err)
+	}
+	if added {
+		t.Error("Expected to *not* add vote from peer, too many catchup rounds.")
+	}
+
+	added, err = hvs.AddVote(vote1001_0, "peer2")
+	if !added || err != nil {
+		t.Error("Expected to successfully add vote from another peer")
+	}
+
+}
+
+func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []types.PrivValidator) *types.Vote {
+	privVal := privVals[valIndex]
+	pubKey, err := privVal.GetPubKey(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	randBytes := tmrand.Bytes(tmhash.Size)
+
+	vote := &types.Vote{
+		ValidatorAddress: pubKey.Address(),
+		ValidatorIndex:   valIndex,
+		Height:           height,
+		Round:            round,
+		Timestamp:        tmtime.Now(),
+		Type:             tmproto.PrecommitType,
+		BlockID:          types.BlockID{Hash: randBytes, PartSetHeader: types.PartSetHeader{}},
+	}
+	chainID := config.ChainID()
+
+	v := vote.ToProto()
+	err = privVal.SignVote(context.Background(), chainID, v)
+	if err != nil {
+		panic(fmt.Sprintf("Error signing vote: %v", err))
+	}
+
+	vote.Signature = v.Signature
+
+	return vote
+}

internal/consensus/types/peer_round_state.go

@@ -0,0 +1,93 @@
+package types
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bits"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+
+// PeerRoundState contains the known state of a peer.
+// NOTE: Read-only when returned by PeerState.GetRoundState().
+type PeerRoundState struct {
+	Height int64         `json:"height"` // Height peer is at
+	Round  int32         `json:"round"`  // Round peer is at, -1 if unknown.
+	Step   RoundStepType `json:"step"`   // Step peer is at
+
+	// Estimated start of round 0 at this height
+	StartTime time.Time `json:"start_time"`
+
+	// True if peer has proposal for this round
+	Proposal                   bool                `json:"proposal"`
+	ProposalBlockPartSetHeader types.PartSetHeader `json:"proposal_block_part_set_header"`
+	ProposalBlockParts         *bits.BitArray      `json:"proposal_block_parts"`
+	// Proposal's POL round. -1 if none.
+	ProposalPOLRound int32 `json:"proposal_pol_round"`
+
+	// nil until ProposalPOLMessage received.
+	ProposalPOL     *bits.BitArray `json:"proposal_pol"`
+	Prevotes        *bits.BitArray `json:"prevotes"`          // All votes peer has for this round
+	Precommits      *bits.BitArray `json:"precommits"`        // All precommits peer has for this round
+	LastCommitRound int32          `json:"last_commit_round"` // Round of commit for last height. -1 if none.
+	LastCommit      *bits.BitArray `json:"last_commit"`       // All commit precommits of commit for last height.
+
+	// Round that we have commit for. Not necessarily unique. -1 if none.
+	CatchupCommitRound int32 `json:"catchup_commit_round"`
+
+	// All commit precommits peer has for this height & CatchupCommitRound
+	CatchupCommit *bits.BitArray `json:"catchup_commit"`
+}
+
+// String returns a string representation of the PeerRoundState
+func (prs PeerRoundState) String() string {
+	return prs.StringIndented("")
+}
+
+// Copy provides a deep copy operation. Because many of the fields in
+// the PeerRound struct are pointers, we need an explicit deep copy
+// operation to avoid a non-obvious shared data situation.
+func (prs PeerRoundState) Copy() PeerRoundState {
+	// this works because it's not a pointer receiver so it's
+	// already, effectively a copy.
+
+	headerHash := prs.ProposalBlockPartSetHeader.Hash.Bytes()
+
+	hashCopy := make([]byte, len(headerHash))
+	copy(hashCopy, headerHash)
+	prs.ProposalBlockPartSetHeader = types.PartSetHeader{
+		Total: prs.ProposalBlockPartSetHeader.Total,
+		Hash:  hashCopy,
+	}
+	prs.ProposalBlockParts = prs.ProposalBlockParts.Copy()
+	prs.ProposalPOL = prs.ProposalPOL.Copy()
+	prs.Prevotes = prs.Prevotes.Copy()
+	prs.Precommits = prs.Precommits.Copy()
+	prs.LastCommit = prs.LastCommit.Copy()
+	prs.CatchupCommit = prs.CatchupCommit.Copy()
+
+	return prs
+}
+
+// StringIndented returns a string representation of the PeerRoundState
+func (prs PeerRoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`PeerRoundState{
+%s  %v/%v/%v @%v
+%s  Proposal %v -> %v
+%s  POL      %v (round %v)
+%s  Prevotes   %v
+%s  Precommits %v
+%s  LastCommit %v (round %v)
+%s  Catchup    %v (round %v)
+%s}`,
+		indent, prs.Height, prs.Round, prs.Step, prs.StartTime,
+		indent, prs.ProposalBlockPartSetHeader, prs.ProposalBlockParts,
+		indent, prs.ProposalPOL, prs.ProposalPOLRound,
+		indent, prs.Prevotes,
+		indent, prs.Precommits,
+		indent, prs.LastCommit, prs.LastCommitRound,
+		indent, prs.CatchupCommit, prs.CatchupCommitRound,
+		indent)
+}

internal/consensus/types/peer_round_state_test.go

@@ -0,0 +1,29 @@
+package types
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/libs/bits"
+)
+
+func TestCopy(t *testing.T) {
+	t.Run("VerifyShallowCopy", func(t *testing.T) {
+		prsOne := PeerRoundState{}
+		prsOne.Prevotes = bits.NewBitArray(12)
+		prsTwo := prsOne
+
+		prsOne.Prevotes.SetIndex(1, true)
+
+		require.Equal(t, prsOne.Prevotes, prsTwo.Prevotes)
+	})
+	t.Run("DeepCopy", func(t *testing.T) {
+		prsOne := PeerRoundState{}
+		prsOne.Prevotes = bits.NewBitArray(12)
+		prsTwo := prsOne.Copy()
+
+		prsOne.Prevotes.SetIndex(1, true)
+
+		require.NotEqual(t, prsOne.Prevotes, prsTwo.Prevotes)
+	})
+}

internal/consensus/types/round_state.go

@@ -0,0 +1,215 @@
+package types
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/bytes"
+	"github.com/tendermint/tendermint/types"
+)
+
+//-----------------------------------------------------------------------------
+// RoundStepType enum type
+
+// RoundStepType enumerates the state of the consensus state machine
+type RoundStepType uint8 // These must be numeric, ordered.
+
+// RoundStepType
+const (
+	RoundStepNewHeight     = RoundStepType(0x01) // Wait til CommitTime + timeoutCommit
+	RoundStepNewRound      = RoundStepType(0x02) // Setup new round and go to RoundStepPropose
+	RoundStepPropose       = RoundStepType(0x03) // Did propose, gossip proposal
+	RoundStepPrevote       = RoundStepType(0x04) // Did prevote, gossip prevotes
+	RoundStepPrevoteWait   = RoundStepType(0x05) // Did receive any +2/3 prevotes, start timeout
+	RoundStepPrecommit     = RoundStepType(0x06) // Did precommit, gossip precommits
+	RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout
+	RoundStepCommit        = RoundStepType(0x08) // Entered commit state machine
+	// NOTE: RoundStepNewHeight acts as RoundStepCommitWait.
+
+	// NOTE: Update IsValid method if you change this!
+)
+
+// IsValid returns true if the step is valid, false if unknown/undefined.
+func (rs RoundStepType) IsValid() bool {
+	return uint8(rs) >= 0x01 && uint8(rs) <= 0x08
+}
+
+// String returns a string
+func (rs RoundStepType) String() string {
+	switch rs {
+	case RoundStepNewHeight:
+		return "RoundStepNewHeight"
+	case RoundStepNewRound:
+		return "RoundStepNewRound"
+	case RoundStepPropose:
+		return "RoundStepPropose"
+	case RoundStepPrevote:
+		return "RoundStepPrevote"
+	case RoundStepPrevoteWait:
+		return "RoundStepPrevoteWait"
+	case RoundStepPrecommit:
+		return "RoundStepPrecommit"
+	case RoundStepPrecommitWait:
+		return "RoundStepPrecommitWait"
+	case RoundStepCommit:
+		return "RoundStepCommit"
+	default:
+		return "RoundStepUnknown" // Cannot panic.
+	}
+}
+
+//-----------------------------------------------------------------------------
+
+// RoundState defines the internal consensus state.
+// NOTE: Not thread safe. Should only be manipulated by functions downstream
+// of the cs.receiveRoutine
+type RoundState struct {
+	Height    int64         `json:"height"` // Height we are working on
+	Round     int32         `json:"round"`
+	Step      RoundStepType `json:"step"`
+	StartTime time.Time     `json:"start_time"`
+
+	// Subjective time when +2/3 precommits for Block at Round were found
+	CommitTime         time.Time           `json:"commit_time"`
+	Validators         *types.ValidatorSet `json:"validators"`
+	Proposal           *types.Proposal     `json:"proposal"`
+	ProposalBlock      *types.Block        `json:"proposal_block"`
+	ProposalBlockParts *types.PartSet      `json:"proposal_block_parts"`
+	LockedRound        int32               `json:"locked_round"`
+	LockedBlock        *types.Block        `json:"locked_block"`
+	LockedBlockParts   *types.PartSet      `json:"locked_block_parts"`
+
+	// Last known round with POL for non-nil valid block.
+	ValidRound int32        `json:"valid_round"`
+	ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above.
+
+	// Last known block parts of POL mentioned above.
+	ValidBlockParts           *types.PartSet      `json:"valid_block_parts"`
+	Votes                     *HeightVoteSet      `json:"votes"`
+	CommitRound               int32               `json:"commit_round"` //
+	LastCommit                *types.VoteSet      `json:"last_commit"`  // Last precommits at Height-1
+	LastValidators            *types.ValidatorSet `json:"last_validators"`
+	TriggeredTimeoutPrecommit bool                `json:"triggered_timeout_precommit"`
+}
+
+// Compressed version of the RoundState for use in RPC
+type RoundStateSimple struct {
+	HeightRoundStep   string              `json:"height/round/step"`
+	StartTime         time.Time           `json:"start_time"`
+	ProposalBlockHash bytes.HexBytes      `json:"proposal_block_hash"`
+	LockedBlockHash   bytes.HexBytes      `json:"locked_block_hash"`
+	ValidBlockHash    bytes.HexBytes      `json:"valid_block_hash"`
+	Votes             json.RawMessage     `json:"height_vote_set"`
+	Proposer          types.ValidatorInfo `json:"proposer"`
+}
+
+// Compress the RoundState to RoundStateSimple
+func (rs *RoundState) RoundStateSimple() RoundStateSimple {
+	votesJSON, err := rs.Votes.MarshalJSON()
+	if err != nil {
+		panic(err)
+	}
+
+	addr := rs.Validators.GetProposer().Address
+	idx, _ := rs.Validators.GetByAddress(addr)
+
+	return RoundStateSimple{
+		HeightRoundStep:   fmt.Sprintf("%d/%d/%d", rs.Height, rs.Round, rs.Step),
+		StartTime:         rs.StartTime,
+		ProposalBlockHash: rs.ProposalBlock.Hash(),
+		LockedBlockHash:   rs.LockedBlock.Hash(),
+		ValidBlockHash:    rs.ValidBlock.Hash(),
+		Votes:             votesJSON,
+		Proposer: types.ValidatorInfo{
+			Address: addr,
+			Index:   idx,
+		},
+	}
+}
+
+// NewRoundEvent returns the RoundState with proposer information as an event.
+func (rs *RoundState) NewRoundEvent() types.EventDataNewRound {
+	addr := rs.Validators.GetProposer().Address
+	idx, _ := rs.Validators.GetByAddress(addr)
+
+	return types.EventDataNewRound{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
+		Proposer: types.ValidatorInfo{
+			Address: addr,
+			Index:   idx,
+		},
+	}
+}
+
+// CompleteProposalEvent returns information about a proposed block as an event.
+func (rs *RoundState) CompleteProposalEvent() types.EventDataCompleteProposal {
+	// We must construct BlockID from ProposalBlock and ProposalBlockParts
+	// cs.Proposal is not guaranteed to be set when this function is called
+	blockID := types.BlockID{
+		Hash:          rs.ProposalBlock.Hash(),
+		PartSetHeader: rs.ProposalBlockParts.Header(),
+	}
+
+	return types.EventDataCompleteProposal{
+		Height:  rs.Height,
+		Round:   rs.Round,
+		Step:    rs.Step.String(),
+		BlockID: blockID,
+	}
+}
+
+// RoundStateEvent returns the H/R/S of the RoundState as an event.
+func (rs *RoundState) RoundStateEvent() types.EventDataRoundState {
+	return types.EventDataRoundState{
+		Height: rs.Height,
+		Round:  rs.Round,
+		Step:   rs.Step.String(),
+	}
+}
+
+// String returns a string
+func (rs *RoundState) String() string {
+	return rs.StringIndented("")
+}
+
+// StringIndented returns a string
+func (rs *RoundState) StringIndented(indent string) string {
+	return fmt.Sprintf(`RoundState{
+%s  H:%v R:%v S:%v
+%s  StartTime:     %v
+%s  CommitTime:    %v
+%s  Validators:    %v
+%s  Proposal:      %v
+%s  ProposalBlock: %v %v
+%s  LockedRound:   %v
+%s  LockedBlock:   %v %v
+%s  ValidRound:   %v
+%s  ValidBlock:   %v %v
+%s  Votes:         %v
+%s  LastCommit:    %v
+%s  LastValidators:%v
+%s}`,
+		indent, rs.Height, rs.Round, rs.Step,
+		indent, rs.StartTime,
+		indent, rs.CommitTime,
+		indent, rs.Validators.StringIndented(indent+"  "),
+		indent, rs.Proposal,
+		indent, rs.ProposalBlockParts.StringShort(), rs.ProposalBlock.StringShort(),
+		indent, rs.LockedRound,
+		indent, rs.LockedBlockParts.StringShort(), rs.LockedBlock.StringShort(),
+		indent, rs.ValidRound,
+		indent, rs.ValidBlockParts.StringShort(), rs.ValidBlock.StringShort(),
+		indent, rs.Votes.StringIndented(indent+"  "),
+		indent, rs.LastCommit.StringShort(),
+		indent, rs.LastValidators.StringIndented(indent+"  "),
+		indent)
+}
+
+// StringShort returns a string
+func (rs *RoundState) StringShort() string {
+	return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`,
+		rs.Height, rs.Round, rs.Step, rs.StartTime)
+}

internal/consensus/wal.go

@@ -0,0 +1,433 @@
+package consensus
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"path/filepath"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+
+	auto "github.com/tendermint/tendermint/internal/libs/autofile"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/libs/log"
+	tmos "github.com/tendermint/tendermint/libs/os"
+	"github.com/tendermint/tendermint/libs/service"
+	tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	// time.Time + max consensus msg size
+	maxMsgSizeBytes = maxMsgSize + 24
+
+	// how often the WAL should be sync'd during period sync'ing
+	walDefaultFlushInterval = 2 * time.Second
+)
+
+//--------------------------------------------------------
+// types and functions for savings consensus messages
+
+// TimedWALMessage wraps WALMessage and adds Time for debugging purposes.
+type TimedWALMessage struct {
+	Time time.Time  `json:"time"`
+	Msg  WALMessage `json:"msg"`
+}
+
+// EndHeightMessage marks the end of the given height inside WAL.
+// @internal used by scripts/wal2json util.
+type EndHeightMessage struct {
+	Height int64 `json:"height"`
+}
+
+type WALMessage interface{}
+
+func init() {
+	tmjson.RegisterType(msgInfo{}, "tendermint/wal/MsgInfo")
+	tmjson.RegisterType(timeoutInfo{}, "tendermint/wal/TimeoutInfo")
+	tmjson.RegisterType(EndHeightMessage{}, "tendermint/wal/EndHeightMessage")
+}
+
+//--------------------------------------------------------
+// Simple write-ahead logger
+
+// WAL is an interface for any write-ahead logger.
+type WAL interface {
+	Write(WALMessage) error
+	WriteSync(WALMessage) error
+	FlushAndSync() error
+
+	SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error)
+
+	// service methods
+	Start() error
+	Stop() error
+	Wait()
+}
+
+// Write ahead logger writes msgs to disk before they are processed.
+// Can be used for crash-recovery and deterministic replay.
+// TODO: currently the wal is overwritten during replay catchup, give it a mode
+// so it's either reading or appending - must read to end to start appending
+// again.
+type BaseWAL struct {
+	service.BaseService
+
+	group *auto.Group
+
+	enc *WALEncoder
+
+	flushTicker   *time.Ticker
+	flushInterval time.Duration
+}
+
+var _ WAL = &BaseWAL{}
+
+// NewWAL returns a new write-ahead logger based on `baseWAL`, which implements
+// WAL. It's flushed and synced to disk every 2s and once when stopped.
+func NewWAL(walFile string, groupOptions ...func(*auto.Group)) (*BaseWAL, error) {
+	err := tmos.EnsureDir(filepath.Dir(walFile), 0700)
+	if err != nil {
+		return nil, fmt.Errorf("failed to ensure WAL directory is in place: %w", err)
+	}
+
+	group, err := auto.OpenGroup(walFile, groupOptions...)
+	if err != nil {
+		return nil, err
+	}
+	wal := &BaseWAL{
+		group:         group,
+		enc:           NewWALEncoder(group),
+		flushInterval: walDefaultFlushInterval,
+	}
+	wal.BaseService = *service.NewBaseService(nil, "baseWAL", wal)
+	return wal, nil
+}
+
+// SetFlushInterval allows us to override the periodic flush interval for the WAL.
+func (wal *BaseWAL) SetFlushInterval(i time.Duration) {
+	wal.flushInterval = i
+}
+
+func (wal *BaseWAL) Group() *auto.Group {
+	return wal.group
+}
+
+func (wal *BaseWAL) SetLogger(l log.Logger) {
+	wal.BaseService.Logger = l
+	wal.group.SetLogger(l)
+}
+
+func (wal *BaseWAL) OnStart() error {
+	size, err := wal.group.Head.Size()
+	if err != nil {
+		return err
+	} else if size == 0 {
+		if err := wal.WriteSync(EndHeightMessage{0}); err != nil {
+			return err
+		}
+	}
+	err = wal.group.Start()
+	if err != nil {
+		return err
+	}
+	wal.flushTicker = time.NewTicker(wal.flushInterval)
+	go wal.processFlushTicks()
+	return nil
+}
+
+func (wal *BaseWAL) processFlushTicks() {
+	for {
+		select {
+		case <-wal.flushTicker.C:
+			if err := wal.FlushAndSync(); err != nil {
+				wal.Logger.Error("Periodic WAL flush failed", "err", err)
+			}
+		case <-wal.Quit():
+			return
+		}
+	}
+}
+
+// FlushAndSync flushes and fsync's the underlying group's data to disk.
+// See auto#FlushAndSync
+func (wal *BaseWAL) FlushAndSync() error {
+	return wal.group.FlushAndSync()
+}
+
+// Stop the underlying autofile group.
+// Use Wait() to ensure it's finished shutting down
+// before cleaning up files.
+func (wal *BaseWAL) OnStop() {
+	wal.flushTicker.Stop()
+	if err := wal.FlushAndSync(); err != nil {
+		wal.Logger.Error("error on flush data to disk", "error", err)
+	}
+	if err := wal.group.Stop(); err != nil {
+		wal.Logger.Error("error trying to stop wal", "error", err)
+	}
+	wal.group.Close()
+}
+
+// Wait for the underlying autofile group to finish shutting down
+// so it's safe to cleanup files.
+func (wal *BaseWAL) Wait() {
+	wal.group.Wait()
+}
+
+// Write is called in newStep and for each receive on the
+// peerMsgQueue and the timeoutTicker.
+// NOTE: does not call fsync()
+func (wal *BaseWAL) Write(msg WALMessage) error {
+	if wal == nil {
+		return nil
+	}
+
+	if err := wal.enc.Encode(&TimedWALMessage{tmtime.Now(), msg}); err != nil {
+		wal.Logger.Error("Error writing msg to consensus wal. WARNING: recover may not be possible for the current height",
+			"err", err, "msg", msg)
+		return err
+	}
+
+	return nil
+}
+
+// WriteSync is called when we receive a msg from ourselves
+// so that we write to disk before sending signed messages.
+// NOTE: calls fsync()
+func (wal *BaseWAL) WriteSync(msg WALMessage) error {
+	if wal == nil {
+		return nil
+	}
+
+	if err := wal.Write(msg); err != nil {
+		return err
+	}
+
+	if err := wal.FlushAndSync(); err != nil {
+		wal.Logger.Error(`WriteSync failed to flush consensus wal.
+		WARNING: may result in creating alternative proposals / votes for the current height iff the node restarted`,
+			"err", err)
+		return err
+	}
+
+	return nil
+}
+
+// WALSearchOptions are optional arguments to SearchForEndHeight.
+type WALSearchOptions struct {
+	// IgnoreDataCorruptionErrors set to true will result in skipping data corruption errors.
+	IgnoreDataCorruptionErrors bool
+}
+
+// SearchForEndHeight searches for the EndHeightMessage with the given height
+// and returns an auto.GroupReader, whenever it was found or not and an error.
+// Group reader will be nil if found equals false.
+//
+// CONTRACT: caller must close group reader.
+func (wal *BaseWAL) SearchForEndHeight(
+	height int64,
+	options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	var (
+		msg *TimedWALMessage
+		gr  *auto.GroupReader
+	)
+	lastHeightFound := int64(-1)
+
+	// NOTE: starting from the last file in the group because we're usually
+	// searching for the last height. See replay.go
+	min, max := wal.group.MinIndex(), wal.group.MaxIndex()
+	wal.Logger.Info("Searching for height", "height", height, "min", min, "max", max)
+	for index := max; index >= min; index-- {
+		gr, err = wal.group.NewReader(index)
+		if err != nil {
+			return nil, false, err
+		}
+
+		dec := NewWALDecoder(gr)
+		for {
+			msg, err = dec.Decode()
+			if err == io.EOF {
+				// OPTIMISATION: no need to look for height in older files if we've seen h < height
+				if lastHeightFound > 0 && lastHeightFound < height {
+					gr.Close()
+					return nil, false, nil
+				}
+				// check next file
+				break
+			}
+			if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) {
+				wal.Logger.Error("Corrupted entry. Skipping...", "err", err)
+				// do nothing
+				continue
+			} else if err != nil {
+				gr.Close()
+				return nil, false, err
+			}
+
+			if m, ok := msg.Msg.(EndHeightMessage); ok {
+				lastHeightFound = m.Height
+				if m.Height == height { // found
+					wal.Logger.Info("Found", "height", height, "index", index)
+					return gr, true, nil
+				}
+			}
+		}
+		gr.Close()
+	}
+
+	return nil, false, nil
+}
+
+// A WALEncoder writes custom-encoded WAL messages to an output stream.
+//
+// Format: 4 bytes CRC sum + 4 bytes length + arbitrary-length value
+type WALEncoder struct {
+	wr io.Writer
+}
+
+// NewWALEncoder returns a new encoder that writes to wr.
+func NewWALEncoder(wr io.Writer) *WALEncoder {
+	return &WALEncoder{wr}
+}
+
+// Encode writes the custom encoding of v to the stream. It returns an error if
+// the encoded size of v is greater than 1MB. Any error encountered
+// during the write is also returned.
+func (enc *WALEncoder) Encode(v *TimedWALMessage) error {
+	pbMsg, err := WALToProto(v.Msg)
+	if err != nil {
+		return err
+	}
+	pv := tmcons.TimedWALMessage{
+		Time: v.Time,
+		Msg:  pbMsg,
+	}
+
+	data, err := proto.Marshal(&pv)
+	if err != nil {
+		panic(fmt.Errorf("encode timed wall message failure: %w", err))
+	}
+
+	crc := crc32.Checksum(data, crc32c)
+	length := uint32(len(data))
+	if length > maxMsgSizeBytes {
+		return fmt.Errorf("msg is too big: %d bytes, max: %d bytes", length, maxMsgSizeBytes)
+	}
+	totalLength := 8 + int(length)
+
+	msg := make([]byte, totalLength)
+	binary.BigEndian.PutUint32(msg[0:4], crc)
+	binary.BigEndian.PutUint32(msg[4:8], length)
+	copy(msg[8:], data)
+
+	_, err = enc.wr.Write(msg)
+	return err
+}
+
+// IsDataCorruptionError returns true if data has been corrupted inside WAL.
+func IsDataCorruptionError(err error) bool {
+	_, ok := err.(DataCorruptionError)
+	return ok
+}
+
+// DataCorruptionError is an error that occures if data on disk was corrupted.
+type DataCorruptionError struct {
+	cause error
+}
+
+func (e DataCorruptionError) Error() string {
+	return fmt.Sprintf("DataCorruptionError[%v]", e.cause)
+}
+
+func (e DataCorruptionError) Cause() error {
+	return e.cause
+}
+
+// A WALDecoder reads and decodes custom-encoded WAL messages from an input
+// stream. See WALEncoder for the format used.
+//
+// It will also compare the checksums and make sure data size is equal to the
+// length from the header. If that is not the case, error will be returned.
+type WALDecoder struct {
+	rd io.Reader
+}
+
+// NewWALDecoder returns a new decoder that reads from rd.
+func NewWALDecoder(rd io.Reader) *WALDecoder {
+	return &WALDecoder{rd}
+}
+
+// Decode reads the next custom-encoded value from its reader and returns it.
+func (dec *WALDecoder) Decode() (*TimedWALMessage, error) {
+	b := make([]byte, 4)
+
+	_, err := dec.rd.Read(b)
+	if errors.Is(err, io.EOF) {
+		return nil, err
+	}
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read checksum: %v", err)}
+	}
+	crc := binary.BigEndian.Uint32(b)
+
+	b = make([]byte, 4)
+	_, err = dec.rd.Read(b)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read length: %v", err)}
+	}
+	length := binary.BigEndian.Uint32(b)
+
+	if length > maxMsgSizeBytes {
+		return nil, DataCorruptionError{fmt.Errorf(
+			"length %d exceeded maximum possible value of %d bytes",
+			length,
+			maxMsgSizeBytes)}
+	}
+
+	data := make([]byte, length)
+	n, err := dec.rd.Read(data)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to read data: %v (read: %d, wanted: %d)", err, n, length)}
+	}
+
+	// check checksum before decoding data
+	actualCRC := crc32.Checksum(data, crc32c)
+	if actualCRC != crc {
+		return nil, DataCorruptionError{fmt.Errorf("checksums do not match: read: %v, actual: %v", crc, actualCRC)}
+	}
+
+	var res = new(tmcons.TimedWALMessage)
+	err = proto.Unmarshal(data, res)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to decode data: %v", err)}
+	}
+
+	walMsg, err := WALFromProto(res.Msg)
+	if err != nil {
+		return nil, DataCorruptionError{fmt.Errorf("failed to convert from proto: %w", err)}
+	}
+	tMsgWal := &TimedWALMessage{
+		Time: res.Time,
+		Msg:  walMsg,
+	}
+
+	return tMsgWal, err
+}
+
+type nilWAL struct{}
+
+var _ WAL = nilWAL{}
+
+func (nilWAL) Write(m WALMessage) error     { return nil }
+func (nilWAL) WriteSync(m WALMessage) error { return nil }
+func (nilWAL) FlushAndSync() error          { return nil }
+func (nilWAL) SearchForEndHeight(height int64, options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	return nil, false, nil
+}
+func (nilWAL) Start() error { return nil }
+func (nilWAL) Stop() error  { return nil }
+func (nilWAL) Wait()        {}

internal/consensus/wal_fuzz.go

@@ -0,0 +1,31 @@
+// +build gofuzz
+
+package consensus
+
+import (
+	"bytes"
+	"io"
+)
+
+func Fuzz(data []byte) int {
+	dec := NewWALDecoder(bytes.NewReader(data))
+	for {
+		msg, err := dec.Decode()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			if msg != nil {
+				panic("msg != nil on error")
+			}
+			return 0
+		}
+		var w bytes.Buffer
+		enc := NewWALEncoder(&w)
+		err = enc.Encode(msg)
+		if err != nil {
+			panic(err)
+		}
+	}
+	return 1
+}

internal/consensus/wal_generator.go

@@ -0,0 +1,232 @@
+package consensus
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	mrand "math/rand"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	db "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/abci/example/kvstore"
+	cfg "github.com/tendermint/tendermint/config"
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/privval"
+	"github.com/tendermint/tendermint/proxy"
+	sm "github.com/tendermint/tendermint/state"
+	"github.com/tendermint/tendermint/store"
+	"github.com/tendermint/tendermint/types"
+)
+
+// WALGenerateNBlocks generates a consensus WAL. It does this by spinning up a
+// stripped down version of node (proxy app, event bus, consensus state) with a
+// persistent kvstore application and special consensus wal instance
+// (byteBufferWAL) and waits until numBlocks are created.
+// If the node fails to produce given numBlocks, it returns an error.
+func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
+	config := getConfig(t)
+
+	app := kvstore.NewPersistentKVStoreApplication(filepath.Join(config.DBDir(), "wal_generator"))
+	t.Cleanup(func() { require.NoError(t, app.Close()) })
+
+	logger := log.TestingLogger().With("wal_generator", "wal_generator")
+	logger.Info("generating WAL (last height msg excluded)", "numBlocks", numBlocks)
+
+	// COPY PASTE FROM node.go WITH A FEW MODIFICATIONS
+	// NOTE: we can't import node package because of circular dependency.
+	// NOTE: we don't do handshake so need to set state.Version.Consensus.App directly.
+	privValidatorKeyFile := config.PrivValidatorKeyFile()
+	privValidatorStateFile := config.PrivValidatorStateFile()
+	privValidator, err := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile)
+	if err != nil {
+		return err
+	}
+	genDoc, err := types.GenesisDocFromFile(config.GenesisFile())
+	if err != nil {
+		return fmt.Errorf("failed to read genesis file: %w", err)
+	}
+	blockStoreDB := db.NewMemDB()
+	stateDB := blockStoreDB
+	stateStore := sm.NewStore(stateDB)
+	state, err := sm.MakeGenesisState(genDoc)
+	if err != nil {
+		return fmt.Errorf("failed to make genesis state: %w", err)
+	}
+	state.Version.Consensus.App = kvstore.ProtocolVersion
+	if err = stateStore.Save(state); err != nil {
+		t.Error(err)
+	}
+
+	blockStore := store.NewBlockStore(blockStoreDB)
+
+	proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app))
+	proxyApp.SetLogger(logger.With("module", "proxy"))
+	if err := proxyApp.Start(); err != nil {
+		return fmt.Errorf("failed to start proxy app connections: %w", err)
+	}
+	t.Cleanup(func() {
+		if err := proxyApp.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+
+	eventBus := types.NewEventBus()
+	eventBus.SetLogger(logger.With("module", "events"))
+	if err := eventBus.Start(); err != nil {
+		return fmt.Errorf("failed to start event bus: %w", err)
+	}
+	t.Cleanup(func() {
+		if err := eventBus.Stop(); err != nil {
+			t.Error(err)
+		}
+	})
+	mempool := emptyMempool{}
+	evpool := sm.EmptyEvidencePool{}
+	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
+	consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
+	consensusState.SetLogger(logger)
+	consensusState.SetEventBus(eventBus)
+	if privValidator != nil && privValidator != (*privval.FilePV)(nil) {
+		consensusState.SetPrivValidator(privValidator)
+	}
+	// END OF COPY PASTE
+
+	// set consensus wal to buffered WAL, which will write all incoming msgs to buffer
+	numBlocksWritten := make(chan struct{})
+	wal := newByteBufferWAL(logger, NewWALEncoder(wr), int64(numBlocks), numBlocksWritten)
+	// see wal.go#103
+	if err := wal.Write(EndHeightMessage{0}); err != nil {
+		t.Error(err)
+	}
+
+	consensusState.wal = wal
+
+	if err := consensusState.Start(); err != nil {
+		return fmt.Errorf("failed to start consensus state: %w", err)
+	}
+
+	select {
+	case <-numBlocksWritten:
+		if err := consensusState.Stop(); err != nil {
+			t.Error(err)
+		}
+		return nil
+	case <-time.After(1 * time.Minute):
+		if err := consensusState.Stop(); err != nil {
+			t.Error(err)
+		}
+		return fmt.Errorf("waited too long for tendermint to produce %d blocks (grep logs for `wal_generator`)", numBlocks)
+	}
+}
+
+// WALWithNBlocks returns a WAL content with numBlocks.
+func WALWithNBlocks(t *testing.T, numBlocks int) (data []byte, err error) {
+	var b bytes.Buffer
+	wr := bufio.NewWriter(&b)
+
+	if err := WALGenerateNBlocks(t, wr, numBlocks); err != nil {
+		return []byte{}, err
+	}
+
+	wr.Flush()
+	return b.Bytes(), nil
+}
+
+func randPort() int {
+	// returns between base and base + spread
+	base, spread := 20000, 20000
+	// nolint:gosec // G404: Use of weak random number generator
+	return base + mrand.Intn(spread)
+}
+
+func makeAddrs() (string, string, string) {
+	start := randPort()
+	return fmt.Sprintf("tcp://127.0.0.1:%d", start),
+		fmt.Sprintf("tcp://127.0.0.1:%d", start+1),
+		fmt.Sprintf("tcp://127.0.0.1:%d", start+2)
+}
+
+// getConfig returns a config for test cases
+func getConfig(t *testing.T) *cfg.Config {
+	c := cfg.ResetTestRoot(t.Name())
+
+	// and we use random ports to run in parallel
+	tm, rpc, grpc := makeAddrs()
+	c.P2P.ListenAddress = tm
+	c.RPC.ListenAddress = rpc
+	c.RPC.GRPCListenAddress = grpc
+	return c
+}
+
+// byteBufferWAL is a WAL which writes all msgs to a byte buffer. Writing stops
+// when the heightToStop is reached. Client will be notified via
+// signalWhenStopsTo channel.
+type byteBufferWAL struct {
+	enc               *WALEncoder
+	stopped           bool
+	heightToStop      int64
+	signalWhenStopsTo chan<- struct{}
+
+	logger log.Logger
+}
+
+// needed for determinism
+var fixedTime, _ = time.Parse(time.RFC3339, "2017-01-02T15:04:05Z")
+
+func newByteBufferWAL(logger log.Logger, enc *WALEncoder, nBlocks int64, signalStop chan<- struct{}) *byteBufferWAL {
+	return &byteBufferWAL{
+		enc:               enc,
+		heightToStop:      nBlocks,
+		signalWhenStopsTo: signalStop,
+		logger:            logger,
+	}
+}
+
+// Save writes message to the internal buffer except when heightToStop is
+// reached, in which case it will signal the caller via signalWhenStopsTo and
+// skip writing.
+func (w *byteBufferWAL) Write(m WALMessage) error {
+	if w.stopped {
+		w.logger.Debug("WAL already stopped. Not writing message", "msg", m)
+		return nil
+	}
+
+	if endMsg, ok := m.(EndHeightMessage); ok {
+		w.logger.Debug("WAL write end height message", "height", endMsg.Height, "stopHeight", w.heightToStop)
+		if endMsg.Height == w.heightToStop {
+			w.logger.Debug("Stopping WAL at height", "height", endMsg.Height)
+			w.signalWhenStopsTo <- struct{}{}
+			w.stopped = true
+			return nil
+		}
+	}
+
+	w.logger.Debug("WAL Write Message", "msg", m)
+	err := w.enc.Encode(&TimedWALMessage{fixedTime, m})
+	if err != nil {
+		panic(fmt.Sprintf("failed to encode the msg %v", m))
+	}
+
+	return nil
+}
+
+func (w *byteBufferWAL) WriteSync(m WALMessage) error {
+	return w.Write(m)
+}
+
+func (w *byteBufferWAL) FlushAndSync() error { return nil }
+
+func (w *byteBufferWAL) SearchForEndHeight(
+	height int64,
+	options *WALSearchOptions) (rd io.ReadCloser, found bool, err error) {
+	return nil, false, nil
+}
+
+func (w *byteBufferWAL) Start() error { return nil }
+func (w *byteBufferWAL) Stop() error  { return nil }
+func (w *byteBufferWAL) Wait()        {}

internal/consensus/wal_test.go

@@ -0,0 +1,276 @@
+package consensus
+
+import (
+	"bytes"
+	"crypto/rand"
+	"path/filepath"
+
+	// "sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/tendermint/tendermint/crypto/merkle"
+	"github.com/tendermint/tendermint/internal/consensus/types"
+	"github.com/tendermint/tendermint/internal/libs/autofile"
+	"github.com/tendermint/tendermint/libs/log"
+	tmtypes "github.com/tendermint/tendermint/types"
+	tmtime "github.com/tendermint/tendermint/types/time"
+)
+
+const (
+	walTestFlushInterval = time.Duration(100) * time.Millisecond
+)
+
+func TestWALTruncate(t *testing.T) {
+	walDir := t.TempDir()
+	walFile := filepath.Join(walDir, "wal")
+
+	// this magic number 4K can truncate the content when RotateFile.
+	// defaultHeadSizeLimit(10M) is hard to simulate.
+	// this magic number 1 * time.Millisecond make RotateFile check frequently.
+	// defaultGroupCheckDuration(5s) is hard to simulate.
+	wal, err := NewWAL(walFile,
+		autofile.GroupHeadSizeLimit(4096),
+		autofile.GroupCheckDuration(1*time.Millisecond),
+	)
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
+	err = wal.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	})
+
+	// 60 block's size nearly 70K, greater than group's headBuf size(4096 * 10),
+	// when headBuf is full, truncate content will Flush to the file. at this
+	// time, RotateFile is called, truncate content exist in each file.
+	err = WALGenerateNBlocks(t, wal.Group(), 60)
+	require.NoError(t, err)
+
+	time.Sleep(1 * time.Millisecond) // wait groupCheckDuration, make sure RotateFile run
+
+	if err := wal.FlushAndSync(); err != nil {
+		t.Error(err)
+	}
+
+	h := int64(50)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	t.Cleanup(func() { _ = gr.Close() })
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, "wrong height")
+}
+
+func TestWALEncoderDecoder(t *testing.T) {
+	now := tmtime.Now()
+	msgs := []TimedWALMessage{
+		{Time: now, Msg: EndHeightMessage{0}},
+		{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
+		{Time: now, Msg: tmtypes.EventDataRoundState{Height: 1, Round: 1, Step: ""}},
+	}
+
+	b := new(bytes.Buffer)
+
+	for _, msg := range msgs {
+		msg := msg
+
+		b.Reset()
+
+		enc := NewWALEncoder(b)
+		err := enc.Encode(&msg)
+		require.NoError(t, err)
+
+		dec := NewWALDecoder(b)
+		decoded, err := dec.Decode()
+		require.NoError(t, err)
+		assert.Equal(t, msg.Time.UTC(), decoded.Time)
+		assert.Equal(t, msg.Msg, decoded.Msg)
+	}
+}
+
+func TestWALWrite(t *testing.T) {
+	walDir := t.TempDir()
+	walFile := filepath.Join(walDir, "wal")
+
+	wal, err := NewWAL(walFile)
+	require.NoError(t, err)
+	err = wal.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		// wait for the wal to finish shutting down so we
+		// can safely remove the directory
+		wal.Wait()
+	})
+
+	// 1) Write returns an error if msg is too big
+	msg := &BlockPartMessage{
+		Height: 1,
+		Round:  1,
+		Part: &tmtypes.Part{
+			Index: 1,
+			Bytes: make([]byte, 1),
+			Proof: merkle.Proof{
+				Total:    1,
+				Index:    1,
+				LeafHash: make([]byte, maxMsgSizeBytes-30),
+			},
+		},
+	}
+
+	err = wal.Write(msgInfo{
+		Msg: msg,
+	})
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "msg is too big")
+	}
+}
+
+func TestWALSearchForEndHeight(t *testing.T) {
+	walBody, err := WALWithNBlocks(t, 6)
+	if err != nil {
+		t.Fatal(err)
+	}
+	walFile := tempWALWithData(walBody)
+
+	wal, err := NewWAL(walFile)
+	require.NoError(t, err)
+	wal.SetLogger(log.TestingLogger())
+
+	h := int64(3)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	t.Cleanup(func() { _ = gr.Close() })
+
+	dec := NewWALDecoder(gr)
+	msg, err := dec.Decode()
+	assert.NoError(t, err, "expected to decode a message")
+	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
+	assert.True(t, ok, "expected message of type EventDataRoundState")
+	assert.Equal(t, rs.Height, h+1, "wrong height")
+}
+
+func TestWALPeriodicSync(t *testing.T) {
+	walDir := t.TempDir()
+	walFile := filepath.Join(walDir, "wal")
+	wal, err := NewWAL(walFile, autofile.GroupCheckDuration(1*time.Millisecond))
+
+	require.NoError(t, err)
+
+	wal.SetFlushInterval(walTestFlushInterval)
+	wal.SetLogger(log.TestingLogger())
+
+	// Generate some data
+	err = WALGenerateNBlocks(t, wal.Group(), 5)
+	require.NoError(t, err)
+
+	// We should have data in the buffer now
+	assert.NotZero(t, wal.Group().Buffered())
+
+	require.NoError(t, wal.Start())
+	t.Cleanup(func() {
+		if err := wal.Stop(); err != nil {
+			t.Error(err)
+		}
+		wal.Wait()
+	})
+
+	time.Sleep(walTestFlushInterval + (10 * time.Millisecond))
+
+	// The data should have been flushed by the periodic sync
+	assert.Zero(t, wal.Group().Buffered())
+
+	h := int64(4)
+	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
+	assert.NoError(t, err, "expected not to err on height %d", h)
+	assert.True(t, found, "expected to find end height for %d", h)
+	assert.NotNil(t, gr)
+	if gr != nil {
+		gr.Close()
+	}
+}
+
+/*
+var initOnce sync.Once
+
+func registerInterfacesOnce() {
+	initOnce.Do(func() {
+		var _ = wire.RegisterInterface(
+			struct{ WALMessage }{},
+			wire.ConcreteType{[]byte{}, 0x10},
+		)
+	})
+}
+*/
+
+func nBytes(n int) []byte {
+	buf := make([]byte, n)
+	n, _ = rand.Read(buf)
+	return buf[:n]
+}
+
+func benchmarkWalDecode(b *testing.B, n int) {
+	// registerInterfacesOnce()
+	buf := new(bytes.Buffer)
+	enc := NewWALEncoder(buf)
+
+	data := nBytes(n)
+	if err := enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second).UTC()}); err != nil {
+		b.Error(err)
+	}
+
+	encoded := buf.Bytes()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		buf.Reset()
+		buf.Write(encoded)
+		dec := NewWALDecoder(buf)
+		if _, err := dec.Decode(); err != nil {
+			b.Fatal(err)
+		}
+	}
+	b.ReportAllocs()
+}
+
+func BenchmarkWalDecode512B(b *testing.B) {
+	benchmarkWalDecode(b, 512)
+}
+
+func BenchmarkWalDecode10KB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024)
+}
+func BenchmarkWalDecode100KB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024)
+}
+func BenchmarkWalDecode1MB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024)
+}
+func BenchmarkWalDecode10MB(b *testing.B) {
+	benchmarkWalDecode(b, 10*1024*1024)
+}
+func BenchmarkWalDecode100MB(b *testing.B) {
+	benchmarkWalDecode(b, 100*1024*1024)
+}
+func BenchmarkWalDecode1GB(b *testing.B) {
+	benchmarkWalDecode(b, 1024*1024*1024)
+}