diff --git a/consensus/README.md b/consensus/README.md index 3f32d7e46..44a36012f 100644 --- a/consensus/README.md +++ b/consensus/README.md @@ -1,3 +1,3 @@ -# Consensus +# Consensus -See the [consensus spec](https://github.com/tendermint/spec/tree/master/spec/consensus). +See the [consensus spec](https://github.com/tendermint/spec/tree/master/spec/consensus) and the [reactor consensus spec](https://github.com/tendermint/spec/tree/master/spec/reactors/consensus) for more information. diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 7fed81395..b16a98c6a 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -7,47 +7,49 @@ import ( "path" "sync" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + dbm "github.com/tendermint/tm-db" + abcicli "github.com/tendermint/tendermint/abci/client" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/evidence" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/service" tmsync "github.com/tendermint/tendermint/libs/sync" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" - tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tm-db" ) -// Byzantine node sends two different prevotes (nil and blockID) to the same -// validator. -func TestByzantinePrevoteEquivocation(t *testing.T) { - configSetup(t) +//---------------------------------------------- +// byzantine failures - nValidators := 4 - prevoteHeight := int64(2) +// Byzantine node sends two different prevotes (nil and blockID) to the same validator +func TestByzantinePrevoteEquivocation(t *testing.T) { + const nValidators = 4 + const byzantineNode = 0 + const prevoteHeight = int64(2) testName := "consensus_byzantine_test" tickerFunc := newMockTickerFunc(true) appFunc := newCounter genDoc, privVals := randGenesisDoc(nValidators, false, 30) - states := make([]*State, nValidators) + css := make([]*State, nValidators) for i := 0; i < nValidators; i++ { logger := consensusLogger().With("test", "byzantine", "validator", i) stateDB := dbm.NewMemDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) - thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) defer os.RemoveAll(thisConfig.RootDir) - ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal app := appFunc() vals := types.TM2PB.ValidatorUpdates(state.Validators) @@ -90,174 +92,183 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { cs.SetTimeoutTicker(tickerFunc()) cs.SetLogger(logger) - states[i] = cs + css[i] = cs } - rts := setup(t, nValidators, states, 100) // buffer must be large enough to not deadlock + // initialize the reactors for each of the validators + reactors := make([]*Reactor, nValidators) + blocksSubs := make([]types.Subscription, 0) + eventBuses := make([]*types.EventBus, nValidators) + for i := 0; i < nValidators; i++ { + reactors[i] = NewReactor(css[i], true) // so we dont start the consensus states + reactors[i].SetLogger(css[i].Logger) - var bzNodeID p2p.NodeID + // eventBus is already started with the cs + eventBuses[i] = css[i].eventBus + reactors[i].SetEventBus(eventBuses[i]) - // Set the first state's reactor as the dedicated byzantine reactor and grab - // the NodeID that corresponds to the state so we can reference the reactor. - bzNodeState := states[0] - for nID, s := range rts.states { - if s == bzNodeState { - bzNodeID = nID - break + blocksSub, err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, 100) + require.NoError(t, err) + blocksSubs = append(blocksSubs, blocksSub) + + if css[i].state.LastBlockHeight == 0 { // simulate handle initChain in handshake + err = css[i].blockExec.Store().Save(css[i].state) + require.NoError(t, err) } } + // make connected switches and start all reactors + p2p.MakeConnectedSwitches(config.P2P, nValidators, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + s.SetLogger(reactors[i].conS.Logger.With("module", "p2p")) + return s + }, p2p.Connect2Switches) - bzReactor := rts.reactors[bzNodeID] + // create byzantine validator + bcs := css[byzantineNode] // alter prevote so that the byzantine node double votes when height is 2 - bzNodeState.doPrevote = func(height int64, round int32) { + bcs.doPrevote = func(height int64, round int32) { // allow first height to happen normally so that byzantine validator is no longer proposer if height == prevoteHeight { - prevote1, err := bzNodeState.signVote( - tmproto.PrevoteType, - bzNodeState.ProposalBlock.Hash(), - bzNodeState.ProposalBlockParts.Header(), - ) + bcs.Logger.Info("Sending two votes") + prevote1, err := bcs.signVote(tmproto.PrevoteType, bcs.ProposalBlock.Hash(), bcs.ProposalBlockParts.Header()) require.NoError(t, err) - - prevote2, err := bzNodeState.signVote(tmproto.PrevoteType, nil, types.PartSetHeader{}) + prevote2, err := bcs.signVote(tmproto.PrevoteType, nil, types.PartSetHeader{}) require.NoError(t, err) - + peerList := reactors[byzantineNode].Switch.Peers().List() + bcs.Logger.Info("Getting peer list", "peers", peerList) // send two votes to all peers (1st to one half, 2nd to another half) - i := 0 - for _, ps := range bzReactor.peers { - if i < len(bzReactor.peers)/2 { - bzNodeState.Logger.Info("signed and pushed vote", "vote", prevote1, "peer", ps.peerID) - bzReactor.voteCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.Vote{ - Vote: prevote1.ToProto(), - }, - } + for i, peer := range peerList { + if i < len(peerList)/2 { + bcs.Logger.Info("Signed and pushed vote", "vote", prevote1, "peer", peer) + peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote1})) } else { - bzNodeState.Logger.Info("signed and pushed vote", "vote", prevote2, "peer", ps.peerID) - bzReactor.voteCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.Vote{ - Vote: prevote2.ToProto(), - }, - } + bcs.Logger.Info("Signed and pushed vote", "vote", prevote2, "peer", peer) + peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote2})) } - - i++ } } else { - bzNodeState.Logger.Info("behaving normally") - bzNodeState.defaultDoPrevote(height, round) + bcs.Logger.Info("Behaving normally") + bcs.defaultDoPrevote(height, round) } } - // Introducing a lazy proposer means that the time of the block committed is - // different to the timestamp that the other nodes have. This tests to ensure - // that the evidence that finally gets proposed will have a valid timestamp. - // lazyProposer := states[1] - lazyNodeState := states[1] + // introducing a lazy proposer means that the time of the block committed is different to the + // timestamp that the other nodes have. This tests to ensure that the evidence that finally gets + // proposed will have a valid timestamp + lazyProposer := css[1] - lazyNodeState.decideProposal = func(height int64, round int32) { - lazyNodeState.Logger.Info("Lazy Proposer proposing condensed commit") - require.NotNil(t, lazyNodeState.privValidator) + lazyProposer.decideProposal = func(height int64, round int32) { + lazyProposer.Logger.Info("Lazy Proposer proposing condensed commit") + if lazyProposer.privValidator == nil { + panic("entered createProposalBlock with privValidator being nil") + } var commit *types.Commit switch { - case lazyNodeState.Height == lazyNodeState.state.InitialHeight: + case lazyProposer.Height == lazyProposer.state.InitialHeight: // We're creating a proposal for the first block. // The commit is empty, but not nil. commit = types.NewCommit(0, 0, types.BlockID{}, nil) - case lazyNodeState.LastCommit.HasTwoThirdsMajority(): + case lazyProposer.LastCommit.HasTwoThirdsMajority(): // Make the commit from LastCommit - commit = lazyNodeState.LastCommit.MakeCommit() + commit = lazyProposer.LastCommit.MakeCommit() default: // This shouldn't happen. - lazyNodeState.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block") + lazyProposer.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block") return } // omit the last signature in the commit commit.Signatures[len(commit.Signatures)-1] = types.NewCommitSigAbsent() - if lazyNodeState.privValidatorPubKey == nil { + if lazyProposer.privValidatorPubKey == nil { // If this node is a validator & proposer in the current round, it will // miss the opportunity to create a block. - lazyNodeState.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) + lazyProposer.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) return } - proposerAddr := lazyNodeState.privValidatorPubKey.Address() + proposerAddr := lazyProposer.privValidatorPubKey.Address() - block, blockParts := lazyNodeState.blockExec.CreateProposalBlock( - lazyNodeState.Height, lazyNodeState.state, commit, proposerAddr, + block, blockParts := lazyProposer.blockExec.CreateProposalBlock( + lazyProposer.Height, lazyProposer.state, commit, proposerAddr, ) // Flush the WAL. Otherwise, we may not recompute the same proposal to sign, // and the privValidator will refuse to sign anything. - if err := lazyNodeState.wal.FlushAndSync(); err != nil { - lazyNodeState.Logger.Error("Error flushing to disk") + if err := lazyProposer.wal.FlushAndSync(); err != nil { + lazyProposer.Logger.Error("Error flushing to disk") } // Make proposal propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()} - proposal := types.NewProposal(height, round, lazyNodeState.ValidRound, propBlockID) + proposal := types.NewProposal(height, round, lazyProposer.ValidRound, propBlockID) p := proposal.ToProto() - if err := lazyNodeState.privValidator.SignProposal(context.Background(), lazyNodeState.state.ChainID, p); err == nil { + if err := lazyProposer.privValidator.SignProposal(lazyProposer.state.ChainID, p); err == nil { proposal.Signature = p.Signature // send proposal and block parts on internal msg queue - lazyNodeState.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) + lazyProposer.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) for i := 0; i < int(blockParts.Total()); i++ { part := blockParts.GetPart(i) - lazyNodeState.sendInternalMessage(msgInfo{&BlockPartMessage{lazyNodeState.Height, lazyNodeState.Round, part}, ""}) + lazyProposer.sendInternalMessage(msgInfo{&BlockPartMessage{lazyProposer.Height, lazyProposer.Round, part}, ""}) } - lazyNodeState.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal) - lazyNodeState.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block)) - } else if !lazyNodeState.replayMode { - lazyNodeState.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) + lazyProposer.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal) + lazyProposer.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block)) + } else if !lazyProposer.replayMode { + lazyProposer.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) } } - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) + // start the consensus reactors + for i := 0; i < nValidators; i++ { + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s, false) } + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) // Evidence should be submitted and committed at the third height but // we will check the first six just in case evidenceFromEachValidator := make([]types.Evidence, nValidators) wg := new(sync.WaitGroup) - i := 0 - for _, sub := range rts.subs { + for i := 0; i < nValidators; i++ { wg.Add(1) - - go func(j int, s types.Subscription) { + go func(i int) { defer wg.Done() - - for msg := range s.Out() { + for msg := range blocksSubs[i].Out() { block := msg.Data().(types.EventDataNewBlock).Block if len(block.Evidence.Evidence) != 0 { - evidenceFromEachValidator[j] = block.Evidence.Evidence[0] + evidenceFromEachValidator[i] = block.Evidence.Evidence[0] return } } - }(i, sub) - - i++ + }(i) } - wg.Wait() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() - pubkey, err := bzNodeState.privValidator.GetPubKey(context.Background()) + pubkey, err := bcs.privValidator.GetPubKey() require.NoError(t, err) - for idx, ev := range evidenceFromEachValidator { - if assert.NotNil(t, ev, idx) { - ev, ok := ev.(*types.DuplicateVoteEvidence) - assert.True(t, ok) - assert.Equal(t, pubkey.Address(), ev.VoteA.ValidatorAddress) - assert.Equal(t, prevoteHeight, ev.Height()) + select { + case <-done: + for idx, ev := range evidenceFromEachValidator { + if assert.NotNil(t, ev, idx) { + ev, ok := ev.(*types.DuplicateVoteEvidence) + assert.True(t, ok) + assert.Equal(t, pubkey.Address(), ev.VoteA.ValidatorAddress) + assert.Equal(t, prevoteHeight, ev.Height()) + } } + case <-time.After(20 * time.Second): + for i, reactor := range reactors { + t.Logf("Consensus Reactor %d\n%v", i, reactor) + } + t.Fatalf("Timed out waiting for validators to commit evidence") } } @@ -267,260 +278,259 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { // B sees a commit, A doesn't. // Heal partition and ensure A sees the commit func TestByzantineConflictingProposalsWithPartition(t *testing.T) { - // TODO: https://github.com/tendermint/tendermint/issues/6092 - t.SkipNow() + N := 4 + logger := consensusLogger().With("test", "byzantine") + app := newCounter + css, cleanup := randConsensusNet(N, "consensus_byzantine_test", newMockTickerFunc(false), app) + defer cleanup() - // n := 4 - // logger := consensusLogger().With("test", "byzantine") - // app := newCounter + // give the byzantine validator a normal ticker + ticker := NewTimeoutTicker() + ticker.SetLogger(css[0].Logger) + css[0].SetTimeoutTicker(ticker) - // states, cleanup := randConsensusState(n, "consensus_byzantine_test", newMockTickerFunc(false), app) - // t.Cleanup(cleanup) + p2pLogger := logger.With("module", "p2p") - // // give the byzantine validator a normal ticker - // ticker := NewTimeoutTicker() - // ticker.SetLogger(states[0].Logger) - // states[0].SetTimeoutTicker(ticker) + blocksSubs := make([]types.Subscription, N) + reactors := make([]p2p.Reactor, N) + for i := 0; i < N; i++ { - // p2pLogger := logger.With("module", "p2p") + // enable txs so we can create different proposals + assertMempool(css[i].txNotifier).EnableTxsAvailable() - // blocksSubs := make([]types.Subscription, n) - // reactors := make([]p2p.Reactor, n) - // for i := 0; i < n; i++ { - // // enable txs so we can create different proposals - // assertMempool(states[i].txNotifier).EnableTxsAvailable() + eventBus := css[i].eventBus + eventBus.SetLogger(logger.With("module", "events", "validator", i)) - // eventBus := states[i].eventBus - // eventBus.SetLogger(logger.With("module", "events", "validator", i)) + var err error + blocksSubs[i], err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) + require.NoError(t, err) - // var err error - // blocksSubs[i], err = eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) - // require.NoError(t, err) + conR := NewReactor(css[i], true) // so we don't start the consensus states + conR.SetLogger(logger.With("validator", i)) + conR.SetEventBus(eventBus) - // conR := NewReactor(states[i], true) // so we don't start the consensus states - // conR.SetLogger(logger.With("validator", i)) - // conR.SetEventBus(eventBus) + var conRI p2p.Reactor = conR - // var conRI p2p.Reactor = conR + // make first val byzantine + if i == 0 { + conRI = NewByzantineReactor(conR) + } - // // make first val byzantine - // if i == 0 { - // conRI = NewByzantineReactor(conR) - // } + reactors[i] = conRI + err = css[i].blockExec.Store().Save(css[i].state) // for save height 1's validators info + require.NoError(t, err) + } - // reactors[i] = conRI - // err = states[i].blockExec.Store().Save(states[i].state) // for save height 1's validators info - // require.NoError(t, err) - // } + switches := p2p.MakeConnectedSwitches(config.P2P, N, func(i int, sw *p2p.Switch) *p2p.Switch { + sw.SetLogger(p2pLogger.With("validator", i)) + sw.AddReactor("CONSENSUS", reactors[i]) + return sw + }, func(sws []*p2p.Switch, i, j int) { + // the network starts partitioned with globally active adversary + if i != 0 { + return + } + p2p.Connect2Switches(sws, i, j) + }) - // switches := p2p.MakeConnectedSwitches(config.P2P, N, func(i int, sw *p2p.Switch) *p2p.Switch { - // sw.SetLogger(p2pLogger.With("validator", i)) - // sw.AddReactor("CONSENSUS", reactors[i]) - // return sw - // }, func(sws []*p2p.Switch, i, j int) { - // // the network starts partitioned with globally active adversary - // if i != 0 { - // return - // } - // p2p.Connect2Switches(sws, i, j) - // }) + // make first val byzantine + // NOTE: Now, test validators are MockPV, which by default doesn't + // do any safety checks. + css[0].privValidator.(types.MockPV).DisableChecks() + css[0].decideProposal = func(j int32) func(int64, int32) { + return func(height int64, round int32) { + byzantineDecideProposalFunc(t, height, round, css[j], switches[j]) + } + }(int32(0)) + // We are setting the prevote function to do nothing because the prevoting + // and precommitting are done alongside the proposal. + css[0].doPrevote = func(height int64, round int32) {} - // // make first val byzantine - // // NOTE: Now, test validators are MockPV, which by default doesn't - // // do any safety checks. - // states[0].privValidator.(types.MockPV).DisableChecks() - // states[0].decideProposal = func(j int32) func(int64, int32) { - // return func(height int64, round int32) { - // byzantineDecideProposalFunc(t, height, round, states[j], switches[j]) - // } - // }(int32(0)) - // // We are setting the prevote function to do nothing because the prevoting - // // and precommitting are done alongside the proposal. - // states[0].doPrevote = func(height int64, round int32) {} + defer func() { + for _, sw := range switches { + err := sw.Stop() + require.NoError(t, err) + } + }() - // defer func() { - // for _, sw := range switches { - // err := sw.Stop() - // require.NoError(t, err) - // } - // }() + // start the non-byz state machines. + // note these must be started before the byz + for i := 1; i < N; i++ { + cr := reactors[i].(*Reactor) + cr.SwitchToConsensus(cr.conS.GetState(), false) + } - // // start the non-byz state machines. - // // note these must be started before the byz - // for i := 1; i < n; i++ { - // cr := reactors[i].(*Reactor) - // cr.SwitchToConsensus(cr.conS.GetState(), false) - // } + // start the byzantine state machine + byzR := reactors[0].(*ByzantineReactor) + s := byzR.reactor.conS.GetState() + byzR.reactor.SwitchToConsensus(s, false) - // // start the byzantine state machine - // byzR := reactors[0].(*ByzantineReactor) - // s := byzR.reactor.conS.GetState() - // byzR.reactor.SwitchToConsensus(s, false) + // byz proposer sends one block to peers[0] + // and the other block to peers[1] and peers[2]. + // note peers and switches order don't match. + peers := switches[0].Peers().List() - // // byz proposer sends one block to peers[0] - // // and the other block to peers[1] and peers[2]. - // // note peers and switches order don't match. - // peers := switches[0].Peers().List() + // partition A + ind0 := getSwitchIndex(switches, peers[0]) - // // partition A - // ind0 := getSwitchIndex(switches, peers[0]) + // partition B + ind1 := getSwitchIndex(switches, peers[1]) + ind2 := getSwitchIndex(switches, peers[2]) + p2p.Connect2Switches(switches, ind1, ind2) - // // partition B - // ind1 := getSwitchIndex(switches, peers[1]) - // ind2 := getSwitchIndex(switches, peers[2]) - // p2p.Connect2Switches(switches, ind1, ind2) + // wait for someone in the big partition (B) to make a block + <-blocksSubs[ind2].Out() - // // wait for someone in the big partition (B) to make a block - // <-blocksSubs[ind2].Out() + t.Log("A block has been committed. Healing partition") + p2p.Connect2Switches(switches, ind0, ind1) + p2p.Connect2Switches(switches, ind0, ind2) - // t.Log("A block has been committed. Healing partition") - // p2p.Connect2Switches(switches, ind0, ind1) - // p2p.Connect2Switches(switches, ind0, ind2) + // wait till everyone makes the first new block + // (one of them already has) + wg := new(sync.WaitGroup) + for i := 1; i < N-1; i++ { + wg.Add(1) + go func(j int) { + <-blocksSubs[j].Out() + wg.Done() + }(i) + } - // // wait till everyone makes the first new block - // // (one of them already has) - // wg := new(sync.WaitGroup) - // for i := 1; i < N-1; i++ { - // wg.Add(1) - // go func(j int) { - // <-blocksSubs[j].Out() - // wg.Done() - // }(i) - // } + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() - // done := make(chan struct{}) - // go func() { - // wg.Wait() - // close(done) - // }() - - // tick := time.NewTicker(time.Second * 10) - // select { - // case <-done: - // case <-tick.C: - // for i, reactor := range reactors { - // t.Log(fmt.Sprintf("Consensus Reactor %v", i)) - // t.Log(fmt.Sprintf("%v", reactor)) - // } - // t.Fatalf("Timed out waiting for all validators to commit first block") - // } + tick := time.NewTicker(time.Second * 10) + select { + case <-done: + case <-tick.C: + for i, reactor := range reactors { + t.Log(fmt.Sprintf("Consensus Reactor %v", i)) + t.Log(fmt.Sprintf("%v", reactor)) + } + t.Fatalf("Timed out waiting for all validators to commit first block") + } } -// func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch) { -// // byzantine user should create two proposals and try to split the vote. -// // Avoid sending on internalMsgQueue and running consensus state. +//------------------------------- +// byzantine consensus functions -// // Create a new proposal block from state/txs from the mempool. -// block1, blockParts1 := cs.createProposalBlock() -// polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()} -// proposal1 := types.NewProposal(height, round, polRound, propBlockID) -// p1 := proposal1.ToProto() -// if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil { -// t.Error(err) -// } +func byzantineDecideProposalFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch) { + // byzantine user should create two proposals and try to split the vote. + // Avoid sending on internalMsgQueue and running consensus state. -// proposal1.Signature = p1.Signature + // Create a new proposal block from state/txs from the mempool. + block1, blockParts1 := cs.createProposalBlock() + polRound, propBlockID := cs.ValidRound, types.BlockID{Hash: block1.Hash(), PartSetHeader: blockParts1.Header()} + proposal1 := types.NewProposal(height, round, polRound, propBlockID) + p1 := proposal1.ToProto() + if err := cs.privValidator.SignProposal(cs.state.ChainID, p1); err != nil { + t.Error(err) + } -// // some new transactions come in (this ensures that the proposals are different) -// deliverTxsRange(cs, 0, 1) + proposal1.Signature = p1.Signature -// // Create a new proposal block from state/txs from the mempool. -// block2, blockParts2 := cs.createProposalBlock() -// polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()} -// proposal2 := types.NewProposal(height, round, polRound, propBlockID) -// p2 := proposal2.ToProto() -// if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil { -// t.Error(err) -// } + // some new transactions come in (this ensures that the proposals are different) + deliverTxsRange(cs, 0, 1) -// proposal2.Signature = p2.Signature + // Create a new proposal block from state/txs from the mempool. + block2, blockParts2 := cs.createProposalBlock() + polRound, propBlockID = cs.ValidRound, types.BlockID{Hash: block2.Hash(), PartSetHeader: blockParts2.Header()} + proposal2 := types.NewProposal(height, round, polRound, propBlockID) + p2 := proposal2.ToProto() + if err := cs.privValidator.SignProposal(cs.state.ChainID, p2); err != nil { + t.Error(err) + } -// block1Hash := block1.Hash() -// block2Hash := block2.Hash() + proposal2.Signature = p2.Signature -// // broadcast conflicting proposals/block parts to peers -// peers := sw.Peers().List() -// t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers)) -// for i, peer := range peers { -// if i < len(peers)/2 { -// go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1) -// } else { -// go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2) -// } -// } -// } + block1Hash := block1.Hash() + block2Hash := block2.Hash() -// func sendProposalAndParts( -// height int64, -// round int32, -// cs *State, -// peer p2p.Peer, -// proposal *types.Proposal, -// blockHash []byte, -// parts *types.PartSet, -// ) { -// // proposal -// msg := &ProposalMessage{Proposal: proposal} -// peer.Send(DataChannel, MustEncode(msg)) + // broadcast conflicting proposals/block parts to peers + peers := sw.Peers().List() + t.Logf("Byzantine: broadcasting conflicting proposals to %d peers", len(peers)) + for i, peer := range peers { + if i < len(peers)/2 { + go sendProposalAndParts(height, round, cs, peer, proposal1, block1Hash, blockParts1) + } else { + go sendProposalAndParts(height, round, cs, peer, proposal2, block2Hash, blockParts2) + } + } +} -// // parts -// for i := 0; i < int(parts.Total()); i++ { -// part := parts.GetPart(i) -// msg := &BlockPartMessage{ -// Height: height, // This tells peer that this part applies to us. -// Round: round, // This tells peer that this part applies to us. -// Part: part, -// } -// peer.Send(DataChannel, MustEncode(msg)) -// } +func sendProposalAndParts( + height int64, + round int32, + cs *State, + peer p2p.Peer, + proposal *types.Proposal, + blockHash []byte, + parts *types.PartSet, +) { + // proposal + msg := &ProposalMessage{Proposal: proposal} + peer.Send(DataChannel, MustEncode(msg)) -// // votes -// cs.mtx.Lock() -// prevote, _ := cs.signVote(tmproto.PrevoteType, blockHash, parts.Header()) -// precommit, _ := cs.signVote(tmproto.PrecommitType, blockHash, parts.Header()) -// cs.mtx.Unlock() + // parts + for i := 0; i < int(parts.Total()); i++ { + part := parts.GetPart(i) + msg := &BlockPartMessage{ + Height: height, // This tells peer that this part applies to us. + Round: round, // This tells peer that this part applies to us. + Part: part, + } + peer.Send(DataChannel, MustEncode(msg)) + } -// peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote})) -// peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit})) -// } + // votes + cs.mtx.Lock() + prevote, _ := cs.signVote(tmproto.PrevoteType, blockHash, parts.Header()) + precommit, _ := cs.signVote(tmproto.PrecommitType, blockHash, parts.Header()) + cs.mtx.Unlock() -// type ByzantineReactor struct { -// service.Service -// reactor *Reactor -// } + peer.Send(VoteChannel, MustEncode(&VoteMessage{prevote})) + peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit})) +} -// func NewByzantineReactor(conR *Reactor) *ByzantineReactor { -// return &ByzantineReactor{ -// Service: conR, -// reactor: conR, -// } -// } +//---------------------------------------- +// byzantine consensus reactor -// func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } -// func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } +type ByzantineReactor struct { + service.Service + reactor *Reactor +} -// func (br *ByzantineReactor) AddPeer(peer p2p.Peer) { -// if !br.reactor.IsRunning() { -// return -// } +func NewByzantineReactor(conR *Reactor) *ByzantineReactor { + return &ByzantineReactor{ + Service: conR, + reactor: conR, + } +} -// // Create peerState for peer -// peerState := NewPeerState(peer).SetLogger(br.reactor.Logger) -// peer.Set(types.PeerStateKey, peerState) +func (br *ByzantineReactor) SetSwitch(s *p2p.Switch) { br.reactor.SetSwitch(s) } +func (br *ByzantineReactor) GetChannels() []*p2p.ChannelDescriptor { return br.reactor.GetChannels() } +func (br *ByzantineReactor) AddPeer(peer p2p.Peer) { + if !br.reactor.IsRunning() { + return + } -// // Send our state to peer. -// // If we're syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). -// if !br.reactor.waitSync { -// br.reactor.sendNewRoundStepMessage(peer) -// } -// } + // Create peerState for peer + peerState := NewPeerState(peer).SetLogger(br.reactor.Logger) + peer.Set(types.PeerStateKey, peerState) -// func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) { -// br.reactor.RemovePeer(peer, reason) -// } - -// func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { -// br.reactor.Receive(chID, peer, msgBytes) -// } - -// func (br *ByzantineReactor) InitPeer(peer p2p.Peer) p2p.Peer { return peer } + // Send our state to peer. + // If we're syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). + if !br.reactor.waitSync { + br.reactor.sendNewRoundStepMessage(peer) + } +} +func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) { + br.reactor.RemovePeer(peer, reason) +} +func (br *ByzantineReactor) Receive(chID byte, peer p2p.Peer, msgBytes []byte) { + br.reactor.Receive(chID, peer, msgBytes) +} +func (br *ByzantineReactor) InitPeer(peer p2p.Peer) p2p.Peer { return peer } diff --git a/consensus/common_test.go b/consensus/common_test.go index bd844982c..8502883e7 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -31,6 +31,7 @@ import ( tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmsync "github.com/tendermint/tendermint/libs/sync" mempl "github.com/tendermint/tendermint/mempool" + "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" @@ -54,24 +55,6 @@ var ( ensureTimeout = time.Millisecond * 200 ) -func configSetup(t *testing.T) { - t.Helper() - - config = ResetConfig("consensus_reactor_test") - consensusReplayConfig = ResetConfig("consensus_replay_test") - configStateTest := ResetConfig("consensus_state_test") - configMempoolTest := ResetConfig("consensus_mempool_test") - configByzantineTest := ResetConfig("consensus_byzantine_test") - - t.Cleanup(func() { - os.RemoveAll(config.RootDir) - os.RemoveAll(consensusReplayConfig.RootDir) - os.RemoveAll(configStateTest.RootDir) - os.RemoveAll(configMempoolTest.RootDir) - os.RemoveAll(configByzantineTest.RootDir) - }) -} - func ensureDir(dir string, mode os.FileMode) { if err := tmos.EnsureDir(dir, mode); err != nil { panic(err) @@ -108,7 +91,7 @@ func (vs *validatorStub) signVote( hash []byte, header types.PartSetHeader) (*types.Vote, error) { - pubKey, err := vs.PrivValidator.GetPubKey(context.Background()) + pubKey, err := vs.PrivValidator.GetPubKey() if err != nil { return nil, fmt.Errorf("can't get pubkey: %w", err) } @@ -123,7 +106,7 @@ func (vs *validatorStub) signVote( BlockID: types.BlockID{Hash: hash, PartSetHeader: header}, } v := vote.ToProto() - err = vs.PrivValidator.SignVote(context.Background(), config.ChainID(), v) + err = vs.PrivValidator.SignVote(config.ChainID(), v) vote.Signature = v.Signature return vote, err @@ -169,11 +152,11 @@ func (vss ValidatorStubsByPower) Len() int { } func (vss ValidatorStubsByPower) Less(i, j int) bool { - vssi, err := vss[i].GetPubKey(context.Background()) + vssi, err := vss[i].GetPubKey() if err != nil { panic(err) } - vssj, err := vss[j].GetPubKey(context.Background()) + vssj, err := vss[j].GetPubKey() if err != nil { panic(err) } @@ -220,7 +203,7 @@ func decideProposal( polRound, propBlockID := validRound, types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()} proposal = types.NewProposal(height, round, polRound, propBlockID) p := proposal.ToProto() - if err := vs.SignProposal(context.Background(), chainID, p); err != nil { + if err := vs.SignProposal(chainID, p); err != nil { panic(err) } @@ -248,7 +231,7 @@ func signAddVotes( func validatePrevote(t *testing.T, cs *State, round int32, privVal *validatorStub, blockHash []byte) { prevotes := cs.Votes.Prevotes(round) - pubKey, err := privVal.GetPubKey(context.Background()) + pubKey, err := privVal.GetPubKey() require.NoError(t, err) address := pubKey.Address() var vote *types.Vote @@ -268,7 +251,7 @@ func validatePrevote(t *testing.T, cs *State, round int32, privVal *validatorStu func validateLastPrecommit(t *testing.T, cs *State, privVal *validatorStub, blockHash []byte) { votes := cs.LastCommit - pv, err := privVal.GetPubKey(context.Background()) + pv, err := privVal.GetPubKey() require.NoError(t, err) address := pv.Address() var vote *types.Vote @@ -290,7 +273,7 @@ func validatePrecommit( lockedBlockHash []byte, ) { precommits := cs.Votes.Precommits(thisRound) - pv, err := privVal.GetPubKey(context.Background()) + pv, err := privVal.GetPubKey() require.NoError(t, err) address := pv.Address() var vote *types.Vote @@ -692,18 +675,11 @@ func consensusLogger() log.Logger { }).With("module", "consensus") } -func randConsensusState( - nValidators int, - testName string, - tickerFunc func() TimeoutTicker, - appFunc func() abci.Application, - configOpts ...func(*cfg.Config), -) ([]*State, cleanupFunc) { - +func randConsensusNet(nValidators int, testName string, tickerFunc func() TimeoutTicker, + appFunc func() abci.Application, configOpts ...func(*cfg.Config)) ([]*State, cleanupFunc) { genDoc, privVals := randGenesisDoc(nValidators, false, 30) css := make([]*State, nValidators) logger := consensusLogger() - configRootDirs := make([]string, 0, nValidators) for i := 0; i < nValidators; i++ { stateDB := dbm.NewMemDB() // each state needs its own db @@ -711,13 +687,10 @@ func randConsensusState( state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) configRootDirs = append(configRootDirs, thisConfig.RootDir) - for _, opt := range configOpts { opt(thisConfig) } - ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal - app := appFunc() vals := types.TM2PB.ValidatorUpdates(state.Validators) app.InitChain(abci.RequestInitChain{Validators: vals}) @@ -726,7 +699,6 @@ func randConsensusState( css[i].SetTimeoutTicker(tickerFunc()) css[i].SetLogger(logger.With("validator", i, "module", "consensus")) } - return css, func() { for _, dir := range configRootDirs { os.RemoveAll(dir) @@ -796,6 +768,18 @@ func randConsensusNetWithPeers( } } +func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { + for i, s := range switches { + if peer.NodeInfo().ID() == s.NodeInfo().ID() { + return i + } + } + panic("didnt find peer in switches") +} + +//------------------------------------------------------------------------------- +// genesis + func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) { validators := make([]types.GenesisValidator, numValidators) privValidators := make([]types.PrivValidator, numValidators) @@ -823,6 +807,9 @@ func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.Sta return s0, privValidators } +//------------------------------------ +// mock ticker + func newMockTickerFunc(onlyOnce bool) func() TimeoutTicker { return func() TimeoutTicker { return &mockTicker{ @@ -868,6 +855,8 @@ func (m *mockTicker) Chan() <-chan timeoutInfo { func (*mockTicker) SetLogger(log.Logger) {} +//------------------------------------ + func newCounter() abci.Application { return counter.NewApplication(true) } diff --git a/consensus/invalid_test.go b/consensus/invalid_test.go index 31a4db4f5..5b161dd71 100644 --- a/consensus/invalid_test.go +++ b/consensus/invalid_test.go @@ -1,73 +1,60 @@ package consensus import ( - "context" - "sync" "testing" - "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/bytes" + "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/p2p" - tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/types" ) -func TestReactorInvalidPrecommit(t *testing.T) { - configSetup(t) +//---------------------------------------------- +// byzantine failures - n := 4 - states, cleanup := randConsensusState(n, "consensus_reactor_test", newMockTickerFunc(true), newCounter) +// one byz val sends a precommit for a random block at each height +// Ensure a testnet makes blocks +func TestReactorInvalidPrecommit(t *testing.T) { + N := 4 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) t.Cleanup(cleanup) for i := 0; i < 4; i++ { ticker := NewTimeoutTicker() - ticker.SetLogger(states[i].Logger) - states[i].SetTimeoutTicker(ticker) + ticker.SetLogger(css[i].Logger) + css[i].SetTimeoutTicker(ticker) + } - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N) // this val sends a random precommit at each height - node := rts.network.RandomNode() + byzValIdx := 0 + byzVal := css[byzValIdx] + byzR := reactors[byzValIdx] - byzState := rts.states[node.NodeID] - byzReactor := rts.reactors[node.NodeID] - - // Update the doPrevote function to just send a valid precommit for a random - // block and otherwise disable the priv validator. - byzState.mtx.Lock() - privVal := byzState.privValidator - byzState.doPrevote = func(height int64, round int32) { - invalidDoPrevoteFunc(t, height, round, byzState, byzReactor, privVal) + // update the doPrevote function to just send a valid precommit for a random block + // and otherwise disable the priv validator + byzVal.mtx.Lock() + pv := byzVal.privValidator + byzVal.doPrevote = func(height int64, round int32) { + invalidDoPrevoteFunc(t, height, round, byzVal, byzR.Switch, pv) } - byzState.mtx.Unlock() + byzVal.mtx.Unlock() + t.Cleanup(func() { stopConsensusNet(log.TestingLogger(), reactors, eventBuses) }) // wait for a bunch of blocks - // - // TODO: Make this tighter by ensuring the halt happens by block 2. - var wg sync.WaitGroup + // TODO: make this tighter by ensuring the halt happens by block 2 for i := 0; i < 10; i++ { - for _, sub := range rts.subs { - wg.Add(1) - - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) - } + timeoutWaitGroup(t, N, func(j int) { + <-blocksSubs[j].Out() + }, css) } - - wg.Wait() } -func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, r *Reactor, pv types.PrivValidator) { +func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, sw *p2p.Switch, pv types.PrivValidator) { // routine to: // - precommit for a random block // - send precommit to all peers @@ -75,10 +62,10 @@ func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, r go func() { cs.mtx.Lock() cs.privValidator = pv - - pubKey, err := cs.privValidator.GetPubKey(context.Background()) - require.NoError(t, err) - + pubKey, err := cs.privValidator.GetPubKey() + if err != nil { + panic(err) + } addr := pubKey.Address() valIndex, _ := cs.Validators.GetByAddress(addr) @@ -95,24 +82,19 @@ func invalidDoPrevoteFunc(t *testing.T, height int64, round int32, cs *State, r Hash: blockHash, PartSetHeader: types.PartSetHeader{Total: 1, Hash: tmrand.Bytes(32)}}, } - p := precommit.ToProto() - err = cs.privValidator.SignVote(context.Background(), cs.state.ChainID, p) - require.NoError(t, err) - + err = cs.privValidator.SignVote(cs.state.ChainID, p) + if err != nil { + t.Error(err) + } precommit.Signature = p.Signature cs.privValidator = nil // disable priv val so we don't do normal votes cs.mtx.Unlock() - for _, ps := range r.peers { - cs.Logger.Info("sending bad vote", "block", blockHash, "peer", ps.peerID) - - r.voteCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.Vote{ - Vote: precommit.ToProto(), - }, - } + peers := sw.Peers().List() + for _, peer := range peers { + cs.Logger.Info("Sending bad vote", "block", blockHash, "peer", peer) + peer.Send(VoteChannel, MustEncode(&VoteMessage{precommit})) } }() } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 85005f9eb..c53185048 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -25,8 +25,6 @@ func assertMempool(txn txNotifier) mempl.Mempool { } func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) { - configSetup(t) - config := ResetConfig("consensus_mempool_txs_available_test") t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) }) @@ -47,8 +45,6 @@ func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) { } func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) { - configSetup(t) - config := ResetConfig("consensus_mempool_txs_available_test") t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) }) @@ -67,8 +63,6 @@ func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) { } func TestMempoolProgressInHigherRound(t *testing.T) { - configSetup(t) - config := ResetConfig("consensus_mempool_txs_available_test") t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) }) @@ -119,8 +113,6 @@ func deliverTxsRange(cs *State, start, end int) { } func TestMempoolTxConcurrentWithCommit(t *testing.T) { - configSetup(t) - state, privVals := randGenesisState(1, false, 10) blockDB := dbm.NewMemDB() stateStore := sm.NewStore(blockDB) @@ -145,8 +137,6 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) { } func TestMempoolRmBadTx(t *testing.T) { - configSetup(t) - state, privVals := randGenesisState(1, false, 10) app := NewCounterApplication() blockDB := dbm.NewMemDB() diff --git a/consensus/msgs.go b/consensus/msgs.go index 60f869123..65bf81aa0 100644 --- a/consensus/msgs.go +++ b/consensus/msgs.go @@ -4,9 +4,10 @@ import ( "errors" "fmt" + "github.com/gogo/protobuf/proto" + cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/libs/bits" - tmjson "github.com/tendermint/tendermint/libs/json" tmmath "github.com/tendermint/tendermint/libs/math" "github.com/tendermint/tendermint/p2p" tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" @@ -14,309 +15,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// Message defines an interface that the consensus domain types implement. When -// a proto message is received on a consensus p2p Channel, it is wrapped and then -// converted to a Message via MsgFromProto. -type Message interface { - ValidateBasic() error -} - -func init() { - tmjson.RegisterType(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage") - tmjson.RegisterType(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage") - tmjson.RegisterType(&ProposalMessage{}, "tendermint/Proposal") - tmjson.RegisterType(&ProposalPOLMessage{}, "tendermint/ProposalPOL") - tmjson.RegisterType(&BlockPartMessage{}, "tendermint/BlockPart") - tmjson.RegisterType(&VoteMessage{}, "tendermint/Vote") - tmjson.RegisterType(&HasVoteMessage{}, "tendermint/HasVote") - tmjson.RegisterType(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23") - tmjson.RegisterType(&VoteSetBitsMessage{}, "tendermint/VoteSetBits") -} - -// NewRoundStepMessage is sent for every step taken in the ConsensusState. -// For every height/round/step transition -type NewRoundStepMessage struct { - Height int64 - Round int32 - Step cstypes.RoundStepType - SecondsSinceStartTime int64 - LastCommitRound int32 -} - -// ValidateBasic performs basic validation. -func (m *NewRoundStepMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.Round < 0 { - return errors.New("negative Round") - } - if !m.Step.IsValid() { - return errors.New("invalid Step") - } - - // NOTE: SecondsSinceStartTime may be negative - - // LastCommitRound will be -1 for the initial height, but we don't know what height this is - // since it can be specified in genesis. The reactor will have to validate this via - // ValidateHeight(). - if m.LastCommitRound < -1 { - return errors.New("invalid LastCommitRound (cannot be < -1)") - } - - return nil -} - -// ValidateHeight validates the height given the chain's initial height. -func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error { - if m.Height < initialHeight { - return fmt.Errorf("invalid Height %v (lower than initial height %v)", - m.Height, initialHeight) - } - if m.Height == initialHeight && m.LastCommitRound != -1 { - return fmt.Errorf("invalid LastCommitRound %v (must be -1 for initial height %v)", - m.LastCommitRound, initialHeight) - } - if m.Height > initialHeight && m.LastCommitRound < 0 { - return fmt.Errorf("LastCommitRound can only be negative for initial height %v", // nolint - initialHeight) - } - return nil -} - -// String returns a string representation. -func (m *NewRoundStepMessage) String() string { - return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]", - m.Height, m.Round, m.Step, m.LastCommitRound) -} - -// NewValidBlockMessage is sent when a validator observes a valid block B in some round r, -// i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. -// In case the block is also committed, then IsCommit flag is set to true. -type NewValidBlockMessage struct { - Height int64 - Round int32 - BlockPartSetHeader types.PartSetHeader - BlockParts *bits.BitArray - IsCommit bool -} - -// ValidateBasic performs basic validation. -func (m *NewValidBlockMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.Round < 0 { - return errors.New("negative Round") - } - if err := m.BlockPartSetHeader.ValidateBasic(); err != nil { - return fmt.Errorf("wrong BlockPartSetHeader: %v", err) - } - if m.BlockParts.Size() == 0 { - return errors.New("empty blockParts") - } - if m.BlockParts.Size() != int(m.BlockPartSetHeader.Total) { - return fmt.Errorf("blockParts bit array size %d not equal to BlockPartSetHeader.Total %d", - m.BlockParts.Size(), - m.BlockPartSetHeader.Total) - } - if m.BlockParts.Size() > int(types.MaxBlockPartsCount) { - return fmt.Errorf("blockParts bit array is too big: %d, max: %d", m.BlockParts.Size(), types.MaxBlockPartsCount) - } - return nil -} - -// String returns a string representation. -func (m *NewValidBlockMessage) String() string { - return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]", - m.Height, m.Round, m.BlockPartSetHeader, m.BlockParts, m.IsCommit) -} - -// ProposalMessage is sent when a new block is proposed. -type ProposalMessage struct { - Proposal *types.Proposal -} - -// ValidateBasic performs basic validation. -func (m *ProposalMessage) ValidateBasic() error { - return m.Proposal.ValidateBasic() -} - -// String returns a string representation. -func (m *ProposalMessage) String() string { - return fmt.Sprintf("[Proposal %v]", m.Proposal) -} - -// ProposalPOLMessage is sent when a previous proposal is re-proposed. -type ProposalPOLMessage struct { - Height int64 - ProposalPOLRound int32 - ProposalPOL *bits.BitArray -} - -// ValidateBasic performs basic validation. -func (m *ProposalPOLMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.ProposalPOLRound < 0 { - return errors.New("negative ProposalPOLRound") - } - if m.ProposalPOL.Size() == 0 { - return errors.New("empty ProposalPOL bit array") - } - if m.ProposalPOL.Size() > types.MaxVotesCount { - return fmt.Errorf("proposalPOL bit array is too big: %d, max: %d", m.ProposalPOL.Size(), types.MaxVotesCount) - } - return nil -} - -// String returns a string representation. -func (m *ProposalPOLMessage) String() string { - return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL) -} - -// BlockPartMessage is sent when gossipping a piece of the proposed block. -type BlockPartMessage struct { - Height int64 - Round int32 - Part *types.Part -} - -// ValidateBasic performs basic validation. -func (m *BlockPartMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.Round < 0 { - return errors.New("negative Round") - } - if err := m.Part.ValidateBasic(); err != nil { - return fmt.Errorf("wrong Part: %v", err) - } - return nil -} - -// String returns a string representation. -func (m *BlockPartMessage) String() string { - return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part) -} - -// VoteMessage is sent when voting for a proposal (or lack thereof). -type VoteMessage struct { - Vote *types.Vote -} - -// ValidateBasic performs basic validation. -func (m *VoteMessage) ValidateBasic() error { - return m.Vote.ValidateBasic() -} - -// String returns a string representation. -func (m *VoteMessage) String() string { - return fmt.Sprintf("[Vote %v]", m.Vote) -} - -// HasVoteMessage is sent to indicate that a particular vote has been received. -type HasVoteMessage struct { - Height int64 - Round int32 - Type tmproto.SignedMsgType - Index int32 -} - -// ValidateBasic performs basic validation. -func (m *HasVoteMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.Round < 0 { - return errors.New("negative Round") - } - if !types.IsVoteTypeValid(m.Type) { - return errors.New("invalid Type") - } - if m.Index < 0 { - return errors.New("negative Index") - } - return nil -} - -// String returns a string representation. -func (m *HasVoteMessage) String() string { - return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type) -} - -// VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes. -type VoteSetMaj23Message struct { - Height int64 - Round int32 - Type tmproto.SignedMsgType - BlockID types.BlockID -} - -// ValidateBasic performs basic validation. -func (m *VoteSetMaj23Message) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if m.Round < 0 { - return errors.New("negative Round") - } - if !types.IsVoteTypeValid(m.Type) { - return errors.New("invalid Type") - } - if err := m.BlockID.ValidateBasic(); err != nil { - return fmt.Errorf("wrong BlockID: %v", err) - } - - return nil -} - -// String returns a string representation. -func (m *VoteSetMaj23Message) String() string { - return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) -} - -// VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the -// BlockID. -type VoteSetBitsMessage struct { - Height int64 - Round int32 - Type tmproto.SignedMsgType - BlockID types.BlockID - Votes *bits.BitArray -} - -// ValidateBasic performs basic validation. -func (m *VoteSetBitsMessage) ValidateBasic() error { - if m.Height < 0 { - return errors.New("negative Height") - } - if !types.IsVoteTypeValid(m.Type) { - return errors.New("invalid Type") - } - if err := m.BlockID.ValidateBasic(); err != nil { - return fmt.Errorf("wrong BlockID: %v", err) - } - - // NOTE: Votes.Size() can be zero if the node does not have any - if m.Votes.Size() > types.MaxVotesCount { - return fmt.Errorf("votes bit array is too big: %d, max: %d", m.Votes.Size(), types.MaxVotesCount) - } - - return nil -} - -// String returns a string representation. -func (m *VoteSetBitsMessage) String() string { - return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) -} - -// MsgToProto takes a consensus message type and returns the proto defined -// consensus message. -// -// TODO: This needs to be removed, but WALToProto depends on this. +// MsgToProto takes a consensus message type and returns the proto defined consensus message func MsgToProto(msg Message) (*tmcons.Message, error) { if msg == nil { return nil, errors.New("consensus: message is nil") @@ -444,7 +143,7 @@ func MsgToProto(msg Message) (*tmcons.Message, error) { return &pb, nil } -// MsgFromProto takes a consensus proto message and returns the native go type. +// MsgFromProto takes a consensus proto message and returns the native go type func MsgFromProto(msg *tmcons.Message) (Message, error) { if msg == nil { return nil, errors.New("consensus: nil message") @@ -570,7 +269,21 @@ func MsgFromProto(msg *tmcons.Message) (Message, error) { return pb, nil } -// WALToProto takes a WAL message and return a proto walMessage and error. +// MustEncode takes the reactors msg, makes it proto and marshals it +// this mimics `MustMarshalBinaryBare` in that is panics on error +func MustEncode(msg Message) []byte { + pb, err := MsgToProto(msg) + if err != nil { + panic(err) + } + enc, err := proto.Marshal(pb) + if err != nil { + panic(err) + } + return enc +} + +// WALToProto takes a WAL message and return a proto walMessage and error func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) { var pb tmcons.WALMessage @@ -598,7 +311,6 @@ func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) { }, }, } - case timeoutInfo: pb = tmcons.WALMessage{ Sum: &tmcons.WALMessage_TimeoutInfo{ @@ -610,7 +322,6 @@ func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) { }, }, } - case EndHeightMessage: pb = tmcons.WALMessage{ Sum: &tmcons.WALMessage_EndHeight{ @@ -619,7 +330,6 @@ func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) { }, }, } - default: return nil, fmt.Errorf("to proto: wal message not recognized: %T", msg) } @@ -627,13 +337,11 @@ func WALToProto(msg WALMessage) (*tmcons.WALMessage, error) { return &pb, nil } -// WALFromProto takes a proto wal message and return a consensus walMessage and -// error. +// WALFromProto takes a proto wal message and return a consensus walMessage and error func WALFromProto(msg *tmcons.WALMessage) (WALMessage, error) { if msg == nil { return nil, errors.New("nil WAL message") } - var pb WALMessage switch msg := msg.Sum.(type) { @@ -643,7 +351,6 @@ func WALFromProto(msg *tmcons.WALMessage) (WALMessage, error) { Round: msg.EventDataRoundState.Round, Step: msg.EventDataRoundState.Step, } - case *tmcons.WALMessage_MsgInfo: walMsg, err := MsgFromProto(&msg.MsgInfo.Msg) if err != nil { @@ -660,26 +367,20 @@ func WALFromProto(msg *tmcons.WALMessage) (WALMessage, error) { if err != nil { return nil, fmt.Errorf("denying message due to possible overflow: %w", err) } - pb = timeoutInfo{ Duration: msg.TimeoutInfo.Duration, Height: msg.TimeoutInfo.Height, Round: msg.TimeoutInfo.Round, Step: cstypes.RoundStepType(tis), } - return pb, nil - case *tmcons.WALMessage_EndHeight: pb := EndHeightMessage{ Height: msg.EndHeight.Height, } - return pb, nil - default: return nil, fmt.Errorf("from proto: wal message not recognized: %T", msg) } - return pb, nil } diff --git a/consensus/msgs_test.go b/consensus/msgs_test.go index 24c56d4a7..52aea7e27 100644 --- a/consensus/msgs_test.go +++ b/consensus/msgs_test.go @@ -1,9 +1,7 @@ package consensus import ( - "context" "encoding/hex" - "fmt" "math" "testing" "time" @@ -12,11 +10,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/crypto/merkle" - "github.com/tendermint/tendermint/crypto/tmhash" "github.com/tendermint/tendermint/libs/bits" - "github.com/tendermint/tendermint/libs/bytes" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/p2p" tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" @@ -63,7 +58,7 @@ func TestMsgToProto(t *testing.T) { pbProposal := proposal.ToProto() pv := types.NewMockPV() - pk, err := pv.GetPubKey(context.Background()) + pk, err := pv.GetPubKey() require.NoError(t, err) val := types.NewValidator(pk, 100) @@ -430,309 +425,3 @@ func TestConsMsgsVectors(t *testing.T) { }) } } - -func TestVoteSetMaj23MessageValidateBasic(t *testing.T) { - const ( - validSignedMsgType tmproto.SignedMsgType = 0x01 - invalidSignedMsgType tmproto.SignedMsgType = 0x03 - ) - - validBlockID := types.BlockID{} - invalidBlockID := types.BlockID{ - Hash: bytes.HexBytes{}, - PartSetHeader: types.PartSetHeader{ - Total: 1, - Hash: []byte{0}, - }, - } - - testCases := []struct { // nolint: maligned - expectErr bool - messageRound int32 - messageHeight int64 - testName string - messageType tmproto.SignedMsgType - messageBlockID types.BlockID - }{ - {false, 0, 0, "Valid Message", validSignedMsgType, validBlockID}, - {true, -1, 0, "Invalid Message", validSignedMsgType, validBlockID}, - {true, 0, -1, "Invalid Message", validSignedMsgType, validBlockID}, - {true, 0, 0, "Invalid Message", invalidSignedMsgType, validBlockID}, - {true, 0, 0, "Invalid Message", validSignedMsgType, invalidBlockID}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - message := VoteSetMaj23Message{ - Height: tc.messageHeight, - Round: tc.messageRound, - Type: tc.messageType, - BlockID: tc.messageBlockID, - } - - assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") - }) - } -} - -func TestVoteSetBitsMessageValidateBasic(t *testing.T) { - testCases := []struct { - malleateFn func(*VoteSetBitsMessage) - expErr string - }{ - {func(msg *VoteSetBitsMessage) {}, ""}, - {func(msg *VoteSetBitsMessage) { msg.Height = -1 }, "negative Height"}, - {func(msg *VoteSetBitsMessage) { msg.Type = 0x03 }, "invalid Type"}, - {func(msg *VoteSetBitsMessage) { - msg.BlockID = types.BlockID{ - Hash: bytes.HexBytes{}, - PartSetHeader: types.PartSetHeader{ - Total: 1, - Hash: []byte{0}, - }, - } - }, "wrong BlockID: wrong PartSetHeader: wrong Hash:"}, - {func(msg *VoteSetBitsMessage) { msg.Votes = bits.NewBitArray(types.MaxVotesCount + 1) }, - "votes bit array is too big: 10001, max: 10000"}, - } - - for i, tc := range testCases { - tc := tc - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - msg := &VoteSetBitsMessage{ - Height: 1, - Round: 0, - Type: 0x01, - Votes: bits.NewBitArray(1), - BlockID: types.BlockID{}, - } - - tc.malleateFn(msg) - err := msg.ValidateBasic() - if tc.expErr != "" && assert.Error(t, err) { - assert.Contains(t, err.Error(), tc.expErr) - } - }) - } -} - -func TestNewRoundStepMessageValidateBasic(t *testing.T) { - testCases := []struct { // nolint: maligned - expectErr bool - messageRound int32 - messageLastCommitRound int32 - messageHeight int64 - testName string - messageStep cstypes.RoundStepType - }{ - {false, 0, 0, 0, "Valid Message", cstypes.RoundStepNewHeight}, - {true, -1, 0, 0, "Negative round", cstypes.RoundStepNewHeight}, - {true, 0, 0, -1, "Negative height", cstypes.RoundStepNewHeight}, - {true, 0, 0, 0, "Invalid Step", cstypes.RoundStepCommit + 1}, - // The following cases will be handled by ValidateHeight - {false, 0, 0, 1, "H == 1 but LCR != -1 ", cstypes.RoundStepNewHeight}, - {false, 0, -1, 2, "H > 1 but LCR < 0", cstypes.RoundStepNewHeight}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - message := NewRoundStepMessage{ - Height: tc.messageHeight, - Round: tc.messageRound, - Step: tc.messageStep, - LastCommitRound: tc.messageLastCommitRound, - } - - err := message.ValidateBasic() - if tc.expectErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestNewRoundStepMessageValidateHeight(t *testing.T) { - initialHeight := int64(10) - testCases := []struct { // nolint: maligned - expectErr bool - messageLastCommitRound int32 - messageHeight int64 - testName string - }{ - {false, 0, 11, "Valid Message"}, - {true, 0, -1, "Negative height"}, - {true, 0, 0, "Zero height"}, - {true, 0, 10, "Initial height but LCR != -1 "}, - {true, -1, 11, "Normal height but LCR < 0"}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - message := NewRoundStepMessage{ - Height: tc.messageHeight, - Round: 0, - Step: cstypes.RoundStepNewHeight, - LastCommitRound: tc.messageLastCommitRound, - } - - err := message.ValidateHeight(initialHeight) - if tc.expectErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) - } -} - -func TestNewValidBlockMessageValidateBasic(t *testing.T) { - testCases := []struct { - malleateFn func(*NewValidBlockMessage) - expErr string - }{ - {func(msg *NewValidBlockMessage) {}, ""}, - {func(msg *NewValidBlockMessage) { msg.Height = -1 }, "negative Height"}, - {func(msg *NewValidBlockMessage) { msg.Round = -1 }, "negative Round"}, - { - func(msg *NewValidBlockMessage) { msg.BlockPartSetHeader.Total = 2 }, - "blockParts bit array size 1 not equal to BlockPartSetHeader.Total 2", - }, - { - func(msg *NewValidBlockMessage) { - msg.BlockPartSetHeader.Total = 0 - msg.BlockParts = bits.NewBitArray(0) - }, - "empty blockParts", - }, - { - func(msg *NewValidBlockMessage) { msg.BlockParts = bits.NewBitArray(int(types.MaxBlockPartsCount) + 1) }, - "blockParts bit array size 1602 not equal to BlockPartSetHeader.Total 1", - }, - } - - for i, tc := range testCases { - tc := tc - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - msg := &NewValidBlockMessage{ - Height: 1, - Round: 0, - BlockPartSetHeader: types.PartSetHeader{ - Total: 1, - }, - BlockParts: bits.NewBitArray(1), - } - - tc.malleateFn(msg) - err := msg.ValidateBasic() - if tc.expErr != "" && assert.Error(t, err) { - assert.Contains(t, err.Error(), tc.expErr) - } - }) - } -} - -func TestProposalPOLMessageValidateBasic(t *testing.T) { - testCases := []struct { - malleateFn func(*ProposalPOLMessage) - expErr string - }{ - {func(msg *ProposalPOLMessage) {}, ""}, - {func(msg *ProposalPOLMessage) { msg.Height = -1 }, "negative Height"}, - {func(msg *ProposalPOLMessage) { msg.ProposalPOLRound = -1 }, "negative ProposalPOLRound"}, - {func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(0) }, "empty ProposalPOL bit array"}, - {func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(types.MaxVotesCount + 1) }, - "proposalPOL bit array is too big: 10001, max: 10000"}, - } - - for i, tc := range testCases { - tc := tc - t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { - msg := &ProposalPOLMessage{ - Height: 1, - ProposalPOLRound: 1, - ProposalPOL: bits.NewBitArray(1), - } - - tc.malleateFn(msg) - err := msg.ValidateBasic() - if tc.expErr != "" && assert.Error(t, err) { - assert.Contains(t, err.Error(), tc.expErr) - } - }) - } -} - -func TestBlockPartMessageValidateBasic(t *testing.T) { - testPart := new(types.Part) - testPart.Proof.LeafHash = tmhash.Sum([]byte("leaf")) - testCases := []struct { - testName string - messageHeight int64 - messageRound int32 - messagePart *types.Part - expectErr bool - }{ - {"Valid Message", 0, 0, testPart, false}, - {"Invalid Message", -1, 0, testPart, true}, - {"Invalid Message", 0, -1, testPart, true}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - message := BlockPartMessage{ - Height: tc.messageHeight, - Round: tc.messageRound, - Part: tc.messagePart, - } - - assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") - }) - } - - message := BlockPartMessage{Height: 0, Round: 0, Part: new(types.Part)} - message.Part.Index = 1 - - assert.Equal(t, true, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") -} - -func TestHasVoteMessageValidateBasic(t *testing.T) { - const ( - validSignedMsgType tmproto.SignedMsgType = 0x01 - invalidSignedMsgType tmproto.SignedMsgType = 0x03 - ) - - testCases := []struct { // nolint: maligned - expectErr bool - messageRound int32 - messageIndex int32 - messageHeight int64 - testName string - messageType tmproto.SignedMsgType - }{ - {false, 0, 0, 0, "Valid Message", validSignedMsgType}, - {true, -1, 0, 0, "Invalid Message", validSignedMsgType}, - {true, 0, -1, 0, "Invalid Message", validSignedMsgType}, - {true, 0, 0, 0, "Invalid Message", invalidSignedMsgType}, - {true, 0, 0, -1, "Invalid Message", validSignedMsgType}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.testName, func(t *testing.T) { - message := HasVoteMessage{ - Height: tc.messageHeight, - Round: tc.messageRound, - Type: tc.messageType, - Index: tc.messageIndex, - } - - assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") - }) - } -} diff --git a/consensus/peer_state.go b/consensus/peer_state.go deleted file mode 100644 index 31406a025..000000000 --- a/consensus/peer_state.go +++ /dev/null @@ -1,528 +0,0 @@ -package consensus - -import ( - "errors" - "fmt" - "sync" - "time" - - cstypes "github.com/tendermint/tendermint/consensus/types" - "github.com/tendermint/tendermint/libs/bits" - tmjson "github.com/tendermint/tendermint/libs/json" - "github.com/tendermint/tendermint/libs/log" - tmsync "github.com/tendermint/tendermint/libs/sync" - "github.com/tendermint/tendermint/p2p" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - "github.com/tendermint/tendermint/types" - tmtime "github.com/tendermint/tendermint/types/time" -) - -var ( - ErrPeerStateHeightRegression = errors.New("peer state height regression") - ErrPeerStateInvalidStartTime = errors.New("peer state invalid startTime") -) - -// peerStateStats holds internal statistics for a peer. -type peerStateStats struct { - Votes int `json:"votes"` - BlockParts int `json:"block_parts"` -} - -func (pss peerStateStats) String() string { - return fmt.Sprintf("peerStateStats{votes: %d, blockParts: %d}", pss.Votes, pss.BlockParts) -} - -// PeerState contains the known state of a peer, including its connection and -// threadsafe access to its PeerRoundState. -// NOTE: THIS GETS DUMPED WITH rpc/core/consensus.go. -// Be mindful of what you Expose. -type PeerState struct { - peerID p2p.NodeID - logger log.Logger - - // NOTE: Modify below using setters, never directly. - mtx tmsync.RWMutex - running bool - PRS cstypes.PeerRoundState `json:"round_state"` - Stats *peerStateStats `json:"stats"` - - broadcastWG sync.WaitGroup - closer *tmsync.Closer -} - -// NewPeerState returns a new PeerState for the given node ID. -func NewPeerState(logger log.Logger, peerID p2p.NodeID) *PeerState { - return &PeerState{ - peerID: peerID, - logger: logger, - closer: tmsync.NewCloser(), - PRS: cstypes.PeerRoundState{ - Round: -1, - ProposalPOLRound: -1, - LastCommitRound: -1, - CatchupCommitRound: -1, - }, - Stats: &peerStateStats{}, - } -} - -// SetRunning sets the running state of the peer. -func (ps *PeerState) SetRunning(v bool) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - ps.running = v -} - -// IsRunning returns true if a PeerState is considered running where multiple -// broadcasting goroutines exist for the peer. -func (ps *PeerState) IsRunning() bool { - ps.mtx.RLock() - defer ps.mtx.RUnlock() - - return ps.running -} - -// GetRoundState returns a shallow copy of the PeerRoundState. There's no point -// in mutating it since it won't change PeerState. -func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - prs := ps.PRS // copy - return &prs -} - -// ToJSON returns a json of PeerState. -func (ps *PeerState) ToJSON() ([]byte, error) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - return tmjson.Marshal(ps) -} - -// GetHeight returns an atomic snapshot of the PeerRoundState's height used by -// the mempool to ensure peers are caught up before broadcasting new txs. -func (ps *PeerState) GetHeight() int64 { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - return ps.PRS.Height -} - -// SetHasProposal sets the given proposal as known for the peer. -func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round { - return - } - - if ps.PRS.Proposal { - return - } - - ps.PRS.Proposal = true - - // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage - if ps.PRS.ProposalBlockParts != nil { - return - } - - ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader - ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) - ps.PRS.ProposalPOLRound = proposal.POLRound - ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. -} - -// InitProposalBlockParts initializes the peer's proposal block parts header -// and bit array. -func (ps *PeerState) InitProposalBlockParts(partSetHeader types.PartSetHeader) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.ProposalBlockParts != nil { - return - } - - ps.PRS.ProposalBlockPartSetHeader = partSetHeader - ps.PRS.ProposalBlockParts = bits.NewBitArray(int(partSetHeader.Total)) -} - -// SetHasProposalBlockPart sets the given block part index as known for the peer. -func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index int) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.Height != height || ps.PRS.Round != round { - return - } - - ps.PRS.ProposalBlockParts.SetIndex(index, true) -} - -// PickVoteToSend picks a vote to send to the peer. It will return true if a -// vote was picked. -// -// NOTE: `votes` must be the correct Size() for the Height(). -func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (*types.Vote, bool) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if votes.Size() == 0 { - return nil, false - } - - var ( - height = votes.GetHeight() - round = votes.GetRound() - votesType = tmproto.SignedMsgType(votes.Type()) - size = votes.Size() - ) - - // lazily set data using 'votes' - if votes.IsCommit() { - ps.ensureCatchupCommitRound(height, round, size) - } - - ps.ensureVoteBitArrays(height, size) - - psVotes := ps.getVoteBitArray(height, round, votesType) - if psVotes == nil { - return nil, false // not something worth sending - } - - if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { - return votes.GetByIndex(int32(index)), true - } - - return nil, false -} - -func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmproto.SignedMsgType) *bits.BitArray { - if !types.IsVoteTypeValid(votesType) { - return nil - } - - if ps.PRS.Height == height { - if ps.PRS.Round == round { - switch votesType { - case tmproto.PrevoteType: - return ps.PRS.Prevotes - - case tmproto.PrecommitType: - return ps.PRS.Precommits - } - } - - if ps.PRS.CatchupCommitRound == round { - switch votesType { - case tmproto.PrevoteType: - return nil - - case tmproto.PrecommitType: - return ps.PRS.CatchupCommit - } - } - - if ps.PRS.ProposalPOLRound == round { - switch votesType { - case tmproto.PrevoteType: - return ps.PRS.ProposalPOL - - case tmproto.PrecommitType: - return nil - } - } - - return nil - } - if ps.PRS.Height == height+1 { - if ps.PRS.LastCommitRound == round { - switch votesType { - case tmproto.PrevoteType: - return nil - - case tmproto.PrecommitType: - return ps.PRS.LastCommit - } - } - - return nil - } - - return nil -} - -// 'round': A round for which we have a +2/3 commit. -func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) { - if ps.PRS.Height != height { - return - } - - /* - NOTE: This is wrong, 'round' could change. - e.g. if orig round is not the same as block LastCommit round. - if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round { - panic(fmt.Sprintf( - "Conflicting CatchupCommitRound. Height: %v, - Orig: %v, - New: %v", - height, - ps.CatchupCommitRound, - round)) - } - */ - - if ps.PRS.CatchupCommitRound == round { - return // Nothing to do! - } - - ps.PRS.CatchupCommitRound = round - if round == ps.PRS.Round { - ps.PRS.CatchupCommit = ps.PRS.Precommits - } else { - ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) - } -} - -// EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking -// what votes this peer has received. -// NOTE: It's important to make sure that numValidators actually matches -// what the node sees as the number of validators for height. -func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - ps.ensureVoteBitArrays(height, numValidators) -} - -func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { - if ps.PRS.Height == height { - if ps.PRS.Prevotes == nil { - ps.PRS.Prevotes = bits.NewBitArray(numValidators) - } - if ps.PRS.Precommits == nil { - ps.PRS.Precommits = bits.NewBitArray(numValidators) - } - if ps.PRS.CatchupCommit == nil { - ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) - } - if ps.PRS.ProposalPOL == nil { - ps.PRS.ProposalPOL = bits.NewBitArray(numValidators) - } - } else if ps.PRS.Height == height+1 { - if ps.PRS.LastCommit == nil { - ps.PRS.LastCommit = bits.NewBitArray(numValidators) - } - } -} - -// RecordVote increments internal votes related statistics for this peer. -// It returns the total number of added votes. -func (ps *PeerState) RecordVote() int { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - ps.Stats.Votes++ - - return ps.Stats.Votes -} - -// VotesSent returns the number of blocks for which peer has been sending us -// votes. -func (ps *PeerState) VotesSent() int { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - return ps.Stats.Votes -} - -// RecordBlockPart increments internal block part related statistics for this peer. -// It returns the total number of added block parts. -func (ps *PeerState) RecordBlockPart() int { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - ps.Stats.BlockParts++ - return ps.Stats.BlockParts -} - -// BlockPartsSent returns the number of useful block parts the peer has sent us. -func (ps *PeerState) BlockPartsSent() int { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - return ps.Stats.BlockParts -} - -// SetHasVote sets the given vote as known by the peer -func (ps *PeerState) SetHasVote(vote *types.Vote) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) -} - -func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { - logger := ps.logger.With( - "peerH/R", fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round), - "H/R", fmt.Sprintf("%d/%d", height, round), - ) - - logger.Debug("setHasVote", "type", voteType, "index", index) - - // NOTE: some may be nil BitArrays -> no side effects - psVotes := ps.getVoteBitArray(height, round, voteType) - if psVotes != nil { - psVotes.SetIndex(int(index), true) - } -} - -// ApplyNewRoundStepMessage updates the peer state for the new round. -func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - // ignore duplicates or decreases - if CompareHRS(msg.Height, msg.Round, msg.Step, ps.PRS.Height, ps.PRS.Round, ps.PRS.Step) <= 0 { - return - } - - var ( - psHeight = ps.PRS.Height - psRound = ps.PRS.Round - psCatchupCommitRound = ps.PRS.CatchupCommitRound - psCatchupCommit = ps.PRS.CatchupCommit - startTime = tmtime.Now().Add(-1 * time.Duration(msg.SecondsSinceStartTime) * time.Second) - ) - - ps.PRS.Height = msg.Height - ps.PRS.Round = msg.Round - ps.PRS.Step = msg.Step - ps.PRS.StartTime = startTime - - if psHeight != msg.Height || psRound != msg.Round { - ps.PRS.Proposal = false - ps.PRS.ProposalBlockPartSetHeader = types.PartSetHeader{} - ps.PRS.ProposalBlockParts = nil - ps.PRS.ProposalPOLRound = -1 - ps.PRS.ProposalPOL = nil - - // we'll update the BitArray capacity later - ps.PRS.Prevotes = nil - ps.PRS.Precommits = nil - } - - if psHeight == msg.Height && psRound != msg.Round && msg.Round == psCatchupCommitRound { - // Peer caught up to CatchupCommitRound. - // Preserve psCatchupCommit! - // NOTE: We prefer to use prs.Precommits if - // pr.Round matches pr.CatchupCommitRound. - ps.PRS.Precommits = psCatchupCommit - } - - if psHeight != msg.Height { - // shift Precommits to LastCommit - if psHeight+1 == msg.Height && psRound == msg.LastCommitRound { - ps.PRS.LastCommitRound = msg.LastCommitRound - ps.PRS.LastCommit = ps.PRS.Precommits - } else { - ps.PRS.LastCommitRound = msg.LastCommitRound - ps.PRS.LastCommit = nil - } - - // we'll update the BitArray capacity later - ps.PRS.CatchupCommitRound = -1 - ps.PRS.CatchupCommit = nil - } -} - -// ApplyNewValidBlockMessage updates the peer state for the new valid block. -func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.Height != msg.Height { - return - } - - if ps.PRS.Round != msg.Round && !msg.IsCommit { - return - } - - ps.PRS.ProposalBlockPartSetHeader = msg.BlockPartSetHeader - ps.PRS.ProposalBlockParts = msg.BlockParts -} - -// ApplyProposalPOLMessage updates the peer state for the new proposal POL. -func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.Height != msg.Height { - return - } - if ps.PRS.ProposalPOLRound != msg.ProposalPOLRound { - return - } - - // TODO: Merge onto existing ps.PRS.ProposalPOL? - // We might have sent some prevotes in the meantime. - ps.PRS.ProposalPOL = msg.ProposalPOL -} - -// ApplyHasVoteMessage updates the peer state for the new vote. -func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - if ps.PRS.Height != msg.Height { - return - } - - ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) -} - -// ApplyVoteSetBitsMessage updates the peer state for the bit-array of votes -// it claims to have for the corresponding BlockID. -// `ourVotes` is a BitArray of votes we have for msg.BlockID -// NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height), -// we conservatively overwrite ps's votes w/ msg.Votes. -func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *bits.BitArray) { - ps.mtx.Lock() - defer ps.mtx.Unlock() - - votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type) - if votes != nil { - if ourVotes == nil { - votes.Update(msg.Votes) - } else { - otherVotes := votes.Sub(ourVotes) - hasVotes := otherVotes.Or(msg.Votes) - votes.Update(hasVotes) - } - } -} - -// String returns a string representation of the PeerState -func (ps *PeerState) String() string { - return ps.StringIndented("") -} - -// StringIndented returns a string representation of the PeerState -func (ps *PeerState) StringIndented(indent string) string { - ps.mtx.Lock() - defer ps.mtx.Unlock() - return fmt.Sprintf(`PeerState{ -%s Key %v -%s RoundState %v -%s Stats %v -%s}`, - indent, ps.peerID, - indent, ps.PRS.StringIndented(indent+" "), - indent, ps.Stats, - indent, - ) -} diff --git a/consensus/reactor.go b/consensus/reactor.go index 9e3551d03..83de4af2a 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -1,26 +1,41 @@ package consensus import ( + "errors" "fmt" + "reflect" + "sync" "time" + "github.com/gogo/protobuf/proto" + cstypes "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/libs/bits" tmevents "github.com/tendermint/tendermint/libs/events" + tmjson "github.com/tendermint/tendermint/libs/json" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/libs/service" tmsync "github.com/tendermint/tendermint/libs/sync" "github.com/tendermint/tendermint/p2p" tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + tmtime "github.com/tendermint/tendermint/types/time" +) + +const ( + StateChannel = byte(0x20) + DataChannel = byte(0x21) + VoteChannel = byte(0x22) + VoteSetBitsChannel = byte(0x23) + + maxMsgSize = 1048576 // 1MB; NOTE/TODO: keep in sync with types.PartSet sizes. + + blocksToContributeToBecomeGoodPeer = 10000 + votesToContributeToBecomeGoodPeer = 10000 ) var ( - _ service.Service = (*Reactor)(nil) - _ p2p.Wrapper = (*tmcons.Message)(nil) - // ChannelShims contains a map of ChannelDescriptorShim objects, where each // object wraps a reference to a legacy p2p ChannelDescriptor and the corresponding // p2p proto.Message the new p2p Channel is responsible for handling. @@ -29,7 +44,7 @@ var ( // TODO: Remove once p2p refactor is complete. // ref: https://github.com/tendermint/tendermint/issues/5670 ChannelShims = map[p2p.ChannelID]*p2p.ChannelDescriptorShim{ - StateChannel: { + p2p.ChannelID(StateChannel): { MsgType: new(tmcons.Message), Descriptor: &p2p.ChannelDescriptor{ ID: byte(StateChannel), @@ -38,7 +53,7 @@ var ( RecvMessageCapacity: maxMsgSize, }, }, - DataChannel: { + p2p.ChannelID(DataChannel): { MsgType: new(tmcons.Message), Descriptor: &p2p.ChannelDescriptor{ // TODO: Consider a split between gossiping current block and catchup @@ -51,7 +66,7 @@ var ( RecvMessageCapacity: maxMsgSize, }, }, - VoteChannel: { + p2p.ChannelID(VoteChannel): { MsgType: new(tmcons.Message), Descriptor: &p2p.ChannelDescriptor{ ID: byte(VoteChannel), @@ -61,7 +76,7 @@ var ( RecvMessageCapacity: maxMsgSize, }, }, - VoteSetBitsChannel: { + p2p.ChannelID(VoteSetBitsChannel): { MsgType: new(tmcons.Message), Descriptor: &p2p.ChannelDescriptor{ ID: byte(VoteSetBitsChannel), @@ -74,488 +89,517 @@ var ( } ) -const ( - StateChannel = p2p.ChannelID(0x20) - DataChannel = p2p.ChannelID(0x21) - VoteChannel = p2p.ChannelID(0x22) - VoteSetBitsChannel = p2p.ChannelID(0x23) - - maxMsgSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. - - blocksToContributeToBecomeGoodPeer = 10000 - votesToContributeToBecomeGoodPeer = 10000 - - listenerIDConsensus = "consensus-reactor" -) - -type ReactorOption func(*Reactor) +//----------------------------------------------------------------------------- // Reactor defines a reactor for the consensus service. type Reactor struct { - service.BaseService + p2p.BaseReactor // BaseService + p2p.Switch - state *State - eventBus *types.EventBus - Metrics *Metrics + conS *State mtx tmsync.RWMutex - peers map[p2p.NodeID]*PeerState waitSync bool + eventBus *types.EventBus - stateCh *p2p.Channel - dataCh *p2p.Channel - voteCh *p2p.Channel - voteSetBitsCh *p2p.Channel - peerUpdates *p2p.PeerUpdates - - // NOTE: We need a dedicated stateCloseCh channel for signaling closure of - // the StateChannel due to the fact that the StateChannel message handler - // performs a send on the VoteSetBitsChannel. This is an antipattern, so having - // this dedicated channel,stateCloseCh, is necessary in order to avoid data races. - stateCloseCh chan struct{} - closeCh chan struct{} + Metrics *Metrics } -// NewReactor returns a reference to a new consensus reactor, which implements -// the service.Service interface. It accepts a logger, consensus state, references -// to relevant p2p Channels and a channel to listen for peer updates on. The -// reactor will close all p2p Channels when stopping. -func NewReactor( - logger log.Logger, - cs *State, - stateCh *p2p.Channel, - dataCh *p2p.Channel, - voteCh *p2p.Channel, - voteSetBitsCh *p2p.Channel, - peerUpdates *p2p.PeerUpdates, - waitSync bool, - options ...ReactorOption, -) *Reactor { +type ReactorOption func(*Reactor) - r := &Reactor{ - state: cs, - waitSync: waitSync, - peers: make(map[p2p.NodeID]*PeerState), - Metrics: NopMetrics(), - stateCh: stateCh, - dataCh: dataCh, - voteCh: voteCh, - voteSetBitsCh: voteSetBitsCh, - peerUpdates: peerUpdates, - stateCloseCh: make(chan struct{}), - closeCh: make(chan struct{}), +// NewReactor returns a new Reactor with the given +// consensusState. +func NewReactor(consensusState *State, waitSync bool, options ...ReactorOption) *Reactor { + conR := &Reactor{ + conS: consensusState, + waitSync: waitSync, + Metrics: NopMetrics(), } - r.BaseService = *service.NewBaseService(logger, "Consensus", r) + conR.BaseReactor = *p2p.NewBaseReactor("Consensus", conR) - for _, opt := range options { - opt(r) + for _, option := range options { + option(conR) } - return r + return conR } -// OnStart starts separate go routines for each p2p Channel and listens for -// envelopes on each. In addition, it also listens for peer updates and handles -// messages on that p2p channel accordingly. The caller must be sure to execute -// OnStop to ensure the outbound p2p Channels are closed. -func (r *Reactor) OnStart() error { - r.Logger.Debug("consensus wait sync", "wait_sync", r.WaitSync()) +// OnStart implements BaseService by subscribing to events, which later will be +// broadcasted to other peers and starting state if we're not in fast sync. +func (conR *Reactor) OnStart() error { + conR.Logger.Info("Reactor ", "waitSync", conR.WaitSync()) // start routine that computes peer statistics for evaluating peer quality - // - // TODO: Evaluate if we need this to be synchronized via WaitGroup as to not - // leak the goroutine when stopping the reactor. - go r.peerStatsRoutine() + go conR.peerStatsRoutine() - r.subscribeToBroadcastEvents() + conR.subscribeToBroadcastEvents() - if !r.WaitSync() { - if err := r.state.Start(); err != nil { + if !conR.WaitSync() { + err := conR.conS.Start() + if err != nil { return err } } - go r.processStateCh() - go r.processDataCh() - go r.processVoteCh() - go r.processVoteSetBitsCh() - go r.processPeerUpdates() - return nil } -// OnStop stops the reactor by signaling to all spawned goroutines to exit and -// blocking until they all exit, as well as unsubscribing from events and stopping +// OnStop implements BaseService by unsubscribing from events and stopping // state. -func (r *Reactor) OnStop() { - r.unsubscribeFromBroadcastEvents() - - if err := r.state.Stop(); err != nil { - r.Logger.Error("failed to stop consensus state", "err", err) +func (conR *Reactor) OnStop() { + conR.unsubscribeFromBroadcastEvents() + if err := conR.conS.Stop(); err != nil { + conR.Logger.Error("Error stopping consensus state", "err", err) } - - if !r.WaitSync() { - r.state.Wait() + if !conR.WaitSync() { + conR.conS.Wait() } - - r.mtx.Lock() - peers := r.peers - r.mtx.Unlock() - - // wait for all spawned peer goroutines to gracefully exit - for _, ps := range peers { - ps.closer.Close() - } - for _, ps := range peers { - ps.broadcastWG.Wait() - } - - // Close the StateChannel goroutine separately since it uses its own channel - // to signal closure. - close(r.stateCloseCh) - <-r.stateCh.Done() - - // Close closeCh to signal to all spawned goroutines to gracefully exit. All - // p2p Channels should execute Close(). - close(r.closeCh) - - // Wait for all p2p Channels to be closed before returning. This ensures we - // can easily reason about synchronization of all p2p Channels and ensure no - // panics will occur. - <-r.voteSetBitsCh.Done() - <-r.dataCh.Done() - <-r.voteCh.Done() - <-r.peerUpdates.Done() } -// SetEventBus sets the reactor's event bus. -func (r *Reactor) SetEventBus(b *types.EventBus) { - r.eventBus = b - r.state.SetEventBus(b) -} +// SwitchToConsensus switches from fast_sync mode to consensus mode. +// It resets the state, turns off fast_sync, and starts the consensus state-machine +func (conR *Reactor) SwitchToConsensus(state sm.State, skipWAL bool) { + conR.Logger.Info("SwitchToConsensus") -// WaitSync returns whether the consensus reactor is waiting for state/fast sync. -func (r *Reactor) WaitSync() bool { - r.mtx.RLock() - defer r.mtx.RUnlock() - - return r.waitSync -} - -// ReactorMetrics sets the reactor's metrics as an option function. -func ReactorMetrics(metrics *Metrics) ReactorOption { - return func(r *Reactor) { r.Metrics = metrics } -} - -// SwitchToConsensus switches from fast-sync mode to consensus mode. It resets -// the state, turns off fast-sync, and starts the consensus state-machine. -func (r *Reactor) SwitchToConsensus(state sm.State, skipWAL bool) { - r.Logger.Info("switching to consensus") - - // we have no votes, so reconstruct LastCommit from SeenCommit + // We have no votes, so reconstruct LastCommit from SeenCommit. if state.LastBlockHeight > 0 { - r.state.reconstructLastCommit(state) + conR.conS.reconstructLastCommit(state) } // NOTE: The line below causes broadcastNewRoundStepRoutine() to broadcast a // NewRoundStepMessage. - r.state.updateToState(state) + conR.conS.updateToState(state) - r.mtx.Lock() - r.waitSync = false - r.mtx.Unlock() - - r.Metrics.FastSyncing.Set(0) - r.Metrics.StateSyncing.Set(0) + conR.mtx.Lock() + conR.waitSync = false + conR.mtx.Unlock() + conR.Metrics.FastSyncing.Set(0) + conR.Metrics.StateSyncing.Set(0) if skipWAL { - r.state.doWALCatchup = false + conR.conS.doWALCatchup = false } - - if err := r.state.Start(); err != nil { - panic(fmt.Sprintf(`failed to start consensus state: %v + err := conR.conS.Start() + if err != nil { + panic(fmt.Sprintf(`Failed to start consensus state: %v conS: %+v conR: -%+v`, err, r.state, r)) +%+v`, err, conR.conS, conR)) } } -// String returns a string representation of the Reactor. -// -// NOTE: For now, it is just a hard-coded string to avoid accessing unprotected -// shared variables. -// -// TODO: improve! -func (r *Reactor) String() string { - return "ConsensusReactor" -} - -// StringIndented returns an indented string representation of the Reactor. -func (r *Reactor) StringIndented(indent string) string { - r.mtx.RLock() - defer r.mtx.RUnlock() - - s := "ConsensusReactor{\n" - s += indent + " " + r.state.StringIndented(indent+" ") + "\n" - - for _, ps := range r.peers { - s += indent + " " + ps.StringIndented(indent+" ") + "\n" - } - - s += indent + "}" - return s -} - -// GetPeerState returns PeerState for a given NodeID. -func (r *Reactor) GetPeerState(peerID p2p.NodeID) (*PeerState, bool) { - r.mtx.RLock() - defer r.mtx.RUnlock() - - ps, ok := r.peers[peerID] - return ps, ok -} - -func (r *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { - r.stateCh.Out <- p2p.Envelope{ - Broadcast: true, - Message: makeRoundStepMessage(rs), - } -} - -func (r *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { - psHeader := rs.ProposalBlockParts.Header() - r.stateCh.Out <- p2p.Envelope{ - Broadcast: true, - Message: &tmcons.NewValidBlock{ - Height: rs.Height, - Round: rs.Round, - BlockPartSetHeader: psHeader.ToProto(), - BlockParts: rs.ProposalBlockParts.BitArray().ToProto(), - IsCommit: rs.Step == cstypes.RoundStepCommit, +// GetChannels implements Reactor +func (conR *Reactor) GetChannels() []*p2p.ChannelDescriptor { + // TODO optimize + return []*p2p.ChannelDescriptor{ + { + ID: StateChannel, + Priority: 6, + SendQueueCapacity: 100, + RecvMessageCapacity: maxMsgSize, + }, + { + ID: DataChannel, // maybe split between gossiping current block and catchup stuff + // once we gossip the whole block there's nothing left to send until next height or round + Priority: 10, + SendQueueCapacity: 100, + RecvBufferCapacity: 50 * 4096, + RecvMessageCapacity: maxMsgSize, + }, + { + ID: VoteChannel, + Priority: 7, + SendQueueCapacity: 100, + RecvBufferCapacity: 100 * 100, + RecvMessageCapacity: maxMsgSize, + }, + { + ID: VoteSetBitsChannel, + Priority: 1, + SendQueueCapacity: 2, + RecvBufferCapacity: 1024, + RecvMessageCapacity: maxMsgSize, }, } } -func (r *Reactor) broadcastHasVoteMessage(vote *types.Vote) { - r.stateCh.Out <- p2p.Envelope{ - Broadcast: true, - Message: &tmcons.HasVote{ - Height: vote.Height, - Round: vote.Round, - Type: vote.Type, - Index: vote.ValidatorIndex, - }, - } +// InitPeer implements Reactor by creating a state for the peer. +func (conR *Reactor) InitPeer(peer p2p.Peer) p2p.Peer { + peerState := NewPeerState(peer).SetLogger(conR.Logger) + peer.Set(types.PeerStateKey, peerState) + return peer } -// subscribeToBroadcastEvents subscribes for new round steps and votes using the -// internal pubsub defined in the consensus state to broadcast them to peers -// upon receiving. -func (r *Reactor) subscribeToBroadcastEvents() { - err := r.state.evsw.AddListenerForEvent( - listenerIDConsensus, - types.EventNewRoundStep, - func(data tmevents.EventData) { - r.broadcastNewRoundStepMessage(data.(*cstypes.RoundState)) - }, - ) - if err != nil { - r.Logger.Error("failed to add listener for events", "err", err) - } - - err = r.state.evsw.AddListenerForEvent( - listenerIDConsensus, - types.EventValidBlock, - func(data tmevents.EventData) { - r.broadcastNewValidBlockMessage(data.(*cstypes.RoundState)) - }, - ) - if err != nil { - r.Logger.Error("failed to add listener for events", "err", err) - } - - err = r.state.evsw.AddListenerForEvent( - listenerIDConsensus, - types.EventVote, - func(data tmevents.EventData) { - r.broadcastHasVoteMessage(data.(*types.Vote)) - }, - ) - if err != nil { - r.Logger.Error("failed to add listener for events", "err", err) - } -} - -func (r *Reactor) unsubscribeFromBroadcastEvents() { - r.state.evsw.RemoveListener(listenerIDConsensus) -} - -func makeRoundStepMessage(rs *cstypes.RoundState) *tmcons.NewRoundStep { - return &tmcons.NewRoundStep{ - Height: rs.Height, - Round: rs.Round, - Step: uint32(rs.Step), - SecondsSinceStartTime: int64(time.Since(rs.StartTime).Seconds()), - LastCommitRound: rs.LastCommit.GetRound(), - } -} - -func (r *Reactor) sendNewRoundStepMessage(peerID p2p.NodeID) { - rs := r.state.GetRoundState() - msg := makeRoundStepMessage(rs) - r.stateCh.Out <- p2p.Envelope{ - To: peerID, - Message: msg, - } -} - -func (r *Reactor) gossipDataForCatchup(rs *cstypes.RoundState, prs *cstypes.PeerRoundState, ps *PeerState) { - logger := r.Logger.With("height", prs.Height).With("peer", ps.peerID) - - if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { - // ensure that the peer's PartSetHeader is correct - blockMeta := r.state.blockStore.LoadBlockMeta(prs.Height) - if blockMeta == nil { - logger.Error( - "failed to load block meta", - "our_height", rs.Height, - "blockstore_base", r.state.blockStore.Base(), - "blockstore_height", r.state.blockStore.Height(), - ) - - time.Sleep(r.state.config.PeerGossipSleepDuration) - return - } else if !blockMeta.BlockID.PartSetHeader.Equals(prs.ProposalBlockPartSetHeader) { - logger.Info( - "peer ProposalBlockPartSetHeader mismatch; sleeping", - "block_part_set_header", blockMeta.BlockID.PartSetHeader, - "peer_block_part_set_header", prs.ProposalBlockPartSetHeader, - ) - - time.Sleep(r.state.config.PeerGossipSleepDuration) - return - } - - part := r.state.blockStore.LoadBlockPart(prs.Height, index) - if part == nil { - logger.Error( - "failed to load block part", - "index", index, - "block_part_set_header", blockMeta.BlockID.PartSetHeader, - "peer_block_part_set_header", prs.ProposalBlockPartSetHeader, - ) - - time.Sleep(r.state.config.PeerGossipSleepDuration) - return - } - - partProto, err := part.ToProto() - if err != nil { - logger.Error("failed to convert block part to proto", "err", err) - - time.Sleep(r.state.config.PeerGossipSleepDuration) - return - } - - logger.Debug("sending block part for catchup", "round", prs.Round, "index", index) - r.dataCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.BlockPart{ - Height: prs.Height, // not our height, so it does not matter. - Round: prs.Round, // not our height, so it does not matter - Part: *partProto, - }, - } - +// AddPeer implements Reactor by spawning multiple gossiping goroutines for the +// peer. +func (conR *Reactor) AddPeer(peer p2p.Peer) { + if !conR.IsRunning() { return } - time.Sleep(r.state.config.PeerGossipSleepDuration) + peerState, ok := peer.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("peer %v has no state", peer)) + } + // Begin routines for this peer. + go conR.gossipDataRoutine(peer, peerState) + go conR.gossipVotesRoutine(peer, peerState) + go conR.queryMaj23Routine(peer, peerState) + + // Send our state to peer. + // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). + if !conR.WaitSync() { + conR.sendNewRoundStepMessage(peer) + } } -func (r *Reactor) gossipDataRoutine(ps *PeerState) { - logger := r.Logger.With("peer", ps.peerID) +// RemovePeer is a noop. +func (conR *Reactor) RemovePeer(peer p2p.Peer, reason interface{}) { + if !conR.IsRunning() { + return + } + // TODO + // ps, ok := peer.Get(PeerStateKey).(*PeerState) + // if !ok { + // panic(fmt.Sprintf("Peer %v has no state", peer)) + // } + // ps.Disconnect() +} - defer ps.broadcastWG.Done() +// Receive implements Reactor +// NOTE: We process these messages even when we're fast_syncing. +// Messages affect either a peer state or the consensus state. +// Peer state updates can happen in parallel, but processing of +// proposals, block parts, and votes are ordered by the receiveRoutine +// NOTE: blocks on consensus state for proposals, block parts, and votes +// XXX: do not call any methods that can block or incur heavy processing. +// https://github.com/tendermint/tendermint/issues/2888 +func (conR *Reactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { + if !conR.IsRunning() { + conR.Logger.Debug("Receive", "src", src, "chId", chID, "bytes", msgBytes) + return + } + + msg, err := decodeMsg(msgBytes) + if err != nil { + conR.Logger.Error("Error decoding message", "src", src, "chId", chID, "err", err) + conR.Switch.StopPeerForError(src, err) + return + } + + if err = msg.ValidateBasic(); err != nil { + conR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) + conR.Switch.StopPeerForError(src, err) + return + } + + conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) + + // Get peer states + ps, ok := src.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", src)) + } + + switch chID { + case StateChannel: + switch msg := msg.(type) { + case *NewRoundStepMessage: + conR.conS.mtx.Lock() + initialHeight := conR.conS.state.InitialHeight + conR.conS.mtx.Unlock() + if err = msg.ValidateHeight(initialHeight); err != nil { + conR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) + conR.Switch.StopPeerForError(src, err) + return + } + ps.ApplyNewRoundStepMessage(msg) + case *NewValidBlockMessage: + ps.ApplyNewValidBlockMessage(msg) + case *HasVoteMessage: + ps.ApplyHasVoteMessage(msg) + case *VoteSetMaj23Message: + cs := conR.conS + cs.mtx.Lock() + height, votes := cs.Height, cs.Votes + cs.mtx.Unlock() + if height != msg.Height { + return + } + // Peer claims to have a maj23 for some BlockID at H,R,S, + err := votes.SetPeerMaj23(msg.Round, msg.Type, ps.peer.ID(), msg.BlockID) + if err != nil { + conR.Switch.StopPeerForError(src, err) + return + } + // Respond with a VoteSetBitsMessage showing which votes we have. + // (and consequently shows which we don't have) + var ourVotes *bits.BitArray + switch msg.Type { + case tmproto.PrevoteType: + ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) + case tmproto.PrecommitType: + ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) + default: + panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") + } + src.TrySend(VoteSetBitsChannel, MustEncode(&VoteSetBitsMessage{ + Height: msg.Height, + Round: msg.Round, + Type: msg.Type, + BlockID: msg.BlockID, + Votes: ourVotes, + })) + default: + conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) + } + + case DataChannel: + if conR.WaitSync() { + conR.Logger.Info("Ignoring message received during sync", "msg", msg) + return + } + switch msg := msg.(type) { + case *ProposalMessage: + ps.SetHasProposal(msg.Proposal) + conR.conS.peerMsgQueue <- msgInfo{msg, src.ID()} + case *ProposalPOLMessage: + ps.ApplyProposalPOLMessage(msg) + case *BlockPartMessage: + ps.SetHasProposalBlockPart(msg.Height, msg.Round, int(msg.Part.Index)) + conR.Metrics.BlockParts.With("peer_id", string(src.ID())).Add(1) + conR.conS.peerMsgQueue <- msgInfo{msg, src.ID()} + default: + conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) + } + + case VoteChannel: + if conR.WaitSync() { + conR.Logger.Info("Ignoring message received during sync", "msg", msg) + return + } + switch msg := msg.(type) { + case *VoteMessage: + cs := conR.conS + cs.mtx.RLock() + height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size() + cs.mtx.RUnlock() + ps.EnsureVoteBitArrays(height, valSize) + ps.EnsureVoteBitArrays(height-1, lastCommitSize) + ps.SetHasVote(msg.Vote) + + cs.peerMsgQueue <- msgInfo{msg, src.ID()} + + default: + // don't punish (leave room for soft upgrades) + conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) + } + + case VoteSetBitsChannel: + if conR.WaitSync() { + conR.Logger.Info("Ignoring message received during sync", "msg", msg) + return + } + switch msg := msg.(type) { + case *VoteSetBitsMessage: + cs := conR.conS + cs.mtx.Lock() + height, votes := cs.Height, cs.Votes + cs.mtx.Unlock() + + if height == msg.Height { + var ourVotes *bits.BitArray + switch msg.Type { + case tmproto.PrevoteType: + ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) + case tmproto.PrecommitType: + ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) + default: + panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") + } + ps.ApplyVoteSetBitsMessage(msg, ourVotes) + } else { + ps.ApplyVoteSetBitsMessage(msg, nil) + } + default: + // don't punish (leave room for soft upgrades) + conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) + } + + default: + conR.Logger.Error(fmt.Sprintf("Unknown chId %X", chID)) + } +} + +// SetEventBus sets event bus. +func (conR *Reactor) SetEventBus(b *types.EventBus) { + conR.eventBus = b + conR.conS.SetEventBus(b) +} + +// WaitSync returns whether the consensus reactor is waiting for state/fast sync. +func (conR *Reactor) WaitSync() bool { + conR.mtx.RLock() + defer conR.mtx.RUnlock() + return conR.waitSync +} + +//-------------------------------------- + +// subscribeToBroadcastEvents subscribes for new round steps and votes +// using internal pubsub defined on state to broadcast +// them to peers upon receiving. +func (conR *Reactor) subscribeToBroadcastEvents() { + const subscriber = "consensus-reactor" + if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventNewRoundStep, + func(data tmevents.EventData) { + conR.broadcastNewRoundStepMessage(data.(*cstypes.RoundState)) + }); err != nil { + conR.Logger.Error("Error adding listener for events", "err", err) + } + + if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventValidBlock, + func(data tmevents.EventData) { + conR.broadcastNewValidBlockMessage(data.(*cstypes.RoundState)) + }); err != nil { + conR.Logger.Error("Error adding listener for events", "err", err) + } + + if err := conR.conS.evsw.AddListenerForEvent(subscriber, types.EventVote, + func(data tmevents.EventData) { + conR.broadcastHasVoteMessage(data.(*types.Vote)) + }); err != nil { + conR.Logger.Error("Error adding listener for events", "err", err) + } + +} + +func (conR *Reactor) unsubscribeFromBroadcastEvents() { + const subscriber = "consensus-reactor" + conR.conS.evsw.RemoveListener(subscriber) +} + +func (conR *Reactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { + nrsMsg := makeRoundStepMessage(rs) + conR.Switch.Broadcast(StateChannel, MustEncode(nrsMsg)) +} + +func (conR *Reactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { + csMsg := &NewValidBlockMessage{ + Height: rs.Height, + Round: rs.Round, + BlockPartSetHeader: rs.ProposalBlockParts.Header(), + BlockParts: rs.ProposalBlockParts.BitArray(), + IsCommit: rs.Step == cstypes.RoundStepCommit, + } + conR.Switch.Broadcast(StateChannel, MustEncode(csMsg)) +} + +// Broadcasts HasVoteMessage to peers that care. +func (conR *Reactor) broadcastHasVoteMessage(vote *types.Vote) { + msg := &HasVoteMessage{ + Height: vote.Height, + Round: vote.Round, + Type: vote.Type, + Index: vote.ValidatorIndex, + } + conR.Switch.Broadcast(StateChannel, MustEncode(msg)) + /* + // TODO: Make this broadcast more selective. + for _, peer := range conR.Switch.Peers().List() { + ps, ok := peer.Get(PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } + prs := ps.GetRoundState() + if prs.Height == vote.Height { + // TODO: Also filter on round? + peer.TrySend(StateChannel, struct{ ConsensusMessage }{msg}) + } else { + // Height doesn't match + // TODO: check a field, maybe CatchupCommitRound? + // TODO: But that requires changing the struct field comment. + } + } + */ +} + +func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage) { + nrsMsg = &NewRoundStepMessage{ + Height: rs.Height, + Round: rs.Round, + Step: rs.Step, + SecondsSinceStartTime: int64(time.Since(rs.StartTime).Seconds()), + LastCommitRound: rs.LastCommit.GetRound(), + } + return +} + +func (conR *Reactor) sendNewRoundStepMessage(peer p2p.Peer) { + rs := conR.conS.GetRoundState() + nrsMsg := makeRoundStepMessage(rs) + peer.Send(StateChannel, MustEncode(nrsMsg)) +} + +func (conR *Reactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) { + logger := conR.Logger.With("peer", peer) OUTER_LOOP: for { - if !r.IsRunning() { + // Manage disconnects from self or peer. + if !peer.IsRunning() || !conR.IsRunning() { + logger.Info("Stopping gossipDataRoutine for peer") return } - - select { - case <-ps.closer.Done(): - // The peer is marked for removal via a PeerUpdate as the doneCh was - // explicitly closed to signal we should exit. - return - - default: - } - - rs := r.state.GetRoundState() + rs := conR.conS.GetRoundState() prs := ps.GetRoundState() // Send proposal Block parts? if rs.ProposalBlockParts.HasHeader(prs.ProposalBlockPartSetHeader) { if index, ok := rs.ProposalBlockParts.BitArray().Sub(prs.ProposalBlockParts.Copy()).PickRandom(); ok { part := rs.ProposalBlockParts.GetPart(index) - partProto, err := part.ToProto() - if err != nil { - logger.Error("failed to convert block part to proto", "err", err) - return + msg := &BlockPartMessage{ + Height: rs.Height, // This tells peer that this part applies to us. + Round: rs.Round, // This tells peer that this part applies to us. + Part: part, } - - logger.Debug("sending block part", "height", prs.Height, "round", prs.Round) - r.dataCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.BlockPart{ - Height: rs.Height, // this tells peer that this part applies to us - Round: rs.Round, // this tells peer that this part applies to us - Part: *partProto, - }, + logger.Debug("Sending block part", "height", prs.Height, "round", prs.Round) + if peer.Send(DataChannel, MustEncode(msg)) { + ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) } - - ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) continue OUTER_LOOP } } - // if the peer is on a previous height that we have, help catch up - blockStoreBase := r.state.blockStore.Base() + // If the peer is on a previous height that we have, help catch up. + blockStoreBase := conR.conS.blockStore.Base() if blockStoreBase > 0 && 0 < prs.Height && prs.Height < rs.Height && prs.Height >= blockStoreBase { heightLogger := logger.With("height", prs.Height) - // If we never received the commit message from the peer, the block parts - // will not be initialized. + // if we never received the commit message from the peer, the block parts wont be initialized if prs.ProposalBlockParts == nil { - blockMeta := r.state.blockStore.LoadBlockMeta(prs.Height) + blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) if blockMeta == nil { - heightLogger.Error( - "failed to load block meta", - "blockstoreBase", blockStoreBase, - "blockstoreHeight", r.state.blockStore.Height(), - ) - time.Sleep(r.state.config.PeerGossipSleepDuration) + heightLogger.Error("Failed to load block meta", + "blockstoreBase", blockStoreBase, "blockstoreHeight", conR.conS.blockStore.Height()) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) } else { ps.InitProposalBlockParts(blockMeta.BlockID.PartSetHeader) } - - // Continue the loop since prs is a copy and not effected by this - // initialization. + // continue the loop since prs is a copy and not effected by this initialization continue OUTER_LOOP } - - r.gossipDataForCatchup(rs, prs, ps) + conR.gossipDataForCatchup(heightLogger, rs, prs, ps, peer) continue OUTER_LOOP } - // if height and round don't match, sleep + // If height and round don't match, sleep. if (rs.Height != prs.Height) || (rs.Round != prs.Round) { - time.Sleep(r.state.config.PeerGossipSleepDuration) + // logger.Info("Peer Height|Round mismatch, sleeping", + // "peerHeight", prs.Height, "peerRound", prs.Round, "peer", peer) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) continue OUTER_LOOP } @@ -568,821 +612,1171 @@ OUTER_LOOP: if rs.Proposal != nil && !prs.Proposal { // Proposal: share the proposal metadata with peer. { - propProto := rs.Proposal.ToProto() - - logger.Debug("sending proposal", "height", prs.Height, "round", prs.Round) - r.dataCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.Proposal{ - Proposal: *propProto, - }, + msg := &ProposalMessage{Proposal: rs.Proposal} + logger.Debug("Sending proposal", "height", prs.Height, "round", prs.Round) + if peer.Send(DataChannel, MustEncode(msg)) { + // NOTE[ZM]: A peer might have received different proposal msg so this Proposal msg will be rejected! + ps.SetHasProposal(rs.Proposal) } - - // NOTE: A peer might have received a different proposal message, so - // this Proposal msg will be rejected! - ps.SetHasProposal(rs.Proposal) } - - // ProposalPOL: lets peer know which POL votes we have so far. The peer - // must receive ProposalMessage first. Note, rs.Proposal was validated, - // so rs.Proposal.POLRound <= rs.Round, so we definitely have - // rs.Votes.Prevotes(rs.Proposal.POLRound). + // ProposalPOL: lets peer know which POL votes we have so far. + // Peer must receive ProposalMessage first. + // rs.Proposal was validated, so rs.Proposal.POLRound <= rs.Round, + // so we definitely have rs.Votes.Prevotes(rs.Proposal.POLRound). if 0 <= rs.Proposal.POLRound { - pPol := rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray() - pPolProto := pPol.ToProto() - - logger.Debug("sending POL", "height", prs.Height, "round", prs.Round) - r.dataCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.ProposalPOL{ - Height: rs.Height, - ProposalPolRound: rs.Proposal.POLRound, - ProposalPol: *pPolProto, - }, + msg := &ProposalPOLMessage{ + Height: rs.Height, + ProposalPOLRound: rs.Proposal.POLRound, + ProposalPOL: rs.Votes.Prevotes(rs.Proposal.POLRound).BitArray(), } + logger.Debug("Sending POL", "height", prs.Height, "round", prs.Round) + peer.Send(DataChannel, MustEncode(msg)) } - continue OUTER_LOOP } - // nothing to do -- sleep - time.Sleep(r.state.config.PeerGossipSleepDuration) + // Nothing to do. Sleep. + time.Sleep(conR.conS.config.PeerGossipSleepDuration) continue OUTER_LOOP } } -// pickSendVote picks a vote and sends it to the peer. It will return true if -// there is a vote to send and false otherwise. -func (r *Reactor) pickSendVote(ps *PeerState, votes types.VoteSetReader) bool { - if vote, ok := ps.PickVoteToSend(votes); ok { - r.Logger.Debug("sending vote message", "ps", ps, "vote", vote) - r.voteCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.Vote{ - Vote: vote.ToProto(), - }, +func (conR *Reactor) gossipDataForCatchup(logger log.Logger, rs *cstypes.RoundState, + prs *cstypes.PeerRoundState, ps *PeerState, peer p2p.Peer) { + + if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok { + // Ensure that the peer's PartSetHeader is correct + blockMeta := conR.conS.blockStore.LoadBlockMeta(prs.Height) + if blockMeta == nil { + logger.Error("Failed to load block meta", "ourHeight", rs.Height, + "blockstoreBase", conR.conS.blockStore.Base(), "blockstoreHeight", conR.conS.blockStore.Height()) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) + return + } else if !blockMeta.BlockID.PartSetHeader.Equals(prs.ProposalBlockPartSetHeader) { + logger.Info("Peer ProposalBlockPartSetHeader mismatch, sleeping", + "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) + return } - - ps.SetHasVote(vote) - return true + // Load the part + part := conR.conS.blockStore.LoadBlockPart(prs.Height, index) + if part == nil { + logger.Error("Could not load part", "index", index, + "blockPartSetHeader", blockMeta.BlockID.PartSetHeader, "peerBlockPartSetHeader", prs.ProposalBlockPartSetHeader) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) + return + } + // Send the part + msg := &BlockPartMessage{ + Height: prs.Height, // Not our height, so it doesn't matter. + Round: prs.Round, // Not our height, so it doesn't matter. + Part: part, + } + logger.Debug("Sending block part for catchup", "round", prs.Round, "index", index) + if peer.Send(DataChannel, MustEncode(msg)) { + ps.SetHasProposalBlockPart(prs.Height, prs.Round, index) + } else { + logger.Debug("Sending block part for catchup failed") + } + return } - - return false + // logger.Info("No parts to send in catch-up, sleeping") + time.Sleep(conR.conS.config.PeerGossipSleepDuration) } -func (r *Reactor) gossipVotesForHeight(rs *cstypes.RoundState, prs *cstypes.PeerRoundState, ps *PeerState) bool { - logger := r.Logger.With("height", prs.Height).With("peer", ps.peerID) +func (conR *Reactor) gossipVotesRoutine(peer p2p.Peer, ps *PeerState) { + logger := conR.Logger.With("peer", peer) - // if there are lastCommits to send... - if prs.Step == cstypes.RoundStepNewHeight { - if r.pickSendVote(ps, rs.LastCommit) { - logger.Debug("picked rs.LastCommit to send") - return true - } - } - - // if there are POL prevotes to send... - if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 { - if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { - if r.pickSendVote(ps, polPrevotes) { - logger.Debug("picked rs.Prevotes(prs.ProposalPOLRound) to send", "round", prs.ProposalPOLRound) - return true - } - } - } - - // if there are prevotes to send... - if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round { - if r.pickSendVote(ps, rs.Votes.Prevotes(prs.Round)) { - logger.Debug("picked rs.Prevotes(prs.Round) to send", "round", prs.Round) - return true - } - } - - // if there are precommits to send... - if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round { - if r.pickSendVote(ps, rs.Votes.Precommits(prs.Round)) { - logger.Debug("picked rs.Precommits(prs.Round) to send", "round", prs.Round) - return true - } - } - - // if there are prevotes to send...(which are needed because of validBlock mechanism) - if prs.Round != -1 && prs.Round <= rs.Round { - if r.pickSendVote(ps, rs.Votes.Prevotes(prs.Round)) { - logger.Debug("picked rs.Prevotes(prs.Round) to send", "round", prs.Round) - return true - } - } - - // if there are POLPrevotes to send... - if prs.ProposalPOLRound != -1 { - if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { - if r.pickSendVote(ps, polPrevotes) { - logger.Debug("picked rs.Prevotes(prs.ProposalPOLRound) to send", "round", prs.ProposalPOLRound) - return true - } - } - } - - return false -} - -func (r *Reactor) gossipVotesRoutine(ps *PeerState) { - logger := r.Logger.With("peer", ps.peerID) - - defer ps.broadcastWG.Done() - - // XXX: simple hack to throttle logs upon sleep - logThrottle := 0 + // Simple hack to throttle logs upon sleep. + var sleeping = 0 OUTER_LOOP: for { - if !r.IsRunning() { + // Manage disconnects from self or peer. + if !peer.IsRunning() || !conR.IsRunning() { + logger.Info("Stopping gossipVotesRoutine for peer") return } - - select { - case <-ps.closer.Done(): - // The peer is marked for removal via a PeerUpdate as the doneCh was - // explicitly closed to signal we should exit. - return - - default: - } - - rs := r.state.GetRoundState() + rs := conR.conS.GetRoundState() prs := ps.GetRoundState() - switch logThrottle { - case 1: // first sleep - logThrottle = 2 - case 2: // no more sleep - logThrottle = 0 + switch sleeping { + case 1: // First sleep + sleeping = 2 + case 2: // No more sleep + sleeping = 0 } - // if height matches, then send LastCommit, Prevotes, and Precommits + // logger.Debug("gossipVotesRoutine", "rsHeight", rs.Height, "rsRound", rs.Round, + // "prsHeight", prs.Height, "prsRound", prs.Round, "prsStep", prs.Step) + + // If height matches, then send LastCommit, Prevotes, Precommits. if rs.Height == prs.Height { - if r.gossipVotesForHeight(rs, prs, ps) { + heightLogger := logger.With("height", prs.Height) + if conR.gossipVotesForHeight(heightLogger, rs, prs, ps) { continue OUTER_LOOP } } - // special catchup logic -- if peer is lagging by height 1, send LastCommit + // Special catchup logic. + // If peer is lagging by height 1, send LastCommit. if prs.Height != 0 && rs.Height == prs.Height+1 { - if r.pickSendVote(ps, rs.LastCommit) { - logger.Debug("picked rs.LastCommit to send", "height", prs.Height) + if ps.PickSendVote(rs.LastCommit) { + logger.Debug("Picked rs.LastCommit to send", "height", prs.Height) continue OUTER_LOOP } } - // catchup logic -- if peer is lagging by more than 1, send Commit - blockStoreBase := r.state.blockStore.Base() + // Catchup logic + // If peer is lagging by more than 1, send Commit. + blockStoreBase := conR.conS.blockStore.Base() if blockStoreBase > 0 && prs.Height != 0 && rs.Height >= prs.Height+2 && prs.Height >= blockStoreBase { - // Load the block commit for prs.Height, which contains precommit - // signatures for prs.Height. - if commit := r.state.blockStore.LoadBlockCommit(prs.Height); commit != nil { - if r.pickSendVote(ps, commit) { - logger.Debug("picked Catchup commit to send", "height", prs.Height) + // Load the block commit for prs.Height, + // which contains precommit signatures for prs.Height. + if commit := conR.conS.blockStore.LoadBlockCommit(prs.Height); commit != nil { + if ps.PickSendVote(commit) { + logger.Debug("Picked Catchup commit to send", "height", prs.Height) continue OUTER_LOOP } } } - if logThrottle == 0 { - // we sent nothing -- sleep - logThrottle = 1 - logger.Debug( - "no votes to send; sleeping", - "rs.Height", rs.Height, - "prs.Height", prs.Height, + if sleeping == 0 { + // We sent nothing. Sleep... + sleeping = 1 + logger.Debug("No votes to send, sleeping", "rs.Height", rs.Height, "prs.Height", prs.Height, "localPV", rs.Votes.Prevotes(rs.Round).BitArray(), "peerPV", prs.Prevotes, - "localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits, - ) - } else if logThrottle == 2 { - logThrottle = 1 + "localPC", rs.Votes.Precommits(rs.Round).BitArray(), "peerPC", prs.Precommits) + } else if sleeping == 2 { + // Continued sleep... + sleeping = 1 } - time.Sleep(r.state.config.PeerGossipSleepDuration) + time.Sleep(conR.conS.config.PeerGossipSleepDuration) continue OUTER_LOOP } } +func (conR *Reactor) gossipVotesForHeight( + logger log.Logger, + rs *cstypes.RoundState, + prs *cstypes.PeerRoundState, + ps *PeerState, +) bool { + + // If there are lastCommits to send... + if prs.Step == cstypes.RoundStepNewHeight { + if ps.PickSendVote(rs.LastCommit) { + logger.Debug("Picked rs.LastCommit to send") + return true + } + } + // If there are POL prevotes to send... + if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 { + if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { + if ps.PickSendVote(polPrevotes) { + logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", + "round", prs.ProposalPOLRound) + return true + } + } + } + // If there are prevotes to send... + if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round { + if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { + logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) + return true + } + } + // If there are precommits to send... + if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round { + if ps.PickSendVote(rs.Votes.Precommits(prs.Round)) { + logger.Debug("Picked rs.Precommits(prs.Round) to send", "round", prs.Round) + return true + } + } + // If there are prevotes to send...Needed because of validBlock mechanism + if prs.Round != -1 && prs.Round <= rs.Round { + if ps.PickSendVote(rs.Votes.Prevotes(prs.Round)) { + logger.Debug("Picked rs.Prevotes(prs.Round) to send", "round", prs.Round) + return true + } + } + // If there are POLPrevotes to send... + if prs.ProposalPOLRound != -1 { + if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil { + if ps.PickSendVote(polPrevotes) { + logger.Debug("Picked rs.Prevotes(prs.ProposalPOLRound) to send", + "round", prs.ProposalPOLRound) + return true + } + } + } + + return false +} + // NOTE: `queryMaj23Routine` has a simple crude design since it only comes // into play for liveness when there's a signature DDoS attack happening. -func (r *Reactor) queryMaj23Routine(ps *PeerState) { - defer ps.broadcastWG.Done() +func (conR *Reactor) queryMaj23Routine(peer p2p.Peer, ps *PeerState) { + logger := conR.Logger.With("peer", peer) OUTER_LOOP: for { - if !r.IsRunning() { + // Manage disconnects from self or peer. + if !peer.IsRunning() || !conR.IsRunning() { + logger.Info("Stopping queryMaj23Routine for peer") return } - select { - case <-ps.closer.Done(): - // The peer is marked for removal via a PeerUpdate as the doneCh was - // explicitly closed to signal we should exit. - return - - default: - } - - // maybe send Height/Round/Prevotes + // Maybe send Height/Round/Prevotes { - rs := r.state.GetRoundState() + rs := conR.conS.GetRoundState() prs := ps.GetRoundState() - if rs.Height == prs.Height { if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok { - r.stateCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.VoteSetMaj23{ - Height: prs.Height, - Round: prs.Round, - Type: tmproto.PrevoteType, - BlockID: maj23.ToProto(), - }, - } - - time.Sleep(r.state.config.PeerQueryMaj23SleepDuration) + peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.Round, + Type: tmproto.PrevoteType, + BlockID: maj23, + })) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) } } } - // maybe send Height/Round/Precommits + // Maybe send Height/Round/Precommits { - rs := r.state.GetRoundState() + rs := conR.conS.GetRoundState() prs := ps.GetRoundState() - if rs.Height == prs.Height { if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok { - r.stateCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.VoteSetMaj23{ - Height: prs.Height, - Round: prs.Round, - Type: tmproto.PrecommitType, - BlockID: maj23.ToProto(), - }, - } - - time.Sleep(r.state.config.PeerQueryMaj23SleepDuration) + peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.Round, + Type: tmproto.PrecommitType, + BlockID: maj23, + })) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) } } } - // maybe send Height/Round/ProposalPOL + // Maybe send Height/Round/ProposalPOL { - rs := r.state.GetRoundState() + rs := conR.conS.GetRoundState() prs := ps.GetRoundState() - if rs.Height == prs.Height && prs.ProposalPOLRound >= 0 { if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok { - r.stateCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.VoteSetMaj23{ - Height: prs.Height, - Round: prs.ProposalPOLRound, - Type: tmproto.PrevoteType, - BlockID: maj23.ToProto(), - }, - } - - time.Sleep(r.state.config.PeerQueryMaj23SleepDuration) + peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ + Height: prs.Height, + Round: prs.ProposalPOLRound, + Type: tmproto.PrevoteType, + BlockID: maj23, + })) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) } } } - // Little point sending LastCommitRound/LastCommit, these are fleeting and - // non-blocking. + // Little point sending LastCommitRound/LastCommit, + // These are fleeting and non-blocking. - // maybe send Height/CatchupCommitRound/CatchupCommit + // Maybe send Height/CatchupCommitRound/CatchupCommit. { prs := ps.GetRoundState() - - if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= r.state.blockStore.Height() && - prs.Height >= r.state.blockStore.Base() { - if commit := r.state.LoadCommit(prs.Height); commit != nil { - r.stateCh.Out <- p2p.Envelope{ - To: ps.peerID, - Message: &tmcons.VoteSetMaj23{ - Height: prs.Height, - Round: commit.Round, - Type: tmproto.PrecommitType, - BlockID: commit.BlockID.ToProto(), - }, - } - - time.Sleep(r.state.config.PeerQueryMaj23SleepDuration) + if prs.CatchupCommitRound != -1 && prs.Height > 0 && prs.Height <= conR.conS.blockStore.Height() && + prs.Height >= conR.conS.blockStore.Base() { + if commit := conR.conS.LoadCommit(prs.Height); commit != nil { + peer.TrySend(StateChannel, MustEncode(&VoteSetMaj23Message{ + Height: prs.Height, + Round: commit.Round, + Type: tmproto.PrecommitType, + BlockID: commit.BlockID, + })) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) } } } - time.Sleep(r.state.config.PeerQueryMaj23SleepDuration) + time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) + continue OUTER_LOOP } } -// processPeerUpdate process a peer update message. For new or reconnected peers, -// we create a peer state if one does not exist for the peer, which should always -// be the case, and we spawn all the relevant goroutine to broadcast messages to -// the peer. During peer removal, we remove the peer for our set of peers and -// signal to all spawned goroutines to gracefully exit in a non-blocking manner. -func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) { - r.Logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status) - - r.mtx.Lock() - defer r.mtx.Unlock() - - switch peerUpdate.Status { - case p2p.PeerStatusUp: - // Do not allow starting new broadcasting goroutines after reactor shutdown - // has been initiated. This can happen after we've manually closed all - // peer goroutines and closed r.closeCh, but the router still sends in-flight - // peer updates. - if !r.IsRunning() { - return - } - - var ( - ps *PeerState - ok bool - ) - - ps, ok = r.peers[peerUpdate.NodeID] - if !ok { - ps = NewPeerState(r.Logger, peerUpdate.NodeID) - r.peers[peerUpdate.NodeID] = ps - } - - if !ps.IsRunning() { - // Set the peer state's closer to signal to all spawned goroutines to exit - // when the peer is removed. We also set the running state to ensure we - // do not spawn multiple instances of the same goroutines and finally we - // set the waitgroup counter so we know when all goroutines have exited. - ps.broadcastWG.Add(3) - ps.SetRunning(true) - - // start goroutines for this peer - go r.gossipDataRoutine(ps) - go r.gossipVotesRoutine(ps) - go r.queryMaj23Routine(ps) - - // Send our state to the peer. If we're fast-syncing, broadcast a - // RoundStepMessage later upon SwitchToConsensus(). - if !r.waitSync { - go r.sendNewRoundStepMessage(ps.peerID) - } - } - - case p2p.PeerStatusDown: - ps, ok := r.peers[peerUpdate.NodeID] - if ok && ps.IsRunning() { - // signal to all spawned goroutines for the peer to gracefully exit - ps.closer.Close() - - go func() { - // Wait for all spawned broadcast goroutines to exit before marking the - // peer state as no longer running and removal from the peers map. - ps.broadcastWG.Wait() - - r.mtx.Lock() - delete(r.peers, peerUpdate.NodeID) - r.mtx.Unlock() - - ps.SetRunning(false) - }() - } - } -} - -// handleStateMessage handles envelopes sent from peers on the StateChannel. -// An error is returned if the message is unrecognized or if validation fails. -// If we fail to find the peer state for the envelope sender, we perform a no-op -// and return. This can happen when we process the envelope after the peer is -// removed. -func (r *Reactor) handleStateMessage(envelope p2p.Envelope, msgI Message) error { - ps, ok := r.GetPeerState(envelope.From) - if !ok || ps == nil { - r.Logger.Debug("failed to find peer state", "peer", envelope.From, "ch_id", "StateChannel") - return nil - } - - switch msg := envelope.Message.(type) { - case *tmcons.NewRoundStep: - r.state.mtx.RLock() - initialHeight := r.state.state.InitialHeight - r.state.mtx.RUnlock() - - if err := msgI.(*NewRoundStepMessage).ValidateHeight(initialHeight); err != nil { - r.Logger.Error("peer sent us an invalid msg", "msg", msg, "err", err) - return err - } - - ps.ApplyNewRoundStepMessage(msgI.(*NewRoundStepMessage)) - - case *tmcons.NewValidBlock: - ps.ApplyNewValidBlockMessage(msgI.(*NewValidBlockMessage)) - - case *tmcons.HasVote: - ps.ApplyHasVoteMessage(msgI.(*HasVoteMessage)) - - case *tmcons.VoteSetMaj23: - r.state.mtx.RLock() - height, votes := r.state.Height, r.state.Votes - r.state.mtx.RUnlock() - - if height != msg.Height { - return nil - } - - vsmMsg := msgI.(*VoteSetMaj23Message) - - // peer claims to have a maj23 for some BlockID at - err := votes.SetPeerMaj23(msg.Round, msg.Type, ps.peerID, vsmMsg.BlockID) - if err != nil { - return err - } - - // Respond with a VoteSetBitsMessage showing which votes we have and - // consequently shows which we don't have. - var ourVotes *bits.BitArray - switch vsmMsg.Type { - case tmproto.PrevoteType: - ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(vsmMsg.BlockID) - - case tmproto.PrecommitType: - ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(vsmMsg.BlockID) - - default: - panic("bad VoteSetBitsMessage field type; forgot to add a check in ValidateBasic?") - } - - eMsg := &tmcons.VoteSetBits{ - Height: msg.Height, - Round: msg.Round, - Type: msg.Type, - BlockID: msg.BlockID, - } - - if votesProto := ourVotes.ToProto(); votesProto != nil { - eMsg.Votes = *votesProto - } - - r.voteSetBitsCh.Out <- p2p.Envelope{ - To: envelope.From, - Message: eMsg, - } - - default: - return fmt.Errorf("received unknown message on StateChannel: %T", msg) - } - - return nil -} - -// handleDataMessage handles envelopes sent from peers on the DataChannel. If we -// fail to find the peer state for the envelope sender, we perform a no-op and -// return. This can happen when we process the envelope after the peer is -// removed. -func (r *Reactor) handleDataMessage(envelope p2p.Envelope, msgI Message) error { - logger := r.Logger.With("peer", envelope.From, "ch_id", "DataChannel") - - ps, ok := r.GetPeerState(envelope.From) - if !ok || ps == nil { - r.Logger.Debug("failed to find peer state") - return nil - } - - if r.WaitSync() { - logger.Info("ignoring message received during sync", "msg", msgI) - return nil - } - - switch msg := envelope.Message.(type) { - case *tmcons.Proposal: - pMsg := msgI.(*ProposalMessage) - - ps.SetHasProposal(pMsg.Proposal) - r.state.peerMsgQueue <- msgInfo{pMsg, envelope.From} - - case *tmcons.ProposalPOL: - ps.ApplyProposalPOLMessage(msgI.(*ProposalPOLMessage)) - - case *tmcons.BlockPart: - bpMsg := msgI.(*BlockPartMessage) - - ps.SetHasProposalBlockPart(bpMsg.Height, bpMsg.Round, int(bpMsg.Part.Index)) - r.Metrics.BlockParts.With("peer_id", string(envelope.From)).Add(1) - r.state.peerMsgQueue <- msgInfo{bpMsg, envelope.From} - - default: - return fmt.Errorf("received unknown message on DataChannel: %T", msg) - } - - return nil -} - -// handleVoteMessage handles envelopes sent from peers on the VoteChannel. If we -// fail to find the peer state for the envelope sender, we perform a no-op and -// return. This can happen when we process the envelope after the peer is -// removed. -func (r *Reactor) handleVoteMessage(envelope p2p.Envelope, msgI Message) error { - logger := r.Logger.With("peer", envelope.From, "ch_id", "VoteChannel") - - ps, ok := r.GetPeerState(envelope.From) - if !ok || ps == nil { - r.Logger.Debug("failed to find peer state") - return nil - } - - if r.WaitSync() { - logger.Info("ignoring message received during sync", "msg", msgI) - return nil - } - - switch msg := envelope.Message.(type) { - case *tmcons.Vote: - r.state.mtx.RLock() - height, valSize, lastCommitSize := r.state.Height, r.state.Validators.Size(), r.state.LastCommit.Size() - r.state.mtx.RUnlock() - - vMsg := msgI.(*VoteMessage) - - ps.EnsureVoteBitArrays(height, valSize) - ps.EnsureVoteBitArrays(height-1, lastCommitSize) - ps.SetHasVote(vMsg.Vote) - - r.state.peerMsgQueue <- msgInfo{vMsg, envelope.From} - - default: - return fmt.Errorf("received unknown message on VoteChannel: %T", msg) - } - - return nil -} - -// handleVoteSetBitsMessage handles envelopes sent from peers on the -// VoteSetBitsChannel. If we fail to find the peer state for the envelope sender, -// we perform a no-op and return. This can happen when we process the envelope -// after the peer is removed. -func (r *Reactor) handleVoteSetBitsMessage(envelope p2p.Envelope, msgI Message) error { - logger := r.Logger.With("peer", envelope.From, "ch_id", "VoteSetBitsChannel") - - ps, ok := r.GetPeerState(envelope.From) - if !ok || ps == nil { - r.Logger.Debug("failed to find peer state") - return nil - } - - if r.WaitSync() { - logger.Info("ignoring message received during sync", "msg", msgI) - return nil - } - - switch msg := envelope.Message.(type) { - case *tmcons.VoteSetBits: - r.state.mtx.RLock() - height, votes := r.state.Height, r.state.Votes - r.state.mtx.RUnlock() - - vsbMsg := msgI.(*VoteSetBitsMessage) - - if height == msg.Height { - var ourVotes *bits.BitArray - - switch msg.Type { - case tmproto.PrevoteType: - ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(vsbMsg.BlockID) - - case tmproto.PrecommitType: - ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(vsbMsg.BlockID) - - default: - panic("bad VoteSetBitsMessage field type; forgot to add a check in ValidateBasic?") - } - - ps.ApplyVoteSetBitsMessage(vsbMsg, ourVotes) - } else { - ps.ApplyVoteSetBitsMessage(vsbMsg, nil) - } - - default: - return fmt.Errorf("received unknown message on VoteSetBitsChannel: %T", msg) - } - - return nil -} - -// handleMessage handles an Envelope sent from a peer on a specific p2p Channel. -// It will handle errors and any possible panics gracefully. A caller can handle -// any error returned by sending a PeerError on the respective channel. -// -// NOTE: We process these messages even when we're fast_syncing. Messages affect -// either a peer state or the consensus state. Peer state updates can happen in -// parallel, but processing of proposals, block parts, and votes are ordered by -// the p2p channel. -// -// NOTE: We block on consensus state for proposals, block parts, and votes. -func (r *Reactor) handleMessage(chID p2p.ChannelID, envelope p2p.Envelope) (err error) { - defer func() { - if e := recover(); e != nil { - err = fmt.Errorf("panic in processing message: %v", e) - r.Logger.Error("recovering from processing message panic", "err", err) - } - }() - - // We wrap the envelope's message in a Proto wire type so we can convert back - // the domain type that individual channel message handlers can work with. We - // do this here once to avoid having to do it for each individual message type. - // and because a large part of the core business logic depends on these - // domain types opposed to simply working with the Proto types. - protoMsg := new(tmcons.Message) - if err := protoMsg.Wrap(envelope.Message); err != nil { - return err - } - - msgI, err := MsgFromProto(protoMsg) - if err != nil { - return err - } - - r.Logger.Debug("received message", "ch_id", chID, "message", msgI, "peer", envelope.From) - - switch chID { - case StateChannel: - err = r.handleStateMessage(envelope, msgI) - - case DataChannel: - err = r.handleDataMessage(envelope, msgI) - - case VoteChannel: - err = r.handleVoteMessage(envelope, msgI) - - case VoteSetBitsChannel: - err = r.handleVoteSetBitsMessage(envelope, msgI) - - default: - err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope) - } - - return err -} - -// processStateCh initiates a blocking process where we listen for and handle -// envelopes on the StateChannel. Any error encountered during message -// execution will result in a PeerError being sent on the StateChannel. When -// the reactor is stopped, we will catch the signal and close the p2p Channel -// gracefully. -func (r *Reactor) processStateCh() { - defer r.stateCh.Close() - +func (conR *Reactor) peerStatsRoutine() { for { - select { - case envelope := <-r.stateCh.In: - if err := r.handleMessage(r.stateCh.ID, envelope); err != nil { - r.Logger.Error("failed to process message", "ch_id", r.stateCh.ID, "envelope", envelope, "err", err) - r.stateCh.Error <- p2p.PeerError{ - NodeID: envelope.From, - Err: err, - } - } - - case <-r.stateCloseCh: - r.Logger.Debug("stopped listening on StateChannel; closing...") - return - } - } -} - -// processDataCh initiates a blocking process where we listen for and handle -// envelopes on the DataChannel. Any error encountered during message -// execution will result in a PeerError being sent on the DataChannel. When -// the reactor is stopped, we will catch the signal and close the p2p Channel -// gracefully. -func (r *Reactor) processDataCh() { - defer r.dataCh.Close() - - for { - select { - case envelope := <-r.dataCh.In: - if err := r.handleMessage(r.dataCh.ID, envelope); err != nil { - r.Logger.Error("failed to process message", "ch_id", r.dataCh.ID, "envelope", envelope, "err", err) - r.dataCh.Error <- p2p.PeerError{ - NodeID: envelope.From, - Err: err, - } - } - - case <-r.closeCh: - r.Logger.Debug("stopped listening on DataChannel; closing...") - return - } - } -} - -// processVoteCh initiates a blocking process where we listen for and handle -// envelopes on the VoteChannel. Any error encountered during message -// execution will result in a PeerError being sent on the VoteChannel. When -// the reactor is stopped, we will catch the signal and close the p2p Channel -// gracefully. -func (r *Reactor) processVoteCh() { - defer r.voteCh.Close() - - for { - select { - case envelope := <-r.voteCh.In: - if err := r.handleMessage(r.voteCh.ID, envelope); err != nil { - r.Logger.Error("failed to process message", "ch_id", r.voteCh.ID, "envelope", envelope, "err", err) - r.voteCh.Error <- p2p.PeerError{ - NodeID: envelope.From, - Err: err, - } - } - - case <-r.closeCh: - r.Logger.Debug("stopped listening on VoteChannel; closing...") - return - } - } -} - -// processVoteCh initiates a blocking process where we listen for and handle -// envelopes on the VoteSetBitsChannel. Any error encountered during message -// execution will result in a PeerError being sent on the VoteSetBitsChannel. -// When the reactor is stopped, we will catch the signal and close the p2p -// Channel gracefully. -func (r *Reactor) processVoteSetBitsCh() { - defer r.voteSetBitsCh.Close() - - for { - select { - case envelope := <-r.voteSetBitsCh.In: - if err := r.handleMessage(r.voteSetBitsCh.ID, envelope); err != nil { - r.Logger.Error("failed to process message", "ch_id", r.voteSetBitsCh.ID, "envelope", envelope, "err", err) - r.voteSetBitsCh.Error <- p2p.PeerError{ - NodeID: envelope.From, - Err: err, - } - } - - case <-r.closeCh: - r.Logger.Debug("stopped listening on VoteSetBitsChannel; closing...") - return - } - } -} - -// processPeerUpdates initiates a blocking process where we listen for and handle -// PeerUpdate messages. When the reactor is stopped, we will catch the signal and -// close the p2p PeerUpdatesCh gracefully. -func (r *Reactor) processPeerUpdates() { - defer r.peerUpdates.Close() - - for { - select { - case peerUpdate := <-r.peerUpdates.Updates(): - r.processPeerUpdate(peerUpdate) - - case <-r.closeCh: - r.Logger.Debug("stopped listening on peer updates channel; closing...") - return - } - } -} - -func (r *Reactor) peerStatsRoutine() { - for { - if !r.IsRunning() { - r.Logger.Info("stopping peerStatsRoutine") + if !conR.IsRunning() { + conR.Logger.Info("Stopping peerStatsRoutine") return } select { - case msg := <-r.state.statsMsgQueue: - ps, ok := r.GetPeerState(msg.PeerID) - if !ok || ps == nil { - r.Logger.Debug("attempt to update stats for non-existent peer", "peer", msg.PeerID) + case msg := <-conR.conS.statsMsgQueue: + // Get peer + peer := conR.Switch.Peers().Get(msg.PeerID) + if peer == nil { + conR.Logger.Debug("Attempt to update stats for non-existent peer", + "peer", msg.PeerID) continue } - + // Get peer state + ps, ok := peer.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } switch msg.Msg.(type) { case *VoteMessage: - if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 { // nolint: staticcheck - // TODO: Handle peer quality via the peer manager. - // r.Switch.MarkPeerAsGood(peer) + if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 { + conR.Switch.MarkPeerAsGood(peer) } - case *BlockPartMessage: - if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 { // nolint: staticcheck - // TODO: Handle peer quality via the peer manager. - // r.Switch.MarkPeerAsGood(peer) + if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 { + conR.Switch.MarkPeerAsGood(peer) } } + case <-conR.conS.Quit(): + return - case <-r.closeCh: + case <-conR.Quit(): return } } } + +// String returns a string representation of the Reactor. +// NOTE: For now, it is just a hard-coded string to avoid accessing unprotected shared variables. +// TODO: improve! +func (conR *Reactor) String() string { + // better not to access shared variables + return "ConsensusReactor" // conR.StringIndented("") +} + +// StringIndented returns an indented string representation of the Reactor +func (conR *Reactor) StringIndented(indent string) string { + s := "ConsensusReactor{\n" + s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" + for _, peer := range conR.Switch.Peers().List() { + ps, ok := peer.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } + s += indent + " " + ps.StringIndented(indent+" ") + "\n" + } + s += indent + "}" + return s +} + +// ReactorMetrics sets the metrics +func ReactorMetrics(metrics *Metrics) ReactorOption { + return func(conR *Reactor) { conR.Metrics = metrics } +} + +//----------------------------------------------------------------------------- + +var ( + ErrPeerStateHeightRegression = errors.New("error peer state height regression") + ErrPeerStateInvalidStartTime = errors.New("error peer state invalid startTime") +) + +// PeerState contains the known state of a peer, including its connection and +// threadsafe access to its PeerRoundState. +// NOTE: THIS GETS DUMPED WITH rpc/core/consensus.go. +// Be mindful of what you Expose. +type PeerState struct { + peer p2p.Peer + logger log.Logger + + mtx sync.Mutex // NOTE: Modify below using setters, never directly. + PRS cstypes.PeerRoundState `json:"round_state"` // Exposed. + Stats *peerStateStats `json:"stats"` // Exposed. +} + +// peerStateStats holds internal statistics for a peer. +type peerStateStats struct { + Votes int `json:"votes"` + BlockParts int `json:"block_parts"` +} + +func (pss peerStateStats) String() string { + return fmt.Sprintf("peerStateStats{votes: %d, blockParts: %d}", + pss.Votes, pss.BlockParts) +} + +// NewPeerState returns a new PeerState for the given Peer +func NewPeerState(peer p2p.Peer) *PeerState { + return &PeerState{ + peer: peer, + logger: log.NewNopLogger(), + PRS: cstypes.PeerRoundState{ + Round: -1, + ProposalPOLRound: -1, + LastCommitRound: -1, + CatchupCommitRound: -1, + }, + Stats: &peerStateStats{}, + } +} + +// SetLogger allows to set a logger on the peer state. Returns the peer state +// itself. +func (ps *PeerState) SetLogger(logger log.Logger) *PeerState { + ps.logger = logger + return ps +} + +// GetRoundState returns an shallow copy of the PeerRoundState. +// There's no point in mutating it since it won't change PeerState. +func (ps *PeerState) GetRoundState() *cstypes.PeerRoundState { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + prs := ps.PRS // copy + return &prs +} + +// ToJSON returns a json of PeerState. +func (ps *PeerState) ToJSON() ([]byte, error) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + return tmjson.Marshal(ps) +} + +// GetHeight returns an atomic snapshot of the PeerRoundState's height +// used by the mempool to ensure peers are caught up before broadcasting new txs +func (ps *PeerState) GetHeight() int64 { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return ps.PRS.Height +} + +// SetHasProposal sets the given proposal as known for the peer. +func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round { + return + } + + if ps.PRS.Proposal { + return + } + + ps.PRS.Proposal = true + + // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage + if ps.PRS.ProposalBlockParts != nil { + return + } + + ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader + ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) + ps.PRS.ProposalPOLRound = proposal.POLRound + ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. +} + +// InitProposalBlockParts initializes the peer's proposal block parts header and bit array. +func (ps *PeerState) InitProposalBlockParts(partSetHeader types.PartSetHeader) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.ProposalBlockParts != nil { + return + } + + ps.PRS.ProposalBlockPartSetHeader = partSetHeader + ps.PRS.ProposalBlockParts = bits.NewBitArray(int(partSetHeader.Total)) +} + +// SetHasProposalBlockPart sets the given block part index as known for the peer. +func (ps *PeerState) SetHasProposalBlockPart(height int64, round int32, index int) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.Height != height || ps.PRS.Round != round { + return + } + + ps.PRS.ProposalBlockParts.SetIndex(index, true) +} + +// PickSendVote picks a vote and sends it to the peer. +// Returns true if vote was sent. +func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool { + if vote, ok := ps.PickVoteToSend(votes); ok { + msg := &VoteMessage{vote} + ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote) + if ps.peer.Send(VoteChannel, MustEncode(msg)) { + ps.SetHasVote(vote) + return true + } + return false + } + return false +} + +// PickVoteToSend picks a vote to send to the peer. +// Returns true if a vote was picked. +// NOTE: `votes` must be the correct Size() for the Height(). +func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote, ok bool) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if votes.Size() == 0 { + return nil, false + } + + height, round, votesType, size := + votes.GetHeight(), votes.GetRound(), tmproto.SignedMsgType(votes.Type()), votes.Size() + + // Lazily set data using 'votes'. + if votes.IsCommit() { + ps.ensureCatchupCommitRound(height, round, size) + } + ps.ensureVoteBitArrays(height, size) + + psVotes := ps.getVoteBitArray(height, round, votesType) + if psVotes == nil { + return nil, false // Not something worth sending + } + if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { + return votes.GetByIndex(int32(index)), true + } + return nil, false +} + +func (ps *PeerState) getVoteBitArray(height int64, round int32, votesType tmproto.SignedMsgType) *bits.BitArray { + if !types.IsVoteTypeValid(votesType) { + return nil + } + + if ps.PRS.Height == height { + if ps.PRS.Round == round { + switch votesType { + case tmproto.PrevoteType: + return ps.PRS.Prevotes + case tmproto.PrecommitType: + return ps.PRS.Precommits + } + } + if ps.PRS.CatchupCommitRound == round { + switch votesType { + case tmproto.PrevoteType: + return nil + case tmproto.PrecommitType: + return ps.PRS.CatchupCommit + } + } + if ps.PRS.ProposalPOLRound == round { + switch votesType { + case tmproto.PrevoteType: + return ps.PRS.ProposalPOL + case tmproto.PrecommitType: + return nil + } + } + return nil + } + if ps.PRS.Height == height+1 { + if ps.PRS.LastCommitRound == round { + switch votesType { + case tmproto.PrevoteType: + return nil + case tmproto.PrecommitType: + return ps.PRS.LastCommit + } + } + return nil + } + return nil +} + +// 'round': A round for which we have a +2/3 commit. +func (ps *PeerState) ensureCatchupCommitRound(height int64, round int32, numValidators int) { + if ps.PRS.Height != height { + return + } + /* + NOTE: This is wrong, 'round' could change. + e.g. if orig round is not the same as block LastCommit round. + if ps.CatchupCommitRound != -1 && ps.CatchupCommitRound != round { + panic(fmt.Sprintf( + "Conflicting CatchupCommitRound. Height: %v, + Orig: %v, + New: %v", + height, + ps.CatchupCommitRound, + round)) + } + */ + if ps.PRS.CatchupCommitRound == round { + return // Nothing to do! + } + ps.PRS.CatchupCommitRound = round + if round == ps.PRS.Round { + ps.PRS.CatchupCommit = ps.PRS.Precommits + } else { + ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) + } +} + +// EnsureVoteBitArrays ensures the bit-arrays have been allocated for tracking +// what votes this peer has received. +// NOTE: It's important to make sure that numValidators actually matches +// what the node sees as the number of validators for height. +func (ps *PeerState) EnsureVoteBitArrays(height int64, numValidators int) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + ps.ensureVoteBitArrays(height, numValidators) +} + +func (ps *PeerState) ensureVoteBitArrays(height int64, numValidators int) { + if ps.PRS.Height == height { + if ps.PRS.Prevotes == nil { + ps.PRS.Prevotes = bits.NewBitArray(numValidators) + } + if ps.PRS.Precommits == nil { + ps.PRS.Precommits = bits.NewBitArray(numValidators) + } + if ps.PRS.CatchupCommit == nil { + ps.PRS.CatchupCommit = bits.NewBitArray(numValidators) + } + if ps.PRS.ProposalPOL == nil { + ps.PRS.ProposalPOL = bits.NewBitArray(numValidators) + } + } else if ps.PRS.Height == height+1 { + if ps.PRS.LastCommit == nil { + ps.PRS.LastCommit = bits.NewBitArray(numValidators) + } + } +} + +// RecordVote increments internal votes related statistics for this peer. +// It returns the total number of added votes. +func (ps *PeerState) RecordVote() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + ps.Stats.Votes++ + + return ps.Stats.Votes +} + +// VotesSent returns the number of blocks for which peer has been sending us +// votes. +func (ps *PeerState) VotesSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + return ps.Stats.Votes +} + +// RecordBlockPart increments internal block part related statistics for this peer. +// It returns the total number of added block parts. +func (ps *PeerState) RecordBlockPart() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + ps.Stats.BlockParts++ + return ps.Stats.BlockParts +} + +// BlockPartsSent returns the number of useful block parts the peer has sent us. +func (ps *PeerState) BlockPartsSent() int { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + return ps.Stats.BlockParts +} + +// SetHasVote sets the given vote as known by the peer +func (ps *PeerState) SetHasVote(vote *types.Vote) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) +} + +func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { + logger := ps.logger.With( + "peerH/R", + fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round), + "H/R", + fmt.Sprintf("%d/%d", height, round)) + logger.Debug("setHasVote", "type", voteType, "index", index) + + // NOTE: some may be nil BitArrays -> no side effects. + psVotes := ps.getVoteBitArray(height, round, voteType) + if psVotes != nil { + psVotes.SetIndex(int(index), true) + } +} + +// ApplyNewRoundStepMessage updates the peer state for the new round. +func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + // Ignore duplicates or decreases + if CompareHRS(msg.Height, msg.Round, msg.Step, ps.PRS.Height, ps.PRS.Round, ps.PRS.Step) <= 0 { + return + } + + // Just remember these values. + psHeight := ps.PRS.Height + psRound := ps.PRS.Round + psCatchupCommitRound := ps.PRS.CatchupCommitRound + psCatchupCommit := ps.PRS.CatchupCommit + + startTime := tmtime.Now().Add(-1 * time.Duration(msg.SecondsSinceStartTime) * time.Second) + ps.PRS.Height = msg.Height + ps.PRS.Round = msg.Round + ps.PRS.Step = msg.Step + ps.PRS.StartTime = startTime + if psHeight != msg.Height || psRound != msg.Round { + ps.PRS.Proposal = false + ps.PRS.ProposalBlockPartSetHeader = types.PartSetHeader{} + ps.PRS.ProposalBlockParts = nil + ps.PRS.ProposalPOLRound = -1 + ps.PRS.ProposalPOL = nil + // We'll update the BitArray capacity later. + ps.PRS.Prevotes = nil + ps.PRS.Precommits = nil + } + if psHeight == msg.Height && psRound != msg.Round && msg.Round == psCatchupCommitRound { + // Peer caught up to CatchupCommitRound. + // Preserve psCatchupCommit! + // NOTE: We prefer to use prs.Precommits if + // pr.Round matches pr.CatchupCommitRound. + ps.PRS.Precommits = psCatchupCommit + } + if psHeight != msg.Height { + // Shift Precommits to LastCommit. + if psHeight+1 == msg.Height && psRound == msg.LastCommitRound { + ps.PRS.LastCommitRound = msg.LastCommitRound + ps.PRS.LastCommit = ps.PRS.Precommits + } else { + ps.PRS.LastCommitRound = msg.LastCommitRound + ps.PRS.LastCommit = nil + } + // We'll update the BitArray capacity later. + ps.PRS.CatchupCommitRound = -1 + ps.PRS.CatchupCommit = nil + } +} + +// ApplyNewValidBlockMessage updates the peer state for the new valid block. +func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.Height != msg.Height { + return + } + + if ps.PRS.Round != msg.Round && !msg.IsCommit { + return + } + + ps.PRS.ProposalBlockPartSetHeader = msg.BlockPartSetHeader + ps.PRS.ProposalBlockParts = msg.BlockParts +} + +// ApplyProposalPOLMessage updates the peer state for the new proposal POL. +func (ps *PeerState) ApplyProposalPOLMessage(msg *ProposalPOLMessage) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.Height != msg.Height { + return + } + if ps.PRS.ProposalPOLRound != msg.ProposalPOLRound { + return + } + + // TODO: Merge onto existing ps.PRS.ProposalPOL? + // We might have sent some prevotes in the meantime. + ps.PRS.ProposalPOL = msg.ProposalPOL +} + +// ApplyHasVoteMessage updates the peer state for the new vote. +func (ps *PeerState) ApplyHasVoteMessage(msg *HasVoteMessage) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if ps.PRS.Height != msg.Height { + return + } + + ps.setHasVote(msg.Height, msg.Round, msg.Type, msg.Index) +} + +// ApplyVoteSetBitsMessage updates the peer state for the bit-array of votes +// it claims to have for the corresponding BlockID. +// `ourVotes` is a BitArray of votes we have for msg.BlockID +// NOTE: if ourVotes is nil (e.g. msg.Height < rs.Height), +// we conservatively overwrite ps's votes w/ msg.Votes. +func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes *bits.BitArray) { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + votes := ps.getVoteBitArray(msg.Height, msg.Round, msg.Type) + if votes != nil { + if ourVotes == nil { + votes.Update(msg.Votes) + } else { + otherVotes := votes.Sub(ourVotes) + hasVotes := otherVotes.Or(msg.Votes) + votes.Update(hasVotes) + } + } +} + +// String returns a string representation of the PeerState +func (ps *PeerState) String() string { + return ps.StringIndented("") +} + +// StringIndented returns a string representation of the PeerState +func (ps *PeerState) StringIndented(indent string) string { + ps.mtx.Lock() + defer ps.mtx.Unlock() + return fmt.Sprintf(`PeerState{ +%s Key %v +%s RoundState %v +%s Stats %v +%s}`, + indent, ps.peer.ID(), + indent, ps.PRS.StringIndented(indent+" "), + indent, ps.Stats, + indent) +} + +//----------------------------------------------------------------------------- +// Messages + +// Message is a message that can be sent and received on the Reactor +type Message interface { + ValidateBasic() error +} + +func init() { + tmjson.RegisterType(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage") + tmjson.RegisterType(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage") + tmjson.RegisterType(&ProposalMessage{}, "tendermint/Proposal") + tmjson.RegisterType(&ProposalPOLMessage{}, "tendermint/ProposalPOL") + tmjson.RegisterType(&BlockPartMessage{}, "tendermint/BlockPart") + tmjson.RegisterType(&VoteMessage{}, "tendermint/Vote") + tmjson.RegisterType(&HasVoteMessage{}, "tendermint/HasVote") + tmjson.RegisterType(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23") + tmjson.RegisterType(&VoteSetBitsMessage{}, "tendermint/VoteSetBits") +} + +func decodeMsg(bz []byte) (msg Message, err error) { + pb := &tmcons.Message{} + if err = proto.Unmarshal(bz, pb); err != nil { + return msg, err + } + + return MsgFromProto(pb) +} + +//------------------------------------- + +// NewRoundStepMessage is sent for every step taken in the ConsensusState. +// For every height/round/step transition +type NewRoundStepMessage struct { + Height int64 + Round int32 + Step cstypes.RoundStepType + SecondsSinceStartTime int64 + LastCommitRound int32 +} + +// ValidateBasic performs basic validation. +func (m *NewRoundStepMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.Round < 0 { + return errors.New("negative Round") + } + if !m.Step.IsValid() { + return errors.New("invalid Step") + } + + // NOTE: SecondsSinceStartTime may be negative + + // LastCommitRound will be -1 for the initial height, but we don't know what height this is + // since it can be specified in genesis. The reactor will have to validate this via + // ValidateHeight(). + if m.LastCommitRound < -1 { + return errors.New("invalid LastCommitRound (cannot be < -1)") + } + + return nil +} + +// ValidateHeight validates the height given the chain's initial height. +func (m *NewRoundStepMessage) ValidateHeight(initialHeight int64) error { + if m.Height < initialHeight { + return fmt.Errorf("invalid Height %v (lower than initial height %v)", + m.Height, initialHeight) + } + if m.Height == initialHeight && m.LastCommitRound != -1 { + return fmt.Errorf("invalid LastCommitRound %v (must be -1 for initial height %v)", + m.LastCommitRound, initialHeight) + } + if m.Height > initialHeight && m.LastCommitRound < 0 { + return fmt.Errorf("LastCommitRound can only be negative for initial height %v", // nolint + initialHeight) + } + return nil +} + +// String returns a string representation. +func (m *NewRoundStepMessage) String() string { + return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]", + m.Height, m.Round, m.Step, m.LastCommitRound) +} + +//------------------------------------- + +// NewValidBlockMessage is sent when a validator observes a valid block B in some round r, +// i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. +// In case the block is also committed, then IsCommit flag is set to true. +type NewValidBlockMessage struct { + Height int64 + Round int32 + BlockPartSetHeader types.PartSetHeader + BlockParts *bits.BitArray + IsCommit bool +} + +// ValidateBasic performs basic validation. +func (m *NewValidBlockMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.Round < 0 { + return errors.New("negative Round") + } + if err := m.BlockPartSetHeader.ValidateBasic(); err != nil { + return fmt.Errorf("wrong BlockPartSetHeader: %v", err) + } + if m.BlockParts.Size() == 0 { + return errors.New("empty blockParts") + } + if m.BlockParts.Size() != int(m.BlockPartSetHeader.Total) { + return fmt.Errorf("blockParts bit array size %d not equal to BlockPartSetHeader.Total %d", + m.BlockParts.Size(), + m.BlockPartSetHeader.Total) + } + if m.BlockParts.Size() > int(types.MaxBlockPartsCount) { + return fmt.Errorf("blockParts bit array is too big: %d, max: %d", m.BlockParts.Size(), types.MaxBlockPartsCount) + } + return nil +} + +// String returns a string representation. +func (m *NewValidBlockMessage) String() string { + return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]", + m.Height, m.Round, m.BlockPartSetHeader, m.BlockParts, m.IsCommit) +} + +//------------------------------------- + +// ProposalMessage is sent when a new block is proposed. +type ProposalMessage struct { + Proposal *types.Proposal +} + +// ValidateBasic performs basic validation. +func (m *ProposalMessage) ValidateBasic() error { + return m.Proposal.ValidateBasic() +} + +// String returns a string representation. +func (m *ProposalMessage) String() string { + return fmt.Sprintf("[Proposal %v]", m.Proposal) +} + +//------------------------------------- + +// ProposalPOLMessage is sent when a previous proposal is re-proposed. +type ProposalPOLMessage struct { + Height int64 + ProposalPOLRound int32 + ProposalPOL *bits.BitArray +} + +// ValidateBasic performs basic validation. +func (m *ProposalPOLMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.ProposalPOLRound < 0 { + return errors.New("negative ProposalPOLRound") + } + if m.ProposalPOL.Size() == 0 { + return errors.New("empty ProposalPOL bit array") + } + if m.ProposalPOL.Size() > types.MaxVotesCount { + return fmt.Errorf("proposalPOL bit array is too big: %d, max: %d", m.ProposalPOL.Size(), types.MaxVotesCount) + } + return nil +} + +// String returns a string representation. +func (m *ProposalPOLMessage) String() string { + return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL) +} + +//------------------------------------- + +// BlockPartMessage is sent when gossipping a piece of the proposed block. +type BlockPartMessage struct { + Height int64 + Round int32 + Part *types.Part +} + +// ValidateBasic performs basic validation. +func (m *BlockPartMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.Round < 0 { + return errors.New("negative Round") + } + if err := m.Part.ValidateBasic(); err != nil { + return fmt.Errorf("wrong Part: %v", err) + } + return nil +} + +// String returns a string representation. +func (m *BlockPartMessage) String() string { + return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part) +} + +//------------------------------------- + +// VoteMessage is sent when voting for a proposal (or lack thereof). +type VoteMessage struct { + Vote *types.Vote +} + +// ValidateBasic performs basic validation. +func (m *VoteMessage) ValidateBasic() error { + return m.Vote.ValidateBasic() +} + +// String returns a string representation. +func (m *VoteMessage) String() string { + return fmt.Sprintf("[Vote %v]", m.Vote) +} + +//------------------------------------- + +// HasVoteMessage is sent to indicate that a particular vote has been received. +type HasVoteMessage struct { + Height int64 + Round int32 + Type tmproto.SignedMsgType + Index int32 +} + +// ValidateBasic performs basic validation. +func (m *HasVoteMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.Round < 0 { + return errors.New("negative Round") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("invalid Type") + } + if m.Index < 0 { + return errors.New("negative Index") + } + return nil +} + +// String returns a string representation. +func (m *HasVoteMessage) String() string { + return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type) +} + +//------------------------------------- + +// VoteSetMaj23Message is sent to indicate that a given BlockID has seen +2/3 votes. +type VoteSetMaj23Message struct { + Height int64 + Round int32 + Type tmproto.SignedMsgType + BlockID types.BlockID +} + +// ValidateBasic performs basic validation. +func (m *VoteSetMaj23Message) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if m.Round < 0 { + return errors.New("negative Round") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("invalid Type") + } + if err := m.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("wrong BlockID: %v", err) + } + return nil +} + +// String returns a string representation. +func (m *VoteSetMaj23Message) String() string { + return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) +} + +//------------------------------------- + +// VoteSetBitsMessage is sent to communicate the bit-array of votes seen for the BlockID. +type VoteSetBitsMessage struct { + Height int64 + Round int32 + Type tmproto.SignedMsgType + BlockID types.BlockID + Votes *bits.BitArray +} + +// ValidateBasic performs basic validation. +func (m *VoteSetBitsMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("negative Height") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("invalid Type") + } + if err := m.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("wrong BlockID: %v", err) + } + // NOTE: Votes.Size() can be zero if the node does not have any + if m.Votes.Size() > types.MaxVotesCount { + return fmt.Errorf("votes bit array is too big: %d, max: %d", m.Votes.Size(), types.MaxVotesCount) + } + return nil +} + +// String returns a string representation. +func (m *VoteSetBitsMessage) String() string { + return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) +} + +//------------------------------------- diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index c0346d903..f23ec727d 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -5,300 +5,150 @@ import ( "fmt" "os" "path" + "runtime" + "runtime/pprof" "sync" "testing" "time" - "github.com/fortytw2/leaktest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + abcicli "github.com/tendermint/tendermint/abci/client" "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" + cstypes "github.com/tendermint/tendermint/consensus/types" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/libs/bits" + "github.com/tendermint/tendermint/libs/bytes" "github.com/tendermint/tendermint/libs/log" tmsync "github.com/tendermint/tendermint/libs/sync" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/p2p/p2ptest" - tmcons "github.com/tendermint/tendermint/proto/tendermint/consensus" + p2pmock "github.com/tendermint/tendermint/p2p/mock" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" statemocks "github.com/tendermint/tendermint/state/mocks" "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" - dbm "github.com/tendermint/tm-db" ) -var ( - defaultTestTime = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC) -) +//---------------------------------------------- +// in-process testnets -type reactorTestSuite struct { - network *p2ptest.Network - states map[p2p.NodeID]*State - reactors map[p2p.NodeID]*Reactor - subs map[p2p.NodeID]types.Subscription - stateChannels map[p2p.NodeID]*p2p.Channel - dataChannels map[p2p.NodeID]*p2p.Channel - voteChannels map[p2p.NodeID]*p2p.Channel - voteSetBitsChannels map[p2p.NodeID]*p2p.Channel -} +var defaultTestTime = time.Date(2019, 1, 1, 0, 0, 0, 0, time.UTC) -func setup(t *testing.T, numNodes int, states []*State, size int) *reactorTestSuite { - t.Helper() +func startConsensusNet(t *testing.T, css []*State, n int) ( + []*Reactor, + []types.Subscription, + []*types.EventBus, +) { + reactors := make([]*Reactor, n) + blocksSubs := make([]types.Subscription, 0) + eventBuses := make([]*types.EventBus, n) + for i := 0; i < n; i++ { + /*logger, err := tmflags.ParseLogLevel("consensus:info,*:error", logger, "info") + if err != nil { t.Fatal(err)}*/ + reactors[i] = NewReactor(css[i], true) // so we dont start the consensus states + reactors[i].SetLogger(css[i].Logger) - rts := &reactorTestSuite{ - network: p2ptest.MakeNetwork(t, numNodes), - states: make(map[p2p.NodeID]*State), - reactors: make(map[p2p.NodeID]*Reactor, numNodes), - subs: make(map[p2p.NodeID]types.Subscription, numNodes), - } + // eventBus is already started with the cs + eventBuses[i] = css[i].eventBus + reactors[i].SetEventBus(eventBuses[i]) - rts.stateChannels = rts.network.MakeChannelsNoCleanup(t, StateChannel, new(tmcons.Message), size) - rts.dataChannels = rts.network.MakeChannelsNoCleanup(t, DataChannel, new(tmcons.Message), size) - rts.voteChannels = rts.network.MakeChannelsNoCleanup(t, VoteChannel, new(tmcons.Message), size) - rts.voteSetBitsChannels = rts.network.MakeChannelsNoCleanup(t, VoteSetBitsChannel, new(tmcons.Message), size) - - i := 0 - for nodeID, node := range rts.network.Nodes { - state := states[i] - - reactor := NewReactor( - state.Logger.With("node", nodeID), - state, - rts.stateChannels[nodeID], - rts.dataChannels[nodeID], - rts.voteChannels[nodeID], - rts.voteSetBitsChannels[nodeID], - node.MakePeerUpdates(t), - true, - ) - - reactor.SetEventBus(state.eventBus) - - blocksSub, err := state.eventBus.Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock, size) + blocksSub, err := eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryNewBlock) require.NoError(t, err) + blocksSubs = append(blocksSubs, blocksSub) - rts.states[nodeID] = state - rts.subs[nodeID] = blocksSub - rts.reactors[nodeID] = reactor - - // simulate handle initChain in handshake - if state.state.LastBlockHeight == 0 { - require.NoError(t, state.blockExec.Store().Save(state.state)) - } - - require.NoError(t, reactor.Start()) - require.True(t, reactor.IsRunning()) - - i++ - } - - require.Len(t, rts.reactors, numNodes) - - // start the in-memory network and connect all peers with each other - rts.network.Start(t) - - t.Cleanup(func() { - for nodeID, r := range rts.reactors { - require.NoError(t, rts.states[nodeID].eventBus.Stop()) - require.NoError(t, r.Stop()) - require.False(t, r.IsRunning()) - } - - leaktest.Check(t) - }) - - return rts -} - -func validateBlock(block *types.Block, activeVals map[string]struct{}) error { - if block.LastCommit.Size() != len(activeVals) { - return fmt.Errorf( - "commit size doesn't match number of active validators. Got %d, expected %d", - block.LastCommit.Size(), len(activeVals), - ) - } - - for _, commitSig := range block.LastCommit.Signatures { - if _, ok := activeVals[string(commitSig.ValidatorAddress)]; !ok { - return fmt.Errorf("found vote for inactive validator %X", commitSig.ValidatorAddress) - } - } - - return nil -} - -func waitForAndValidateBlock( - t *testing.T, - n int, - activeVals map[string]struct{}, - blocksSubs []types.Subscription, - states []*State, - txs ...[]byte, -) { - - fn := func(j int) { - msg := <-blocksSubs[j].Out() - newBlock := msg.Data().(types.EventDataNewBlock).Block - - require.NoError(t, validateBlock(newBlock, activeVals)) - - for _, tx := range txs { - require.NoError(t, assertMempool(states[j].txNotifier).CheckTx(tx, nil, mempl.TxInfo{})) - } - } - - var wg sync.WaitGroup - wg.Add(n) - - for i := 0; i < n; i++ { - go func(j int) { - fn(j) - wg.Done() - }(i) - } - - wg.Wait() -} - -func waitForAndValidateBlockWithTx( - t *testing.T, - n int, - activeVals map[string]struct{}, - blocksSubs []types.Subscription, - states []*State, - txs ...[]byte, -) { - - fn := func(j int) { - ntxs := 0 - BLOCK_TX_LOOP: - for { - msg := <-blocksSubs[j].Out() - newBlock := msg.Data().(types.EventDataNewBlock).Block - - require.NoError(t, validateBlock(newBlock, activeVals)) - - // check that txs match the txs we're waiting for. - // note they could be spread over multiple blocks, - // but they should be in order. - for _, tx := range newBlock.Data.Txs { - require.EqualValues(t, txs[ntxs], tx) - ntxs++ + if css[i].state.LastBlockHeight == 0 { // simulate handle initChain in handshake + if err := css[i].blockExec.Store().Save(css[i].state); err != nil { + t.Error(err) } - if ntxs == len(txs) { - break BLOCK_TX_LOOP - } } } + // make connected switches and start all reactors + p2p.MakeConnectedSwitches(config.P2P, n, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("CONSENSUS", reactors[i]) + s.SetLogger(reactors[i].conS.Logger.With("module", "p2p")) + return s + }, p2p.Connect2Switches) - var wg sync.WaitGroup - wg.Add(n) - + // now that everyone is connected, start the state machines + // If we started the state machines before everyone was connected, + // we'd block when the cs fires NewBlockEvent and the peers are trying to start their reactors + // TODO: is this still true with new pubsub? for i := 0; i < n; i++ { - go func(j int) { - fn(j) - wg.Done() - }(i) + s := reactors[i].conS.GetState() + reactors[i].SwitchToConsensus(s, false) } - - wg.Wait() + return reactors, blocksSubs, eventBuses } -func waitForBlockWithUpdatedValsAndValidateIt( - t *testing.T, - n int, - updatedVals map[string]struct{}, - blocksSubs []types.Subscription, - css []*State, -) { - - fn := func(j int) { - var newBlock *types.Block - - LOOP: - for { - msg := <-blocksSubs[j].Out() - newBlock = msg.Data().(types.EventDataNewBlock).Block - if newBlock.LastCommit.Size() == len(updatedVals) { - break LOOP - } +func stopConsensusNet(logger log.Logger, reactors []*Reactor, eventBuses []*types.EventBus) { + logger.Info("stopConsensusNet", "n", len(reactors)) + for i, r := range reactors { + logger.Info("stopConsensusNet: Stopping Reactor", "i", i) + if err := r.Switch.Stop(); err != nil { + logger.Error("error trying to stop switch", "error", err) } - - require.NoError(t, validateBlock(newBlock, updatedVals)) } - - var wg sync.WaitGroup - wg.Add(n) - - for i := 0; i < n; i++ { - go func(j int) { - fn(j) - wg.Done() - }(i) + for i, b := range eventBuses { + logger.Info("stopConsensusNet: Stopping eventBus", "i", i) + if err := b.Stop(); err != nil { + logger.Error("error trying to stop eventbus", "error", err) + } } - - wg.Wait() + logger.Info("stopConsensusNet: DONE", "n", len(reactors)) } +// Ensure a testnet makes blocks func TestReactorBasic(t *testing.T) { - configSetup(t) - - n := 4 - states, cleanup := randConsensusState(n, "consensus_reactor_test", newMockTickerFunc(true), newCounter) - t.Cleanup(cleanup) - - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } - - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) - - // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) - } - - wg.Wait() + N := 4 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) + defer cleanup() + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) + // wait till everyone makes the first new block + timeoutWaitGroup(t, N, func(j int) { + <-blocksSubs[j].Out() + }, css) } +// Ensure we can process blocks with evidence func TestReactorWithEvidence(t *testing.T) { - configSetup(t) - - n := 4 + nValidators := 4 testName := "consensus_reactor_test" tickerFunc := newMockTickerFunc(true) appFunc := newCounter - genDoc, privVals := randGenesisDoc(n, false, 30) - states := make([]*State, n) - logger := consensusLogger() + // heed the advice from https://www.sandimetz.com/blog/2016/1/20/the-wrong-abstraction + // to unroll unwieldy abstractions. Here we duplicate the code from: + // css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) - for i := 0; i < n; i++ { + genDoc, privVals := randGenesisDoc(nValidators, false, 30) + css := make([]*State, nValidators) + logger := consensusLogger() + for i := 0; i < nValidators; i++ { stateDB := dbm.NewMemDB() // each state needs its own db stateStore := sm.NewStore(stateDB) state, _ := stateStore.LoadFromDBOrGenesisDoc(genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) - defer os.RemoveAll(thisConfig.RootDir) - ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal app := appFunc() vals := types.TM2PB.ValidatorUpdates(state.Validators) app.InitChain(abci.RequestInitChain{Validators: vals}) pv := privVals[i] + // duplicate code from: + // css[i] = newStateWithConfig(thisConfig, state, privVals[i], app) + blockDB := dbm.NewMemDB() blockStore := store.NewBlockStore(blockDB) @@ -307,6 +157,7 @@ func TestReactorWithEvidence(t *testing.T) { proxyAppConnMem := abcicli.NewLocalClient(mtx, app) proxyAppConnCon := abcicli.NewLocalClient(mtx, app) + // Make Mempool mempool := mempl.NewCListMempool(thisConfig.Mempool, proxyAppConnMem, 0) mempool.SetLogger(log.TestingLogger().With("module", "mempool")) if thisConfig.Consensus.WaitForTxs() { @@ -315,8 +166,7 @@ func TestReactorWithEvidence(t *testing.T) { // mock the evidence pool // everyone includes evidence of another double signing - vIdx := (i + 1) % n - + vIdx := (i + 1) % nValidators ev := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID()) evpool := &statemocks.EvidencePool{} evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil) @@ -326,6 +176,7 @@ func TestReactorWithEvidence(t *testing.T) { evpool2 := sm.EmptyEvidencePool{} + // Make State blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2) cs.SetLogger(log.TestingLogger().With("module", "consensus")) @@ -340,360 +191,784 @@ func TestReactorWithEvidence(t *testing.T) { cs.SetTimeoutTicker(tickerFunc()) cs.SetLogger(logger.With("validator", i, "module", "consensus")) - states[i] = cs + css[i] = cs } - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nValidators) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } - - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) - - // We expect for each validator that is the proposer to propose one piece of - // evidence. - go func(s types.Subscription) { - msg := <-s.Out() + // we expect for each validator that is the proposer to propose one piece of evidence. + for i := 0; i < nValidators; i++ { + timeoutWaitGroup(t, nValidators, func(j int) { + msg := <-blocksSubs[j].Out() block := msg.Data().(types.EventDataNewBlock).Block - - require.Len(t, block.Evidence.Evidence, 1) - wg.Done() - }(sub) + assert.Len(t, block.Evidence.Evidence, 1) + }, css) } - - wg.Wait() } -func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) { - configSetup(t) +//------------------------------------ - n := 4 - states, cleanup := randConsensusState( - n, - "consensus_reactor_test", - newMockTickerFunc(true), - newCounter, +// Ensure a testnet makes blocks when there are txs +func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) { + N := 4 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter, func(c *cfg.Config) { c.Consensus.CreateEmptyBlocks = false - }, - ) - - t.Cleanup(cleanup) - - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } + }) + defer cleanup() + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) // send a tx - require.NoError(t, assertMempool(states[3].txNotifier).CheckTx([]byte{1, 2, 3}, nil, mempl.TxInfo{})) - - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) - - // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) + if err := assertMempool(css[3].txNotifier).CheckTx([]byte{1, 2, 3}, nil, mempl.TxInfo{}); err != nil { + t.Error(err) } - wg.Wait() + // wait till everyone makes the first new block + timeoutWaitGroup(t, N, func(j int) { + <-blocksSubs[j].Out() + }, css) } -func TestReactorRecordsVotesAndBlockParts(t *testing.T) { - configSetup(t) +func TestReactorReceiveDoesNotPanicIfAddPeerHasntBeenCalledYet(t *testing.T) { + N := 1 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) + defer cleanup() + reactors, _, eventBuses := startConsensusNet(t, css, N) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) - n := 4 - states, cleanup := randConsensusState(n, "consensus_reactor_test", newMockTickerFunc(true), newCounter) - t.Cleanup(cleanup) - - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } - - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) - - // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) - } - - wg.Wait() - - // Require at least one node to have sent block parts, but we can't know which - // peer sent it. - require.Eventually( - t, - func() bool { - for _, reactor := range rts.reactors { - for _, ps := range reactor.peers { - if ps.BlockPartsSent() > 0 { - return true - } - } - } - - return false - }, - time.Second, - 10*time.Millisecond, - "number of block parts sent should've increased", + var ( + reactor = reactors[0] + peer = p2pmock.NewPeer(nil) + msg = MustEncode(&HasVoteMessage{Height: 1, + Round: 1, Index: 1, Type: tmproto.PrevoteType}) ) - nodeID := rts.network.RandomNode().NodeID - reactor := rts.reactors[nodeID] - peers := rts.network.Peers(nodeID) + reactor.InitPeer(peer) - ps, ok := reactor.GetPeerState(peers[0].NodeID) - require.True(t, ok) - require.NotNil(t, ps) - require.Greater(t, ps.VotesSent(), 0, "number of votes sent should've increased") + // simulate switch calling Receive before AddPeer + assert.NotPanics(t, func() { + reactor.Receive(StateChannel, peer, msg) + reactor.AddPeer(peer) + }) } +func TestReactorReceivePanicsIfInitPeerHasntBeenCalledYet(t *testing.T) { + N := 1 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) + defer cleanup() + reactors, _, eventBuses := startConsensusNet(t, css, N) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) + + var ( + reactor = reactors[0] + peer = p2pmock.NewPeer(nil) + msg = MustEncode(&HasVoteMessage{Height: 1, + Round: 1, Index: 1, Type: tmproto.PrevoteType}) + ) + + // we should call InitPeer here + + // simulate switch calling Receive before AddPeer + assert.Panics(t, func() { + reactor.Receive(StateChannel, peer, msg) + }) +} + +// Test we record stats about votes and block parts from other peers. +func TestReactorRecordsVotesAndBlockParts(t *testing.T) { + N := 4 + css, cleanup := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter) + defer cleanup() + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) + + // wait till everyone makes the first new block + timeoutWaitGroup(t, N, func(j int) { + <-blocksSubs[j].Out() + }, css) + + // Get peer + peer := reactors[1].Switch.Peers().List()[0] + // Get peer state + ps := peer.Get(types.PeerStateKey).(*PeerState) + + assert.Equal(t, true, ps.VotesSent() > 0, "number of votes sent should have increased") + assert.Equal(t, true, ps.BlockPartsSent() > 0, "number of votes sent should have increased") +} + +//------------------------------------------------------------- +// ensure we can make blocks despite cycling a validator set + func TestReactorVotingPowerChange(t *testing.T) { - configSetup(t) - - n := 4 - states, cleanup := randConsensusState( - n, + nVals := 4 + logger := log.TestingLogger() + css, cleanup := randConsensusNet( + nVals, "consensus_voting_power_changes_test", newMockTickerFunc(true), - newPersistentKVStore, - ) - - t.Cleanup(cleanup) - - rts := setup(t, n, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } - - // map of active validators - activeVals := make(map[string]struct{}) - for i := 0; i < n; i++ { - pubKey, err := states[i].privValidator.GetPubKey(context.Background()) - require.NoError(t, err) - - addr := pubKey.Address() - activeVals[string(addr)] = struct{}{} - } - - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) - - // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) - } - - wg.Wait() - - blocksSubs := []types.Subscription{} - for _, sub := range rts.subs { - blocksSubs = append(blocksSubs, sub) - } - - val1PubKey, err := states[0].privValidator.GetPubKey(context.Background()) - require.NoError(t, err) - - val1PubKeyABCI, err := cryptoenc.PubKeyToProto(val1PubKey) - require.NoError(t, err) - - updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25) - previousTotalVotingPower := states[0].GetRoundState().LastValidators.TotalVotingPower() - - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - - require.NotEqualf( - t, previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - states[0].GetRoundState().LastValidators.TotalVotingPower(), - ) - - updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2) - previousTotalVotingPower = states[0].GetRoundState().LastValidators.TotalVotingPower() - - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - - require.NotEqualf( - t, states[0].GetRoundState().LastValidators.TotalVotingPower(), previousTotalVotingPower, - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), - ) - - updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26) - previousTotalVotingPower = states[0].GetRoundState().LastValidators.TotalVotingPower() - - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlockWithTx(t, n, activeVals, blocksSubs, states, updateValidatorTx) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - waitForAndValidateBlock(t, n, activeVals, blocksSubs, states) - - require.NotEqualf( - t, previousTotalVotingPower, states[0].GetRoundState().LastValidators.TotalVotingPower(), - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, - states[0].GetRoundState().LastValidators.TotalVotingPower(), - ) -} - -func TestReactorValidatorSetChanges(t *testing.T) { - configSetup(t) - - nPeers := 7 - nVals := 4 - states, _, _, cleanup := randConsensusNetWithPeers( - nVals, - nPeers, - "consensus_val_set_changes_test", - newMockTickerFunc(true), - newPersistentKVStoreWithPath, - ) - t.Cleanup(cleanup) - - rts := setup(t, nPeers, states, 100) // buffer must be large enough to not deadlock - - for _, reactor := range rts.reactors { - state := reactor.state.GetState() - reactor.SwitchToConsensus(state, false) - } + newPersistentKVStore) + defer cleanup() + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nVals) + defer stopConsensusNet(logger, reactors, eventBuses) // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - pubKey, err := states[i].privValidator.GetPubKey(context.Background()) + pubKey, err := css[i].privValidator.GetPubKey() require.NoError(t, err) + addr := pubKey.Address() + activeVals[string(addr)] = struct{}{} + } + // wait till everyone makes block 1 + timeoutWaitGroup(t, nVals, func(j int) { + <-blocksSubs[j].Out() + }, css) + + //--------------------------------------------------------------------------- + logger.Debug("---------------------------- Testing changing the voting power of one validator a few times") + + val1PubKey, err := css[0].privValidator.GetPubKey() + require.NoError(t, err) + + val1PubKeyABCI, err := cryptoenc.PubKeyToProto(val1PubKey) + require.NoError(t, err) + updateValidatorTx := kvstore.MakeValSetChangeTx(val1PubKeyABCI, 25) + previousTotalVotingPower := css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf( + "expected voting power to change (before: %d, after: %d)", + previousTotalVotingPower, + css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 2) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf( + "expected voting power to change (before: %d, after: %d)", + previousTotalVotingPower, + css[0].GetRoundState().LastValidators.TotalVotingPower()) + } + + updateValidatorTx = kvstore.MakeValSetChangeTx(val1PubKeyABCI, 26) + previousTotalVotingPower = css[0].GetRoundState().LastValidators.TotalVotingPower() + + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlockWithTx(t, nVals, activeVals, blocksSubs, css, updateValidatorTx) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + waitForAndValidateBlock(t, nVals, activeVals, blocksSubs, css) + + if css[0].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Fatalf( + "expected voting power to change (before: %d, after: %d)", + previousTotalVotingPower, + css[0].GetRoundState().LastValidators.TotalVotingPower()) + } +} + +func TestReactorValidatorSetChanges(t *testing.T) { + nPeers := 7 + nVals := 4 + css, _, _, cleanup := randConsensusNetWithPeers( + nVals, + nPeers, + "consensus_val_set_changes_test", + newMockTickerFunc(true), + newPersistentKVStoreWithPath) + + defer cleanup() + logger := log.TestingLogger() + + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nPeers) + defer stopConsensusNet(logger, reactors, eventBuses) + + // map of active validators + activeVals := make(map[string]struct{}) + for i := 0; i < nVals; i++ { + pubKey, err := css[i].privValidator.GetPubKey() + require.NoError(t, err) activeVals[string(pubKey.Address())] = struct{}{} } - var wg sync.WaitGroup - for _, sub := range rts.subs { - wg.Add(1) + // wait till everyone makes block 1 + timeoutWaitGroup(t, nPeers, func(j int) { + <-blocksSubs[j].Out() + }, css) - // wait till everyone makes the first new block - go func(s types.Subscription) { - <-s.Out() - wg.Done() - }(sub) - } - - wg.Wait() - - newValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background()) - require.NoError(t, err) + //--------------------------------------------------------------------------- + logger.Info("---------------------------- Testing adding one validator") + newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() + assert.NoError(t, err) valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1) - require.NoError(t, err) - + assert.NoError(t, err) newValidatorTx1 := kvstore.MakeValSetChangeTx(valPubKey1ABCI, testMinPower) - blocksSubs := []types.Subscription{} - for _, sub := range rts.subs { - blocksSubs = append(blocksSubs, sub) - } - // wait till everyone makes block 2 // ensure the commit includes all validators // send newValTx to change vals in block 3 - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, newValidatorTx1) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1) // wait till everyone makes block 3. // it includes the commit for block 2, which is by the original validator set - waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, newValidatorTx1) + waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx1) // wait till everyone makes block 4. // it includes the commit for block 3, which is by the original validator set - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) // the commits for block 4 should be with the updated validator set activeVals[string(newValidatorPubKey1.Address())] = struct{}{} // wait till everyone makes block 5 // it includes the commit for block 4, which should have the updated validator set - waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states) + waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) - updateValidatorPubKey1, err := states[nVals].privValidator.GetPubKey(context.Background()) + //--------------------------------------------------------------------------- + logger.Info("---------------------------- Testing changing the voting power of one validator") + + updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() require.NoError(t, err) - updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1) require.NoError(t, err) - updateValidatorTx1 := kvstore.MakeValSetChangeTx(updatePubKey1ABCI, 25) - previousTotalVotingPower := states[nVals].GetRoundState().LastValidators.TotalVotingPower() + previousTotalVotingPower := css[nVals].GetRoundState().LastValidators.TotalVotingPower() - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, updateValidatorTx1) - waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, updateValidatorTx1) - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states) - waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) + waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, updateValidatorTx1) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) + waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) - require.NotEqualf( - t, states[nVals].GetRoundState().LastValidators.TotalVotingPower(), previousTotalVotingPower, - "expected voting power to change (before: %d, after: %d)", - previousTotalVotingPower, states[nVals].GetRoundState().LastValidators.TotalVotingPower(), - ) + if css[nVals].GetRoundState().LastValidators.TotalVotingPower() == previousTotalVotingPower { + t.Errorf( + "expected voting power to change (before: %d, after: %d)", + previousTotalVotingPower, + css[nVals].GetRoundState().LastValidators.TotalVotingPower()) + } - newValidatorPubKey2, err := states[nVals+1].privValidator.GetPubKey(context.Background()) + //--------------------------------------------------------------------------- + logger.Info("---------------------------- Testing adding two validators at once") + + newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey() require.NoError(t, err) - newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2) require.NoError(t, err) - newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) - newValidatorPubKey3, err := states[nVals+2].privValidator.GetPubKey(context.Background()) + newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey() require.NoError(t, err) - newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3) require.NoError(t, err) - newValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, testMinPower) - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, newValidatorTx2, newValidatorTx3) - waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, newValidatorTx2, newValidatorTx3) - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states) - + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3) + waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, newValidatorTx2, newValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) activeVals[string(newValidatorPubKey2.Address())] = struct{}{} activeVals[string(newValidatorPubKey3.Address())] = struct{}{} + waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) - waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states) + //--------------------------------------------------------------------------- + logger.Info("---------------------------- Testing removing two validators at once") removeValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, 0) removeValidatorTx3 := kvstore.MakeValSetChangeTx(newVal3ABCI, 0) - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states, removeValidatorTx2, removeValidatorTx3) - waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, states, removeValidatorTx2, removeValidatorTx3) - waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, states) - + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3) + waitForAndValidateBlockWithTx(t, nPeers, activeVals, blocksSubs, css, removeValidatorTx2, removeValidatorTx3) + waitForAndValidateBlock(t, nPeers, activeVals, blocksSubs, css) delete(activeVals, string(newValidatorPubKey2.Address())) delete(activeVals, string(newValidatorPubKey3.Address())) - - waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, states) + waitForBlockWithUpdatedValsAndValidateIt(t, nPeers, activeVals, blocksSubs, css) +} + +// Check we can make blocks with skip_timeout_commit=false +func TestReactorWithTimeoutCommit(t *testing.T) { + N := 4 + css, cleanup := randConsensusNet(N, "consensus_reactor_with_timeout_commit_test", newMockTickerFunc(false), newCounter) + defer cleanup() + // override default SkipTimeoutCommit == true for tests + for i := 0; i < N; i++ { + css[i].config.SkipTimeoutCommit = false + } + + reactors, blocksSubs, eventBuses := startConsensusNet(t, css, N-1) + defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) + + // wait till everyone makes the first new block + timeoutWaitGroup(t, N-1, func(j int) { + <-blocksSubs[j].Out() + }, css) +} + +func waitForAndValidateBlock( + t *testing.T, + n int, + activeVals map[string]struct{}, + blocksSubs []types.Subscription, + css []*State, + txs ...[]byte, +) { + timeoutWaitGroup(t, n, func(j int) { + css[j].Logger.Debug("waitForAndValidateBlock") + msg := <-blocksSubs[j].Out() + newBlock := msg.Data().(types.EventDataNewBlock).Block + css[j].Logger.Debug("waitForAndValidateBlock: Got block", "height", newBlock.Height) + err := validateBlock(newBlock, activeVals) + assert.Nil(t, err) + for _, tx := range txs { + err := assertMempool(css[j].txNotifier).CheckTx(tx, nil, mempl.TxInfo{}) + assert.Nil(t, err) + } + }, css) +} + +func waitForAndValidateBlockWithTx( + t *testing.T, + n int, + activeVals map[string]struct{}, + blocksSubs []types.Subscription, + css []*State, + txs ...[]byte, +) { + timeoutWaitGroup(t, n, func(j int) { + ntxs := 0 + BLOCK_TX_LOOP: + for { + css[j].Logger.Debug("waitForAndValidateBlockWithTx", "ntxs", ntxs) + msg := <-blocksSubs[j].Out() + newBlock := msg.Data().(types.EventDataNewBlock).Block + css[j].Logger.Debug("waitForAndValidateBlockWithTx: Got block", "height", newBlock.Height) + err := validateBlock(newBlock, activeVals) + assert.Nil(t, err) + + // check that txs match the txs we're waiting for. + // note they could be spread over multiple blocks, + // but they should be in order. + for _, tx := range newBlock.Data.Txs { + assert.EqualValues(t, txs[ntxs], tx) + ntxs++ + } + + if ntxs == len(txs) { + break BLOCK_TX_LOOP + } + } + + }, css) +} + +func waitForBlockWithUpdatedValsAndValidateIt( + t *testing.T, + n int, + updatedVals map[string]struct{}, + blocksSubs []types.Subscription, + css []*State, +) { + timeoutWaitGroup(t, n, func(j int) { + + var newBlock *types.Block + LOOP: + for { + css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt") + msg := <-blocksSubs[j].Out() + newBlock = msg.Data().(types.EventDataNewBlock).Block + if newBlock.LastCommit.Size() == len(updatedVals) { + css[j].Logger.Debug("waitForBlockWithUpdatedValsAndValidateIt: Got block", "height", newBlock.Height) + break LOOP + } else { + css[j].Logger.Debug( + "waitForBlockWithUpdatedValsAndValidateIt: Got block with no new validators. Skipping", + "height", + newBlock.Height) + } + } + + err := validateBlock(newBlock, updatedVals) + assert.Nil(t, err) + }, css) +} + +// expects high synchrony! +func validateBlock(block *types.Block, activeVals map[string]struct{}) error { + if block.LastCommit.Size() != len(activeVals) { + return fmt.Errorf( + "commit size doesn't match number of active validators. Got %d, expected %d", + block.LastCommit.Size(), + len(activeVals)) + } + + for _, commitSig := range block.LastCommit.Signatures { + if _, ok := activeVals[string(commitSig.ValidatorAddress)]; !ok { + return fmt.Errorf("found vote for inactive validator %X", commitSig.ValidatorAddress) + } + } + return nil +} + +func timeoutWaitGroup(t *testing.T, n int, f func(int), css []*State) { + wg := new(sync.WaitGroup) + wg.Add(n) + for i := 0; i < n; i++ { + go func(j int) { + f(j) + wg.Done() + }(i) + } + + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + + // we're running many nodes in-process, possibly in in a virtual machine, + // and spewing debug messages - making a block could take a while, + timeout := time.Second * 120 + + select { + case <-done: + case <-time.After(timeout): + for i, cs := range css { + t.Log("#################") + t.Log("Validator", i) + t.Log(cs.GetRoundState()) + t.Log("") + } + os.Stdout.Write([]byte("pprof.Lookup('goroutine'):\n")) + err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + require.NoError(t, err) + capture() + panic("Timed out waiting for all validators to commit a block") + } +} + +func capture() { + trace := make([]byte, 10240000) + count := runtime.Stack(trace, true) + fmt.Printf("Stack of %d bytes: %s\n", count, trace) +} + +//------------------------------------------------------------- +// Ensure basic validation of structs is functioning + +func TestNewRoundStepMessageValidateBasic(t *testing.T) { + testCases := []struct { // nolint: maligned + expectErr bool + messageRound int32 + messageLastCommitRound int32 + messageHeight int64 + testName string + messageStep cstypes.RoundStepType + }{ + {false, 0, 0, 0, "Valid Message", cstypes.RoundStepNewHeight}, + {true, -1, 0, 0, "Negative round", cstypes.RoundStepNewHeight}, + {true, 0, 0, -1, "Negative height", cstypes.RoundStepNewHeight}, + {true, 0, 0, 0, "Invalid Step", cstypes.RoundStepCommit + 1}, + // The following cases will be handled by ValidateHeight + {false, 0, 0, 1, "H == 1 but LCR != -1 ", cstypes.RoundStepNewHeight}, + {false, 0, -1, 2, "H > 1 but LCR < 0", cstypes.RoundStepNewHeight}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + message := NewRoundStepMessage{ + Height: tc.messageHeight, + Round: tc.messageRound, + Step: tc.messageStep, + LastCommitRound: tc.messageLastCommitRound, + } + + err := message.ValidateBasic() + if tc.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestNewRoundStepMessageValidateHeight(t *testing.T) { + initialHeight := int64(10) + testCases := []struct { // nolint: maligned + expectErr bool + messageLastCommitRound int32 + messageHeight int64 + testName string + }{ + {false, 0, 11, "Valid Message"}, + {true, 0, -1, "Negative height"}, + {true, 0, 0, "Zero height"}, + {true, 0, 10, "Initial height but LCR != -1 "}, + {true, -1, 11, "Normal height but LCR < 0"}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + message := NewRoundStepMessage{ + Height: tc.messageHeight, + Round: 0, + Step: cstypes.RoundStepNewHeight, + LastCommitRound: tc.messageLastCommitRound, + } + + err := message.ValidateHeight(initialHeight) + if tc.expectErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestNewValidBlockMessageValidateBasic(t *testing.T) { + testCases := []struct { + malleateFn func(*NewValidBlockMessage) + expErr string + }{ + {func(msg *NewValidBlockMessage) {}, ""}, + {func(msg *NewValidBlockMessage) { msg.Height = -1 }, "negative Height"}, + {func(msg *NewValidBlockMessage) { msg.Round = -1 }, "negative Round"}, + { + func(msg *NewValidBlockMessage) { msg.BlockPartSetHeader.Total = 2 }, + "blockParts bit array size 1 not equal to BlockPartSetHeader.Total 2", + }, + { + func(msg *NewValidBlockMessage) { + msg.BlockPartSetHeader.Total = 0 + msg.BlockParts = bits.NewBitArray(0) + }, + "empty blockParts", + }, + { + func(msg *NewValidBlockMessage) { msg.BlockParts = bits.NewBitArray(int(types.MaxBlockPartsCount) + 1) }, + "blockParts bit array size 1602 not equal to BlockPartSetHeader.Total 1", + }, + } + + for i, tc := range testCases { + tc := tc + t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + msg := &NewValidBlockMessage{ + Height: 1, + Round: 0, + BlockPartSetHeader: types.PartSetHeader{ + Total: 1, + }, + BlockParts: bits.NewBitArray(1), + } + + tc.malleateFn(msg) + err := msg.ValidateBasic() + if tc.expErr != "" && assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expErr) + } + }) + } +} + +func TestProposalPOLMessageValidateBasic(t *testing.T) { + testCases := []struct { + malleateFn func(*ProposalPOLMessage) + expErr string + }{ + {func(msg *ProposalPOLMessage) {}, ""}, + {func(msg *ProposalPOLMessage) { msg.Height = -1 }, "negative Height"}, + {func(msg *ProposalPOLMessage) { msg.ProposalPOLRound = -1 }, "negative ProposalPOLRound"}, + {func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(0) }, "empty ProposalPOL bit array"}, + {func(msg *ProposalPOLMessage) { msg.ProposalPOL = bits.NewBitArray(types.MaxVotesCount + 1) }, + "proposalPOL bit array is too big: 10001, max: 10000"}, + } + + for i, tc := range testCases { + tc := tc + t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + msg := &ProposalPOLMessage{ + Height: 1, + ProposalPOLRound: 1, + ProposalPOL: bits.NewBitArray(1), + } + + tc.malleateFn(msg) + err := msg.ValidateBasic() + if tc.expErr != "" && assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expErr) + } + }) + } +} + +func TestBlockPartMessageValidateBasic(t *testing.T) { + testPart := new(types.Part) + testPart.Proof.LeafHash = tmhash.Sum([]byte("leaf")) + testCases := []struct { + testName string + messageHeight int64 + messageRound int32 + messagePart *types.Part + expectErr bool + }{ + {"Valid Message", 0, 0, testPart, false}, + {"Invalid Message", -1, 0, testPart, true}, + {"Invalid Message", 0, -1, testPart, true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + message := BlockPartMessage{ + Height: tc.messageHeight, + Round: tc.messageRound, + Part: tc.messagePart, + } + + assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } + + message := BlockPartMessage{Height: 0, Round: 0, Part: new(types.Part)} + message.Part.Index = 1 + + assert.Equal(t, true, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") +} + +func TestHasVoteMessageValidateBasic(t *testing.T) { + const ( + validSignedMsgType tmproto.SignedMsgType = 0x01 + invalidSignedMsgType tmproto.SignedMsgType = 0x03 + ) + + testCases := []struct { // nolint: maligned + expectErr bool + messageRound int32 + messageIndex int32 + messageHeight int64 + testName string + messageType tmproto.SignedMsgType + }{ + {false, 0, 0, 0, "Valid Message", validSignedMsgType}, + {true, -1, 0, 0, "Invalid Message", validSignedMsgType}, + {true, 0, -1, 0, "Invalid Message", validSignedMsgType}, + {true, 0, 0, 0, "Invalid Message", invalidSignedMsgType}, + {true, 0, 0, -1, "Invalid Message", validSignedMsgType}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + message := HasVoteMessage{ + Height: tc.messageHeight, + Round: tc.messageRound, + Type: tc.messageType, + Index: tc.messageIndex, + } + + assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} + +func TestVoteSetMaj23MessageValidateBasic(t *testing.T) { + const ( + validSignedMsgType tmproto.SignedMsgType = 0x01 + invalidSignedMsgType tmproto.SignedMsgType = 0x03 + ) + + validBlockID := types.BlockID{} + invalidBlockID := types.BlockID{ + Hash: bytes.HexBytes{}, + PartSetHeader: types.PartSetHeader{ + Total: 1, + Hash: []byte{0}, + }, + } + + testCases := []struct { // nolint: maligned + expectErr bool + messageRound int32 + messageHeight int64 + testName string + messageType tmproto.SignedMsgType + messageBlockID types.BlockID + }{ + {false, 0, 0, "Valid Message", validSignedMsgType, validBlockID}, + {true, -1, 0, "Invalid Message", validSignedMsgType, validBlockID}, + {true, 0, -1, "Invalid Message", validSignedMsgType, validBlockID}, + {true, 0, 0, "Invalid Message", invalidSignedMsgType, validBlockID}, + {true, 0, 0, "Invalid Message", validSignedMsgType, invalidBlockID}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.testName, func(t *testing.T) { + message := VoteSetMaj23Message{ + Height: tc.messageHeight, + Round: tc.messageRound, + Type: tc.messageType, + BlockID: tc.messageBlockID, + } + + assert.Equal(t, tc.expectErr, message.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} + +func TestVoteSetBitsMessageValidateBasic(t *testing.T) { + testCases := []struct { + malleateFn func(*VoteSetBitsMessage) + expErr string + }{ + {func(msg *VoteSetBitsMessage) {}, ""}, + {func(msg *VoteSetBitsMessage) { msg.Height = -1 }, "negative Height"}, + {func(msg *VoteSetBitsMessage) { msg.Type = 0x03 }, "invalid Type"}, + {func(msg *VoteSetBitsMessage) { + msg.BlockID = types.BlockID{ + Hash: bytes.HexBytes{}, + PartSetHeader: types.PartSetHeader{ + Total: 1, + Hash: []byte{0}, + }, + } + }, "wrong BlockID: wrong PartSetHeader: wrong Hash:"}, + {func(msg *VoteSetBitsMessage) { msg.Votes = bits.NewBitArray(types.MaxVotesCount + 1) }, + "votes bit array is too big: 10001, max: 10000"}, + } + + for i, tc := range testCases { + tc := tc + t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) { + msg := &VoteSetBitsMessage{ + Height: 1, + Round: 0, + Type: 0x01, + Votes: bits.NewBitArray(1), + BlockID: types.BlockID{}, + } + + tc.malleateFn(msg) + err := msg.ValidateBasic() + if tc.expErr != "" && assert.Error(t, err) { + assert.Contains(t, err.Error(), tc.expErr) + } + }) + } } diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 0a7430ab8..778f74b31 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -34,6 +34,21 @@ import ( "github.com/tendermint/tendermint/types" ) +func TestMain(m *testing.M) { + config = ResetConfig("consensus_reactor_test") + consensusReplayConfig = ResetConfig("consensus_replay_test") + configStateTest := ResetConfig("consensus_state_test") + configMempoolTest := ResetConfig("consensus_mempool_test") + configByzantineTest := ResetConfig("consensus_byzantine_test") + code := m.Run() + os.RemoveAll(config.RootDir) + os.RemoveAll(consensusReplayConfig.RootDir) + os.RemoveAll(configStateTest.RootDir) + os.RemoveAll(configMempoolTest.RootDir) + os.RemoveAll(configByzantineTest.RootDir) + os.Exit(code) +} + // These tests ensure we can always recover from failure at any part of the consensus process. // There are two general failure scenarios: failure during consensus, and failure while applying the block. // Only the latter interacts with the app and store, @@ -306,8 +321,6 @@ var modes = []uint{0, 1, 2, 3} // This is actually not a test, it's for storing validator change tx data for testHandshakeReplay func TestSimulateValidatorsChange(t *testing.T) { - configSetup(t) - nPeers := 7 nVals := 4 css, genDoc, config, cleanup := randConsensusNetWithPeers( @@ -343,7 +356,7 @@ func TestSimulateValidatorsChange(t *testing.T) { // HEIGHT 2 height++ incrementHeight(vss...) - newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background()) + newValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() require.NoError(t, err) valPubKey1ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey1) require.NoError(t, err) @@ -356,7 +369,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal := types.NewProposal(vss[1].Height, round, -1, blockID) p := proposal.ToProto() - if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vss[1].SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -373,7 +386,7 @@ func TestSimulateValidatorsChange(t *testing.T) { // HEIGHT 3 height++ incrementHeight(vss...) - updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey(context.Background()) + updateValidatorPubKey1, err := css[nVals].privValidator.GetPubKey() require.NoError(t, err) updatePubKey1ABCI, err := cryptoenc.PubKeyToProto(updateValidatorPubKey1) require.NoError(t, err) @@ -386,7 +399,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal = types.NewProposal(vss[2].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[2].SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vss[2].SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -403,14 +416,14 @@ func TestSimulateValidatorsChange(t *testing.T) { // HEIGHT 4 height++ incrementHeight(vss...) - newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey(context.Background()) + newValidatorPubKey2, err := css[nVals+1].privValidator.GetPubKey() require.NoError(t, err) newVal2ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey2) require.NoError(t, err) newValidatorTx2 := kvstore.MakeValSetChangeTx(newVal2ABCI, testMinPower) err = assertMempool(css[0].txNotifier).CheckTx(newValidatorTx2, nil, mempl.TxInfo{}) assert.Nil(t, err) - newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey(context.Background()) + newValidatorPubKey3, err := css[nVals+2].privValidator.GetPubKey() require.NoError(t, err) newVal3ABCI, err := cryptoenc.PubKeyToProto(newValidatorPubKey3) require.NoError(t, err) @@ -426,10 +439,10 @@ func TestSimulateValidatorsChange(t *testing.T) { valIndexFn := func(cssIdx int) int { for i, vs := range newVss { - vsPubKey, err := vs.GetPubKey(context.Background()) + vsPubKey, err := vs.GetPubKey() require.NoError(t, err) - cssPubKey, err := css[cssIdx].privValidator.GetPubKey(context.Background()) + cssPubKey, err := css[cssIdx].privValidator.GetPubKey() require.NoError(t, err) if vsPubKey.Equals(cssPubKey) { @@ -443,7 +456,7 @@ func TestSimulateValidatorsChange(t *testing.T) { proposal = types.NewProposal(vss[3].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[3].SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vss[3].SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -502,7 +515,7 @@ func TestSimulateValidatorsChange(t *testing.T) { selfIndex = valIndexFn(0) proposal = types.NewProposal(vss[1].Height, round, -1, blockID) p = proposal.ToProto() - if err := vss[1].SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vss[1].SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -531,8 +544,6 @@ func TestSimulateValidatorsChange(t *testing.T) { // Sync from scratch func TestHandshakeReplayAll(t *testing.T) { - configSetup(t) - for _, m := range modes { testHandshakeReplay(t, config, 0, m, false) } @@ -543,8 +554,6 @@ func TestHandshakeReplayAll(t *testing.T) { // Sync many, not from scratch func TestHandshakeReplaySome(t *testing.T) { - configSetup(t) - for _, m := range modes { testHandshakeReplay(t, config, 2, m, false) } @@ -555,8 +564,6 @@ func TestHandshakeReplaySome(t *testing.T) { // Sync from lagging by one func TestHandshakeReplayOne(t *testing.T) { - configSetup(t) - for _, m := range modes { testHandshakeReplay(t, config, numBlocks-1, m, false) } @@ -567,8 +574,6 @@ func TestHandshakeReplayOne(t *testing.T) { // Sync from caught up func TestHandshakeReplayNone(t *testing.T) { - configSetup(t) - for _, m := range modes { testHandshakeReplay(t, config, numBlocks, m, false) } @@ -579,8 +584,6 @@ func TestHandshakeReplayNone(t *testing.T) { // Test mockProxyApp should not panic when app return ABCIResponses with some empty ResponseDeliverTx func TestMockProxyApp(t *testing.T) { - configSetup(t) - sim.CleanupFunc() // clean the test env created in TestSimulateValidatorsChange logger := log.TestingLogger() var validTxs, invalidTxs = 0, 0 @@ -672,8 +675,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) - privVal, err := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - require.NoError(t, err) + privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) wal, err := NewWAL(walFile) require.NoError(t, err) @@ -687,7 +689,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin }) chain, commits, err = makeBlockchainFromWAL(wal) require.NoError(t, err) - pubKey, err := privVal.GetPubKey(context.Background()) + pubKey, err := privVal.GetPubKey() require.NoError(t, err) stateDB, genesisState, store = stateAndStore(config, pubKey, kvstore.ProtocolVersion) @@ -885,10 +887,9 @@ func TestHandshakePanicsIfAppReturnsWrongAppHash(t *testing.T) { // - 0x03 config := ResetConfig("handshake_test_") t.Cleanup(func() { os.RemoveAll(config.RootDir) }) - privVal, err := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - require.NoError(t, err) + privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) const appVersion = 0x0 - pubKey, err := privVal.GetPubKey(context.Background()) + pubKey, err := privVal.GetPubKey() require.NoError(t, err) stateDB, state, store := stateAndStore(config, pubKey, appVersion) stateStore := sm.NewStore(stateDB) @@ -1222,9 +1223,8 @@ func TestHandshakeUpdatesValidators(t *testing.T) { config := ResetConfig("handshake_test_") t.Cleanup(func() { _ = os.RemoveAll(config.RootDir) }) - privVal, err := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) - require.NoError(t, err) - pubKey, err := privVal.GetPubKey(context.Background()) + privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + pubKey, err := privVal.GetPubKey() require.NoError(t, err) stateDB, state, store := stateAndStore(config, pubKey, 0x0) stateStore := sm.NewStore(stateDB) diff --git a/consensus/state.go b/consensus/state.go index 780fc3af5..385214489 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -2,11 +2,11 @@ package consensus import ( "bytes" - "context" "errors" "fmt" "io/ioutil" "os" + "reflect" "runtime/debug" "time" @@ -30,7 +30,9 @@ import ( tmtime "github.com/tendermint/tendermint/types/time" ) -// Consensus sentinel errors +//----------------------------------------------------------------------------- +// Errors + var ( ErrInvalidProposalSignature = errors.New("error invalid proposal signature") ErrInvalidProposalPOLRound = errors.New("error invalid proposal POL round") @@ -40,7 +42,11 @@ var ( errPubKeyIsNotSet = errors.New("pubkey is not set. Look for \"Can't get private validator pubkey\" errors") ) -var msgQueueSize = 1000 +//----------------------------------------------------------------------------- + +var ( + msgQueueSize = 1000 +) // msgs from the reactor which may update the state type msgInfo struct { @@ -171,7 +177,6 @@ func NewState( evsw: tmevents.NewEventSwitch(), metrics: NopMetrics(), } - // set function defaults (may be overwritten before calling Start) cs.decideProposal = cs.defaultDecideProposal cs.doPrevote = cs.defaultDoPrevote @@ -184,16 +189,19 @@ func NewState( cs.updateToState(state) - // NOTE: we do not call scheduleRound0 yet, we do that upon Start() + // Don't call scheduleRound0 yet. + // We do that upon Start(). cs.BaseService = *service.NewBaseService(nil, "State", cs) for _, option := range options { option(cs) } - return cs } +//---------------------------------------- +// Public interface + // SetLogger implements Service. func (cs *State) SetLogger(l log.Logger) { cs.BaseService.Logger = l @@ -270,12 +278,11 @@ func (cs *State) SetPrivValidator(priv types.PrivValidator) { cs.privValidator = priv if err := cs.updatePrivValidatorPubKey(); err != nil { - cs.Logger.Error("failed to get private validator pubkey", "err", err) + cs.Logger.Error("Can't get private validator pubkey", "err", err) } } -// SetTimeoutTicker sets the local timer. It may be useful to overwrite for -// testing. +// SetTimeoutTicker sets the local timer. It may be useful to overwrite for testing. func (cs *State) SetTimeoutTicker(timeoutTicker TimeoutTicker) { cs.mtx.Lock() cs.timeoutTicker = timeoutTicker @@ -286,11 +293,9 @@ func (cs *State) SetTimeoutTicker(timeoutTicker TimeoutTicker) { func (cs *State) LoadCommit(height int64) *types.Commit { cs.mtx.RLock() defer cs.mtx.RUnlock() - if height == cs.blockStore.Height() { return cs.blockStore.LoadSeenCommit(height) } - return cs.blockStore.LoadBlockCommit(height) } @@ -309,29 +314,25 @@ func (cs *State) OnStart() error { // log to catchup. if cs.doWALCatchup { repairAttempted := false - LOOP: for { err := cs.catchupReplay(cs.Height) switch { case err == nil: break LOOP - case !IsDataCorruptionError(err): - cs.Logger.Error("error on catchup replay; proceeding to start state anyway", "err", err) + cs.Logger.Error("Error on catchup replay. Proceeding to start State anyway", "err", err) break LOOP - case repairAttempted: return err } - cs.Logger.Error("the WAL file is corrupted; attempting repair", "err", err) + cs.Logger.Error("WAL file is corrupted, attempting repair", "err", err) // 1) prep work if err := cs.wal.Stop(); err != nil { return err } - repairAttempted = true // 2) backup original WAL file @@ -339,16 +340,14 @@ func (cs *State) OnStart() error { if err := tmos.CopyFile(cs.config.WalFile(), corruptedFile); err != nil { return err } - - cs.Logger.Debug("backed up WAL file", "src", cs.config.WalFile(), "dst", corruptedFile) + cs.Logger.Info("Backed up WAL file", "src", cs.config.WalFile(), "dst", corruptedFile) // 3) try to repair (WAL file will be overwritten!) if err := repairWalFile(corruptedFile, cs.config.WalFile()); err != nil { - cs.Logger.Error("the WAL repair failed", "err", err) + cs.Logger.Error("WAL repair failed", "err", err) return err } - - cs.Logger.Info("successful WAL repair") + cs.Logger.Info("Successful repair") // reload WAL file if err := cs.loadWalFile(); err != nil { @@ -390,10 +389,9 @@ func (cs *State) OnStart() error { func (cs *State) startRoutines(maxSteps int) { err := cs.timeoutTicker.Start() if err != nil { - cs.Logger.Error("failed to start timeout ticker", "err", err) + cs.Logger.Error("Error starting timeout ticker", "err", err) return } - go cs.receiveRoutine(maxSteps) } @@ -401,10 +399,9 @@ func (cs *State) startRoutines(maxSteps int) { func (cs *State) loadWalFile() error { wal, err := cs.OpenWAL(cs.config.WalFile()) if err != nil { - cs.Logger.Error("failed to load state WAL", "err", err) + cs.Logger.Error("Error loading State wal", "err", err) return err } - cs.wal = wal return nil } @@ -412,11 +409,10 @@ func (cs *State) loadWalFile() error { // OnStop implements service.Service. func (cs *State) OnStop() { if err := cs.evsw.Stop(); err != nil { - cs.Logger.Error("failed trying to stop eventSwitch", "error", err) + cs.Logger.Error("error trying to stop eventSwitch", "error", err) } - if err := cs.timeoutTicker.Stop(); err != nil { - cs.Logger.Error("failed trying to stop timeoutTicket", "error", err) + cs.Logger.Error("error trying to stop timeoutTicket", "error", err) } // WAL is stopped in receiveRoutine. } @@ -433,17 +429,14 @@ func (cs *State) Wait() { func (cs *State) OpenWAL(walFile string) (WAL, error) { wal, err := NewWAL(walFile) if err != nil { - cs.Logger.Error("failed to open WAL", "file", walFile, "err", err) + cs.Logger.Error("Failed to open WAL", "file", walFile, "err", err) return nil, err } - wal.SetLogger(cs.Logger.With("wal", walFile)) - if err := wal.Start(); err != nil { - cs.Logger.Error("failed to start WAL", "err", err) + cs.Logger.Error("Failed to start WAL", "err", err) return nil, err } - return wal, nil } @@ -499,18 +492,15 @@ func (cs *State) SetProposalAndBlock( parts *types.PartSet, peerID p2p.NodeID, ) error { - if err := cs.SetProposal(proposal, peerID); err != nil { return err } - for i := 0; i < int(parts.Total()); i++ { part := parts.GetPart(i) if err := cs.AddProposalBlockPart(proposal.Height, proposal.Round, part, peerID); err != nil { return err } } - return nil } @@ -548,7 +538,7 @@ func (cs *State) sendInternalMessage(mi msgInfo) { // be processed out of order. // TODO: use CList here for strict determinism and // attempt push to internalMsgQueue in receiveRoutine - cs.Logger.Debug("internal msg queue is full; using a go-routine") + cs.Logger.Info("Internal msg queue is full. Using a go-routine") go func() { cs.internalMsgQueue <- mi }() } } @@ -558,15 +548,13 @@ func (cs *State) sendInternalMessage(mi msgInfo) { func (cs *State) reconstructLastCommit(state sm.State) { seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) if seenCommit == nil { - panic(fmt.Sprintf( - "failed to reconstruct last commit; seen commit for height %v not found", - state.LastBlockHeight, - )) + panic(fmt.Sprintf("Failed to reconstruct LastCommit: seen commit for height %v not found", + state.LastBlockHeight)) } lastPrecommits := types.CommitToVoteSet(state.ChainID, seenCommit, state.LastValidators) if !lastPrecommits.HasTwoThirdsMajority() { - panic("failed to reconstruct last commit; does not have +2/3 maj") + panic("Failed to reconstruct LastCommit: Does not have +2/3 maj") } cs.LastCommit = lastPrecommits @@ -576,26 +564,19 @@ func (cs *State) reconstructLastCommit(state sm.State) { // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight. func (cs *State) updateToState(state sm.State) { if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight { - panic(fmt.Sprintf( - "updateToState() expected state height of %v but found %v", - cs.Height, state.LastBlockHeight, - )) + panic(fmt.Sprintf("updateToState() expected state height of %v but found %v", + cs.Height, state.LastBlockHeight)) } - if !cs.state.IsEmpty() { if cs.state.LastBlockHeight > 0 && cs.state.LastBlockHeight+1 != cs.Height { // This might happen when someone else is mutating cs.state. // Someone forgot to pass in state.Copy() somewhere?! - panic(fmt.Sprintf( - "inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v", - cs.state.LastBlockHeight+1, cs.Height, - )) + panic(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight+1 %v vs cs.Height %v", + cs.state.LastBlockHeight+1, cs.Height)) } if cs.state.LastBlockHeight > 0 && cs.Height == cs.state.InitialHeight { - panic(fmt.Sprintf( - "inconsistent cs.state.LastBlockHeight %v, expected 0 for initial height %v", - cs.state.LastBlockHeight, cs.state.InitialHeight, - )) + panic(fmt.Sprintf("Inconsistent cs.state.LastBlockHeight %v, expected 0 for initial height %v", + cs.state.LastBlockHeight, cs.state.InitialHeight)) } // If state isn't further out than cs.state, just ignore. @@ -604,11 +585,12 @@ func (cs *State) updateToState(state sm.State) { // signal the new round step, because other services (eg. txNotifier) // depend on having an up-to-date peer state! if state.LastBlockHeight <= cs.state.LastBlockHeight { - cs.Logger.Debug( - "ignoring updateToState()", - "new_height", state.LastBlockHeight+1, - "old_height", cs.state.LastBlockHeight+1, - ) + cs.Logger.Info( + "Ignoring updateToState()", + "newHeight", + state.LastBlockHeight+1, + "oldHeight", + cs.state.LastBlockHeight+1) cs.newStep() return } @@ -622,19 +604,16 @@ func (cs *State) updateToState(state sm.State) { cs.LastCommit = (*types.VoteSet)(nil) case cs.CommitRound > -1 && cs.Votes != nil: // Otherwise, use cs.Votes if !cs.Votes.Precommits(cs.CommitRound).HasTwoThirdsMajority() { - panic(fmt.Sprintf( - "wanted to form a commit, but precommits (H/R: %d/%d) didn't have 2/3+: %v", - state.LastBlockHeight, cs.CommitRound, cs.Votes.Precommits(cs.CommitRound), - )) + panic(fmt.Sprintf("Wanted to form a Commit, but Precommits (H/R: %d/%d) didn't have 2/3+: %v", + state.LastBlockHeight, + cs.CommitRound, + cs.Votes.Precommits(cs.CommitRound))) } - cs.LastCommit = cs.Votes.Precommits(cs.CommitRound) - case cs.LastCommit == nil: // NOTE: when Tendermint starts, it has no votes. reconstructLastCommit // must be called to reconstruct LastCommit from SeenCommit. - panic(fmt.Sprintf( - "last commit cannot be empty after initial block (H:%d)", + panic(fmt.Sprintf("LastCommit cannot be empty after initial block (H:%d)", state.LastBlockHeight+1, )) } @@ -648,7 +627,6 @@ func (cs *State) updateToState(state sm.State) { // RoundState fields cs.updateHeight(height) cs.updateRoundStep(0, cstypes.RoundStepNewHeight) - if cs.CommitTime.IsZero() { // "Now" makes it easier to sync up dev nodes. // We add timeoutCommit to allow transactions @@ -684,17 +662,14 @@ func (cs *State) updateToState(state sm.State) { func (cs *State) newStep() { rs := cs.RoundStateEvent() if err := cs.wal.Write(rs); err != nil { - cs.Logger.Error("failed writing to WAL", "err", err) + cs.Logger.Error("Error writing to wal", "err", err) } - cs.nSteps++ - // newStep is called by updateToState in NewState before the eventBus is set! if cs.eventBus != nil { if err := cs.eventBus.PublishEventNewRoundStep(rs); err != nil { - cs.Logger.Error("failed publishing new round step", "err", err) + cs.Logger.Error("Error publishing new round step", "err", err) } - cs.evsw.FireEvent(types.EventNewRoundStep, &cs.RoundState) } } @@ -715,10 +690,10 @@ func (cs *State) receiveRoutine(maxSteps int) { // close wal now that we're done writing to it if err := cs.wal.Stop(); err != nil { - cs.Logger.Error("failed trying to stop WAL", "error", err) + cs.Logger.Error("error trying to stop wal", "error", err) } - cs.wal.Wait() + close(cs.done) } @@ -740,35 +715,28 @@ func (cs *State) receiveRoutine(maxSteps int) { for { if maxSteps > 0 { if cs.nSteps >= maxSteps { - cs.Logger.Debug("reached max steps; exiting receive routine") + cs.Logger.Info("reached max steps. exiting receive routine") cs.nSteps = 0 return } } - rs := cs.RoundState var mi msgInfo select { case <-cs.txNotifier.TxsAvailable(): cs.handleTxsAvailable() - case mi = <-cs.peerMsgQueue: if err := cs.wal.Write(mi); err != nil { - cs.Logger.Error("failed writing to WAL", "err", err) + cs.Logger.Error("Error writing to wal", "err", err) } - // handles proposals, block parts, votes // may generate internal events (votes, complete proposals, 2/3 majorities) cs.handleMsg(mi) - case mi = <-cs.internalMsgQueue: err := cs.wal.WriteSync(mi) // NOTE: fsync if err != nil { - panic(fmt.Sprintf( - "failed to write %v msg to consensus WAL due to %v; check your file system and restart the node", - mi, err, - )) + panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node", mi, err)) } if _, ok := mi.Msg.(*VoteMessage); ok { @@ -781,16 +749,13 @@ func (cs *State) receiveRoutine(maxSteps int) { // handles proposals, block parts, votes cs.handleMsg(mi) - case ti := <-cs.timeoutTicker.Chan(): // tockChan: if err := cs.wal.Write(ti); err != nil { - cs.Logger.Error("failed writing to WAL", "err", err) + cs.Logger.Error("Error writing to wal", "err", err) } - // if the timeout is relevant to the rs // go to the next step cs.handleTimeout(ti, rs) - case <-cs.Quit(): onExit(cs) return @@ -807,15 +772,12 @@ func (cs *State) handleMsg(mi msgInfo) { added bool err error ) - msg, peerID := mi.Msg, mi.PeerID - switch msg := msg.(type) { case *ProposalMessage: // will not cause transition. // once proposal is set, we can receive block parts err = cs.setProposal(msg.Proposal) - case *BlockPartMessage: // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit added, err = cs.addProposalBlockPart(msg, peerID) @@ -825,14 +787,15 @@ func (cs *State) handleMsg(mi msgInfo) { if err != nil && msg.Round != cs.Round { cs.Logger.Debug( - "received block part from wrong round", - "height", cs.Height, - "cs_round", cs.Round, - "block_round", msg.Round, - ) + "Received block part from wrong round", + "height", + cs.Height, + "csRound", + cs.Round, + "blockRound", + msg.Round) err = nil } - case *VoteMessage: // attempt to add the vote and dupeout the validator if its a duplicate signature // if the vote gives us a 2/3-any or 2/3-one, we transition @@ -855,30 +818,23 @@ func (cs *State) handleMsg(mi msgInfo) { // TODO: If rs.Height == vote.Height && rs.Round < vote.Round, // the peer is sending us CatchupCommit precommits. // We could make note of this and help filter in broadcastHasVoteMessage(). - default: - cs.Logger.Error("unknown msg type", "type", fmt.Sprintf("%T", msg)) + cs.Logger.Error("Unknown msg type", "type", reflect.TypeOf(msg)) return } if err != nil { - cs.Logger.Error( - "failed to process message", - "height", cs.Height, - "round", cs.Round, - "peer", peerID, - "err", err, - "msg", msg, - ) + cs.Logger.Error("Error with msg", "height", cs.Height, "round", cs.Round, + "peer", peerID, "err", err, "msg", msg) } } func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { - cs.Logger.Debug("received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) + cs.Logger.Debug("Received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) // timeouts must be for current height, round, step if ti.Height != rs.Height || ti.Round < rs.Round || (ti.Round == rs.Round && ti.Step < rs.Step) { - cs.Logger.Debug("ignoring tock because we are ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step) + cs.Logger.Debug("Ignoring tock because we're ahead", "height", rs.Height, "round", rs.Round, "step", rs.Step) return } @@ -891,34 +847,26 @@ func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { // NewRound event fired from enterNewRound. // XXX: should we fire timeout here (for timeout commit)? cs.enterNewRound(ti.Height, 0) - case cstypes.RoundStepNewRound: cs.enterPropose(ti.Height, 0) - case cstypes.RoundStepPropose: if err := cs.eventBus.PublishEventTimeoutPropose(cs.RoundStateEvent()); err != nil { - cs.Logger.Error("failed publishing timeout propose", "err", err) + cs.Logger.Error("Error publishing timeout propose", "err", err) } - cs.enterPrevote(ti.Height, ti.Round) - case cstypes.RoundStepPrevoteWait: if err := cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()); err != nil { - cs.Logger.Error("failed publishing timeout wait", "err", err) + cs.Logger.Error("Error publishing timeout wait", "err", err) } - cs.enterPrecommit(ti.Height, ti.Round) - case cstypes.RoundStepPrecommitWait: if err := cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()); err != nil { - cs.Logger.Error("failed publishing timeout wait", "err", err) + cs.Logger.Error("Error publishing timeout wait", "err", err) } - cs.enterPrecommit(ti.Height, ti.Round) cs.enterNewRound(ti.Height, ti.Round+1) - default: - panic(fmt.Sprintf("invalid timeout step: %v", ti.Step)) + panic(fmt.Sprintf("Invalid timeout step: %v", ti.Step)) } } @@ -942,7 +890,6 @@ func (cs *State) handleTxsAvailable() { // +1ms to ensure RoundStepNewRound timeout always happens after RoundStepNewHeight timeoutCommit := cs.StartTime.Sub(tmtime.Now()) + 1*time.Millisecond cs.scheduleTimeout(timeoutCommit, cs.Height, 0, cstypes.RoundStepNewRound) - case cstypes.RoundStepNewRound: // after timeoutCommit cs.enterPropose(cs.Height, 0) } @@ -962,20 +909,23 @@ func (cs *State) enterNewRound(height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != cstypes.RoundStepNewHeight) { - logger.Debug( - "entering new round with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + logger.Debug(fmt.Sprintf( + "enterNewRound(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + round, + cs.Height, + cs.Round, + cs.Step)) return } if now := tmtime.Now(); cs.StartTime.After(now) { - logger.Debug("need to set a buffer and log message here for sanity", "start_time", cs.StartTime, "now", now) + logger.Debug("need to set a buffer and log message here for sanity", "startTime", cs.StartTime, "now", now) } - logger.Debug("entering new round", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Info(fmt.Sprintf("enterNewRound(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) - // increment validators if necessary + // Increment validators if necessary validators := cs.Validators if cs.Round < round { validators = validators.Copy() @@ -992,19 +942,17 @@ func (cs *State) enterNewRound(height int64, round int32) { // and meanwhile we might have received a proposal // for round 0. } else { - logger.Debug("resetting proposal info") + logger.Info("Resetting Proposal info") cs.Proposal = nil cs.ProposalBlock = nil cs.ProposalBlockParts = nil } - cs.Votes.SetRound(tmmath.SafeAddInt32(round, 1)) // also track next round (round+1) to allow round-skipping cs.TriggeredTimeoutPrecommit = false if err := cs.eventBus.PublishEventNewRound(cs.NewRoundEvent()); err != nil { - cs.Logger.Error("failed publishing new round", "err", err) + cs.Logger.Error("Error publishing new round", "err", err) } - cs.metrics.Rounds.Set(float64(round)) // Wait for txs to be available in the mempool @@ -1032,7 +980,6 @@ func (cs *State) needProofBlock(height int64) bool { if lastBlockMeta == nil { panic(fmt.Sprintf("needProofBlock: last block meta for height %d not found", height-1)) } - return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash) } @@ -1044,14 +991,16 @@ func (cs *State) enterPropose(height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPropose <= cs.Step) { - logger.Debug( - "entering propose step with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + logger.Debug(fmt.Sprintf( + "enterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + round, + cs.Height, + cs.Round, + cs.Step)) return } - - logger.Debug("entering propose step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Info(fmt.Sprintf("enterPropose(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) defer func() { // Done enterPropose: @@ -1071,38 +1020,35 @@ func (cs *State) enterPropose(height int64, round int32) { // Nothing more to do if we're not a validator if cs.privValidator == nil { - logger.Debug("node is not a validator") + logger.Debug("This node is not a validator") return } - - logger.Debug("node is a validator") + logger.Debug("This node is a validator") if cs.privValidatorPubKey == nil { // If this node is a validator & proposer in the current round, it will // miss the opportunity to create a block. - logger.Error("propose step; empty priv validator public key", "err", errPubKeyIsNotSet) + logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) return } - address := cs.privValidatorPubKey.Address() // if not a validator, we're done if !cs.Validators.HasAddress(address) { - logger.Debug("node is not a validator", "addr", address, "vals", cs.Validators) + logger.Debug("This node is not a validator", "addr", address, "vals", cs.Validators) return } if cs.isProposer(address) { - logger.Debug( - "propose step; our turn to propose", + logger.Debug("enterPropose: our turn to propose", "proposer", address, + "privValidator", cs.privValidator, ) - cs.decideProposal(height, round) } else { - logger.Debug( - "propose step; not our turn to propose", + logger.Debug("enterPropose: not our turn to propose", "proposer", cs.Validators.GetProposer().Address, + "privValidator", cs.privValidator, ) } } @@ -1130,31 +1076,26 @@ func (cs *State) defaultDecideProposal(height int64, round int32) { // Flush the WAL. Otherwise, we may not recompute the same proposal to sign, // and the privValidator will refuse to sign anything. if err := cs.wal.FlushAndSync(); err != nil { - cs.Logger.Error("failed flushing WAL to disk") + cs.Logger.Error("Error flushing to disk") } // Make proposal propBlockID := types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()} proposal := types.NewProposal(height, round, cs.ValidRound, propBlockID) p := proposal.ToProto() - - // wait the max amount we would wait for a proposal - ctx, cancel := context.WithTimeout(context.TODO(), cs.config.TimeoutPropose) - defer cancel() - if err := cs.privValidator.SignProposal(ctx, cs.state.ChainID, p); err == nil { + if err := cs.privValidator.SignProposal(cs.state.ChainID, p); err == nil { proposal.Signature = p.Signature // send proposal and block parts on internal msg queue cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) - for i := 0; i < int(blockParts.Total()); i++ { part := blockParts.GetPart(i) cs.sendInternalMessage(msgInfo{&BlockPartMessage{cs.Height, cs.Round, part}, ""}) } - - cs.Logger.Debug("signed proposal", "height", height, "round", round, "proposal", proposal) + cs.Logger.Info("Signed proposal", "height", height, "round", round, "proposal", proposal) + cs.Logger.Debug(fmt.Sprintf("Signed proposal block: %v", block)) } else if !cs.replayMode { - cs.Logger.Error("propose step; failed signing proposal", "height", height, "round", round, "err", err) + cs.Logger.Error("enterPropose: Error signing proposal", "height", height, "round", round, "err", err) } } @@ -1192,23 +1133,20 @@ func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.Pa // We're creating a proposal for the first block. // The commit is empty, but not nil. commit = types.NewCommit(0, 0, types.BlockID{}, nil) - case cs.LastCommit.HasTwoThirdsMajority(): // Make the commit from LastCommit commit = cs.LastCommit.MakeCommit() - default: // This shouldn't happen. - cs.Logger.Error("propose step; cannot propose anything without commit for the previous block") + cs.Logger.Error("enterPropose: Cannot propose anything: No commit for the previous block") return } if cs.privValidatorPubKey == nil { // If this node is a validator & proposer in the current round, it will // miss the opportunity to create a block. - cs.Logger.Error("propose step; empty priv validator public key", "err", errPubKeyIsNotSet) + cs.Logger.Error(fmt.Sprintf("enterPropose: %v", errPubKeyIsNotSet)) return } - proposerAddr := cs.privValidatorPubKey.Address() return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr) @@ -1219,13 +1157,14 @@ func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.Pa // Prevote for LockedBlock if we're locked, or ProposalBlock if valid. // Otherwise vote nil. func (cs *State) enterPrevote(height int64, round int32) { - logger := cs.Logger.With("height", height, "round", round) - if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevote <= cs.Step) { - logger.Debug( - "entering prevote step with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + cs.Logger.Debug(fmt.Sprintf( + "enterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + round, + cs.Height, + cs.Round, + cs.Step)) return } @@ -1235,7 +1174,7 @@ func (cs *State) enterPrevote(height int64, round int32) { cs.newStep() }() - logger.Debug("entering prevote step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + cs.Logger.Debug(fmt.Sprintf("enterPrevote(%v/%v); current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) // Sign and broadcast vote as necessary cs.doPrevote(height, round) @@ -1249,14 +1188,14 @@ func (cs *State) defaultDoPrevote(height int64, round int32) { // If a block is locked, prevote that. if cs.LockedBlock != nil { - logger.Debug("prevote step; already locked on a block; prevoting locked block") + logger.Debug("enterPrevote: already locked on a block; prevoting locked block") cs.signAddVote(tmproto.PrevoteType, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) return } // If ProposalBlock is nil, prevote nil. if cs.ProposalBlock == nil { - logger.Debug("prevote step: ProposalBlock is nil") + logger.Debug("enterPrevote: ProposalBlock is nil") cs.signAddVote(tmproto.PrevoteType, nil, types.PartSetHeader{}) return } @@ -1265,7 +1204,7 @@ func (cs *State) defaultDoPrevote(height int64, round int32) { err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock) if err != nil { // ProposalBlock is invalid, prevote nil. - logger.Error("prevote step: ProposalBlock is invalid", "err", err) + logger.Error("enterPrevote: ProposalBlock is invalid", "err", err) cs.signAddVote(tmproto.PrevoteType, nil, types.PartSetHeader{}) return } @@ -1273,7 +1212,7 @@ func (cs *State) defaultDoPrevote(height int64, round int32) { // Prevote cs.ProposalBlock // NOTE: the proposal signature is validated when it is received, // and the proposal block parts are validated as they are received (against the merkle hash in the proposal) - logger.Debug("prevote step: ProposalBlock is valid") + logger.Debug("enterPrevote: ProposalBlock is valid") cs.signAddVote(tmproto.PrevoteType, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header()) } @@ -1282,21 +1221,20 @@ func (cs *State) enterPrevoteWait(height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevoteWait <= cs.Step) { - logger.Debug( - "entering prevote wait step with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + logger.Debug(fmt.Sprintf( + "enterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + round, + cs.Height, + cs.Round, + cs.Step)) return } - if !cs.Votes.Prevotes(round).HasTwoThirdsAny() { - panic(fmt.Sprintf( - "entering prevote wait step (%v/%v), but prevotes does not have any +2/3 votes", - height, round, - )) + panic(fmt.Sprintf("enterPrevoteWait(%v/%v), but Prevotes does not have any +2/3 votes", height, round)) } - logger.Debug("entering prevote wait step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Debug(fmt.Sprintf("enterPrevoteWait(%v/%v); current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) defer func() { // Done enterPrevoteWait: @@ -1318,14 +1256,17 @@ func (cs *State) enterPrecommit(height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommit <= cs.Step) { - logger.Debug( - "entering precommit step with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + logger.Debug(fmt.Sprintf( + "enterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + round, + cs.Height, + cs.Round, + cs.Step)) return } - logger.Debug("entering precommit step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Debug(fmt.Sprintf("enterPrecommit(%v/%v); current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) defer func() { // Done enterPrecommit: @@ -1339,41 +1280,38 @@ func (cs *State) enterPrecommit(height int64, round int32) { // If we don't have a polka, we must precommit nil. if !ok { if cs.LockedBlock != nil { - logger.Debug("precommit step; no +2/3 prevotes during enterPrecommit while we are locked; precommitting nil") + logger.Debug("enterPrecommit: no +2/3 prevotes during enterPrecommit while we're locked. Precommitting nil") } else { - logger.Debug("precommit step; no +2/3 prevotes during enterPrecommit; precommitting nil") + logger.Debug("enterPrecommit: no +2/3 prevotes during enterPrecommit. Precommitting nil.") } - cs.signAddVote(tmproto.PrecommitType, nil, types.PartSetHeader{}) return } // At this point +2/3 prevoted for a particular block or nil. if err := cs.eventBus.PublishEventPolka(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing polka", "err", err) + cs.Logger.Error("Error publishing polka", "err", err) } // the latest POLRound should be this round. polRound, _ := cs.Votes.POLInfo() if polRound < round { - panic(fmt.Sprintf("this POLRound should be %v but got %v", round, polRound)) + panic(fmt.Sprintf("This POLRound should be %v but got %v", round, polRound)) } // +2/3 prevoted nil. Unlock and precommit nil. if len(blockID.Hash) == 0 { if cs.LockedBlock == nil { - logger.Debug("precommit step; +2/3 prevoted for nil") + logger.Debug("enterPrecommit: +2/3 prevoted for nil") } else { - logger.Debug("precommit step; +2/3 prevoted for nil; unlocking") + logger.Debug("enterPrecommit: +2/3 prevoted for nil; unlocking") cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing event unlock", "err", err) + cs.Logger.Error("Error publishing event unlock", "err", err) } } - cs.signAddVote(tmproto.PrecommitType, nil, types.PartSetHeader{}) return } @@ -1382,34 +1320,28 @@ func (cs *State) enterPrecommit(height int64, round int32) { // If we're already locked on that block, precommit it, and update the LockedRound if cs.LockedBlock.HashesTo(blockID.Hash) { - logger.Debug("precommit step; +2/3 prevoted locked block; relocking") + logger.Debug("enterPrecommit: +2/3 prevoted locked block; relocking") cs.LockedRound = round - if err := cs.eventBus.PublishEventRelock(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing event relock", "err", err) + cs.Logger.Error("Error publishing event relock", "err", err) } - cs.signAddVote(tmproto.PrecommitType, blockID.Hash, blockID.PartSetHeader) return } // If +2/3 prevoted for proposal block, stage and precommit it if cs.ProposalBlock.HashesTo(blockID.Hash) { - logger.Debug("precommit step; +2/3 prevoted proposal block; locking", "hash", blockID.Hash) - + logger.Debug("enterPrecommit: +2/3 prevoted proposal block; locking", "hash", blockID.Hash) // Validate the block. if err := cs.blockExec.ValidateBlock(cs.state, cs.ProposalBlock); err != nil { - panic(fmt.Sprintf("precommit step; +2/3 prevoted for an invalid block: %v", err)) + panic(fmt.Sprintf("enterPrecommit: +2/3 prevoted for an invalid block: %v", err)) } - cs.LockedRound = round cs.LockedBlock = cs.ProposalBlock cs.LockedBlockParts = cs.ProposalBlockParts - if err := cs.eventBus.PublishEventLock(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing event lock", "err", err) + cs.Logger.Error("Error publishing event lock", "err", err) } - cs.signAddVote(tmproto.PrecommitType, blockID.Hash, blockID.PartSetHeader) return } @@ -1417,21 +1349,17 @@ func (cs *State) enterPrecommit(height int64, round int32) { // There was a polka in this round for a block we don't have. // Fetch that block, unlock, and precommit nil. // The +2/3 prevotes for this round is the POL for our unlock. - logger.Debug("precommit step; +2/3 prevotes for a block we do not have; voting nil", "block_id", blockID) - + logger.Debug("enterPrecommit: +2/3 prevotes for a block we do not have; voting nil", "blockID", blockID) cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil - if !cs.ProposalBlockParts.HasHeader(blockID.PartSetHeader) { cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) } - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing event unlock", "err", err) + cs.Logger.Error("Error publishing event unlock", "err", err) } - cs.signAddVote(tmproto.PrecommitType, nil, types.PartSetHeader{}) } @@ -1441,21 +1369,16 @@ func (cs *State) enterPrecommitWait(height int64, round int32) { if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) { logger.Debug( - "entering precommit wait step with invalid args", - "triggered_timeout", cs.TriggeredTimeoutPrecommit, - "current", fmt.Sprintf("%v/%v", cs.Height, cs.Round), - ) + fmt.Sprintf( + "enterPrecommitWait(%v/%v): Invalid args. "+ + "Current state is Height/Round: %v/%v/, TriggeredTimeoutPrecommit:%v", + height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit)) return } - if !cs.Votes.Precommits(round).HasTwoThirdsAny() { - panic(fmt.Sprintf( - "entering precommit wait step (%v/%v), but precommits does not have any +2/3 votes", - height, round, - )) + panic(fmt.Sprintf("enterPrecommitWait(%v/%v), but Precommits does not have any +2/3 votes", height, round)) } - - logger.Debug("entering precommit wait step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Debug(fmt.Sprintf("enterPrecommitWait(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) defer func() { // Done enterPrecommitWait: @@ -1463,23 +1386,25 @@ func (cs *State) enterPrecommitWait(height int64, round int32) { cs.newStep() }() - // wait for some more precommits; enterNewRound + // Wait for some more precommits; enterNewRound cs.scheduleTimeout(cs.config.Precommit(round), height, round, cstypes.RoundStepPrecommitWait) } // Enter: +2/3 precommits for block func (cs *State) enterCommit(height int64, commitRound int32) { - logger := cs.Logger.With("height", height, "commit_round", commitRound) + logger := cs.Logger.With("height", height, "commitRound", commitRound) if cs.Height != height || cstypes.RoundStepCommit <= cs.Step { - logger.Debug( - "entering commit step with invalid args", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + logger.Debug(fmt.Sprintf( + "enterCommit(%v/%v): Invalid args. Current step: %v/%v/%v", + height, + commitRound, + cs.Height, + cs.Round, + cs.Step)) return } - - logger.Debug("entering commit step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) + logger.Info(fmt.Sprintf("enterCommit(%v/%v). Current: %v/%v/%v", height, commitRound, cs.Height, cs.Round, cs.Step)) defer func() { // Done enterCommit: @@ -1502,7 +1427,7 @@ func (cs *State) enterCommit(height int64, commitRound int32) { // Move them over to ProposalBlock if they match the commit hash, // otherwise they'll be cleared in updateToState. if cs.LockedBlock.HashesTo(blockID.Hash) { - logger.Debug("commit is for a locked block; set ProposalBlock=LockedBlock", "block_hash", blockID.Hash) + logger.Debug("commit is for a locked block; set ProposalBlock=LockedBlock", "blockHash", blockID.Hash) cs.ProposalBlock = cs.LockedBlock cs.ProposalBlockParts = cs.LockedBlockParts } @@ -1520,13 +1445,14 @@ func (cs *State) enterCommit(height int64, commitRound int32) { // Set up ProposalBlockParts and keep waiting. cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) - if err := cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()); err != nil { - logger.Error("failed publishing valid block", "err", err) + cs.Logger.Error("Error publishing valid block", "err", err) } - cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) } + // else { + // We just need to keep waiting. + // } } } @@ -1540,33 +1466,33 @@ func (cs *State) tryFinalizeCommit(height int64) { blockID, ok := cs.Votes.Precommits(cs.CommitRound).TwoThirdsMajority() if !ok || len(blockID.Hash) == 0 { - logger.Error("failed attempt to finalize commit; there was no +2/3 majority or +2/3 was for nil") + logger.Error("Attempt to finalize failed. There was no +2/3 majority, or +2/3 was for .") return } - if !cs.ProposalBlock.HashesTo(blockID.Hash) { // TODO: this happens every time if we're not a validator (ugly logs) // TODO: ^^ wait, why does it matter that we're a validator? logger.Debug( - "failed attempt to finalize commit; we do not have the commit block", - "proposal_block", cs.ProposalBlock.Hash(), - "commit_block", blockID.Hash, + "attempt to finalize failed; we do not have the commit block", + "proposal-block", cs.ProposalBlock.Hash(), + "commit-block", blockID.Hash, ) return } + // go cs.finalizeCommit(height) } // Increment height and goto cstypes.RoundStepNewHeight func (cs *State) finalizeCommit(height int64) { - logger := cs.Logger.With("height", height) - if cs.Height != height || cs.Step != cstypes.RoundStepCommit { - logger.Debug( - "entering finalize commit step", - "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step), - ) + cs.Logger.Debug(fmt.Sprintf( + "finalizeCommit(%v): Invalid args. Current step: %v/%v/%v", + height, + cs.Height, + cs.Round, + cs.Step)) return } @@ -1574,26 +1500,25 @@ func (cs *State) finalizeCommit(height int64) { block, blockParts := cs.ProposalBlock, cs.ProposalBlockParts if !ok { - panic("cannot finalize commit; commit does not have 2/3 majority") + panic("Cannot finalizeCommit, commit does not have two thirds majority") } if !blockParts.HasHeader(blockID.PartSetHeader) { - panic("expected ProposalBlockParts header to be commit header") + panic("Expected ProposalBlockParts header to be commit header") } if !block.HashesTo(blockID.Hash) { - panic("cannot finalize commit; proposal block does not hash to commit hash") + panic("Cannot finalizeCommit, ProposalBlock does not hash to commit hash") } - if err := cs.blockExec.ValidateBlock(cs.state, block); err != nil { panic(fmt.Errorf("+2/3 committed an invalid block: %w", err)) } - logger.Info( - "finalizing commit of block", + cs.Logger.Info("finalizing commit of block with N txs", + "height", block.Height, "hash", block.Hash(), "root", block.AppHash, - "num_txs", len(block.Txs), + "N", len(block.Txs), ) - logger.Debug(fmt.Sprintf("%v", block)) + cs.Logger.Debug(fmt.Sprintf("%v", block)) fail.Fail() // XXX @@ -1606,7 +1531,7 @@ func (cs *State) finalizeCommit(height int64) { cs.blockStore.SaveBlock(block, blockParts, seenCommit) } else { // Happens during replay if we already saved the block but didn't commit - logger.Debug("calling finalizeCommit on already stored block", "height", block.Height) + cs.Logger.Debug("calling finalizeCommit on already stored block", "height", block.Height) } fail.Fail() // XXX @@ -1626,10 +1551,8 @@ func (cs *State) finalizeCommit(height int64) { // restart). endMsg := EndHeightMessage{height} if err := cs.wal.WriteSync(endMsg); err != nil { // NOTE: fsync - panic(fmt.Sprintf( - "failed to write %v msg to consensus WAL due to %v; check your file system and restart the node", - endMsg, err, - )) + panic(fmt.Sprintf("Failed to write %v msg to consensus wal due to %v. Check your FS and restart the node", + endMsg, err)) } fail.Fail() // XXX @@ -1639,21 +1562,14 @@ func (cs *State) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // NOTE The block.AppHash wont reflect these txs until the next block. - var ( - err error - retainHeight int64 - ) - + var err error + var retainHeight int64 stateCopy, retainHeight, err = cs.blockExec.ApplyBlock( stateCopy, - types.BlockID{ - Hash: block.Hash(), - PartSetHeader: blockParts.Header(), - }, - block, - ) + types.BlockID{Hash: block.Hash(), PartSetHeader: blockParts.Header()}, + block) if err != nil { - logger.Error("failed to apply block", "err", err) + cs.Logger.Error("Error on ApplyBlock", "err", err) return } @@ -1663,9 +1579,9 @@ func (cs *State) finalizeCommit(height int64) { if retainHeight > 0 { pruned, err := cs.pruneBlocks(retainHeight) if err != nil { - logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) + cs.Logger.Error("Failed to prune blocks", "retainHeight", retainHeight, "err", err) } else { - logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight) + cs.Logger.Info("Pruned blocks", "pruned", pruned, "retainHeight", retainHeight) } } @@ -1679,7 +1595,7 @@ func (cs *State) finalizeCommit(height int64) { // Private validator might have changed it's key pair => refetch pubkey. if err := cs.updatePrivValidatorPubKey(); err != nil { - logger.Error("failed to get private validator pubkey", "err", err) + cs.Logger.Error("Can't get private validator pubkey", "err", err) } // cs.StartTime is already set. @@ -1846,7 +1762,7 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.NodeID) // Blocks might be reused, so round mismatch is OK if cs.Height != height { - cs.Logger.Debug("received block part from wrong height", "height", height, "round", round) + cs.Logger.Debug("Received block part from wrong height", "height", height, "round", round) return false, nil } @@ -1854,13 +1770,8 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.NodeID) if cs.ProposalBlockParts == nil { // NOTE: this can happen when we've gone to a higher round and // then receive parts from the previous round - not necessarily a bad peer. - cs.Logger.Debug( - "received a block part when we are not expecting any", - "height", height, - "round", round, - "index", part.Index, - "peer", peerID, - ) + cs.Logger.Info("Received a block part when we're not expecting any", + "height", height, "round", round, "index", part.Index, "peer", peerID) return false, nil } @@ -1891,12 +1802,10 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.NodeID) } cs.ProposalBlock = block - // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal - cs.Logger.Info("received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) - + cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) if err := cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent()); err != nil { - cs.Logger.Error("failed publishing event complete proposal", "err", err) + cs.Logger.Error("Error publishing event complete proposal", "err", err) } // Update Valid* if we can. @@ -1904,12 +1813,8 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.NodeID) blockID, hasTwoThirds := prevotes.TwoThirdsMajority() if hasTwoThirds && !blockID.IsZero() && (cs.ValidRound < cs.Round) { if cs.ProposalBlock.HashesTo(blockID.Hash) { - cs.Logger.Debug( - "updating valid block to new proposal block", - "valid_round", cs.Round, - "valid_block_hash", cs.ProposalBlock.Hash(), - ) - + cs.Logger.Info("Updating valid block to new proposal block", + "valid-round", cs.Round, "valid-block-hash", cs.ProposalBlock.Hash()) cs.ValidRound = cs.Round cs.ValidBlock = cs.ProposalBlock cs.ValidBlockParts = cs.ProposalBlockParts @@ -1931,10 +1836,8 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID p2p.NodeID) // If we're waiting on the proposal block... cs.tryFinalizeCommit(height) } - return added, nil } - return added, nil } @@ -1953,47 +1856,52 @@ func (cs *State) tryAddVote(vote *types.Vote, peerID p2p.NodeID) (bool, error) { if bytes.Equal(vote.ValidatorAddress, cs.privValidatorPubKey.Address()) { cs.Logger.Error( - "found conflicting vote from ourselves; did you unsafe_reset a validator?", - "height", vote.Height, - "round", vote.Round, - "type", vote.Type, - ) - + "Found conflicting vote from ourselves. Did you unsafe_reset a validator?", + "height", + vote.Height, + "round", + vote.Round, + "type", + vote.Type) return added, err } - // report conflicting votes to the evidence pool cs.evpool.ReportConflictingVotes(voteErr.VoteA, voteErr.VoteB) - cs.Logger.Debug( - "found and sent conflicting votes to the evidence pool", - "vote_a", voteErr.VoteA, - "vote_b", voteErr.VoteB, + cs.Logger.Info("Found and sent conflicting votes to the evidence pool", + "VoteA", voteErr.VoteA, + "VoteB", voteErr.VoteB, ) - return added, err - } else if errors.Is(err, types.ErrVoteNonDeterministicSignature) { - cs.Logger.Debug("vote has non-deterministic signature", "err", err) + } else if err == types.ErrVoteNonDeterministicSignature { + cs.Logger.Debug("Vote has non-deterministic signature", "err", err) } else { // Either // 1) bad peer OR // 2) not a bad peer? this can also err sometimes with "Unexpected step" OR // 3) tmkms use with multiple validators connecting to a single tmkms instance // (https://github.com/tendermint/tendermint/issues/3839). - cs.Logger.Info("failed attempting to add vote", "err", err) + cs.Logger.Error("Error attempting to add vote", "err", err) return added, ErrAddingVote } } - return added, nil } -func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err error) { +//----------------------------------------------------------------------------- + +func (cs *State) addVote( + vote *types.Vote, + peerID p2p.NodeID) (added bool, err error) { cs.Logger.Debug( - "adding vote", - "vote_height", vote.Height, - "vote_type", vote.Type, - "val_index", vote.ValidatorIndex, - "cs_height", cs.Height, + "addVote", + "voteHeight", + vote.Height, + "voteType", + vote.Type, + "valIndex", + vote.ValidatorIndex, + "csHeight", + cs.Height, ) // A precommit for the previous height? @@ -2001,20 +1909,18 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e if vote.Height+1 == cs.Height && vote.Type == tmproto.PrecommitType { if cs.Step != cstypes.RoundStepNewHeight { // Late precommit at prior height is ignored - cs.Logger.Debug("precommit vote came in after commit timeout and has been ignored", "vote", vote) + cs.Logger.Debug("Precommit vote came in after commit timeout and has been ignored", "vote", vote) return } - added, err = cs.LastCommit.AddVote(vote) if !added { return } - cs.Logger.Debug("added vote to last precommits", "last_commit", cs.LastCommit.StringShort()) + cs.Logger.Info(fmt.Sprintf("Added to lastPrecommits: %v", cs.LastCommit.StringShort())) if err := cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}); err != nil { return added, err } - cs.evsw.FireEvent(types.EventVote, vote) // if we can skip timeoutCommit and have all the votes now, @@ -2030,7 +1936,7 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e // Height mismatch is ignored. // Not necessarily a bad peer, but not favorable behavior. if vote.Height != cs.Height { - cs.Logger.Debug("vote ignored and not added", "vote_height", vote.Height, "cs_height", cs.Height, "peer", peerID) + cs.Logger.Debug("vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "peerID", peerID) return } @@ -2049,10 +1955,11 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e switch vote.Type { case tmproto.PrevoteType: prevotes := cs.Votes.Prevotes(vote.Round) - cs.Logger.Debug("added vote to prevote", "vote", vote, "prevotes", prevotes.StringShort()) + cs.Logger.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort()) // If +2/3 prevotes for a block or nil for *any* round: if blockID, ok := prevotes.TwoThirdsMajority(); ok { + // There was a polka! // If we're locked but this is a recent polka, unlock. // If it matches our ProposalBlock, update the ValidBlock @@ -2064,12 +1971,10 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e (vote.Round <= cs.Round) && !cs.LockedBlock.HashesTo(blockID.Hash) { - cs.Logger.Debug("unlocking because of POL", "locked_round", cs.LockedRound, "pol_round", vote.Round) - + cs.Logger.Info("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round) cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { return added, err } @@ -2078,26 +1983,26 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e // Update Valid* if we can. // NOTE: our proposal block may be nil or not what received a polka.. if len(blockID.Hash) != 0 && (cs.ValidRound < vote.Round) && (vote.Round == cs.Round) { + if cs.ProposalBlock.HashesTo(blockID.Hash) { - cs.Logger.Debug("updating valid block because of POL", "valid_round", cs.ValidRound, "pol_round", vote.Round) + cs.Logger.Info( + "Updating ValidBlock because of POL.", "validRound", cs.ValidRound, "POLRound", vote.Round) cs.ValidRound = vote.Round cs.ValidBlock = cs.ProposalBlock cs.ValidBlockParts = cs.ProposalBlockParts } else { - cs.Logger.Debug( + cs.Logger.Info( "valid block we do not know about; set ProposalBlock=nil", "proposal", cs.ProposalBlock.Hash(), - "block_id", blockID.Hash, + "blockID", blockID.Hash, ) - // we're getting the wrong block + // We're getting the wrong block. cs.ProposalBlock = nil } - if !cs.ProposalBlockParts.HasHeader(blockID.PartSetHeader) { cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) } - cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) if err := cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()); err != nil { return added, err @@ -2110,7 +2015,6 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e case cs.Round < vote.Round && prevotes.HasTwoThirdsAny(): // Round-skip if there is any 2/3+ of votes ahead of us cs.enterNewRound(height, vote.Round) - case cs.Round == vote.Round && cstypes.RoundStepPrevote <= cs.Step: // current round blockID, ok := prevotes.TwoThirdsMajority() if ok && (cs.isProposalComplete() || len(blockID.Hash) == 0) { @@ -2118,7 +2022,6 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e } else if prevotes.HasTwoThirdsAny() { cs.enterPrevoteWait(height, vote.Round) } - case cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round: // If the proposal is now complete, enter prevote of cs.Round. if cs.isProposalComplete() { @@ -2128,14 +2031,13 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e case tmproto.PrecommitType: precommits := cs.Votes.Precommits(vote.Round) - cs.Logger.Debug("added vote to precommit", "vote", vote, "precommits", precommits.StringShort()) + cs.Logger.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort()) blockID, ok := precommits.TwoThirdsMajority() if ok { // Executed as TwoThirdsMajority could be from a higher round cs.enterNewRound(height, vote.Round) cs.enterPrecommit(height, vote.Round) - if len(blockID.Hash) != 0 { cs.enterCommit(height, vote.Round) if cs.config.SkipTimeoutCommit && precommits.HasAll() { @@ -2150,7 +2052,7 @@ func (cs *State) addVote(vote *types.Vote, peerID p2p.NodeID) (added bool, err e } default: - panic(fmt.Sprintf("unexpected vote type %v", vote.Type)) + panic(fmt.Sprintf("Unexpected vote type %v", vote.Type)) } return added, err @@ -2171,7 +2073,6 @@ func (cs *State) signVote( if cs.privValidatorPubKey == nil { return nil, errPubKeyIsNotSet } - addr := cs.privValidatorPubKey.Address() valIdx, _ := cs.Validators.GetByAddress(addr) @@ -2184,26 +2085,8 @@ func (cs *State) signVote( Type: msgType, BlockID: types.BlockID{Hash: hash, PartSetHeader: header}, } - v := vote.ToProto() - - // If the signedMessageType is for precommit, - // use our local precommit Timeout as the max wait time for getting a singed commit. The same goes for prevote. - var timeout time.Duration - - switch msgType { - case tmproto.PrecommitType: - timeout = cs.config.TimeoutPrecommit - case tmproto.PrevoteType: - timeout = cs.config.TimeoutPrevote - default: - timeout = time.Second - } - - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - - err := cs.privValidator.SignVote(ctx, cs.state.ChainID, v) + err := cs.privValidator.SignVote(cs.state.ChainID, v) vote.Signature = v.Signature return vote, err @@ -2254,11 +2137,12 @@ func (cs *State) signAddVote(msgType tmproto.SignedMsgType, hash []byte, header vote, err := cs.signVote(msgType, hash, header) if err == nil { cs.sendInternalMessage(msgInfo{&VoteMessage{vote}, ""}) - cs.Logger.Debug("signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote) + cs.Logger.Info("Signed and pushed vote", "height", cs.Height, "round", cs.Round, "vote", vote) return vote } - - cs.Logger.Error("failed signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) + // if !cs.replayMode { + cs.Logger.Error("Error signing vote", "height", cs.Height, "round", cs.Round, "vote", vote, "err", err) + // } return nil } @@ -2270,17 +2154,7 @@ func (cs *State) updatePrivValidatorPubKey() error { return nil } - var timeout time.Duration - if cs.config.TimeoutPrecommit > cs.config.TimeoutPrevote { - timeout = cs.config.TimeoutPrecommit - } else { - timeout = cs.config.TimeoutPrevote - } - - // set a hard timeout for 2 seconds. This helps in avoiding blocking of the remote signer connection - ctx, cancel := context.WithTimeout(context.TODO(), timeout) - defer cancel() - pubKey, err := cs.privValidator.GetPubKey(ctx) + pubKey, err := cs.privValidator.GetPubKey() if err != nil { return err } @@ -2296,20 +2170,18 @@ func (cs *State) checkDoubleSigningRisk(height int64) error { if doubleSignCheckHeight > height { doubleSignCheckHeight = height } - for i := int64(1); i < doubleSignCheckHeight; i++ { lastCommit := cs.blockStore.LoadSeenCommit(height - i) if lastCommit != nil { for sigIdx, s := range lastCommit.Signatures { if s.BlockIDFlag == types.BlockIDFlagCommit && bytes.Equal(s.ValidatorAddress, valAddr) { - cs.Logger.Info("found signature from the same key", "sig", s, "idx", sigIdx, "height", height-i) + cs.Logger.Info("Found signature from the same key", "sig", s, "idx", sigIdx, "height", height-i) return ErrSignatureFoundInPastBlocks } } } } } - return nil } diff --git a/consensus/state_test.go b/consensus/state_test.go index de0f6cc09..fa2aafb56 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -56,8 +56,6 @@ x * TestHalt1 - if we see +2/3 precommits after timing out into new round, we sh // ProposeSuite func TestStateProposerSelection0(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) height, round := cs1.Height, cs1.Round @@ -71,7 +69,7 @@ func TestStateProposerSelection0(t *testing.T) { // Commit a block and ensure proposer for the next height is correct. prop := cs1.GetRoundState().Validators.GetProposer() - pv, err := cs1.privValidator.GetPubKey(context.Background()) + pv, err := cs1.privValidator.GetPubKey() require.NoError(t, err) address := pv.Address() if !bytes.Equal(prop.Address, address) { @@ -88,7 +86,7 @@ func TestStateProposerSelection0(t *testing.T) { ensureNewRound(newRoundCh, height+1, 0) prop = cs1.GetRoundState().Validators.GetProposer() - pv1, err := vss[1].GetPubKey(context.Background()) + pv1, err := vss[1].GetPubKey() require.NoError(t, err) addr := pv1.Address() if !bytes.Equal(prop.Address, addr) { @@ -98,8 +96,6 @@ func TestStateProposerSelection0(t *testing.T) { // Now let's do it all again, but starting from round 2 instead of 0 func TestStateProposerSelection2(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) // test needs more work for more than 3 validators height := cs1.Height newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) @@ -116,7 +112,7 @@ func TestStateProposerSelection2(t *testing.T) { // everyone just votes nil. we get a new proposer each round for i := int32(0); int(i) < len(vss); i++ { prop := cs1.GetRoundState().Validators.GetProposer() - pvk, err := vss[int(i+round)%len(vss)].GetPubKey(context.Background()) + pvk, err := vss[int(i+round)%len(vss)].GetPubKey() require.NoError(t, err) addr := pvk.Address() correctProposer := addr @@ -137,8 +133,6 @@ func TestStateProposerSelection2(t *testing.T) { // a non-validator should timeout into the prevote round func TestStateEnterProposeNoPrivValidator(t *testing.T) { - configSetup(t) - cs, _ := randState(1) cs.SetPrivValidator(nil) height, round := cs.Height, cs.Round @@ -158,8 +152,6 @@ func TestStateEnterProposeNoPrivValidator(t *testing.T) { // a validator should not timeout of the prevote round (TODO: unless the block is really big!) func TestStateEnterProposeYesPrivValidator(t *testing.T) { - configSetup(t) - cs, _ := randState(1) height, round := cs.Height, cs.Round @@ -190,8 +182,6 @@ func TestStateEnterProposeYesPrivValidator(t *testing.T) { } func TestStateBadProposal(t *testing.T) { - configSetup(t) - cs1, vss := randState(2) height, round := cs1.Height, cs1.Round vs2 := vss[1] @@ -218,7 +208,7 @@ func TestStateBadProposal(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(vs2.Height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vs2.SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -250,8 +240,6 @@ func TestStateBadProposal(t *testing.T) { } func TestStateOversizedBlock(t *testing.T) { - configSetup(t) - cs1, vss := randState(2) cs1.state.ConsensusParams.Block.MaxBytes = 2000 height, round := cs1.Height, cs1.Round @@ -274,7 +262,7 @@ func TestStateOversizedBlock(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vs2.SignProposal(config.ChainID(), p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -314,8 +302,6 @@ func TestStateOversizedBlock(t *testing.T) { // propose, prevote, and precommit a block func TestStateFullRound1(t *testing.T) { - configSetup(t) - cs, vss := randState(1) height, round := cs.Height, cs.Round @@ -356,8 +342,6 @@ func TestStateFullRound1(t *testing.T) { // nil is proposed, so prevote and precommit nil func TestStateFullRoundNil(t *testing.T) { - configSetup(t) - cs, vss := randState(1) height, round := cs.Height, cs.Round @@ -376,8 +360,6 @@ func TestStateFullRoundNil(t *testing.T) { // run through propose, prevote, precommit commit with two validators // where the first validator has to wait for votes from the second func TestStateFullRound2(t *testing.T) { - configSetup(t) - cs1, vss := randState(2) vs2 := vss[1] height, round := cs1.Height, cs1.Round @@ -418,8 +400,6 @@ func TestStateFullRound2(t *testing.T) { // two validators, 4 rounds. // two vals take turns proposing. val1 locks on first one, precommits nil on everything else func TestStateLockNoPOL(t *testing.T) { - configSetup(t) - cs1, vss := randState(2) vs2 := vss[1] height, round := cs1.Height, cs1.Round @@ -606,8 +586,6 @@ func TestStateLockNoPOL(t *testing.T) { // in round two: v1 prevotes the same block that the node is locked on // the others prevote a new block hence v1 changes lock and precommits the new block with the others func TestStateLockPOLRelock(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -616,7 +594,7 @@ func TestStateLockPOLRelock(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -705,8 +683,6 @@ func TestStateLockPOLRelock(t *testing.T) { // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka func TestStateLockPOLUnlock(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -717,7 +693,7 @@ func TestStateLockPOLUnlock(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -799,8 +775,6 @@ func TestStateLockPOLUnlock(t *testing.T) { // v1 should unlock and precommit nil. In the third round another block is proposed, all vals // prevote and now v1 can lock onto the third block and precommit that func TestStateLockPOLUnlockOnUnknownBlock(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -809,7 +783,7 @@ func TestStateLockPOLUnlockOnUnknownBlock(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -927,8 +901,6 @@ func TestStateLockPOLUnlockOnUnknownBlock(t *testing.T) { // then a polka at round 2 that we lock on // then we see the polka from round 1 but shouldn't unlock func TestStateLockPOLSafety1(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -939,7 +911,7 @@ func TestStateLockPOLSafety1(t *testing.T) { timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1048,8 +1020,6 @@ func TestStateLockPOLSafety1(t *testing.T) { // What we want: // dont see P0, lock on P1 at R1, dont unlock using P0 at R2 func TestStateLockPOLSafety2(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1060,7 +1030,7 @@ func TestStateLockPOLSafety2(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1115,7 +1085,7 @@ func TestStateLockPOLSafety2(t *testing.T) { // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, round, 0, propBlockID0) p := newProp.ToProto() - if err := vs3.SignProposal(context.Background(), config.ChainID(), p); err != nil { + if err := vs3.SignProposal(config.ChainID(), p); err != nil { t.Fatal(err) } @@ -1147,8 +1117,6 @@ func TestStateLockPOLSafety2(t *testing.T) { // What we want: // P0 proposes B0 at R3. func TestProposeValidBlock(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1160,7 +1128,7 @@ func TestProposeValidBlock(t *testing.T) { timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1239,8 +1207,6 @@ func TestProposeValidBlock(t *testing.T) { // What we want: // P0 miss to lock B but set valid block to B after receiving delayed prevote. func TestSetValidBlockOnDelayedPrevote(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1251,7 +1217,7 @@ func TestSetValidBlockOnDelayedPrevote(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1303,8 +1269,6 @@ func TestSetValidBlockOnDelayedPrevote(t *testing.T) { // P0 miss to lock B as Proposal Block is missing, but set valid block to B after // receiving delayed Block Proposal. func TestSetValidBlockOnDelayedProposal(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1315,7 +1279,7 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) { timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1361,8 +1325,6 @@ func TestSetValidBlockOnDelayedProposal(t *testing.T) { // What we want: // P0 waits for timeoutPrecommit before starting next round func TestWaitingTimeoutOnNilPolka(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1384,15 +1346,13 @@ func TestWaitingTimeoutOnNilPolka(t *testing.T) { // What we want: // P0 waits for timeoutPropose in the next round before entering prevote func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1422,15 +1382,13 @@ func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { // What we want: // P0 jump to higher round, precommit and start precommit wait func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1460,15 +1418,13 @@ func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) { // What we want: // P0 wait for timeoutPropose to expire before sending prevote. func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, int32(1) timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1489,8 +1445,6 @@ func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) { // What we want: // P0 emit NewValidBlock event upon receiving 2/3+ Precommit for B but hasn't received block B yet func TestEmitNewValidBlockEventOnCommitWithoutBlock(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, int32(1) @@ -1525,8 +1479,6 @@ func TestEmitNewValidBlockEventOnCommitWithoutBlock(t *testing.T) { // P0 receives 2/3+ Precommit for B for round 0, while being in round 1. It emits NewValidBlock event. // After receiving block, it executes block and moves to the next height. func TestCommitFromPreviousRound(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, int32(1) @@ -1580,8 +1532,6 @@ func (n *fakeTxNotifier) Notify() { // and third precommit arrives which leads to the commit of that header and the correct // start of the next round func TestStartNextHeightCorrectlyAfterTimeout(t *testing.T) { - configSetup(t) - config.Consensus.SkipTimeoutCommit = false cs1, vss := randState(4) cs1.txNotifier = &fakeTxNotifier{ch: make(chan struct{})} @@ -1595,7 +1545,7 @@ func TestStartNextHeightCorrectlyAfterTimeout(t *testing.T) { newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1643,8 +1593,6 @@ func TestStartNextHeightCorrectlyAfterTimeout(t *testing.T) { } func TestResetTimeoutPrecommitUponNewHeight(t *testing.T) { - configSetup(t) - config.Consensus.SkipTimeoutCommit = false cs1, vss := randState(4) @@ -1657,7 +1605,7 @@ func TestResetTimeoutPrecommitUponNewHeight(t *testing.T) { newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1787,8 +1735,6 @@ func TestStateSlashingPrecommits(t *testing.T) { // 4 vals. // we receive a final precommit after going into next round, but others might have gone to commit already! func TestStateHalt1(t *testing.T) { - configSetup(t) - cs1, vss := randState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] height, round := cs1.Height, cs1.Round @@ -1798,7 +1744,7 @@ func TestStateHalt1(t *testing.T) { timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock) - pv1, err := cs1.privValidator.GetPubKey(context.Background()) + pv1, err := cs1.privValidator.GetPubKey() require.NoError(t, err) addr := pv1.Address() voteCh := subscribeToVoter(cs1, addr) @@ -1856,8 +1802,6 @@ func TestStateHalt1(t *testing.T) { } func TestStateOutputsBlockPartsStats(t *testing.T) { - configSetup(t) - // create dummy peer cs, _ := randState(1) peer := p2pmock.NewPeer(nil) @@ -1901,8 +1845,6 @@ func TestStateOutputsBlockPartsStats(t *testing.T) { } func TestStateOutputVoteStats(t *testing.T) { - configSetup(t) - cs, vss := randState(2) // create dummy peer peer := p2pmock.NewPeer(nil) diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index e611c3981..68c4d98c0 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -1,7 +1,6 @@ package types import ( - "context" "fmt" "os" "testing" @@ -58,7 +57,7 @@ func TestPeerCatchupRounds(t *testing.T) { func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []types.PrivValidator) *types.Vote { privVal := privVals[valIndex] - pubKey, err := privVal.GetPubKey(context.Background()) + pubKey, err := privVal.GetPubKey() if err != nil { panic(err) } @@ -77,7 +76,7 @@ func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []ty chainID := config.ChainID() v := vote.ToProto() - err = privVal.SignVote(context.Background(), chainID, v) + err = privVal.SignVote(chainID, v) if err != nil { panic(fmt.Sprintf("Error signing vote: %v", err)) } diff --git a/node/node.go b/node/node.go index 752b036b2..7b3119faf 100644 --- a/node/node.go +++ b/node/node.go @@ -516,27 +516,27 @@ func createConsensusReactor( reactorShim := p2p.NewReactorShim(logger, "ConsensusShim", cs.ChannelShims) - var ( - channels map[p2p.ChannelID]*p2p.Channel - peerUpdates *p2p.PeerUpdates - ) + // var ( + // channels map[p2p.ChannelID]*p2p.Channel + // peerUpdates *p2p.PeerUpdates + // ) - if useLegacyP2P { - channels = getChannelsFromShim(reactorShim) - peerUpdates = reactorShim.PeerUpdates - } else { - channels = makeChannelsFromShims(router, cs.ChannelShims) - peerUpdates = peerManager.Subscribe() - } + // if useLegacyP2P { + // channels = getChannelsFromShim(reactorShim) + // peerUpdates = reactorShim.PeerUpdates + // } else { + // channels = makeChannelsFromShims(router, cs.ChannelShims) + // peerUpdates = peerManager.Subscribe() + // } reactor := cs.NewReactor( - logger, + // logger, consensusState, - channels[cs.StateChannel], - channels[cs.DataChannel], - channels[cs.VoteChannel], - channels[cs.VoteSetBitsChannel], - peerUpdates, + // channels[cs.StateChannel], + // channels[cs.DataChannel], + // channels[cs.VoteChannel], + // channels[cs.VoteSetBitsChannel], + // peerUpdates, waitSync, cs.ReactorMetrics(csMetrics), ) diff --git a/proto/tendermint/consensus/message.go b/proto/tendermint/consensus/message.go deleted file mode 100644 index 51feffc22..000000000 --- a/proto/tendermint/consensus/message.go +++ /dev/null @@ -1,80 +0,0 @@ -package consensus - -import ( - fmt "fmt" - - proto "github.com/gogo/protobuf/proto" -) - -// Wrap implements the p2p Wrapper interface and wraps a consensus proto message. -func (m *Message) Wrap(pb proto.Message) error { - switch msg := pb.(type) { - case *NewRoundStep: - m.Sum = &Message_NewRoundStep{NewRoundStep: msg} - - case *NewValidBlock: - m.Sum = &Message_NewValidBlock{NewValidBlock: msg} - - case *Proposal: - m.Sum = &Message_Proposal{Proposal: msg} - - case *ProposalPOL: - m.Sum = &Message_ProposalPol{ProposalPol: msg} - - case *BlockPart: - m.Sum = &Message_BlockPart{BlockPart: msg} - - case *Vote: - m.Sum = &Message_Vote{Vote: msg} - - case *HasVote: - m.Sum = &Message_HasVote{HasVote: msg} - - case *VoteSetMaj23: - m.Sum = &Message_VoteSetMaj23{VoteSetMaj23: msg} - - case *VoteSetBits: - m.Sum = &Message_VoteSetBits{VoteSetBits: msg} - - default: - return fmt.Errorf("unknown message: %T", msg) - } - - return nil -} - -// Unwrap implements the p2p Wrapper interface and unwraps a wrapped consensus -// proto message. -func (m *Message) Unwrap() (proto.Message, error) { - switch msg := m.Sum.(type) { - case *Message_NewRoundStep: - return m.GetNewRoundStep(), nil - - case *Message_NewValidBlock: - return m.GetNewValidBlock(), nil - - case *Message_Proposal: - return m.GetProposal(), nil - - case *Message_ProposalPol: - return m.GetProposalPol(), nil - - case *Message_BlockPart: - return m.GetBlockPart(), nil - - case *Message_Vote: - return m.GetVote(), nil - - case *Message_HasVote: - return m.GetHasVote(), nil - - case *Message_VoteSetMaj23: - return m.GetVoteSetMaj23(), nil - - case *Message_VoteSetBits: - return m.GetVoteSetBits(), nil - - default: - return nil, fmt.Errorf("unknown message: %T", msg) - } -} diff --git a/proto/tendermint/statesync/message.go b/proto/tendermint/statesync/message.go index 71d0b4eb8..fe38bda51 100644 --- a/proto/tendermint/statesync/message.go +++ b/proto/tendermint/statesync/message.go @@ -7,7 +7,7 @@ import ( proto "github.com/gogo/protobuf/proto" ) -// Wrap implements the p2p Wrapper interface and wraps a state sync proto message. +// Wrap implements the p2p Wrapper interface and wraps a state sync messages. func (m *Message) Wrap(pb proto.Message) error { switch msg := pb.(type) { case *ChunkRequest: @@ -30,7 +30,7 @@ func (m *Message) Wrap(pb proto.Message) error { } // Unwrap implements the p2p Wrapper interface and unwraps a wrapped state sync -// proto message. +// message. func (m *Message) Unwrap() (proto.Message, error) { switch msg := m.Sum.(type) { case *Message_ChunkRequest: diff --git a/statesync/reactor.go b/statesync/reactor.go index 5b4d1d95d..a7e536b0f 100644 --- a/statesync/reactor.go +++ b/statesync/reactor.go @@ -68,8 +68,8 @@ const ( chunkMsgSize = int(16e6) ) -// Reactor handles state sync, both restoring snapshots for the local node and -// serving snapshots for other nodes. +// Reactor handles state sync, both restoring snapshots for the local node and serving snapshots +// for other nodes. type Reactor struct { service.BaseService