evidence: introduction of LightClientAttackEvidence and refactor of evidence lifecycle (#5361)

evidence: modify evidence types (#5342)

light: detect light client attacks (#5344)

evidence: refactor evidence pool (#5345)

abci: application evidence prepared by evidence pool (#5354)

consensus/byzantine_test.go

@@ -35,6 +35,7 @@ import (
 func TestByzantinePrevoteEquivocation(t *testing.T) {
 	const nValidators = 4
 	const byzantineNode = 0
+	const prevoteHeight = int64(2)
 	testName := "consensus_byzantine_test"
 	tickerFunc := newMockTickerFunc(true)
 	appFunc := newCounter
@@ -129,7 +130,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 	// alter prevote so that the byzantine node double votes when height is 2
 	bcs.doPrevote = func(height int64, round int32) {
 		// allow first height to happen normally so that byzantine validator is no longer proposer
-		if height == 2 {
+		if height == prevoteHeight {
 			bcs.Logger.Info("Sending two votes")
 			prevote1, err := bcs.signVote(tmproto.PrevoteType, bcs.ProposalBlock.Hash(), bcs.ProposalBlockParts.Header())
 			require.NoError(t, err)
@@ -162,22 +163,46 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
 
 	// Evidence should be submitted and committed at the third height but
 	// we will check the first five just in case
-	var evidence types.Evidence
+	evidenceFromEachValidator := make([]types.Evidence, nValidators)
 
-	for i := 0; i < 5; i++ {
-		msg := <-blocksSubs[nValidators-1].Out()
-		block := msg.Data().(types.EventDataNewBlock).Block
-		if len(block.Evidence.Evidence) > 0 {
-			evidence = block.Evidence.Evidence[0]
-			break
+	wg := new(sync.WaitGroup)
+	wg.Add(4)
+	for height := 1; height < 5; height++ {
+		for i := 0; i < nValidators; i++ {
+			go func(j int) {
+				msg := <-blocksSubs[j].Out()
+				block := msg.Data().(types.EventDataNewBlock).Block
+				if len(block.Evidence.Evidence) != 0 {
+					evidenceFromEachValidator[j] = block.Evidence.Evidence[0]
+					wg.Done()
+				}
+			}(i)
 		}
 	}
 
-	if assert.NotNil(t, evidence) {
-		ev, ok := evidence.(*types.DuplicateVoteEvidence)
-		assert.True(t, ok)
-		pubkey, _ := bcs.privValidator.GetPubKey()
-		assert.Equal(t, []byte(pubkey.Address()), ev.Address())
+	done := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(done)
+	}()
+
+	pubkey, _ := bcs.privValidator.GetPubKey()
+
+	select {
+	case <-done:
+		for idx, ev := range evidenceFromEachValidator {
+			if assert.NotNil(t, ev, idx) {
+				ev, ok := ev.(*types.DuplicateVoteEvidence)
+				assert.True(t, ok)
+				assert.Equal(t, pubkey.Address(), ev.VoteA.ValidatorAddress)
+				assert.Equal(t, prevoteHeight, ev.Height())
+			}
+		}
+	case <-time.After(10 * time.Second):
+		for i, reactor := range reactors {
+			t.Logf("Consensus Reactor %d\n%v", i, reactor)
+		}
+		t.Fatalf("Timed out waiting for all validators to commit first block")
 	}
 }
 

consensus/common_test.go

@@ -386,7 +386,7 @@ func newStateWithConfigAndBlockStore(
 		mempool.EnableTxsAvailable()
 	}
 
-	evpool := emptyEvidencePool{}
+	evpool := sm.EmptyEvidencePool{}
 
 	// Make State
 	stateDB := blockDB

consensus/reactor_test.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	dbm "github.com/tendermint/tm-db"
@@ -29,9 +30,10 @@ import (
 	tmsync "github.com/tendermint/tendermint/libs/sync"
 	mempl "github.com/tendermint/tendermint/mempool"
 	"github.com/tendermint/tendermint/p2p"
-	"github.com/tendermint/tendermint/p2p/mock"
+	p2pmock "github.com/tendermint/tendermint/p2p/mock"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	sm "github.com/tendermint/tendermint/state"
+	statemocks "github.com/tendermint/tendermint/state/mocks"
 	"github.com/tendermint/tendermint/store"
 	"github.com/tendermint/tendermint/types"
 )
@@ -165,11 +167,20 @@ func TestReactorWithEvidence(t *testing.T) {
 		// mock the evidence pool
 		// everyone includes evidence of another double signing
 		vIdx := (i + 1) % nValidators
-		evpool := newMockEvidencePool(privVals[vIdx])
+		evpool := &statemocks.EvidencePool{}
+		evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil)
+		evpool.On("PendingEvidence", mock.AnythingOfType("uint32")).Return([]types.Evidence{
+			types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID()),
+		})
+		evpool.On("Update", mock.AnythingOfType("state.State")).Return()
+		evpool.On("ABCIEvidence", mock.AnythingOfType("int64"), mock.AnythingOfType("[]types.Evidence")).Return(
+			[]abci.Evidence{})
+
+		evpool2 := sm.EmptyEvidencePool{}
 
 		// Make State
 		blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool)
-		cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool)
+		cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2)
 		cs.SetLogger(log.TestingLogger().With("module", "consensus"))
 		cs.SetPrivValidator(pv)
 
@@ -188,53 +199,16 @@ func TestReactorWithEvidence(t *testing.T) {
 	reactors, blocksSubs, eventBuses := startConsensusNet(t, css, nValidators)
 	defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses)
 
-	// wait till everyone makes the first new block with no evidence
-	timeoutWaitGroup(t, nValidators, func(j int) {
-		msg := <-blocksSubs[j].Out()
-		block := msg.Data().(types.EventDataNewBlock).Block
-		assert.True(t, len(block.Evidence.Evidence) == 0)
-	}, css)
-
-	// second block should have evidence
-	timeoutWaitGroup(t, nValidators, func(j int) {
-		msg := <-blocksSubs[j].Out()
-		block := msg.Data().(types.EventDataNewBlock).Block
-		assert.True(t, len(block.Evidence.Evidence) > 0)
-	}, css)
-}
-
-// mock evidence pool returns no evidence for block 1,
-// and returnes one piece for all higher blocks. The one piece
-// is for a given validator at block 1.
-type mockEvidencePool struct {
-	height int
-	ev     []types.Evidence
-}
-
-func newMockEvidencePool(val types.PrivValidator) *mockEvidencePool {
-	return &mockEvidencePool{
-		ev: []types.Evidence{types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, val, config.ChainID())},
+	// we expect for each validator that is the proposer to propose one piece of evidence.
+	for i := 0; i < nValidators; i++ {
+		timeoutWaitGroup(t, nValidators, func(j int) {
+			msg := <-blocksSubs[j].Out()
+			block := msg.Data().(types.EventDataNewBlock).Block
+			assert.Len(t, block.Evidence.Evidence, 1)
+		}, css)
 	}
 }
 
-// NOTE: maxBytes is ignored
-func (m *mockEvidencePool) PendingEvidence(maxBytes uint32) []types.Evidence {
-	if m.height > 0 {
-		return m.ev
-	}
-	return nil
-}
-func (m *mockEvidencePool) AddEvidence(types.Evidence) error { return nil }
-func (m *mockEvidencePool) Update(block *types.Block, state sm.State) {
-	if m.height > 0 {
-		if len(block.Evidence.Evidence) == 0 {
-			panic("block has no evidence")
-		}
-	}
-	m.height++
-}
-func (m *mockEvidencePool) Verify(types.Evidence) error { return nil }
-
 //------------------------------------
 
 // Ensure a testnet makes blocks when there are txs
@@ -268,7 +242,7 @@ func TestReactorReceiveDoesNotPanicIfAddPeerHasntBeenCalledYet(t *testing.T) {
 
 	var (
 		reactor = reactors[0]
-		peer    = mock.NewPeer(nil)
+		peer    = p2pmock.NewPeer(nil)
 		msg     = MustEncode(&HasVoteMessage{Height: 1,
 			Round: 1, Index: 1, Type: tmproto.PrevoteType})
 	)
@@ -291,7 +265,7 @@ func TestReactorReceivePanicsIfInitPeerHasntBeenCalledYet(t *testing.T) {
 
 	var (
 		reactor = reactors[0]
-		peer    = mock.NewPeer(nil)
+		peer    = p2pmock.NewPeer(nil)
 		msg     = MustEncode(&HasVoteMessage{Height: 1,
 			Round: 1, Index: 1, Type: tmproto.PrevoteType})
 	)
@@ -667,7 +641,7 @@ func timeoutWaitGroup(t *testing.T, n int, f func(int), css []*State) {
 
 	// we're running many nodes in-process, possibly in in a virtual machine,
 	// and spewing debug messages - making a block could take a while,
-	timeout := time.Second * 300
+	timeout := time.Second * 120
 
 	select {
 	case <-done:

consensus/replay.go

@@ -496,7 +496,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap
 
 	// Use stubs for both mempool and evidence pool since no transactions nor
 	// evidence are needed here - block already exists.
-	blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, emptyEvidencePool{})
+	blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{})
 	blockExec.SetEventBus(h.eventBus)
 
 	var err error

consensus/replay_file.go

@@ -327,7 +327,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo
 		tmos.Exit(fmt.Sprintf("Error on handshake: %v", err))
 	}
 
-	mempool, evpool := emptyMempool{}, emptyEvidencePool{}
+	mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 
 	consensusState := NewState(csConfig, state.Copy(), blockExec,

consensus/replay_stubs.go

@@ -6,7 +6,6 @@ import (
 	mempl "github.com/tendermint/tendermint/mempool"
 	tmstate "github.com/tendermint/tendermint/proto/tendermint/state"
 	"github.com/tendermint/tendermint/proxy"
-	sm "github.com/tendermint/tendermint/state"
 	"github.com/tendermint/tendermint/types"
 )
 
@@ -45,20 +44,6 @@ func (emptyMempool) TxsWaitChan() <-chan struct{} { return nil }
 func (emptyMempool) InitWAL() error { return nil }
 func (emptyMempool) CloseWAL()      {}
 
-//-----------------------------------------------------------------------------
-
-type emptyEvidencePool struct{}
-
-var _ sm.EvidencePool = emptyEvidencePool{}
-
-func (emptyEvidencePool) PendingEvidence(uint32) []types.Evidence { return nil }
-func (emptyEvidencePool) AddEvidence(types.Evidence) error        { return nil }
-func (emptyEvidencePool) Update(*types.Block, sm.State)           {}
-func (emptyEvidencePool) Verify(types.Evidence) error             { return nil }
-func (emptyEvidencePool) IsCommitted(types.Evidence) bool         { return false }
-func (emptyEvidencePool) IsPending(types.Evidence) bool           { return false }
-func (emptyEvidencePool) Header(int64) *types.Header              { return nil }
-
 //-----------------------------------------------------------------------------
 // mockProxyApp uses ABCIResponses to give the right results.
 //

consensus/replay_test.go

@@ -305,7 +305,7 @@ const (
 
 var (
 	mempool = emptyMempool{}
-	evpool  = emptyEvidencePool{}
+	evpool  = sm.EmptyEvidencePool{}
 
 	sim testSim
 )

consensus/state.go

@@ -73,7 +73,9 @@ type txNotifier interface {
 
 // interface to the evidence pool
 type evidencePool interface {
-	AddEvidence(types.Evidence) error
+	// Adds consensus based evidence to the evidence pool where time is the time
+	// of the block where the offense occurred and the validator set is the current one.
+	AddEvidenceFromConsensus(types.Evidence, time.Time, *types.ValidatorSet) error
 }
 
 // State handles execution of the consensus algorithm.
@@ -1681,13 +1683,20 @@ func (cs *State) recordMetrics(height int64, block *types.Block) {
 	cs.metrics.MissingValidators.Set(float64(missingValidators))
 	cs.metrics.MissingValidatorsPower.Set(float64(missingValidatorsPower))
 
-	cs.metrics.ByzantineValidators.Set(float64(len(block.Evidence.Evidence)))
-	byzantineValidatorsPower := int64(0)
+	// NOTE: byzantine validators power and count is only for consensus evidence i.e. duplicate vote
+	var (
+		byzantineValidatorsPower = int64(0)
+		byzantineValidatorsCount = int64(0)
+	)
 	for _, ev := range block.Evidence.Evidence {
-		if _, val := cs.Validators.GetByAddress(ev.Address()); val != nil {
-			byzantineValidatorsPower += val.VotingPower
+		if dve, ok := ev.(*types.DuplicateVoteEvidence); ok {
+			if _, val := cs.Validators.GetByAddress(dve.VoteA.ValidatorAddress); val != nil {
+				byzantineValidatorsCount++
+				byzantineValidatorsPower += val.VotingPower
+			}
 		}
 	}
+	cs.metrics.ByzantineValidators.Set(float64(byzantineValidatorsCount))
 	cs.metrics.ByzantineValidatorsPower.Set(float64(byzantineValidatorsPower))
 
 	if height > 1 {
@@ -1855,7 +1864,8 @@ func (cs *State) tryAddVote(vote *types.Vote, peerID p2p.ID) (bool, error) {
 			} else {
 				timestamp = sm.MedianTime(cs.LastCommit.MakeCommit(), cs.LastValidators)
 			}
-			evidenceErr := cs.evpool.AddEvidence(types.NewDuplicateVoteEvidence(voteErr.VoteA, voteErr.VoteB, timestamp))
+			evidenceErr := cs.evpool.AddEvidenceFromConsensus(
+				types.NewDuplicateVoteEvidence(voteErr.VoteA, voteErr.VoteB), timestamp, cs.Validators)
 			if evidenceErr != nil {
 				cs.Logger.Error("Failed to add evidence to the evidence pool", "err", evidenceErr)
 			}

consensus/wal_generator.go

@@ -82,7 +82,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) {
 		}
 	})
 	mempool := emptyMempool{}
-	evpool := emptyEvidencePool{}
+	evpool := sm.EmptyEvidencePool{}
 	blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool)
 	consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool)
 	consensusState.SetLogger(logger)