evidence: introduction of LightClientAttackEvidence and refactor of evidence lifecycle (#5361)

evidence: modify evidence types (#5342)

light: detect light client attacks (#5344)

evidence: refactor evidence pool (#5345)

abci: application evidence prepared by evidence pool (#5354)

light/client.go

@@ -402,6 +402,37 @@ func (c *Client) compareWithLatestHeight(height int64) (int64, error) {
 	return height, nil
 }
 
+// Update attempts to advance the state by downloading the latest light
+// block and verifying it. It returns a new light block on a successful
+// update. Otherwise, it returns nil (plus an error, if any).
+func (c *Client) Update(now time.Time) (*types.LightBlock, error) {
+	lastTrustedHeight, err := c.LastTrustedHeight()
+	if err != nil {
+		return nil, fmt.Errorf("can't get last trusted height: %w", err)
+	}
+
+	if lastTrustedHeight == -1 {
+		// no light blocks yet => wait
+		return nil, nil
+	}
+
+	latestBlock, err := c.lightBlockFromPrimary(0)
+	if err != nil {
+		return nil, err
+	}
+
+	if latestBlock.Height > lastTrustedHeight {
+		err = c.verifyLightBlock(latestBlock, now)
+		if err != nil {
+			return nil, err
+		}
+		c.logger.Info("Advanced to new state", "height", latestBlock.Height, "hash", hash2str(latestBlock.Hash()))
+		return latestBlock, nil
+	}
+
+	return nil, nil
+}
+
 // VerifyLightBlockAtHeight fetches the light block at the given height
 // and calls verifyLightBlock. It returns the block immediately if it exists in
 // the trustedStore (no verification is needed).
@@ -579,7 +610,7 @@ func (c *Client) verifySequential(
 			"newHeight", interimBlock.Height,
 			"newHash", hash2str(interimBlock.Hash()))
 
-		err = VerifyAdjacent(c.chainID, verifiedBlock.SignedHeader, interimBlock.SignedHeader, interimBlock.ValidatorSet,
+		err = VerifyAdjacent(verifiedBlock.SignedHeader, interimBlock.SignedHeader, interimBlock.ValidatorSet,
 			c.trustingPeriod, now, c.maxClockDrift)
 		if err != nil {
 			err := ErrVerificationFailed{From: verifiedBlock.Height, To: interimBlock.Height, Reason: err}
@@ -644,13 +675,14 @@ func (c *Client) verifySkipping(
 	source provider.Provider,
 	trustedBlock *types.LightBlock,
 	newLightBlock *types.LightBlock,
-	now time.Time) error {
+	now time.Time) ([]*types.LightBlock, error) {
 
 	var (
 		blockCache = []*types.LightBlock{newLightBlock}
 		depth      = 0
 
 		verifiedBlock = trustedBlock
+		trace         = []*types.LightBlock{trustedBlock}
 	)
 
 	for {
@@ -660,13 +692,14 @@ func (c *Client) verifySkipping(
 			"newHeight", blockCache[depth].Height,
 			"newHash", hash2str(blockCache[depth].Hash()))
 
-		err := Verify(c.chainID, verifiedBlock.SignedHeader, verifiedBlock.ValidatorSet, blockCache[depth].SignedHeader,
+		err := Verify(verifiedBlock.SignedHeader, verifiedBlock.ValidatorSet, blockCache[depth].SignedHeader,
 			blockCache[depth].ValidatorSet, c.trustingPeriod, now, c.maxClockDrift, c.trustLevel)
 		switch err.(type) {
 		case nil:
 			// Have we verified the last header
 			if depth == 0 {
-				return nil
+				trace = append(trace, newLightBlock)
+				return trace, nil
 			}
 			// If not, update the lower bound to the previous upper bound
 			verifiedBlock = blockCache[depth]
@@ -674,35 +707,36 @@ func (c *Client) verifySkipping(
 			blockCache = blockCache[:depth]
 			// Reset the cache depth so that we start from the upper bound again
 			depth = 0
+			// add verifiedBlock to the trace
+			trace = append(trace, verifiedBlock)
 
 		case ErrNewValSetCantBeTrusted:
 			// do add another header to the end of the cache
 			if depth == len(blockCache)-1 {
 				pivotHeight := verifiedBlock.Height + (blockCache[depth].Height-verifiedBlock.
 					Height)*verifySkippingNumerator/verifySkippingDenominator
-				interimBlock, err := source.LightBlock(pivotHeight)
-				if err != nil {
-					return ErrVerificationFailed{From: verifiedBlock.Height, To: pivotHeight, Reason: err}
+				interimBlock, providerErr := source.LightBlock(pivotHeight)
+				if providerErr != nil {
+					return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: pivotHeight, Reason: providerErr}
 				}
 				blockCache = append(blockCache, interimBlock)
 			}
 			depth++
 
 		default:
-			return ErrVerificationFailed{From: verifiedBlock.Height, To: blockCache[depth].Height, Reason: err}
+			return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: blockCache[depth].Height, Reason: err}
 		}
 	}
 }
 
 // verifySkippingAgainstPrimary does verifySkipping plus it compares new header with
-// witnesses and replaces primary if it does not respond after
-// MaxRetryAttempts.
+// witnesses and replaces primary if it sends the light client an invalid header
 func (c *Client) verifySkippingAgainstPrimary(
 	trustedBlock *types.LightBlock,
 	newLightBlock *types.LightBlock,
 	now time.Time) error {
 
-	err := c.verifySkipping(c.primary, trustedBlock, newLightBlock, now)
+	trace, err := c.verifySkipping(c.primary, trustedBlock, newLightBlock, now)
 
 	switch errors.Unwrap(err).(type) {
 	case ErrInvalidHeader:
@@ -746,7 +780,7 @@ func (c *Client) verifySkippingAgainstPrimary(
 		//
 		// CORRECTNESS ASSUMPTION: there's at least 1 correct full node
 		// (primary or one of the witnesses).
-		if cmpErr := c.compareNewHeaderWithWitnesses(newLightBlock, now); cmpErr != nil {
+		if cmpErr := c.detectDivergence(trace, now); cmpErr != nil {
 			return cmpErr
 		}
 	default:
@@ -894,81 +928,6 @@ func (c *Client) backwards(
 	return nil
 }
 
-// compare header with all witnesses provided.
-func (c *Client) compareNewHeaderWithWitnesses(l *types.LightBlock, now time.Time) error {
-	c.providerMutex.Lock()
-	defer c.providerMutex.Unlock()
-
-	// 1. Make sure AT LEAST ONE witness returns the same header.
-	var headerMatched bool
-
-	if len(c.witnesses) == 0 {
-		return errNoWitnesses{}
-	}
-
-	// launch one goroutine per witness
-	errc := make(chan error, len(c.witnesses))
-	for i, witness := range c.witnesses {
-		go c.compareNewHeaderWithWitness(errc, l, witness, i, now)
-	}
-
-	witnessesToRemove := make([]int, 0)
-
-	// handle errors as they come
-	for i := 0; i < cap(errc); i++ {
-		err := <-errc
-
-		switch e := err.(type) {
-		case nil: // at least one header matched
-			headerMatched = true
-		case errBadWitness:
-			c.logger.Info("Requested light block from bad witness", "witness", c.witnesses[e.WitnessIndex], "err", err)
-			// if witness sent us invalid header / vals, remove it
-			if e.Code == invalidLightBlock {
-				c.logger.Info("Witness sent us invalid header / vals -> removing it", "witness", c.witnesses[e.WitnessIndex])
-				witnessesToRemove = append(witnessesToRemove, e.WitnessIndex)
-			}
-		default:
-			c.logger.Info("Requested light block from witness but got error", "error", err)
-		}
-	}
-
-	for _, idx := range witnessesToRemove {
-		c.removeWitness(idx)
-	}
-
-	if headerMatched {
-		return nil
-	}
-
-	return errors.New("awaiting response from all witnesses exceeded dropout time")
-}
-
-func (c *Client) compareNewHeaderWithWitness(errc chan error, l *types.LightBlock,
-	witness provider.Provider, witnessIndex int, now time.Time) {
-
-	altBlock, err := witness.LightBlock(l.Height)
-	if err != nil {
-		if _, ok := err.(provider.ErrBadLightBlock); ok {
-			errc <- errBadWitness{Reason: err, Code: invalidLightBlock, WitnessIndex: witnessIndex}
-		} else {
-			errc <- err
-		}
-		// if not a bad light block, then the witness has either not responded or
-		// doesn't have the block -> we ignore
-		return
-	}
-
-	if !bytes.Equal(l.Hash(), altBlock.Hash()) {
-		if bsErr := c.verifySkipping(witness, c.latestTrustedBlock, altBlock, now); bsErr != nil {
-			errc <- errBadWitness{Reason: bsErr, Code: invalidLightBlock, WitnessIndex: witnessIndex}
-			return
-		}
-	}
-
-	errc <- nil
-}
-
 // NOTE: requires a providerMutex locked.
 func (c *Client) removeWitness(idx int) {
 	switch len(c.witnesses) {
@@ -982,37 +941,6 @@ func (c *Client) removeWitness(idx int) {
 	}
 }
 
-// Update attempts to advance the state by downloading the latest light
-// block and verifying it. It returns a new light block on a successful
-// update. Otherwise, it returns nil (plus an error, if any).
-func (c *Client) Update(now time.Time) (*types.LightBlock, error) {
-	lastTrustedHeight, err := c.LastTrustedHeight()
-	if err != nil {
-		return nil, fmt.Errorf("can't get last trusted height: %w", err)
-	}
-
-	if lastTrustedHeight == -1 {
-		// no light blocks yet => wait
-		return nil, nil
-	}
-
-	latestBlock, err := c.lightBlockFromPrimary(0)
-	if err != nil {
-		return nil, err
-	}
-
-	if latestBlock.Height > lastTrustedHeight {
-		err = c.verifyLightBlock(latestBlock, now)
-		if err != nil {
-			return nil, err
-		}
-		c.logger.Info("Advanced to new state", "height", latestBlock.Height, "hash", hash2str(latestBlock.Hash()))
-		return latestBlock, nil
-	}
-
-	return nil, nil
-}
-
 // replaceProvider takes the first alternative provider and promotes it as the
 // primary provider.
 func (c *Client) replacePrimaryProvider() error {

light/client_benchmark_test.go

@@ -21,7 +21,7 @@ import (
 //
 // Remember that none of these benchmarks account for network latency.
 var (
-	benchmarkFullNode = mockp.New(GenMockNode(chainID, 1000, 100, 1, bTime))
+	benchmarkFullNode = mockp.New(genMockNode(chainID, 1000, 100, 1, bTime))
 	genesisBlock, _   = benchmarkFullNode.LightBlock(1)
 )
 

light/client_test.go

@@ -61,7 +61,7 @@ var (
 		valSet,
 	)
 	deadNode      = mockp.NewDeadMock(chainID)
-	largeFullNode = mockp.New(GenMockNode(chainID, 10, 3, 0, bTime))
+	largeFullNode = mockp.New(genMockNode(chainID, 10, 3, 0, bTime))
 )
 
 func TestValidateTrustOptions(t *testing.T) {
@@ -118,6 +118,7 @@ func TestMock(t *testing.T) {
 func TestClient_SequentialVerification(t *testing.T) {
 	newKeys := genPrivKeys(4)
 	newVals := newKeys.ToValidators(10, 1)
+	differentVals, _ := types.RandValidatorSet(10, 100)
 
 	testCases := []struct {
 		name         string
@@ -146,6 +147,26 @@ func TestClient_SequentialVerification(t *testing.T) {
 			true,
 			false,
 		},
+		{
+			"bad: no first signed header",
+			map[int64]*types.SignedHeader{},
+			map[int64]*types.ValidatorSet{
+				1: differentVals,
+			},
+			true,
+			true,
+		},
+		{
+			"bad: different first validator set",
+			map[int64]*types.SignedHeader{
+				1: h1,
+			},
+			map[int64]*types.ValidatorSet{
+				1: differentVals,
+			},
+			true,
+			true,
+		},
 		{
 			"bad: 1/3 signed interim header",
 			map[int64]*types.SignedHeader{
@@ -356,15 +377,15 @@ func TestClient_SkippingVerification(t *testing.T) {
 // start from a large light block to make sure that the pivot height doesn't select a height outside
 // the appropriate range
 func TestClientLargeBisectionVerification(t *testing.T) {
-	veryLargeFullNode := mockp.New(GenMockNode(chainID, 100, 3, 1, bTime))
-	l1, err := veryLargeFullNode.LightBlock(90)
+	veryLargeFullNode := mockp.New(genMockNode(chainID, 100, 3, 0, bTime))
+	trustedLightBlock, err := veryLargeFullNode.LightBlock(5)
 	require.NoError(t, err)
 	c, err := light.NewClient(
 		chainID,
 		light.TrustOptions{
 			Period: 4 * time.Hour,
-			Height: l1.Height,
-			Hash:   l1.Hash(),
+			Height: trustedLightBlock.Height,
+			Hash:   trustedLightBlock.Hash(),
 		},
 		veryLargeFullNode,
 		[]provider.Provider{veryLargeFullNode},
@@ -896,6 +917,9 @@ func TestClientRemovesWitnessIfItSendsUsIncorrectHeader(t *testing.T) {
 		},
 	)
 
+	lb1, _ := badProvider1.LightBlock(2)
+	require.NotEqual(t, lb1.Hash(), l1.Hash())
+
 	c, err := light.NewClient(
 		chainID,
 		trustOptions,
@@ -919,7 +943,7 @@ func TestClientRemovesWitnessIfItSendsUsIncorrectHeader(t *testing.T) {
 	// remaining witnesses don't have light block -> error
 	_, err = c.VerifyLightBlockAtHeight(3, bTime.Add(2*time.Hour))
 	if assert.Error(t, err) {
-		assert.Equal(t, "awaiting response from all witnesses exceeded dropout time", err.Error())
+		assert.Equal(t, light.ErrFailedHeaderCrossReferencing, err)
 	}
 	// witness does not have a light block -> left in the list
 	assert.EqualValues(t, 1, len(c.Witnesses()))

light/detector.go

@@ -0,0 +1,247 @@
+package light
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/tendermint/tendermint/light/provider"
+	"github.com/tendermint/tendermint/types"
+)
+
+// The detector component of the light client detect and handles attacks on the light client.
+// More info here:
+// tendermint/docs/architecture/adr-047-handling-evidence-from-light-client.md
+
+// detectDivergence is a second wall of defense for the light client and is used
+// only in the case of skipping verification which employs the trust level mechanism.
+//
+// It takes the target verified header and compares it with the headers of a set of
+// witness providers that the light client is connected to. If a conflicting header
+// is returned it verifies and examines the conflicting header against the verified
+// trace that was produced from the primary. If successful it produces two sets of evidence
+// and sends them to the opposite provider before halting.
+//
+// If there are no conflictinge headers, the light client deems the verified target header
+// trusted and saves it to the trusted store.
+func (c *Client) detectDivergence(primaryTrace []*types.LightBlock, now time.Time) error {
+	if primaryTrace == nil || len(primaryTrace) < 2 {
+		return errors.New("nil or single block primary trace")
+	}
+	var (
+		headerMatched      bool
+		lastVerifiedHeader = primaryTrace[len(primaryTrace)-1].SignedHeader
+		witnessesToRemove  = make([]int, 0)
+	)
+	c.logger.Info("Running detector against trace", "endBlockHeight", lastVerifiedHeader.Height,
+		"endBlockHash", lastVerifiedHeader.Hash, "length", len(primaryTrace))
+
+	c.providerMutex.Lock()
+	defer c.providerMutex.Unlock()
+
+	if len(c.witnesses) == 0 {
+		return errNoWitnesses{}
+	}
+
+	// launch one goroutine per witness to retrieve the light block of the target height
+	// and compare it with the header from the primary
+	errc := make(chan error, len(c.witnesses))
+	for i, witness := range c.witnesses {
+		go c.compareNewHeaderWithWitness(errc, lastVerifiedHeader, witness, i)
+	}
+
+	// handle errors from the header comparisons as they come in
+	for i := 0; i < cap(errc); i++ {
+		err := <-errc
+
+		switch e := err.(type) {
+		case nil: // at least one header matched
+			headerMatched = true
+		case errConflictingHeaders:
+			// We have conflicting headers. This could possibly imply an attack on the light client.
+			// First we need to verify the witness's header using the same skipping verification and then we
+			// need to find the point that the headers diverge and examine this for any evidence of an attack.
+			//
+			// We combine these actions together, verifying the witnesses headers and outputting the trace
+			// which captures the bifurcation point and if successful provides the information to create
+			supportingWitness := c.witnesses[e.WitnessIndex]
+			witnessTrace, primaryBlock, err := c.examineConflictingHeaderAgainstTrace(primaryTrace, e.Block.SignedHeader,
+				supportingWitness, now)
+			if err != nil {
+				c.logger.Info("Error validating witness's divergent header", "witness", supportingWitness, "err", err)
+				witnessesToRemove = append(witnessesToRemove, e.WitnessIndex)
+				continue
+			}
+			// if this is an equivocation or amnesia attack, i.e. the validator sets are the same, then we
+			// return the height of the conflicting block else if it is a lunatic attack and the validator sets
+			// are not the same then we send the height of the common header.
+			commonHeight := primaryBlock.Height
+			if isInvalidHeader(witnessTrace[len(witnessTrace)-1].Header, primaryBlock.Header) {
+				// height of the common header
+				commonHeight = witnessTrace[0].Height
+			}
+
+			// We are suspecting that the primary is faulty, hence we hold the witness as the source of truth
+			// and generate evidence against the primary that we can send to the witness
+			ev := &types.LightClientAttackEvidence{
+				ConflictingBlock: primaryBlock,
+				CommonHeight:     commonHeight, // the first block in the bisection is common to both providers
+			}
+			c.logger.Error("Attack detected. Sending evidence againt primary by witness", "ev", ev,
+				"primary", c.primary, "witness", supportingWitness)
+			c.sendEvidence(ev, supportingWitness)
+
+			// This may not be valid because the witness itself is at fault. So now we reverse it, examining the
+			// trace provided by the witness and holding the primary as the source of truth. Note: primary may not
+			// respond but this is okay as we will halt anyway.
+			primaryTrace, witnessBlock, err := c.examineConflictingHeaderAgainstTrace(witnessTrace, primaryBlock.SignedHeader,
+				c.primary, now)
+			if err != nil {
+				c.logger.Info("Error validating primary's divergent header", "primary", c.primary, "err", err)
+				continue
+			}
+			// if this is an equivocation or amnesia attack, i.e. the validator sets are the same, then we
+			// return the height of the conflicting block else if it is a lunatic attack and the validator sets
+			// are not the same then we send the height of the common header.
+			commonHeight = primaryBlock.Height
+			if isInvalidHeader(primaryTrace[len(primaryTrace)-1].Header, witnessBlock.Header) {
+				// height of the common header
+				commonHeight = primaryTrace[0].Height
+			}
+
+			// We now use the primary trace to create evidence against the witness and send it to the primary
+			ev = &types.LightClientAttackEvidence{
+				ConflictingBlock: witnessBlock,
+				CommonHeight:     commonHeight, // the first block in the bisection is common to both providers
+			}
+			c.logger.Error("Sending evidence against witness by primary", "ev", ev,
+				"primary", c.primary, "witness", supportingWitness)
+			c.sendEvidence(ev, c.primary)
+			// We return the error and don't process anymore witnesses
+			return e
+
+		case errBadWitness:
+			c.logger.Info("Witness returned an error during header comparison", "witness", c.witnesses[e.WitnessIndex],
+				"err", err)
+			// if witness sent us an invalid header, then remove it. If it didn't respond or couldn't find the block, then we
+			// ignore it and move on to the next witness
+			if _, ok := e.Reason.(provider.ErrBadLightBlock); ok {
+				c.logger.Info("Witness sent us invalid header / vals -> removing it", "witness", c.witnesses[e.WitnessIndex])
+				witnessesToRemove = append(witnessesToRemove, e.WitnessIndex)
+			}
+		}
+	}
+
+	for _, idx := range witnessesToRemove {
+		c.removeWitness(idx)
+	}
+
+	// 1. If we had at least one witness that returned the same header then we
+	// conclude that we can trust the header
+	if headerMatched {
+		return nil
+	}
+
+	// 2. ELse all witnesses have either not responded, don't have the block or sent invalid blocks.
+	return ErrFailedHeaderCrossReferencing
+}
+
+// compareNewHeaderWithWitness takes the verified header from the primary and compares it with a
+// header from a specified witness. The function can return one of three errors:
+//
+// 1: errConflictingHeaders -> there may have been an attack on this light client
+// 2: errBadWitness -> the witness has either not responded, doesn't have the header or has given us an invalid one
+//    Note: In the case of an invalid header we remove the witness
+// 3: nil -> the hashes of the two headers match
+func (c *Client) compareNewHeaderWithWitness(errc chan error, h *types.SignedHeader,
+	witness provider.Provider, witnessIndex int) {
+
+	lightBlock, err := witness.LightBlock(h.Height)
+	if err != nil {
+		errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex}
+		return
+	}
+
+	if !bytes.Equal(h.Hash(), lightBlock.Hash()) {
+		errc <- errConflictingHeaders{Block: lightBlock, WitnessIndex: witnessIndex}
+	}
+
+	c.logger.Info("Matching header received by witness", "height", h.Height, "witness", witnessIndex)
+	errc <- nil
+}
+
+// sendEvidence sends evidence to a provider on a best effort basis.
+func (c *Client) sendEvidence(ev *types.LightClientAttackEvidence, receiver provider.Provider) {
+	err := receiver.ReportEvidence(ev)
+	if err != nil {
+		c.logger.Error("Failed to report evidence to provider", "ev", ev, "provider", receiver)
+	}
+}
+
+// examineConflictingHeaderAgainstTrace takes a trace from one provider and a divergent header that
+// it has received from another and preforms verifySkipping at the heights of each of the intermediate
+// headers in the trace until it reaches the divergentHeader. 1 of 2 things can happen.
+//
+// 1. The light client verifies a header that is different to the intermediate header in the trace. This
+//    is the bifurcation point and the light client can create evidence from it
+// 2. The source stops responding, doesn't have the block or sends an invalid header in which case we
+//    return the error and remove the witness
+func (c *Client) examineConflictingHeaderAgainstTrace(
+	trace []*types.LightBlock,
+	divergentHeader *types.SignedHeader,
+	source provider.Provider, now time.Time) ([]*types.LightBlock, *types.LightBlock, error) {
+
+	var previouslyVerifiedBlock *types.LightBlock
+
+	for idx, traceBlock := range trace {
+		// The first block in the trace MUST be the same to the light block that the source produces
+		// else we cannot continue with verification.
+		sourceBlock, err := source.LightBlock(traceBlock.Height)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if idx == 0 {
+			if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) {
+				return nil, nil, fmt.Errorf("trusted block is different to the source's first block (%X = %X)",
+					thash, shash)
+			}
+			previouslyVerifiedBlock = sourceBlock
+			continue
+		}
+
+		// we check that the source provider can verify a block at the same height of the
+		// intermediate height
+		trace, err := c.verifySkipping(source, previouslyVerifiedBlock, sourceBlock, now)
+		if err != nil {
+			return nil, nil, fmt.Errorf("verifySkipping of conflicting header failed: %w", err)
+		}
+		// check if the headers verified by the source has diverged from the trace
+		if shash, thash := sourceBlock.Hash(), traceBlock.Hash(); !bytes.Equal(shash, thash) {
+			// Bifurcation point found!
+			return trace, traceBlock, nil
+		}
+
+		// headers are still the same. update the previouslyVerifiedBlock
+		previouslyVerifiedBlock = sourceBlock
+	}
+
+	// We have reached the end of the trace without observing a divergence. The last header  is thus different
+	// from the divergent header that the source originally sent us, then we return an error.
+	return nil, nil, fmt.Errorf("source provided different header to the original header it provided (%X != %X)",
+		previouslyVerifiedBlock.Hash(), divergentHeader.Hash())
+
+}
+
+// isInvalidHeader takes a trusted header and matches it againt a conflicting header
+// to determine whether the conflicting header was the product of a valid state transition
+// or not. If it is then all the deterministic fields of the header should be the same.
+// If not, it is an invalid header and constitutes a lunatic attack.
+func isInvalidHeader(trusted, conflicting *types.Header) bool {
+	return !bytes.Equal(trusted.ValidatorsHash, conflicting.ValidatorsHash) ||
+		!bytes.Equal(trusted.NextValidatorsHash, conflicting.NextValidatorsHash) ||
+		!bytes.Equal(trusted.ConsensusHash, conflicting.ConsensusHash) ||
+		!bytes.Equal(trusted.AppHash, conflicting.AppHash) ||
+		!bytes.Equal(trusted.LastResultsHash, conflicting.LastResultsHash)
+}

light/detector_test.go

@@ -0,0 +1,210 @@
+package light_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	dbm "github.com/tendermint/tm-db"
+
+	"github.com/tendermint/tendermint/libs/log"
+	"github.com/tendermint/tendermint/light"
+	"github.com/tendermint/tendermint/light/provider"
+	mockp "github.com/tendermint/tendermint/light/provider/mock"
+	dbs "github.com/tendermint/tendermint/light/store/db"
+	"github.com/tendermint/tendermint/types"
+)
+
+func TestLightClientAttackEvidence_Lunatic(t *testing.T) {
+	// primary performs a lunatic attack
+	var (
+		latestHeight      = int64(10)
+		valSize           = 5
+		divergenceHeight  = int64(6)
+		primaryHeaders    = make(map[int64]*types.SignedHeader, latestHeight)
+		primaryValidators = make(map[int64]*types.ValidatorSet, latestHeight)
+	)
+
+	witnessHeaders, witnessValidators, chainKeys := genMockNodeWithKeys(chainID, latestHeight, valSize, 2, bTime)
+	witness := mockp.New(chainID, witnessHeaders, witnessValidators)
+	forgedKeys := chainKeys[divergenceHeight-1].ChangeKeys(3) // we change 3 out of the 5 validators (still 2/5 remain)
+	forgedVals := forgedKeys.ToValidators(2, 0)
+
+	for height := int64(1); height <= latestHeight; height++ {
+		if height < divergenceHeight {
+			primaryHeaders[height] = witnessHeaders[height]
+			primaryValidators[height] = witnessValidators[height]
+			continue
+		}
+		primaryHeaders[height] = forgedKeys.GenSignedHeader(chainID, height, bTime.Add(time.Duration(height)*time.Minute),
+			nil, forgedVals, forgedVals, hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(forgedKeys))
+		primaryValidators[height] = forgedVals
+	}
+	primary := mockp.New(chainID, primaryHeaders, primaryValidators)
+
+	c, err := light.NewClient(
+		chainID,
+		light.TrustOptions{
+			Period: 4 * time.Hour,
+			Height: 1,
+			Hash:   primaryHeaders[1].Hash(),
+		},
+		primary,
+		[]provider.Provider{witness},
+		dbs.New(dbm.NewMemDB(), chainID),
+		light.Logger(log.TestingLogger()),
+		light.MaxRetryAttempts(1),
+	)
+	require.NoError(t, err)
+
+	// Check verification returns an error.
+	_, err = c.VerifyLightBlockAtHeight(10, bTime.Add(1*time.Hour))
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "does not match primary")
+	}
+
+	// Check evidence was sent to both full nodes.
+	evAgainstPrimary := &types.LightClientAttackEvidence{
+		// after the divergence height the valset doesn't change so we expect the evidence to be for height 10
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: primaryHeaders[10],
+			ValidatorSet: primaryValidators[10],
+		},
+		CommonHeight: 4,
+	}
+	assert.True(t, witness.HasEvidence(evAgainstPrimary))
+
+	evAgainstWitness := &types.LightClientAttackEvidence{
+		// when forming evidence against witness we learn that the canonical chain continued to change validator sets
+		// hence the conflicting block is at 7
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: witnessHeaders[7],
+			ValidatorSet: witnessValidators[7],
+		},
+		CommonHeight: 4,
+	}
+	assert.True(t, primary.HasEvidence(evAgainstWitness))
+}
+
+func TestLightClientAttackEvidence_Equivocation(t *testing.T) {
+	// primary performs an equivocation attack
+	var (
+		latestHeight      = int64(10)
+		valSize           = 5
+		divergenceHeight  = int64(6)
+		primaryHeaders    = make(map[int64]*types.SignedHeader, latestHeight)
+		primaryValidators = make(map[int64]*types.ValidatorSet, latestHeight)
+	)
+	// validators don't change in this network (however we still use a map just for convenience)
+	witnessHeaders, witnessValidators, chainKeys := genMockNodeWithKeys(chainID, latestHeight+2, valSize, 2, bTime)
+	witness := mockp.New(chainID, witnessHeaders, witnessValidators)
+
+	for height := int64(1); height <= latestHeight; height++ {
+		if height < divergenceHeight {
+			primaryHeaders[height] = witnessHeaders[height]
+			primaryValidators[height] = witnessValidators[height]
+			continue
+		}
+		// we don't have a network partition so we will make 4/5 (greater than 2/3) malicious and vote again for
+		// a different block (which we do by adding txs)
+		primaryHeaders[height] = chainKeys[height].GenSignedHeader(chainID, height,
+			bTime.Add(time.Duration(height)*time.Minute), []types.Tx{[]byte("abcd")},
+			witnessValidators[height], witnessValidators[height+1], hash("app_hash"),
+			hash("cons_hash"), hash("results_hash"), 0, len(chainKeys[height])-1)
+		primaryValidators[height] = witnessValidators[height]
+	}
+	primary := mockp.New(chainID, primaryHeaders, primaryValidators)
+
+	c, err := light.NewClient(
+		chainID,
+		light.TrustOptions{
+			Period: 4 * time.Hour,
+			Height: 1,
+			Hash:   primaryHeaders[1].Hash(),
+		},
+		primary,
+		[]provider.Provider{witness},
+		dbs.New(dbm.NewMemDB(), chainID),
+		light.Logger(log.TestingLogger()),
+		light.MaxRetryAttempts(1),
+	)
+	require.NoError(t, err)
+
+	// Check verification returns an error.
+	_, err = c.VerifyLightBlockAtHeight(10, bTime.Add(1*time.Hour))
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "does not match primary")
+	}
+
+	// Check evidence was sent to both full nodes.
+	// Common height should be set to the height of the divergent header in the instance
+	// of an equivocation attack and the validator sets are the same as what the witness has
+	evAgainstPrimary := &types.LightClientAttackEvidence{
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: primaryHeaders[divergenceHeight],
+			ValidatorSet: primaryValidators[divergenceHeight],
+		},
+		CommonHeight: divergenceHeight,
+	}
+	assert.True(t, witness.HasEvidence(evAgainstPrimary))
+
+	evAgainstWitness := &types.LightClientAttackEvidence{
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: witnessHeaders[divergenceHeight],
+			ValidatorSet: witnessValidators[divergenceHeight],
+		},
+		CommonHeight: divergenceHeight,
+	}
+	assert.True(t, primary.HasEvidence(evAgainstWitness))
+}
+
+func TestClientDivergentTraces(t *testing.T) {
+	primary := mockp.New(genMockNode(chainID, 10, 5, 2, bTime))
+	firstBlock, err := primary.LightBlock(1)
+	require.NoError(t, err)
+	witness := mockp.New(genMockNode(chainID, 10, 5, 2, bTime))
+
+	c, err := light.NewClient(
+		chainID,
+		light.TrustOptions{
+			Height: 1,
+			Hash:   firstBlock.Hash(),
+			Period: 4 * time.Hour,
+		},
+		primary,
+		[]provider.Provider{witness},
+		dbs.New(dbm.NewMemDB(), chainID),
+		light.Logger(log.TestingLogger()),
+		light.MaxRetryAttempts(1),
+	)
+	require.NoError(t, err)
+
+	// 1. Different nodes therefore a divergent header is produced but the
+	// light client can't verify it because it has a different trusted header.
+	_, err = c.VerifyLightBlockAtHeight(10, bTime.Add(1*time.Hour))
+	assert.Error(t, err)
+	assert.Equal(t, 0, len(c.Witnesses()))
+
+	// 2. Two out of three nodes don't respond but the third has a header that matches
+	// verification should be successful and all the witnesses should remain
+
+	c, err = light.NewClient(
+		chainID,
+		light.TrustOptions{
+			Height: 1,
+			Hash:   firstBlock.Hash(),
+			Period: 4 * time.Hour,
+		},
+		primary,
+		[]provider.Provider{deadNode, deadNode, primary},
+		dbs.New(dbm.NewMemDB(), chainID),
+		light.Logger(log.TestingLogger()),
+		light.MaxRetryAttempts(1),
+	)
+	require.NoError(t, err)
+	_, err = c.VerifyLightBlockAtHeight(10, bTime.Add(1*time.Hour))
+	assert.NoError(t, err)
+	assert.Equal(t, 3, len(c.Witnesses()))
+}

light/errors.go

@@ -1,6 +1,7 @@
 package light
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -39,6 +40,12 @@ func (e ErrInvalidHeader) Error() string {
 	return fmt.Sprintf("invalid header: %v", e.Reason)
 }
 
+// ErrFailedHeaderCrossReferencing is returned when the detector was not able to cross reference the header
+// with any of the connected witnesses.
+var ErrFailedHeaderCrossReferencing = errors.New("all witnesses have either not responded, don't have the " +
+	" blocks or sent invalid blocks. You should look to change your witnesses" +
+	"  or review the light client's logs for more information")
+
 // ErrVerificationFailed means either sequential or skipping verification has
 // failed to verify from header #1 to header #2 due to some reason.
 type ErrVerificationFailed struct {
@@ -58,6 +65,20 @@ func (e ErrVerificationFailed) Error() string {
 		e.From, e.To, e.Reason)
 }
 
+// ----------------------------- INTERNAL ERRORS ---------------------------------
+
+// ErrConflictingHeaders is thrown when two conflicting headers are discovered.
+type errConflictingHeaders struct {
+	Block        *types.LightBlock
+	WitnessIndex int
+}
+
+func (e errConflictingHeaders) Error() string {
+	return fmt.Sprintf(
+		"header hash (%X) from witness (%d) does not match primary",
+		e.Block.Hash(), e.WitnessIndex)
+}
+
 // errNoWitnesses means that there are not enough witnesses connected to
 // continue running the light client.
 type errNoWitnesses struct{}
@@ -66,28 +87,13 @@ func (e errNoWitnesses) Error() string {
 	return "no witnesses connected. please reset light client"
 }
 
-type badWitnessCode int
-
-const (
-	noResponse badWitnessCode = iota + 1
-	invalidLightBlock
-)
-
 // errBadWitness is returned when the witness either does not respond or
 // responds with an invalid header.
 type errBadWitness struct {
 	Reason       error
-	Code         badWitnessCode
 	WitnessIndex int
 }
 
 func (e errBadWitness) Error() string {
-	switch e.Code {
-	case noResponse:
-		return fmt.Sprintf("failed to get a header/vals from witness: %v", e.Reason)
-	case invalidLightBlock:
-		return fmt.Sprintf("witness sent us an invalid light block: %v", e.Reason)
-	default:
-		return fmt.Sprintf("unknown code: %d", e.Code)
-	}
+	return fmt.Sprintf("Witness %d returned error: %s", e.WitnessIndex, e.Reason.Error())
 }

light/helpers_test.go

@@ -73,15 +73,12 @@ func (pkz privKeys) ToValidators(init, inc int64) *types.ValidatorSet {
 }
 
 // signHeader properly signs the header with all keys from first to last exclusive.
-func (pkz privKeys) signHeader(header *types.Header, first, last int) *types.Commit {
+func (pkz privKeys) signHeader(header *types.Header, valSet *types.ValidatorSet, first, last int) *types.Commit {
 	commitSigs := make([]types.CommitSig, len(pkz))
 	for i := 0; i < len(pkz); i++ {
 		commitSigs[i] = types.NewCommitSigAbsent()
 	}
 
-	// We need this list to keep the ordering.
-	vset := pkz.ToValidators(1, 1)
-
 	blockID := types.BlockID{
 		Hash:          header.Hash(),
 		PartSetHeader: types.PartSetHeader{Total: 1, Hash: crypto.CRandBytes(32)},
@@ -89,7 +86,7 @@ func (pkz privKeys) signHeader(header *types.Header, first, last int) *types.Com
 
 	// Fill in the votes we want.
 	for i := first; i < last && i < len(pkz); i++ {
-		vote := makeVote(header, vset, pkz[i], blockID)
+		vote := makeVote(header, valSet, pkz[i], blockID)
 		commitSigs[vote.ValidatorIndex] = vote.CommitSig()
 	}
 
@@ -151,7 +148,7 @@ func (pkz privKeys) GenSignedHeader(chainID string, height int64, bTime time.Tim
 	header := genHeader(chainID, height, bTime, txs, valset, nextValset, appHash, consHash, resHash)
 	return &types.SignedHeader{
 		Header: header,
-		Commit: pkz.signHeader(header, first, last),
+		Commit: pkz.signHeader(header, valset, first, last),
 	}
 }
 
@@ -164,7 +161,7 @@ func (pkz privKeys) GenSignedHeaderLastBlockID(chainID string, height int64, bTi
 	header.LastBlockID = lastBlockID
 	return &types.SignedHeader{
 		Header: header,
-		Commit: pkz.signHeader(header, first, last),
+		Commit: pkz.signHeader(header, valset, first, last),
 	}
 }
 
@@ -176,19 +173,20 @@ func (pkz privKeys) ChangeKeys(delta int) privKeys {
 // Generates the header and validator set to create a full entire mock node with blocks to height (
 // blockSize) and with variation in validator sets. BlockIntervals are in per minute.
 // NOTE: Expected to have a large validator set size ~ 100 validators.
-func GenMockNode(
+func genMockNodeWithKeys(
 	chainID string,
 	blockSize int64,
 	valSize int,
 	valVariation float32,
 	bTime time.Time) (
-	string,
 	map[int64]*types.SignedHeader,
-	map[int64]*types.ValidatorSet) {
+	map[int64]*types.ValidatorSet,
+	map[int64]privKeys) {
 
 	var (
 		headers         = make(map[int64]*types.SignedHeader, blockSize)
-		valset          = make(map[int64]*types.ValidatorSet, blockSize)
+		valset          = make(map[int64]*types.ValidatorSet, blockSize+1)
+		keymap          = make(map[int64]privKeys, blockSize+1)
 		keys            = genPrivKeys(valSize)
 		totalVariation  = valVariation
 		valVariationInt int
@@ -198,14 +196,16 @@ func GenMockNode(
 	valVariationInt = int(totalVariation)
 	totalVariation = -float32(valVariationInt)
 	newKeys = keys.ChangeKeys(valVariationInt)
+	keymap[1] = keys
+	keymap[2] = newKeys
 
 	// genesis header and vals
 	lastHeader := keys.GenSignedHeader(chainID, 1, bTime.Add(1*time.Minute), nil,
-		keys.ToValidators(2, 2), newKeys.ToValidators(2, 2), hash("app_hash"), hash("cons_hash"),
+		keys.ToValidators(2, 0), newKeys.ToValidators(2, 0), hash("app_hash"), hash("cons_hash"),
 		hash("results_hash"), 0, len(keys))
 	currentHeader := lastHeader
 	headers[1] = currentHeader
-	valset[1] = keys.ToValidators(2, 2)
+	valset[1] = keys.ToValidators(2, 0)
 	keys = newKeys
 
 	for height := int64(2); height <= blockSize; height++ {
@@ -215,14 +215,28 @@ func GenMockNode(
 		newKeys = keys.ChangeKeys(valVariationInt)
 		currentHeader = keys.GenSignedHeaderLastBlockID(chainID, height, bTime.Add(time.Duration(height)*time.Minute),
 			nil,
-			keys.ToValidators(2, 2), newKeys.ToValidators(2, 2), hash("app_hash"), hash("cons_hash"),
+			keys.ToValidators(2, 0), newKeys.ToValidators(2, 0), hash("app_hash"), hash("cons_hash"),
 			hash("results_hash"), 0, len(keys), types.BlockID{Hash: lastHeader.Hash()})
 		headers[height] = currentHeader
-		valset[height] = keys.ToValidators(2, 2)
+		valset[height] = keys.ToValidators(2, 0)
 		lastHeader = currentHeader
 		keys = newKeys
+		keymap[height+1] = keys
 	}
 
+	return headers, valset, keymap
+}
+
+func genMockNode(
+	chainID string,
+	blockSize int64,
+	valSize int,
+	valVariation float32,
+	bTime time.Time) (
+	string,
+	map[int64]*types.SignedHeader,
+	map[int64]*types.ValidatorSet) {
+	headers, valset, _ := genMockNodeWithKeys(chainID, blockSize, valSize, valVariation, bTime)
 	return chainID, headers, valset
 }
 

light/verifier.go

@@ -30,7 +30,6 @@ var (
 // maxClockDrift defines how much untrustedHeader.Time can drift into the
 // future.
 func VerifyNonAdjacent(
-	chainID string,
 	trustedHeader *types.SignedHeader, // height=X
 	trustedVals *types.ValidatorSet, // height=X or height=X+1
 	untrustedHeader *types.SignedHeader, // height=Y
@@ -49,7 +48,6 @@ func VerifyNonAdjacent(
 	}
 
 	if err := verifyNewHeaderAndVals(
-		chainID,
 		untrustedHeader, untrustedVals,
 		trustedHeader,
 		now, maxClockDrift); err != nil {
@@ -57,7 +55,7 @@ func VerifyNonAdjacent(
 	}
 
 	// Ensure that +`trustLevel` (default 1/3) or more of last trusted validators signed correctly.
-	err := trustedVals.VerifyCommitLightTrusting(chainID, untrustedHeader.Commit, trustLevel)
+	err := trustedVals.VerifyCommitLightTrusting(trustedHeader.ChainID, untrustedHeader.Commit, trustLevel)
 	if err != nil {
 		switch e := err.(type) {
 		case types.ErrNotEnoughVotingPowerSigned:
@@ -72,8 +70,8 @@ func VerifyNonAdjacent(
 	// NOTE: this should always be the last check because untrustedVals can be
 	// intentionally made very large to DOS the light client. not the case for
 	// VerifyAdjacent, where validator set is known in advance.
-	if err := untrustedVals.VerifyCommitLight(chainID, untrustedHeader.Commit.BlockID, untrustedHeader.Height,
-		untrustedHeader.Commit); err != nil {
+	if err := untrustedVals.VerifyCommitLight(trustedHeader.ChainID, untrustedHeader.Commit.BlockID,
+		untrustedHeader.Height, untrustedHeader.Commit); err != nil {
 		return ErrInvalidHeader{err}
 	}
 
@@ -93,7 +91,6 @@ func VerifyNonAdjacent(
 // maxClockDrift defines how much untrustedHeader.Time can drift into the
 // future.
 func VerifyAdjacent(
-	chainID string,
 	trustedHeader *types.SignedHeader, // height=X
 	untrustedHeader *types.SignedHeader, // height=X+1
 	untrustedVals *types.ValidatorSet, // height=X+1
@@ -110,7 +107,6 @@ func VerifyAdjacent(
 	}
 
 	if err := verifyNewHeaderAndVals(
-		chainID,
 		untrustedHeader, untrustedVals,
 		trustedHeader,
 		now, maxClockDrift); err != nil {
@@ -127,8 +123,8 @@ func VerifyAdjacent(
 	}
 
 	// Ensure that +2/3 of new validators signed correctly.
-	if err := untrustedVals.VerifyCommitLight(chainID, untrustedHeader.Commit.BlockID, untrustedHeader.Height,
-		untrustedHeader.Commit); err != nil {
+	if err := untrustedVals.VerifyCommitLight(trustedHeader.ChainID, untrustedHeader.Commit.BlockID,
+		untrustedHeader.Height, untrustedHeader.Commit); err != nil {
 		return ErrInvalidHeader{err}
 	}
 
@@ -137,7 +133,6 @@ func VerifyAdjacent(
 
 // Verify combines both VerifyAdjacent and VerifyNonAdjacent functions.
 func Verify(
-	chainID string,
 	trustedHeader *types.SignedHeader, // height=X
 	trustedVals *types.ValidatorSet, // height=X or height=X+1
 	untrustedHeader *types.SignedHeader, // height=Y
@@ -148,22 +143,21 @@ func Verify(
 	trustLevel tmmath.Fraction) error {
 
 	if untrustedHeader.Height != trustedHeader.Height+1 {
-		return VerifyNonAdjacent(chainID, trustedHeader, trustedVals, untrustedHeader, untrustedVals,
+		return VerifyNonAdjacent(trustedHeader, trustedVals, untrustedHeader, untrustedVals,
 			trustingPeriod, now, maxClockDrift, trustLevel)
 	}
 
-	return VerifyAdjacent(chainID, trustedHeader, untrustedHeader, untrustedVals, trustingPeriod, now, maxClockDrift)
+	return VerifyAdjacent(trustedHeader, untrustedHeader, untrustedVals, trustingPeriod, now, maxClockDrift)
 }
 
 func verifyNewHeaderAndVals(
-	chainID string,
 	untrustedHeader *types.SignedHeader,
 	untrustedVals *types.ValidatorSet,
 	trustedHeader *types.SignedHeader,
 	now time.Time,
 	maxClockDrift time.Duration) error {
 
-	if err := untrustedHeader.ValidateBasic(chainID); err != nil {
+	if err := untrustedHeader.ValidateBasic(trustedHeader.ChainID); err != nil {
 		return fmt.Errorf("untrustedHeader.ValidateBasic failed: %w", err)
 	}
 

light/verifier_test.go

@@ -155,7 +155,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) {
 	for i, tc := range testCases {
 		tc := tc
 		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
-			err := light.VerifyAdjacent(chainID, header, tc.newHeader, tc.newVals, tc.trustingPeriod, tc.now, maxClockDrift)
+			err := light.VerifyAdjacent(header, tc.newHeader, tc.newVals, tc.trustingPeriod, tc.now, maxClockDrift)
 			switch {
 			case tc.expErr != nil && assert.Error(t, err):
 				assert.Equal(t, tc.expErr, err)
@@ -269,7 +269,7 @@ func TestVerifyNonAdjacentHeaders(t *testing.T) {
 	for i, tc := range testCases {
 		tc := tc
 		t.Run(fmt.Sprintf("#%d", i), func(t *testing.T) {
-			err := light.VerifyNonAdjacent(chainID, header, vals, tc.newHeader, tc.newVals, tc.trustingPeriod,
+			err := light.VerifyNonAdjacent(header, vals, tc.newHeader, tc.newVals, tc.trustingPeriod,
 				tc.now, maxClockDrift,
 				light.DefaultTrustLevel)
 
@@ -300,7 +300,7 @@ func TestVerifyReturnsErrorIfTrustLevelIsInvalid(t *testing.T) {
 			hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys))
 	)
 
-	err := light.Verify(chainID, header, vals, header, vals, 2*time.Hour, time.Now(), maxClockDrift,
+	err := light.Verify(header, vals, header, vals, 2*time.Hour, time.Now(), maxClockDrift,
 		tmmath.Fraction{Numerator: 2, Denominator: 1})
 	assert.Error(t, err)
 }