e2e: add evidence tests (#9292)

test/e2e/networks/ci.toml

@@ -3,6 +3,7 @@
 
 ipv6 = true
 initial_height = 1000
+evidence = 5
 initial_state = { initial01 = "a", initial02 = "b", initial03 = "c" }
 prepare_proposal_delay = "100ms"
 process_proposal_delay = "100ms"
@@ -30,11 +31,7 @@ validator05 = 50
 
 [node.seed01]
 mode = "seed"
-seeds = ["seed02"]
-
-[node.seed02]
-mode = "seed"
-seeds = ["seed01"]
+perturb = ["restart"]
 
 [node.validator01]
 seeds = ["seed01"]
@@ -42,7 +39,7 @@ snapshot_interval = 5
 perturb = ["disconnect"]
 
 [node.validator02]
-seeds = ["seed02"]
+seeds = ["seed01"]
 database = "boltdb"
 abci_protocol = "tcp"
 privval_protocol = "tcp"
@@ -56,7 +53,7 @@ database = "badgerdb"
 #abci_protocol = "grpc"
 privval_protocol = "unix"
 persist_interval = 3
-retain_blocks = 3
+retain_blocks = 10
 perturb = ["kill"]
 
 [node.validator04]
@@ -67,7 +64,7 @@ perturb = ["pause"]
 
 [node.validator05]
 start_at = 1005 # Becomes part of the validator set at 1010
-seeds = ["seed02"]
+persistent_peers = ["validator01", "full01"]
 database = "cleveldb"
 fast_sync = true
 mempool_version = "v1"
@@ -81,7 +78,7 @@ start_at = 1010
 mode = "full"
 fast_sync = true
 persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
-retain_blocks = 1
+retain_blocks = 10
 perturb = ["restart"]
 
 [node.full02]

test/e2e/pkg/manifest.go

@@ -52,6 +52,10 @@ type Manifest struct {
 	// Options are ed25519 & secp256k1
 	KeyType string `toml:"key_type"`
 
+	// Evidence indicates the amount of evidence that will be injected into the
+	// testnet via the RPC endpoint of a random node. Default is 0
+	Evidence int `toml:"evidence"`
+
 	// ABCIProtocol specifies the protocol used to communicate with the ABCI
 	// application: "unix", "tcp", "grpc", or "builtin". Defaults to builtin.
 	// builtin will build a complete Tendermint node into the application and

test/e2e/pkg/testnet.go

@@ -46,6 +46,9 @@ const (
 	PerturbationKill       Perturbation = "kill"
 	PerturbationPause      Perturbation = "pause"
 	PerturbationRestart    Perturbation = "restart"
+
+	EvidenceAgeHeight int64         = 7
+	EvidenceAgeTime   time.Duration = 500 * time.Millisecond
 )
 
 // Testnet represents a single testnet.
@@ -60,6 +63,7 @@ type Testnet struct {
 	ValidatorUpdates     map[int64]map[*Node]int64
 	Nodes                []*Node
 	KeyType              string
+	Evidence             int
 	ABCIProtocol         string
 	PrepareProposalDelay time.Duration
 	ProcessProposalDelay time.Duration
@@ -126,6 +130,7 @@ func LoadTestnet(file string) (*Testnet, error) {
 		Validators:           map[*Node]int64{},
 		ValidatorUpdates:     map[int64]map[*Node]int64{},
 		Nodes:                []*Node{},
+		Evidence:             manifest.Evidence,
 		ABCIProtocol:         manifest.ABCIProtocol,
 		PrepareProposalDelay: manifest.PrepareProposalDelay,
 		ProcessProposalDelay: manifest.ProcessProposalDelay,
@@ -335,6 +340,10 @@ func (n Node) Validate(testnet Testnet) error {
 	if n.StateSync && n.StartAt == 0 {
 		return errors.New("state synced nodes cannot start at the initial height")
 	}
+	if n.RetainBlocks != 0 && n.RetainBlocks < uint64(EvidenceAgeHeight) {
+		return fmt.Errorf("retain_blocks must be greater or equal to max evidence age (%d)",
+			EvidenceAgeHeight)
+	}
 	if n.PersistInterval == 0 && n.RetainBlocks > 0 {
 		return errors.New("persist_interval=0 requires retain_blocks=0")
 	}

test/e2e/runner/evidence.go

@@ -0,0 +1,320 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"path/filepath"
+	"time"
+
+	"github.com/tendermint/tendermint/crypto"
+	"github.com/tendermint/tendermint/crypto/tmhash"
+	"github.com/tendermint/tendermint/internal/test"
+	tmjson "github.com/tendermint/tendermint/libs/json"
+	"github.com/tendermint/tendermint/privval"
+	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
+	tmversion "github.com/tendermint/tendermint/proto/tendermint/version"
+	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
+	"github.com/tendermint/tendermint/types"
+	"github.com/tendermint/tendermint/version"
+)
+
+// 1 in 4 evidence is light client evidence, the rest is duplicate vote evidence
+const lightClientEvidenceRatio = 4
+
+// InjectEvidence takes a running testnet and generates an amount of valid
+// evidence and broadcasts it to a random node through the rpc endpoint `/broadcast_evidence`.
+// Evidence is random and can be a mixture of LightClientAttackEvidence and
+// DuplicateVoteEvidence.
+func InjectEvidence(ctx context.Context, r *rand.Rand, testnet *e2e.Testnet, amount int) error {
+	// select a random node
+	var targetNode *e2e.Node
+
+	for _, idx := range r.Perm(len(testnet.Nodes)) {
+		targetNode = testnet.Nodes[idx]
+
+		if targetNode.Mode == e2e.ModeSeed || targetNode.Mode == e2e.ModeLight {
+			targetNode = nil
+			continue
+		}
+
+		break
+	}
+
+	if targetNode == nil {
+		return errors.New("could not find node to inject evidence into")
+	}
+
+	logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount))
+
+	client, err := targetNode.Client()
+	if err != nil {
+		return err
+	}
+
+	// request the latest block and validator set from the node
+	blockRes, err := client.Block(ctx, nil)
+	if err != nil {
+		return err
+	}
+	evidenceHeight := blockRes.Block.Height
+	waitHeight := blockRes.Block.Height + 3
+
+	nValidators := 100
+	valRes, err := client.Validators(ctx, &evidenceHeight, nil, &nValidators)
+	if err != nil {
+		return err
+	}
+
+	valSet, err := types.ValidatorSetFromExistingValidators(valRes.Validators)
+	if err != nil {
+		return err
+	}
+
+	// get the private keys of all the validators in the network
+	privVals, err := getPrivateValidatorKeys(testnet)
+	if err != nil {
+		return err
+	}
+
+	// wait for the node to reach the height above the forged height so that
+	// it is able to validate the evidence
+	_, err = waitForNode(targetNode, waitHeight, time.Minute)
+	if err != nil {
+		return err
+	}
+
+	var ev types.Evidence
+	for i := 1; i <= amount; i++ {
+		if i%lightClientEvidenceRatio == 0 {
+			ev, err = generateLightClientAttackEvidence(
+				ctx, privVals, evidenceHeight, valSet, testnet.Name, blockRes.Block.Time,
+			)
+		} else {
+			ev, err = generateDuplicateVoteEvidence(
+				ctx, privVals, evidenceHeight, valSet, testnet.Name, blockRes.Block.Time,
+			)
+		}
+		if err != nil {
+			return err
+		}
+
+		_, err := client.BroadcastEvidence(ctx, ev)
+		if err != nil {
+			return err
+		}
+	}
+
+	// wait for the node to reach the height above the forged height so that
+	// it is able to validate the evidence
+	_, err = waitForNode(targetNode, blockRes.Block.Height+2, 30*time.Second)
+	if err != nil {
+		return err
+	}
+
+	logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2))
+
+	return nil
+}
+
+func getPrivateValidatorKeys(testnet *e2e.Testnet) ([]types.MockPV, error) {
+	privVals := []types.MockPV{}
+
+	for _, node := range testnet.Nodes {
+		if node.Mode == e2e.ModeValidator {
+			privKeyPath := filepath.Join(testnet.Dir, node.Name, PrivvalKeyFile)
+			privKey, err := readPrivKey(privKeyPath)
+			if err != nil {
+				return nil, err
+			}
+			// Create mock private validators from the validators private key. MockPV is
+			// stateless which means we can double vote and do other funky stuff
+			privVals = append(privVals, types.NewMockPVWithParams(privKey, false, false))
+		}
+	}
+
+	return privVals, nil
+}
+
+// creates evidence of a lunatic attack. The height provided is the common height.
+// The forged height happens 2 blocks later.
+func generateLightClientAttackEvidence(
+	ctx context.Context,
+	privVals []types.MockPV,
+	height int64,
+	vals *types.ValidatorSet,
+	chainID string,
+	evTime time.Time,
+) (*types.LightClientAttackEvidence, error) {
+	// forge a random header
+	forgedHeight := height + 2
+	forgedTime := evTime.Add(1 * time.Second)
+	header := makeHeaderRandom(chainID, forgedHeight)
+	header.Time = forgedTime
+
+	// add a new bogus validator and remove an existing one to
+	// vary the validator set slightly
+	pv, conflictingVals, err := mutateValidatorSet(ctx, privVals, vals)
+	if err != nil {
+		return nil, err
+	}
+
+	header.ValidatorsHash = conflictingVals.Hash()
+
+	// create a commit for the forged header
+	blockID := makeBlockID(header.Hash(), 1000, []byte("partshash"))
+	voteSet := types.NewVoteSet(chainID, forgedHeight, 0, tmproto.SignedMsgType(2), conflictingVals)
+	commit, err := test.MakeCommitFromVoteSet(blockID, voteSet, pv, forgedTime)
+	if err != nil {
+		return nil, err
+	}
+
+	ev := &types.LightClientAttackEvidence{
+		ConflictingBlock: &types.LightBlock{
+			SignedHeader: &types.SignedHeader{
+				Header: header,
+				Commit: commit,
+			},
+			ValidatorSet: conflictingVals,
+		},
+		CommonHeight:     height,
+		TotalVotingPower: vals.TotalVotingPower(),
+		Timestamp:        evTime,
+	}
+	ev.ByzantineValidators = ev.GetByzantineValidators(vals, &types.SignedHeader{
+		Header: makeHeaderRandom(chainID, forgedHeight),
+	})
+	return ev, nil
+}
+
+// generateDuplicateVoteEvidence picks a random validator from the val set and
+// returns duplicate vote evidence against the validator
+func generateDuplicateVoteEvidence(
+	ctx context.Context,
+	privVals []types.MockPV,
+	height int64,
+	vals *types.ValidatorSet,
+	chainID string,
+	time time.Time,
+) (*types.DuplicateVoteEvidence, error) {
+	privVal, valIdx, err := getRandomValidatorIndex(privVals, vals)
+	if err != nil {
+		return nil, err
+	}
+	voteA, err := test.MakeVote(privVal, chainID, valIdx, height, 0, 2, makeRandomBlockID(), time)
+	if err != nil {
+		return nil, err
+	}
+	voteB, err := test.MakeVote(privVal, chainID, valIdx, height, 0, 2, makeRandomBlockID(), time)
+	if err != nil {
+		return nil, err
+	}
+	ev, err := types.NewDuplicateVoteEvidence(voteA, voteB, time, vals)
+	if err != nil {
+		return nil, fmt.Errorf("could not generate evidence: %w", err)
+	}
+
+	return ev, nil
+}
+
+// getRandomValidatorIndex picks a random validator from a slice of mock PrivVals that's
+// also part of the validator set, returning the PrivVal and its index in the validator set
+func getRandomValidatorIndex(privVals []types.MockPV, vals *types.ValidatorSet) (types.MockPV, int32, error) {
+	for _, idx := range rand.Perm(len(privVals)) {
+		pv := privVals[idx]
+		valIdx, _ := vals.GetByAddress(pv.PrivKey.PubKey().Address())
+		if valIdx >= 0 {
+			return pv, valIdx, nil
+		}
+	}
+	return types.MockPV{}, -1, errors.New("no private validator found in validator set")
+}
+
+func readPrivKey(keyFilePath string) (crypto.PrivKey, error) {
+	keyJSONBytes, err := ioutil.ReadFile(keyFilePath)
+	if err != nil {
+		return nil, err
+	}
+	pvKey := privval.FilePVKey{}
+	err = tmjson.Unmarshal(keyJSONBytes, &pvKey)
+	if err != nil {
+		return nil, fmt.Errorf("error reading PrivValidator key from %v: %w", keyFilePath, err)
+	}
+
+	return pvKey.PrivKey, nil
+}
+
+func makeHeaderRandom(chainID string, height int64) *types.Header {
+	return &types.Header{
+		Version:            tmversion.Consensus{Block: version.BlockProtocol, App: 1},
+		ChainID:            chainID,
+		Height:             height,
+		Time:               time.Now(),
+		LastBlockID:        makeBlockID([]byte("headerhash"), 1000, []byte("partshash")),
+		LastCommitHash:     crypto.CRandBytes(tmhash.Size),
+		DataHash:           crypto.CRandBytes(tmhash.Size),
+		ValidatorsHash:     crypto.CRandBytes(tmhash.Size),
+		NextValidatorsHash: crypto.CRandBytes(tmhash.Size),
+		ConsensusHash:      crypto.CRandBytes(tmhash.Size),
+		AppHash:            crypto.CRandBytes(tmhash.Size),
+		LastResultsHash:    crypto.CRandBytes(tmhash.Size),
+		EvidenceHash:       crypto.CRandBytes(tmhash.Size),
+		ProposerAddress:    crypto.CRandBytes(crypto.AddressSize),
+	}
+}
+
+func makeRandomBlockID() types.BlockID {
+	return makeBlockID(crypto.CRandBytes(tmhash.Size), 100, crypto.CRandBytes(tmhash.Size))
+}
+
+func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.BlockID {
+	var (
+		h   = make([]byte, tmhash.Size)
+		psH = make([]byte, tmhash.Size)
+	)
+	copy(h, hash)
+	copy(psH, partSetHash)
+	return types.BlockID{
+		Hash: h,
+		PartSetHeader: types.PartSetHeader{
+			Total: partSetSize,
+			Hash:  psH,
+		},
+	}
+}
+
+func mutateValidatorSet(ctx context.Context, privVals []types.MockPV, vals *types.ValidatorSet,
+) ([]types.PrivValidator, *types.ValidatorSet, error) {
+	newVal, newPrivVal, err := test.Validator(ctx, 10)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var newVals *types.ValidatorSet
+	if vals.Size() > 2 {
+		newVals = types.NewValidatorSet(append(vals.Copy().Validators[:vals.Size()-1], newVal))
+	} else {
+		newVals = types.NewValidatorSet(append(vals.Copy().Validators, newVal))
+	}
+
+	// we need to sort the priv validators with the same index as the validator set
+	pv := make([]types.PrivValidator, newVals.Size())
+	for idx, val := range newVals.Validators {
+		found := false
+		for _, p := range append(privVals, newPrivVal.(types.MockPV)) {
+			if bytes.Equal(p.PrivKey.PubKey().Address(), val.Address) {
+				pv[idx] = p
+				found = true
+				break
+			}
+		}
+		if !found {
+			return nil, nil, fmt.Errorf("missing priv validator for %v", val.Address)
+		}
+	}
+
+	return pv, newVals, nil
+}

test/e2e/runner/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"os"
 	"strconv"
 
@@ -12,9 +13,9 @@ import (
 	e2e "github.com/tendermint/tendermint/test/e2e/pkg"
 )
 
-var (
-	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
-)
+const randomSeed = 2308084734268
+
+var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
 
 func main() {
 	NewCLI().Run()
@@ -56,6 +57,8 @@ func NewCLI() *CLI {
 				return err
 			}
 
+			r := rand.New(rand.NewSource(randomSeed)) // nolint: gosec
+
 			chLoadResult := make(chan error)
 			ctx, loadCancel := context.WithCancel(context.Background())
 			defer loadCancel()
@@ -84,6 +87,15 @@ func NewCLI() *CLI {
 				}
 			}
 
+			if cli.testnet.Evidence > 0 {
+				if err := InjectEvidence(ctx, r, cli.testnet, cli.testnet.Evidence); err != nil {
+					return err
+				}
+				if err := Wait(cli.testnet, 5); err != nil { // ensure chain progress
+					return err
+				}
+			}
+
 			loadCancel()
 			if err := <-chLoadResult; err != nil {
 				return err
@@ -175,6 +187,29 @@ func NewCLI() *CLI {
 		},
 	})
 
+	cli.root.AddCommand(&cobra.Command{
+		Use:   "evidence [amount]",
+		Args:  cobra.MaximumNArgs(1),
+		Short: "Generates and broadcasts evidence to a random node",
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			amount := 1
+
+			if len(args) == 1 {
+				amount, err = strconv.Atoi(args[0])
+				if err != nil {
+					return err
+				}
+			}
+
+			return InjectEvidence(
+				cmd.Context(),
+				rand.New(rand.NewSource(randomSeed)), // nolint: gosec
+				cli.testnet,
+				amount,
+			)
+		},
+	})
+
 	cli.root.AddCommand(&cobra.Command{
 		Use:   "test",
 		Short: "Runs test cases against a running testnet",

test/e2e/tests/evidence_test.go

@@ -0,0 +1,22 @@
+package e2e_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// assert that all nodes that have blocks at the height of a misbehavior has evidence
+// for that misbehavior
+func TestEvidence_Misbehavior(t *testing.T) {
+	blocks := fetchBlockChain(t)
+	testnet := loadTestnet(t)
+	seenEvidence := 0
+	for _, block := range blocks {
+		if len(block.Evidence.Evidence) != 0 {
+			seenEvidence += len(block.Evidence.Evidence)
+		}
+	}
+	require.Equal(t, testnet.Evidence, seenEvidence,
+		"difference between the amount of evidence produced and committed")
+}