Files
tendermint/test/e2e/runner/evidence.go
Callum Waters 74af343f28 statesync: tune backfill process (#6565)
This PR make some tweaks to backfill after running e2e tests:
- Separates sync and backfill as two distinct processes that the node calls. The reason is because if sync fails then the node should fail but if backfill fails it is still possible to proceed.
- Removes peers who don't have the block at a height from the local peer list. As the process goes backwards if a node doesn't have a block at a height they're likely pruning blocks and thus they won't have any prior ones either. 
- Sleep when we've run out of peers, then try again.
2021-06-11 15:26:18 +00:00

275 lines
8.2 KiB
Go

package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"path/filepath"
"time"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/tmhash"
"github.com/tendermint/tendermint/internal/test/factory"
tmjson "github.com/tendermint/tendermint/libs/json"
"github.com/tendermint/tendermint/privval"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
"github.com/tendermint/tendermint/version"
)
// 1 in 4 evidence is light client evidence, the rest is duplicate vote evidence
const lightClientEvidenceRatio = 4
// InjectEvidence takes a running testnet and generates an amount of valid
// evidence and broadcasts it to a random node through the rpc endpoint `/broadcast_evidence`.
// Evidence is random and can be a mixture of LightClientAttackEvidence and
// DuplicateVoteEvidence.
func InjectEvidence(testnet *e2e.Testnet, amount int) error {
// select a random node
targetNode := testnet.RandomNode()
logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount))
client, err := targetNode.Client()
if err != nil {
return err
}
// request the latest block and validator set from the node
blockRes, err := client.Block(context.Background(), nil)
if err != nil {
return err
}
lightEvidenceCommonHeight := blockRes.Block.Height
waitHeight := blockRes.Block.Height + 3
duplicateVoteHeight := waitHeight
nValidators := 100
valRes, err := client.Validators(context.Background(), &lightEvidenceCommonHeight, nil, &nValidators)
if err != nil {
return err
}
valSet, err := types.ValidatorSetFromExistingValidators(valRes.Validators)
if err != nil {
return err
}
// get the private keys of all the validators in the network
privVals, err := getPrivateValidatorKeys(testnet)
if err != nil {
return err
}
// wait for the node to reach the height above the forged height so that
// it is able to validate the evidence
status, err := waitForNode(targetNode, waitHeight, 15*time.Second)
if err != nil {
return err
}
duplicateVoteTime := status.SyncInfo.LatestBlockTime
var ev types.Evidence
for i := 1; i <= amount; i++ {
if i%lightClientEvidenceRatio == 0 {
ev, err = generateLightClientAttackEvidence(
privVals, lightEvidenceCommonHeight, valSet, testnet.Name, blockRes.Block.Time,
)
} else {
ev, err = generateDuplicateVoteEvidence(
privVals, duplicateVoteHeight, valSet, testnet.Name, duplicateVoteTime,
)
}
if err != nil {
return err
}
_, err := client.BroadcastEvidence(context.Background(), ev)
if err != nil {
return err
}
}
logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2))
return nil
}
func getPrivateValidatorKeys(testnet *e2e.Testnet) ([]types.MockPV, error) {
privVals := []types.MockPV{}
for _, node := range testnet.Nodes {
if node.Mode == e2e.ModeValidator {
privKeyPath := filepath.Join(testnet.Dir, node.Name, PrivvalKeyFile)
privKey, err := readPrivKey(privKeyPath)
if err != nil {
return nil, err
}
// Create mock private validators from the validators private key. MockPV is
// stateless which means we can double vote and do other funky stuff
privVals = append(privVals, types.NewMockPVWithParams(privKey, false, false))
}
}
return privVals, nil
}
// creates evidence of a lunatic attack. The height provided is the common height.
// The forged height happens 2 blocks later.
func generateLightClientAttackEvidence(
privVals []types.MockPV,
height int64,
vals *types.ValidatorSet,
chainID string,
evTime time.Time,
) (*types.LightClientAttackEvidence, error) {
// forge a random header
forgedHeight := height + 2
forgedTime := evTime.Add(1 * time.Second)
header := makeHeaderRandom(chainID, forgedHeight)
header.Time = forgedTime
// add a new bogus validator and remove an existing one to
// vary the validator set slightly
pv, conflictingVals, err := mutateValidatorSet(privVals, vals)
if err != nil {
return nil, err
}
header.ValidatorsHash = conflictingVals.Hash()
// create a commit for the forged header
blockID := makeBlockID(header.Hash(), 1000, []byte("partshash"))
voteSet := types.NewVoteSet(chainID, forgedHeight, 0, tmproto.SignedMsgType(2), conflictingVals)
commit, err := factory.MakeCommit(blockID, forgedHeight, 0, voteSet, pv, forgedTime)
if err != nil {
return nil, err
}
ev := &types.LightClientAttackEvidence{
ConflictingBlock: &types.LightBlock{
SignedHeader: &types.SignedHeader{
Header: header,
Commit: commit,
},
ValidatorSet: conflictingVals,
},
CommonHeight: height,
TotalVotingPower: vals.TotalVotingPower(),
Timestamp: evTime,
}
ev.ByzantineValidators = ev.GetByzantineValidators(vals, &types.SignedHeader{
Header: makeHeaderRandom(chainID, forgedHeight),
})
return ev, nil
}
// generateDuplicateVoteEvidence picks a random validator from the val set and
// returns duplicate vote evidence against the validator
func generateDuplicateVoteEvidence(
privVals []types.MockPV,
height int64,
vals *types.ValidatorSet,
chainID string,
time time.Time,
) (*types.DuplicateVoteEvidence, error) {
// nolint:gosec // G404: Use of weak random number generator
privVal := privVals[rand.Intn(len(privVals))]
valIdx, _ := vals.GetByAddress(privVal.PrivKey.PubKey().Address())
voteA, err := factory.MakeVote(privVal, chainID, valIdx, height, 0, 2, makeRandomBlockID(), time)
if err != nil {
return nil, err
}
voteB, err := factory.MakeVote(privVal, chainID, valIdx, height, 0, 2, makeRandomBlockID(), time)
if err != nil {
return nil, err
}
return types.NewDuplicateVoteEvidence(voteA, voteB, time, vals), nil
}
func readPrivKey(keyFilePath string) (crypto.PrivKey, error) {
keyJSONBytes, err := ioutil.ReadFile(keyFilePath)
if err != nil {
return nil, err
}
pvKey := privval.FilePVKey{}
err = tmjson.Unmarshal(keyJSONBytes, &pvKey)
if err != nil {
return nil, fmt.Errorf("error reading PrivValidator key from %v: %w", keyFilePath, err)
}
return pvKey.PrivKey, nil
}
func makeHeaderRandom(chainID string, height int64) *types.Header {
return &types.Header{
Version: version.Consensus{Block: version.BlockProtocol, App: 1},
ChainID: chainID,
Height: height,
Time: time.Now(),
LastBlockID: makeBlockID([]byte("headerhash"), 1000, []byte("partshash")),
LastCommitHash: crypto.CRandBytes(tmhash.Size),
DataHash: crypto.CRandBytes(tmhash.Size),
ValidatorsHash: crypto.CRandBytes(tmhash.Size),
NextValidatorsHash: crypto.CRandBytes(tmhash.Size),
ConsensusHash: crypto.CRandBytes(tmhash.Size),
AppHash: crypto.CRandBytes(tmhash.Size),
LastResultsHash: crypto.CRandBytes(tmhash.Size),
EvidenceHash: crypto.CRandBytes(tmhash.Size),
ProposerAddress: crypto.CRandBytes(crypto.AddressSize),
}
}
func makeRandomBlockID() types.BlockID {
return makeBlockID(crypto.CRandBytes(tmhash.Size), 100, crypto.CRandBytes(tmhash.Size))
}
func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.BlockID {
var (
h = make([]byte, tmhash.Size)
psH = make([]byte, tmhash.Size)
)
copy(h, hash)
copy(psH, partSetHash)
return types.BlockID{
Hash: h,
PartSetHeader: types.PartSetHeader{
Total: partSetSize,
Hash: psH,
},
}
}
func mutateValidatorSet(privVals []types.MockPV, vals *types.ValidatorSet,
) ([]types.PrivValidator, *types.ValidatorSet, error) {
newVal, newPrivVal := factory.RandValidator(false, 10)
var newVals *types.ValidatorSet
if vals.Size() > 2 {
newVals = types.NewValidatorSet(append(vals.Copy().Validators[:vals.Size()-1], newVal))
} else {
newVals = types.NewValidatorSet(append(vals.Copy().Validators, newVal))
}
// we need to sort the priv validators with the same index as the validator set
pv := make([]types.PrivValidator, newVals.Size())
for idx, val := range newVals.Validators {
found := false
for _, p := range append(privVals, newPrivVal.(types.MockPV)) {
if bytes.Equal(p.PrivKey.PubKey().Address(), val.Address) {
pv[idx] = p
found = true
break
}
}
if !found {
return nil, nil, fmt.Errorf("missing priv validator for %v", val.Address)
}
}
return pv, newVals, nil
}