mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-01-10 06:57:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: add evidence generation and testing (#6276)

Browse Source
This commit is contained in:
Callum Waters
2021-03-29 20:00:20 +02:00
committed by GitHub
parent 32ee737d42
commit cbae3613dd
15 changed files with 337 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -38,7 +38,6 @@ terraform.tfstate.d
test/e2e/build
test/e2e/networks/*/
test/logs
test/maverick/maverick
test/p2p/data/
vendor
test/fuzz/**/corpus

9
CONTRIBUTING.md
View File

@@ -366,15 +366,6 @@ cd test/e2e && \
./build/runner -f networks/ci.toml
```
### Maverick
**If you're changing the code in `consensus` package, please make sure to
replicate all the changes in `./test/maverick/consensus`**. Maverick is a
byzantine node used to assert that the validator gets punished for malicious
behavior.
See [README](./test/maverick/README.md) for details.
### Model-based tests (ADVANCED)
*NOTE: if you're just submitting your first PR, you won't need to touch these

7
test/e2e/Makefile
View File

@@ -8,11 +8,6 @@ docker:
# ABCI testing).
app:
go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app
# To be used primarily by the e2e docker instance. If you want to produce this binary
# elsewhere, then run go build in the maverick directory.
maverick:
go build -o build/maverick -tags badgerdb,boltdb,cleveldb,rocksdb ../maverick
generator:
go build -o build/generator ./generator
@@ -20,4 +15,4 @@ generator:
runner:
go build -o build/runner ./runner
.PHONY: all app docker generator maverick runner
.PHONY: all app docker generator runner

41
test/e2e/app/main.go
View File

@@ -83,10 +83,6 @@ func run(configFile string) error {
default:
err = startNode(cfg)
}
// FIXME: Temporarily remove maverick until it is redesigned
// if len(cfg.Misbehaviors) == 0 {
// err = startMaverick(cfg)
// }
default:
err = fmt.Errorf("invalid protocol %q", cfg.Protocol)
}
@@ -227,43 +223,6 @@ func startLightNode(cfg *Config) error {
return nil
}
// FIXME: Temporarily disconnected maverick until it is redesigned
// startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
// configuration is in $TMHOME/config/tendermint.toml.
// func startMaverick(cfg *Config) error {
// app, err := NewApplication(cfg)
// if err != nil {
// return err
// }
// tmcfg, logger, nodeKey, err := setupNode()
// if err != nil {
// return fmt.Errorf("failed to setup config: %w", err)
// }
// misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors))
// for heightString, misbehaviorString := range cfg.Misbehaviors {
// height, _ := strconv.ParseInt(heightString, 10, 64)
// misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
// }
// n, err := maverick.NewNode(tmcfg,
// maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
// *nodeKey,
// proxy.NewLocalClientCreator(app),
// maverick.DefaultGenesisDocProviderFunc(tmcfg),
// maverick.DefaultDBProvider,
// maverick.DefaultMetricsProvider(tmcfg.Instrumentation),
// logger,
// misbehaviors,
// )
// if err != nil {
// return err
// }
// return n.Start()
// }
// startSigner starts a signer server connecting to the given endpoint.
func startSigner(cfg *Config) error {
filePV, err := privval.LoadFilePV(cfg.PrivValKey, cfg.PrivValState)

2
test/e2e/docker/Dockerfile
View File

@@ -19,8 +19,6 @@ COPY . .
RUN make build && cp build/tendermint /usr/bin/tendermint
COPY test/e2e/docker/entrypoint* /usr/bin/
# FIXME: Temporarily disconnect maverick node until it is redesigned
# RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
RUN cd test/e2e && make app && cp build/app /usr/bin/app
# Set up runtime directory. We don't use a separate runtime image since we need

10
test/e2e/docker/entrypoint-maverick
View File

@@ -1,10 +0,0 @@
#!/usr/bin/env bash
# Forcibly remove any stray UNIX sockets left behind from previous runs
rm -rf /var/run/privval.sock /var/run/app.sock
/usr/bin/app /tendermint/config/app.toml &
sleep 1
/usr/bin/maverick "$@"

36
test/e2e/generator/generate.go
View File

@@ -4,7 +4,6 @@ import (
"fmt"
"math/rand"
"sort"
"strconv"
"strings"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -36,20 +35,14 @@ var (
nodeStateSyncs = uniformChoice{false, true}
nodePersistIntervals = uniformChoice{0, 1, 5}
nodeSnapshotIntervals = uniformChoice{0, 3}
nodeRetainBlocks = uniformChoice{0, 1, 5}
nodeRetainBlocks = uniformChoice{0, int(e2e.EvidenceAgeHeight), int(e2e.EvidenceAgeHeight) + 5}
nodePerturbations = probSetChoice{
"disconnect": 0.1,
"pause": 0.1,
"kill": 0.1,
"restart": 0.1,
}
nodeMisbehaviors = weightedChoice{
// FIXME: evidence disabled due to node panicing when not
// having sufficient block history to process evidence.
// https://github.com/tendermint/tendermint/issues/5617
// misbehaviorOption{"double-prevote"}: 1,
misbehaviorOption{}: 9,
}
evidence = uniformChoice{0, 1, 10}
)
// Generate generates random testnets using the given RNG.
@@ -75,6 +68,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
ValidatorUpdates: map[string]map[string]int64{},
Nodes: map[string]*e2e.ManifestNode{},
KeyType: opt["keyType"].(string),
Evidence: evidence.Choose(r).(int),
}
var numSeeds, numValidators, numFulls, numLightClients int
@@ -227,17 +221,6 @@ func generateNode(
node.SnapshotInterval = 3
}
if node.Mode == string(e2e.ModeValidator) {
misbehaveAt := startAt + 5 + int64(r.Intn(10))
if startAt == 0 {
misbehaveAt += initialHeight - 1
}
node.Misbehaviors = nodeMisbehaviors.Choose(r).(misbehaviorOption).atHeight(misbehaveAt)
if len(node.Misbehaviors) != 0 {
node.PrivvalProtocol = "file"
}
}
// If a node which does not persist state also does not retain blocks, randomly
// choose to either persist state or retain all blocks.
if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 {
@@ -276,16 +259,3 @@ func generateLightNode(r *rand.Rand, startAt int64, providers []string) *e2e.Man
func ptrUint64(i uint64) *uint64 {
return &i
}
type misbehaviorOption struct {
misbehavior string
}
func (m misbehaviorOption) atHeight(height int64) map[string]string {
misbehaviorMap := make(map[string]string)
if m.misbehavior == "" {
return misbehaviorMap
}
misbehaviorMap[strconv.Itoa(int(height))] = m.misbehavior
return misbehaviorMap
}

22
test/e2e/generator/random.go
View File

@@ -56,28 +56,6 @@ func (uc uniformChoice) Choose(r *rand.Rand) interface{} {
return uc[r.Intn(len(uc))]
}
// weightedChoice chooses a single random key from a map of keys and weights.
type weightedChoice map[interface{}]uint
func (wc weightedChoice) Choose(r *rand.Rand) interface{} {
total := 0
choices := make([]interface{}, 0, len(wc))
for choice, weight := range wc {
total += int(weight)
choices = append(choices, choice)
}
rem := r.Intn(total)
for _, choice := range choices {
rem -= int(wc[choice])
if rem <= 0 {
return choice
}
}
return nil
}
// probSetChoice picks a set of strings based on each string's probability (0-1).
type probSetChoice map[string]float64

14
test/e2e/networks/ci.toml
View File

@@ -2,6 +2,7 @@
# functionality with a single network.
initial_height = 1000
evidence = 0
initial_state = { initial01 = "a", initial02 = "b", initial03 = "c" }
[validators]
@@ -37,8 +38,6 @@ seeds = ["seed01"]
seeds = ["seed01"]
snapshot_interval = 5
perturb = ["disconnect"]
# FIXME: maverick has been disabled until it is redesigned (https://github.com/tendermint/tendermint/issues/5575)
# misbehaviors = { 1018 = "double-prevote" }
[node.validator02]
seeds = ["seed02"]
@@ -80,7 +79,7 @@ mode = "full"
# FIXME: should be v2, disabled due to flake
fast_sync = "v0"
persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
retain_blocks = 1
retain_blocks = 3
perturb = ["restart"]
[node.full02]
@@ -94,10 +93,5 @@ perturb = ["restart"]
[node.light01]
mode= "light"
start_at= 1005
persistent_peers = ["validator01", "validator02", "validator03"]
[node.light02]
mode= "light"
start_at= 1015
persistent_peers = ["validator04", "full01", "validator05"]
start_at= 1010
persistent_peers = ["validator01", "validator02", "validator03"]

18
test/e2e/pkg/manifest.go
View File

@@ -51,6 +51,10 @@ type Manifest struct {
// Options are ed25519 & secp256k1
KeyType string `toml:"key_type"`
// Evidence indicates the amount of evidence that will be injected into the
// testnet via the RPC endpoint of a random node. Default is 0
Evidence int `toml:"evidence"`
// LogLevel sets the log level of the entire testnet. This can be overridden
// by individual nodes.
LogLevel string `toml:"log_level"`
@@ -113,8 +117,8 @@ type ManifestNode struct {
SnapshotInterval uint64 `toml:"snapshot_interval"`
// RetainBlocks specifies the number of recent blocks to retain. Defaults to
// 0, which retains all blocks. Must be greater that PersistInterval and
// SnapshotInterval.
// 0, which retains all blocks. Must be greater that PersistInterval,
// SnapshotInterval and EvidenceAgeHeight.
RetainBlocks uint64 `toml:"retain_blocks"`
// Perturb lists perturbations to apply to the node after it has been
@@ -126,16 +130,6 @@ type ManifestNode struct {
// restart: restarts the node, shutting it down with SIGTERM
Perturb []string `toml:"perturb"`
// Misbehaviors sets how a validator behaves during consensus at a
// certain height. Multiple misbehaviors at different heights can be used
//
// An example of misbehaviors
// { 10 = "double-prevote", 20 = "double-prevote"}
//
// For more information, look at the readme in the maverick folder.
// A list of all behaviors can be found in ../maverick/consensus/behavior.go
Misbehaviors map[string]string `toml:"misbehaviors"`
// Log level sets the log level of the specific node i.e. "consensus:info,*:error".
// This is helpful when debugging a specific problem. This overrides the network
// level.

57
test/e2e/pkg/testnet.go
View File

@@ -11,6 +11,7 @@ import (
"sort"
"strconv"
"strings"
"time"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/ed25519"
@@ -46,6 +47,9 @@ const (
PerturbationKill Perturbation = "kill"
PerturbationPause Perturbation = "pause"
PerturbationRestart Perturbation = "restart"
EvidenceAgeHeight int64 = 3
EvidenceAgeTime time.Duration = 10 * time.Second
)
// Testnet represents a single testnet.
@@ -60,6 +64,7 @@ type Testnet struct {
ValidatorUpdates map[int64]map[*Node]int64
Nodes []*Node
KeyType string
Evidence int
LogLevel string
}
@@ -84,7 +89,6 @@ type Node struct {
Seeds []*Node
PersistentPeers []*Node
Perturbations []Perturbation
Misbehaviors map[int64]string
LogLevel string
}
@@ -124,6 +128,7 @@ func LoadTestnet(file string) (*Testnet, error) {
Validators: map[*Node]int64{},
ValidatorUpdates: map[int64]map[*Node]int64{},
Nodes: []*Node{},
Evidence: manifest.Evidence,
KeyType: "ed25519",
LogLevel: manifest.LogLevel,
}
@@ -161,7 +166,6 @@ func LoadTestnet(file string) (*Testnet, error) {
SnapshotInterval: nodeManifest.SnapshotInterval,
RetainBlocks: nodeManifest.RetainBlocks,
Perturbations: []Perturbation{},
Misbehaviors: make(map[int64]string),
LogLevel: manifest.LogLevel,
}
if node.StartAt == testnet.InitialHeight {
@@ -185,13 +189,6 @@ func LoadTestnet(file string) (*Testnet, error) {
for _, p := range nodeManifest.Perturb {
node.Perturbations = append(node.Perturbations, Perturbation(p))
}
for heightString, misbehavior := range nodeManifest.Misbehaviors {
height, err := strconv.ParseInt(heightString, 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err)
}
node.Misbehaviors[height] = misbehavior
}
if nodeManifest.LogLevel != "" {
node.LogLevel = nodeManifest.LogLevel
}
@@ -344,6 +341,10 @@ func (n Node) Validate(testnet Testnet) error {
if n.StateSync && n.StartAt == 0 {
return errors.New("state synced nodes cannot start at the initial height")
}
if n.RetainBlocks != 0 && n.RetainBlocks < uint64(EvidenceAgeHeight) {
return fmt.Errorf("retain_blocks must be greater or equal to max evidence age (%d)",
EvidenceAgeHeight)
}
if n.PersistInterval == 0 && n.RetainBlocks > 0 {
return errors.New("persist_interval=0 requires retain_blocks=0")
}
@@ -362,31 +363,6 @@ func (n Node) Validate(testnet Testnet) error {
}
}
if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 {
return errors.New("must be using \"file\" privval protocol to implement misbehaviors")
}
for height, misbehavior := range n.Misbehaviors {
if height < n.StartAt {
return fmt.Errorf("misbehavior height %d is below node start height %d",
height, n.StartAt)
}
if height < testnet.InitialHeight {
return fmt.Errorf("misbehavior height %d is below network initial height %d",
height, testnet.InitialHeight)
}
exists := false
// FIXME: Maverick has been disabled until it is redesigned
// for possibleBehaviors := range mcs.MisbehaviorList {
// if possibleBehaviors == misbehavior {
// exists = true
// }
// }
if !exists {
return fmt.Errorf("misbehavior %s does not exist", misbehavior)
}
}
return nil
}
@@ -438,19 +414,6 @@ func (t Testnet) HasPerturbations() bool {
return false
}
// LastMisbehaviorHeight returns the height of the last misbehavior.
func (t Testnet) LastMisbehaviorHeight() int64 {
lastHeight := int64(0)
for _, node := range t.Nodes {
for height := range node.Misbehaviors {
if height > lastHeight {
lastHeight = height
}
}
}
return lastHeight
}
// Address returns a P2P endpoint address for the node.
func (n Node) AddressP2P(withID bool) string {
ip := n.IP.String()

274
test/e2e/runner/evidence.go Normal file
View File

@@ -0,0 +1,274 @@
package main
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"path/filepath"
"time"
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/tmhash"
tmjson "github.com/tendermint/tendermint/libs/json"
"github.com/tendermint/tendermint/privval"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
"github.com/tendermint/tendermint/version"
)
// 1 in 11 evidence is light client evidence, the rest is duplicate vote
// FIXME: Setting to 11 disables light client attack evidence since nodes
// don't follow a minimum retention height invariant. When we fix this we
// should use a ratio of 4.
const lightClientEvidenceRatio = 11
// InjectEvidence takes a running testnet and generates an amount of valid
// evidence and broadcasts it to a random node through the rpc endpoint `/broadcast_evidence`.
// Evidence is random and can be a mixture of LightClientAttackEvidence and
// DuplicateVoteEvidence.
func InjectEvidence(testnet *e2e.Testnet, amount int) error {
// select a random node
targetNode := testnet.RandomNode()
logger.Info(fmt.Sprintf("Injecting evidence through %v (amount: %d)...", targetNode.Name, amount))
client, err := targetNode.Client()
if err != nil {
return err
}
// request the latest block and validator set from the node
blockRes, err := client.Block(context.Background(), nil)
if err != nil {
return err
}
lightEvidenceCommonHeight := blockRes.Block.Height
waitHeight := blockRes.Block.Height + 3
duplicateVoteHeight := waitHeight
nValidators := 100
valRes, err := client.Validators(context.Background(), &lightEvidenceCommonHeight, nil, &nValidators)
if err != nil {
return err
}
valSet, err := types.ValidatorSetFromExistingValidators(valRes.Validators)
if err != nil {
return err
}
// get the private keys of all the validators in the network
privVals, err := getPrivateValidatorKeys(testnet)
if err != nil {
return err
}
// wait for the node to reach the height above the forged height so that
// it is able to validate the evidence
status, err := waitForNode(targetNode, waitHeight, 10*time.Second)
if err != nil {
return err
}
duplicateVoteTime := status.SyncInfo.LatestBlockTime
var ev types.Evidence
for i := 0; i < amount; i++ {
if i%lightClientEvidenceRatio == 0 {
ev, err = generateLightClientAttackEvidence(
privVals, lightEvidenceCommonHeight, valSet, testnet.Name, blockRes.Block.Time,
)
} else {
ev, err = generateDuplicateVoteEvidence(
privVals, duplicateVoteHeight, valSet, testnet.Name, duplicateVoteTime,
)
}
if err != nil {
return err
}
_, err := client.BroadcastEvidence(context.Background(), ev)
if err != nil {
return err
}
}
logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2))
return nil
}
func getPrivateValidatorKeys(testnet *e2e.Testnet) ([]types.MockPV, error) {
privVals := []types.MockPV{}
for _, node := range testnet.Nodes {
if node.Mode == e2e.ModeValidator {
privKeyPath := filepath.Join(testnet.Dir, node.Name, PrivvalKeyFile)
privKey, err := readPrivKey(privKeyPath)
if err != nil {
return nil, err
}
// Create mock private validators from the validators private key. MockPV is
// stateless which means we can double vote and do other funky stuff
privVals = append(privVals, types.NewMockPVWithParams(privKey, false, false))
}
}
return privVals, nil
}
// creates evidence of a lunatic attack. The height provided is the common height.
// The forged height happens 2 blocks later.
func generateLightClientAttackEvidence(
privVals []types.MockPV,
height int64,
vals *types.ValidatorSet,
chainID string,
evTime time.Time,
) (*types.LightClientAttackEvidence, error) {
// forge a random header
forgedHeight := height + 2
forgedTime := evTime.Add(1 * time.Second)
header := makeHeaderRandom(chainID, forgedHeight)
header.Time = forgedTime
// add a new bogus validator and remove an existing one to
// vary the validator set slightly
pv, conflictingVals, err := mutateValidatorSet(privVals, vals)
if err != nil {
return nil, err
}
header.ValidatorsHash = conflictingVals.Hash()
// create a commit for the forged header
blockID := makeBlockID(header.Hash(), 1000, []byte("partshash"))
voteSet := types.NewVoteSet(chainID, forgedHeight, 0, tmproto.SignedMsgType(2), conflictingVals)
commit, err := types.MakeCommit(blockID, forgedHeight, 0, voteSet, pv, forgedTime)
if err != nil {
return nil, err
}
ev := &types.LightClientAttackEvidence{
ConflictingBlock: &types.LightBlock{
SignedHeader: &types.SignedHeader{
Header: header,
Commit: commit,
},
ValidatorSet: conflictingVals,
},
CommonHeight: height,
TotalVotingPower: vals.TotalVotingPower(),
Timestamp: evTime,
}
ev.ByzantineValidators = ev.GetByzantineValidators(vals, &types.SignedHeader{
Header: makeHeaderRandom(chainID, forgedHeight),
})
return ev, nil
}
// generateDuplicateVoteEvidence picks a random validator from the val set and
// returns duplicate vote evidence against the validator
func generateDuplicateVoteEvidence(
privVals []types.MockPV,
height int64,
vals *types.ValidatorSet,
chainID string,
time time.Time,
) (*types.DuplicateVoteEvidence, error) {
// nolint:gosec // G404: Use of weak random number generator
privVal := privVals[rand.Intn(len(privVals))]
voteA, err := types.MakeVote(height, makeRandomBlockID(), vals, privVal, chainID, time)
if err != nil {
return nil, err
}
voteB, err := types.MakeVote(height, makeRandomBlockID(), vals, privVal, chainID, time)
if err != nil {
return nil, err
}
return types.NewDuplicateVoteEvidence(voteA, voteB, time, vals), nil
}
func readPrivKey(keyFilePath string) (crypto.PrivKey, error) {
keyJSONBytes, err := ioutil.ReadFile(keyFilePath)
if err != nil {
return nil, err
}
pvKey := privval.FilePVKey{}
err = tmjson.Unmarshal(keyJSONBytes, &pvKey)
if err != nil {
return nil, fmt.Errorf("error reading PrivValidator key from %v: %w", keyFilePath, err)
}
return pvKey.PrivKey, nil
}
func makeHeaderRandom(chainID string, height int64) *types.Header {
return &types.Header{
Version: version.Consensus{Block: version.BlockProtocol, App: 1},
ChainID: chainID,
Height: height,
Time: time.Now(),
LastBlockID: makeBlockID([]byte("headerhash"), 1000, []byte("partshash")),
LastCommitHash: crypto.CRandBytes(tmhash.Size),
DataHash: crypto.CRandBytes(tmhash.Size),
ValidatorsHash: crypto.CRandBytes(tmhash.Size),
NextValidatorsHash: crypto.CRandBytes(tmhash.Size),
ConsensusHash: crypto.CRandBytes(tmhash.Size),
AppHash: crypto.CRandBytes(tmhash.Size),
LastResultsHash: crypto.CRandBytes(tmhash.Size),
EvidenceHash: crypto.CRandBytes(tmhash.Size),
ProposerAddress: crypto.CRandBytes(crypto.AddressSize),
}
}
func makeRandomBlockID() types.BlockID {
return makeBlockID(crypto.CRandBytes(tmhash.Size), 100, crypto.CRandBytes(tmhash.Size))
}
func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types.BlockID {
var (
h = make([]byte, tmhash.Size)
psH = make([]byte, tmhash.Size)
)
copy(h, hash)
copy(psH, partSetHash)
return types.BlockID{
Hash: h,
PartSetHeader: types.PartSetHeader{
Total: partSetSize,
Hash: psH,
},
}
}
func mutateValidatorSet(privVals []types.MockPV, vals *types.ValidatorSet,
) ([]types.PrivValidator, *types.ValidatorSet, error) {
newVal, newPrivVal := types.RandValidator(false, 10)
var newVals *types.ValidatorSet
if vals.Size() > 2 {
newVals = types.NewValidatorSet(append(vals.Copy().Validators[:vals.Size()-1], newVal))
} else {
newVals = types.NewValidatorSet(append(vals.Copy().Validators, newVal))
}
// we need to sort the priv validators with the same index as the validator set
pv := make([]types.PrivValidator, newVals.Size())
for idx, val := range newVals.Validators {
found := false
for _, p := range append(privVals, newPrivVal.(types.MockPV)) {
if bytes.Equal(p.PrivKey.PubKey().Address(), val.Address) {
pv[idx] = p
found = true
break
}
}
if !found {
return nil, nil, fmt.Errorf("missing priv validator for %v", val.Address)
}
}
return pv, newVals, nil
}

35
test/e2e/runner/main.go
View File

@@ -71,14 +71,6 @@ func NewCLI() *CLI {
return err
}
if lastMisbehavior := cli.testnet.LastMisbehaviorHeight(); lastMisbehavior > 0 {
// wait for misbehaviors before starting perturbations. We do a separate
// wait for another 5 blocks, since the last misbehavior height may be
// in the past depending on network startup ordering.
if err := WaitUntil(cli.testnet, lastMisbehavior); err != nil {
return err
}
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
return err
}
@@ -92,6 +84,15 @@ func NewCLI() *CLI {
}
}
if cli.testnet.Evidence > 0 {
if err := InjectEvidence(cli.testnet, cli.testnet.Evidence); err != nil {
return err
}
if err := Wait(cli.testnet, 1); err != nil { // ensure chain progress
return err
}
}
loadCancel()
if err := <-chLoadResult; err != nil {
return err
@@ -188,6 +189,24 @@ func NewCLI() *CLI {
},
})
cli.root.AddCommand(&cobra.Command{
Use: "evidence [amount]",
Args: cobra.MaximumNArgs(1),
Short: "Generates and broadcasts evidence to a random node",
RunE: func(cmd *cobra.Command, args []string) (err error) {
amount := 1
if len(args) == 1 {
amount, err = strconv.Atoi(args[0])
if err != nil {
return err
}
}
return InjectEvidence(cli.testnet, amount)
},
})
cli.root.AddCommand(&cobra.Command{
Use: "test",
Short: "Runs test cases against a running testnet",

35
test/e2e/runner/setup.go
View File

@@ -12,7 +12,6 @@ import (
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"text/template"
"time"
@@ -132,28 +131,6 @@ func Setup(testnet *e2e.Testnet) error {
func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
// Must use version 2 Docker Compose format, to support IPv6.
tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
"startCommands": func(misbehaviors map[int64]string, logLevel string) string {
command := "start"
// FIXME: Temporarily disable behaviors until maverick is redesigned
// misbehaviorString := ""
// for height, misbehavior := range misbehaviors {
// // after the first behavior set, a comma must be prepended
// if misbehaviorString != "" {
// misbehaviorString += ","
// }
// heightString := strconv.Itoa(int(height))
// misbehaviorString += misbehavior + "," + heightString
// }
// if misbehaviorString != "" {
// command += " --misbehaviors " + misbehaviorString
// }
if logLevel != "" && logLevel != config.DefaultLogLevel {
command += " --log-level " + logLevel
}
return command
},
"addUint32": func(x, y uint32) uint32 {
return x + y
},
@@ -181,8 +158,8 @@ services:
image: tendermint/e2e-node
{{- if eq .ABCIProtocol "builtin" }}
entrypoint: /usr/bin/entrypoint-builtin
{{- else }}
command: {{ startCommands .Misbehaviors .LogLevel }}
{{- else if .LogLevel }}
command: start --log-level {{ .LogLevel }}
{{- end }}
init: true
ports:
@@ -223,6 +200,8 @@ func MakeGenesis(testnet *e2e.Testnet) (types.GenesisDoc, error) {
default:
return genesis, errors.New("unsupported KeyType")
}
genesis.ConsensusParams.Evidence.MaxAgeNumBlocks = e2e.EvidenceAgeHeight
genesis.ConsensusParams.Evidence.MaxAgeDuration = e2e.EvidenceAgeTime
for validator, power := range testnet.Validators {
genesis.Validators = append(genesis.Validators, types.GenesisValidator{
Name: validator.Name,
@@ -403,12 +382,6 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
}
}
misbehaviors := make(map[string]string)
for height, misbehavior := range node.Misbehaviors {
misbehaviors[strconv.Itoa(int(height))] = misbehavior
}
cfg["misbehaviors"] = misbehaviors
if len(node.Testnet.ValidatorUpdates) > 0 {
validatorUpdates := map[string]map[string]int64{}
for height, validators := range node.Testnet.ValidatorUpdates {

51
test/e2e/tests/evidence_test.go
View File

@@ -1,57 +1,22 @@
package e2e_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
// assert that all nodes that have blocks at the height of a misbehavior has evidence
// for that misbehavior
func TestEvidence_Misbehavior(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
seenEvidence := make(map[int64]struct{})
for _, block := range blocks {
// Find any evidence blaming this node in this block
var nodeEvidence types.Evidence
for _, evidence := range block.Evidence.Evidence {
switch evidence := evidence.(type) {
case *types.DuplicateVoteEvidence:
if bytes.Equal(evidence.VoteA.ValidatorAddress, node.PrivvalKey.PubKey().Address()) {
nodeEvidence = evidence
}
default:
t.Fatalf("unexpected evidence type %T", evidence)
}
}
if nodeEvidence == nil {
continue // no evidence for the node at this height
}
// Check that evidence was as expected
misbehavior, ok := node.Misbehaviors[nodeEvidence.Height()]
require.True(t, ok, "found unexpected evidence %v in height %v",
nodeEvidence, block.Height)
switch misbehavior {
case "double-prevote":
require.IsType(t, &types.DuplicateVoteEvidence{}, nodeEvidence, "unexpected evidence type")
default:
t.Fatalf("unknown misbehavior %v", misbehavior)
}
seenEvidence[nodeEvidence.Height()] = struct{}{}
testnet := loadTestnet(t)
seenEvidence := 0
for _, block := range blocks {
if len(block.Evidence.Evidence) != 0 {
seenEvidence += len(block.Evidence.Evidence)
}
// see if there is any evidence that we were expecting but didn't see
for height, misbehavior := range node.Misbehaviors {
_, ok := seenEvidence[height]
require.True(t, ok, "expected evidence for %v misbehavior at height %v by node but was never found",
misbehavior, height)
}
})
}
require.Equal(t, testnet.Evidence, seenEvidence,
"difference between the amount of evidence produced and committed")
}