test: add evidence e2e tests (#5488)

This commit is contained in:
Callum Waters
2020-10-23 12:33:08 +02:00
committed by Erik Grinaker
parent 75879ab1d7
commit dacbfbe1fe
38 changed files with 8869 additions and 44 deletions

View File

@@ -8,6 +8,11 @@ docker:
# ABCI testing).
app:
go build -o build/app -tags badgerdb,boltdb,cleveldb,rocksdb ./app
# To be used primarily by the e2e docker instance. If you want to produce this binary
# elsewhere, then run go build in the maverick directory.
maverick:
go build -o build/maverick -tags badgerdb,boltdb,cleveldb,rocksdb ../maverick
generator:
go build -o build/generator ./generator
@@ -15,4 +20,4 @@ generator:
runner:
go build -o build/runner ./runner
.PHONY: all app docker generator runner
.PHONY: all app docker generator maverick runner

View File

@@ -21,6 +21,7 @@ type Config struct {
PrivValServer string `toml:"privval_server"`
PrivValKey string `toml:"privval_key"`
PrivValState string `toml:"privval_state"`
Misbehaviors map[string]string `toml:"misbehaviors"`
}
// LoadConfig loads the configuration from disk.

View File

@@ -5,9 +5,11 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"time"
"github.com/spf13/viper"
"github.com/tendermint/tendermint/abci/server"
"github.com/tendermint/tendermint/config"
tmflags "github.com/tendermint/tendermint/libs/cli/flags"
@@ -17,6 +19,8 @@ import (
"github.com/tendermint/tendermint/p2p"
"github.com/tendermint/tendermint/privval"
"github.com/tendermint/tendermint/proxy"
mcs "github.com/tendermint/tendermint/test/maverick/consensus"
maverick "github.com/tendermint/tendermint/test/maverick/node"
)
var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
@@ -60,7 +64,11 @@ func run(configFile string) error {
case "socket", "grpc":
err = startApp(cfg)
case "builtin":
err = startNode(cfg)
if len(cfg.Misbehaviors) == 0 {
err = startNode(cfg)
} else {
err = startMaverick(cfg)
}
default:
err = fmt.Errorf("invalid protocol %q", cfg.Protocol)
}
@@ -102,37 +110,9 @@ func startNode(cfg *Config) error {
return err
}
home := os.Getenv("TMHOME")
if home == "" {
return errors.New("TMHOME not set")
}
viper.AddConfigPath(filepath.Join(home, "config"))
viper.SetConfigName("config")
err = viper.ReadInConfig()
tmcfg, nodeLogger, nodeKey, err := setupNode()
if err != nil {
return err
}
tmcfg := config.DefaultConfig()
err = viper.Unmarshal(tmcfg)
if err != nil {
return err
}
tmcfg.SetRoot(home)
if err = tmcfg.ValidateBasic(); err != nil {
return fmt.Errorf("error in config file: %v", err)
}
if tmcfg.LogFormat == config.LogFormatJSON {
logger = log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout))
}
logger, err = tmflags.ParseLogLevel(tmcfg.LogLevel, logger, config.DefaultLogLevel())
if err != nil {
return err
}
logger = logger.With("module", "main")
nodeKey, err := p2p.LoadOrGenNodeKey(tmcfg.NodeKeyFile())
if err != nil {
return fmt.Errorf("failed to load or gen node key %s: %w", tmcfg.NodeKeyFile(), err)
return fmt.Errorf("failed to setup config: %w", err)
}
n, err := node.NewNode(tmcfg,
@@ -142,7 +122,7 @@ func startNode(cfg *Config) error {
node.DefaultGenesisDocProviderFunc(tmcfg),
node.DefaultDBProvider,
node.DefaultMetricsProvider(tmcfg.Instrumentation),
logger,
nodeLogger,
)
if err != nil {
return err
@@ -150,6 +130,42 @@ func startNode(cfg *Config) error {
return n.Start()
}
// startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
// configuration is in $TMHOME/config/tendermint.toml.
func startMaverick(cfg *Config) error {
app, err := NewApplication(cfg)
if err != nil {
return err
}
tmcfg, logger, nodeKey, err := setupNode()
if err != nil {
return fmt.Errorf("failed to setup config: %w", err)
}
misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors))
for heightString, misbehaviorString := range cfg.Misbehaviors {
height, _ := strconv.ParseInt(heightString, 10, 64)
misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
}
n, err := maverick.NewNode(tmcfg,
maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
nodeKey,
proxy.NewLocalClientCreator(app),
maverick.DefaultGenesisDocProviderFunc(tmcfg),
maverick.DefaultDBProvider,
maverick.DefaultMetricsProvider(tmcfg.Instrumentation),
logger,
misbehaviors,
)
if err != nil {
return err
}
return n.Start()
}
// startSigner starts a signer server connecting to the given endpoint.
func startSigner(cfg *Config) error {
filePV := privval.LoadFilePV(cfg.PrivValKey, cfg.PrivValState)
@@ -175,3 +191,42 @@ func startSigner(cfg *Config) error {
logger.Info(fmt.Sprintf("Remote signer connecting to %v", cfg.PrivValServer))
return nil
}
func setupNode() (*config.Config, log.Logger, *p2p.NodeKey, error) {
var tmcfg *config.Config
home := os.Getenv("TMHOME")
if home == "" {
return nil, nil, nil, errors.New("TMHOME not set")
}
viper.AddConfigPath(filepath.Join(home, "config"))
viper.SetConfigName("config")
err := viper.ReadInConfig()
if err != nil {
return nil, nil, nil, err
}
tmcfg = config.DefaultConfig()
err = viper.Unmarshal(tmcfg)
if err != nil {
return nil, nil, nil, err
}
tmcfg.SetRoot(home)
if err = tmcfg.ValidateBasic(); err != nil {
return nil, nil, nil, fmt.Errorf("error in config file: %w", err)
}
if tmcfg.LogFormat == config.LogFormatJSON {
logger = log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout))
}
nodeLogger, err := tmflags.ParseLogLevel(tmcfg.LogLevel, logger, config.DefaultLogLevel())
if err != nil {
return nil, nil, nil, err
}
nodeLogger = nodeLogger.With("module", "main")
nodeKey, err := p2p.LoadOrGenNodeKey(tmcfg.NodeKeyFile())
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to load or gen node key %s: %w", tmcfg.NodeKeyFile(), err)
}
return tmcfg, nodeLogger, nodeKey, nil
}

View File

@@ -18,6 +18,7 @@ RUN go mod download
COPY . .
RUN make build && cp build/tendermint /usr/bin/tendermint
COPY test/e2e/docker/entrypoint* /usr/bin/
RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
RUN cd test/e2e && make app && cp build/app /usr/bin/app
# Set up runtime directory. We don't use a separate runtime image since we need

View File

@@ -0,0 +1,10 @@
#!/usr/bin/env bash
# Forcibly remove any stray UNIX sockets left behind from previous runs
rm -rf /var/run/privval.sock /var/run/app.sock
/usr/bin/app /tendermint/config/app.toml &
sleep 1
/usr/bin/maverick "$@"

View File

@@ -4,6 +4,7 @@ import (
"fmt"
"math/rand"
"sort"
"strconv"
"strings"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -39,6 +40,10 @@ var (
"kill": 0.1,
"restart": 0.1,
}
nodeMisbehaviors = weightedChoice{
misbehaviorOption{"double-prevote"}: 1,
misbehaviorOption{}: 9,
}
)
// Generate generates random testnets using the given RNG.
@@ -91,7 +96,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
nextStartAt := manifest.InitialHeight + 5
quorum := numValidators*2/3 + 1
for i := 1; i <= numValidators; i++ {
startAt := int64(0)
startAt := manifest.InitialHeight
if i > quorum {
startAt = nextStartAt
nextStartAt += 5
@@ -174,7 +179,8 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er
// generating invalid configurations. We do not set Seeds or PersistentPeers
// here, since we need to know the overall network topology and startup
// sequencing.
func generateNode(r *rand.Rand, mode e2e.Mode, startAt int64, forceArchive bool) *e2e.ManifestNode {
func generateNode(
r *rand.Rand, mode e2e.Mode, startAt int64, forceArchive bool) *e2e.ManifestNode {
node := e2e.ManifestNode{
Mode: string(mode),
StartAt: startAt,
@@ -196,6 +202,14 @@ func generateNode(r *rand.Rand, mode e2e.Mode, startAt int64, forceArchive bool)
node.SnapshotInterval = 3
}
if node.Mode == "validator" {
node.Misbehaviors = nodeMisbehaviors.Choose(r).(misbehaviorOption).
atHeight(startAt + 5 + int64(r.Intn(10)))
if len(node.Misbehaviors) != 0 {
node.PrivvalProtocol = "file"
}
}
// If a node which does not persist state also does not retain blocks, randomly
// choose to either persist state or retain all blocks.
if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 {
@@ -223,3 +237,16 @@ func generateNode(r *rand.Rand, mode e2e.Mode, startAt int64, forceArchive bool)
func ptrUint64(i uint64) *uint64 {
return &i
}
type misbehaviorOption struct {
misbehavior string
}
func (m misbehaviorOption) atHeight(height int64) map[string]string {
misbehaviorMap := make(map[string]string)
if m.misbehavior == "" {
return misbehaviorMap
}
misbehaviorMap[strconv.Itoa(int(height))] = m.misbehavior
return misbehaviorMap
}

View File

@@ -9,6 +9,7 @@ import (
"path/filepath"
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/libs/log"
)

View File

@@ -57,7 +57,7 @@ func (uc uniformChoice) Choose(r *rand.Rand) interface{} {
}
// weightedChoice chooses a single random key from a map of keys and weights.
type weightedChoice map[interface{}]uint // nolint:unused
type weightedChoice map[interface{}]uint
func (wc weightedChoice) Choose(r *rand.Rand) interface{} {
total := 0

View File

@@ -36,6 +36,7 @@ seeds = ["seed01"]
seeds = ["seed01"]
snapshot_interval = 5
perturb = ["disconnect"]
misbehaviors = { 1012 = "double-prevote", 1018 = "double-prevote" }
[node.validator02]
seeds = ["seed02"]

View File

@@ -2,3 +2,4 @@
[node.validator02]
[node.validator03]
[node.validator04]

View File

@@ -115,6 +115,16 @@ type ManifestNode struct {
// pause: temporarily pauses (freezes) the node
// restart: restarts the node, shutting it down with SIGTERM
Perturb []string `toml:"perturb"`
// Misbehaviors sets how a validator behaves during consensus at a
// certain height. Multiple misbehaviors at different heights can be used
//
// An example of misbehaviors
// { 10 = "double-prevote", 20 = "double-prevote"}
//
// For more information, look at the readme in the maverick folder.
// A list of all behaviors can be found in ../maverick/consensus/behavior.go
Misbehaviors map[string]string `toml:"misbehaviors"`
}
// Save saves the testnet manifest to a file.

View File

@@ -15,6 +15,7 @@ import (
"github.com/tendermint/tendermint/crypto"
"github.com/tendermint/tendermint/crypto/ed25519"
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
mcs "github.com/tendermint/tendermint/test/maverick/consensus"
)
const (
@@ -78,6 +79,7 @@ type Node struct {
Seeds []*Node
PersistentPeers []*Node
Perturbations []Perturbation
Misbehaviors map[int64]string
}
// LoadTestnet loads a testnet from a manifest file, using the filename to
@@ -147,6 +149,7 @@ func LoadTestnet(file string) (*Testnet, error) {
SnapshotInterval: nodeManifest.SnapshotInterval,
RetainBlocks: nodeManifest.RetainBlocks,
Perturbations: []Perturbation{},
Misbehaviors: make(map[int64]string),
}
if nodeManifest.Mode != "" {
node.Mode = Mode(nodeManifest.Mode)
@@ -166,6 +169,13 @@ func LoadTestnet(file string) (*Testnet, error) {
for _, p := range nodeManifest.Perturb {
node.Perturbations = append(node.Perturbations, Perturbation(p))
}
for heightString, misbehavior := range nodeManifest.Misbehaviors {
height, err := strconv.ParseInt(heightString, 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err)
}
node.Misbehaviors[height] = misbehavior
}
testnet.Nodes = append(testnet.Nodes, node)
}
@@ -324,6 +334,26 @@ func (n Node) Validate(testnet Testnet) error {
return fmt.Errorf("invalid perturbation %q", perturbation)
}
}
if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 {
return errors.New("must be using \"file\" privval protocol to implement misbehaviors")
}
for height, misbehavior := range n.Misbehaviors {
if height < n.StartAt {
return fmt.Errorf("misbehavior height %d is before start height %d", height, n.StartAt)
}
exists := false
for possibleBehaviors := range mcs.MisbehaviorList {
if possibleBehaviors == misbehavior {
exists = true
}
}
if !exists {
return fmt.Errorf("misbehavior %s does not exist", misbehavior)
}
}
return nil
}

View File

@@ -6,11 +6,14 @@ import (
"os"
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/libs/log"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)
var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
var (
logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
)
func main() {
NewCLI().Run()
@@ -18,8 +21,9 @@ func main() {
// CLI is the Cobra-based command-line interface.
type CLI struct {
root *cobra.Command
testnet *e2e.Testnet
root *cobra.Command
testnet *e2e.Testnet
preserve bool
}
// NewCLI sets up the CLI.
@@ -65,10 +69,13 @@ func NewCLI() *CLI {
if err := Start(cli.testnet); err != nil {
return err
}
if err := waitForAllMisbehaviors(cli.testnet); err != nil {
return err
}
if err := Perturb(cli.testnet); err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
if err := Wait(cli.testnet, interphaseWaitPeriod); err != nil { // allow some txs to go through
return err
}
@@ -76,14 +83,17 @@ func NewCLI() *CLI {
if err := <-chLoadResult; err != nil {
return err
}
if err := Wait(cli.testnet, 5); err != nil { // wait for network to settle before tests
// wait for network to settle before tests
if err := Wait(cli.testnet, interphaseWaitPeriod); err != nil {
return err
}
if err := Test(cli.testnet); err != nil {
return err
}
if err := Cleanup(cli.testnet); err != nil {
return err
if !cli.preserve {
if err := Cleanup(cli.testnet); err != nil {
return err
}
}
return nil
},
@@ -92,6 +102,9 @@ func NewCLI() *CLI {
cli.root.PersistentFlags().StringP("file", "f", "", "Testnet TOML manifest")
_ = cli.root.MarkPersistentFlagRequired("file")
cli.root.Flags().BoolVarP(&cli.preserve, "preserve", "p", false,
"Preserves the running of the test net after tests are completed")
cli.root.AddCommand(&cobra.Command{
Use: "setup",
Short: "Generates the testnet directory and configuration",

View File

@@ -12,11 +12,13 @@ import (
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"text/template"
"time"
"github.com/BurntSushi/toml"
"github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/crypto/ed25519"
"github.com/tendermint/tendermint/p2p"
@@ -118,7 +120,20 @@ func Setup(testnet *e2e.Testnet) error {
// MakeDockerCompose generates a Docker Compose config for a testnet.
func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) {
// Must use version 2 Docker Compose format, to support IPv6.
tmpl, err := template.New("docker-compose").Parse(`version: '2.4'
tmpl, err := template.New("docker-compose").Funcs(template.FuncMap{
"misbehaviorsToString": func(misbehaviors map[int64]string) string {
str := ""
for height, misbehavior := range misbehaviors {
// after the first behavior set, a comma must be prepended
if str != "" {
str += ","
}
heightString := strconv.Itoa(int(height))
str += misbehavior + "," + heightString
}
return str
},
}).Parse(`version: '2.4'
networks:
{{ .Name }}:
@@ -142,6 +157,9 @@ services:
image: tendermint/e2e-node
{{- if eq .ABCIProtocol "builtin" }}
entrypoint: /usr/bin/entrypoint-builtin
{{- else if .Misbehaviors }}
entrypoint: /usr/bin/entrypoint-maverick
command: ["node", "--misbehaviors", "{{ misbehaviorsToString .Misbehaviors }}"]
{{- end }}
init: true
ports:
@@ -330,6 +348,12 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
}
}
misbehaviors := make(map[string]string)
for height, misbehavior := range node.Misbehaviors {
misbehaviors[strconv.Itoa(int(height))] = misbehavior
}
cfg["misbehaviors"] = misbehaviors
if len(node.Testnet.ValidatorUpdates) > 0 {
validatorUpdates := map[string]map[string]int64{}
for height, validators := range node.Testnet.ValidatorUpdates {

View File

@@ -7,6 +7,8 @@ import (
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)
const interphaseWaitPeriod = 5
// Wait waits for a number of blocks to be produced, and for all nodes to catch
// up with it.
func Wait(testnet *e2e.Testnet, blocks int64) error {
@@ -22,3 +24,22 @@ func Wait(testnet *e2e.Testnet, blocks int64) error {
}
return nil
}
// WaitForAllMisbehaviors calculates the height of the last misbehavior and ensures the entire
// testnet has surpassed this height before moving on to the next phase
func waitForAllMisbehaviors(testnet *e2e.Testnet) error {
_, _, err := waitForHeight(testnet, lastMisbehaviorHeight(testnet))
return err
}
func lastMisbehaviorHeight(testnet *e2e.Testnet) int64 {
lastHeight := testnet.InitialHeight
for _, n := range testnet.Nodes {
for height := range n.Misbehaviors {
if height > lastHeight {
lastHeight = height
}
}
}
return lastHeight + interphaseWaitPeriod
}

View File

@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)

View File

@@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)

View File

@@ -8,6 +8,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
rpctypes "github.com/tendermint/tendermint/rpc/core/types"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"

View File

@@ -0,0 +1,50 @@
package e2e_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
// assert that all nodes that have blocks during the height (or height + 1) of a misbehavior has evidence
// for that misbehavior
func TestEvidence_Misbehavior(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
for _, block := range blocks {
// Find any evidence blaming this node in this block
var nodeEvidence types.Evidence
for _, evidence := range block.Evidence.Evidence {
switch evidence := evidence.(type) {
case *types.DuplicateVoteEvidence:
if bytes.Equal(evidence.VoteA.ValidatorAddress, node.Key.PubKey().Address()) {
nodeEvidence = evidence
}
default:
t.Fatalf("unexpected evidence type %T", evidence)
}
}
// Check that evidence was as expected (evidence is submitted in following height)
misbehavior, ok := node.Misbehaviors[block.Height-1]
if !ok {
require.Nil(t, nodeEvidence, "found unexpected evidence %v in height %v",
nodeEvidence, block.Height)
continue
}
require.NotNil(t, nodeEvidence, "no evidence found for misbehavior %v in height %v",
misbehavior, block.Height)
switch misbehavior {
case "double-prevote":
require.IsType(t, &types.DuplicateVoteEvidence{}, nodeEvidence, "unexpected evidence type")
default:
t.Fatalf("unknown misbehavior %v", misbehavior)
}
}
})
}

View File

@@ -4,6 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
)

View File

@@ -5,6 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)