mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-10-01 12:15:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch main into feature/abci++ppp

Browse Source
This commit is contained in:
Sergio Mena
2022-08-12 13:59:19 +02:00
parent cb570f6672 f36999e484
commit 7dc4f934b0
517 changed files with 61641 additions and 20688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/e2e/Makefile
+1 -6
View File
@@ -8,11 +8,6 @@ docker:
# ABCI testing).
node:
go build -o build/node -tags badgerdb,boltdb,cleveldb,rocksdb ./node
# To be used primarily by the e2e docker instance. If you want to produce this binary
# elsewhere, then run go build in the maverick directory.
maverick:
go build -o build/maverick -tags badgerdb,boltdb,cleveldb,rocksdb ../maverick
generator:
go build -o build/generator ./generator
@@ -20,4 +15,4 @@ generator:
runner:
go build -o build/runner ./runner
.PHONY: all node docker generator maverick runner
.PHONY: all node docker generator runner
test/e2e/README.md
+1 -1
View File
@@ -54,7 +54,7 @@ Auxiliary commands:
* `logs`: outputs all node logs.
* `tail`: tails (follows) node logs until cancelled.
* `tail`: tails (follows) node logs until canceled.
## Tests
test/e2e/app/snapshots.go
+4 -5
View File
@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"math"
"os"
"path/filepath"
@@ -45,7 +44,7 @@ func (s *SnapshotStore) loadMetadata() error {
file := filepath.Join(s.dir, "metadata.json")
metadata := []abci.Snapshot{}
bz, err := ioutil.ReadFile(file)
bz, err := os.ReadFile(file)
switch {
case errors.Is(err, os.ErrNotExist):
case err != nil:
@@ -72,7 +71,7 @@ func (s *SnapshotStore) saveMetadata() error {
// save the file to a new file and move it to make saving atomic.
newFile := filepath.Join(s.dir, "metadata.json.new")
file := filepath.Join(s.dir, "metadata.json")
err = ioutil.WriteFile(newFile, bz, 0644) // nolint: gosec
err = os.WriteFile(newFile, bz, 0644) // nolint: gosec
if err != nil {
return err
}
@@ -93,7 +92,7 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
Hash: hashItems(state.Values),
Chunks: byteChunks(bz),
}
err = ioutil.WriteFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", state.Height)), bz, 0644)
err = os.WriteFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", state.Height)), bz, 0644)
if err != nil {
return abci.Snapshot{}, err
}
@@ -122,7 +121,7 @@ func (s *SnapshotStore) LoadChunk(height uint64, format uint32, chunk uint32) ([
defer s.RUnlock()
for _, snapshot := range s.metadata {
if snapshot.Height == height && snapshot.Format == format {
bz, err := ioutil.ReadFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", height)))
bz, err := os.ReadFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", height)))
if err != nil {
return nil, err
}
test/e2e/app/state.go
+4 -5
View File
@@ -6,7 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
@@ -52,11 +51,11 @@ func NewState(dir string, persistInterval uint64) (*State, error) {
// load loads state from disk. It does not take out a lock, since it is called
// during construction.
func (s *State) load() error {
bz, err := ioutil.ReadFile(s.currentFile)
bz, err := os.ReadFile(s.currentFile)
if err != nil {
// if the current state doesn't exist then we try recover from the previous state
if errors.Is(err, os.ErrNotExist) {
bz, err = ioutil.ReadFile(s.previousFile)
bz, err = os.ReadFile(s.previousFile)
if err != nil {
return fmt.Errorf("failed to read both current and previous state (%q): %w",
s.previousFile, err)
@@ -82,7 +81,7 @@ func (s *State) save() error {
// We write the state to a separate file and move it to the destination, to
// make it atomic.
newFile := fmt.Sprintf("%v.new", s.currentFile)
err = ioutil.WriteFile(newFile, bz, 0644)
err = os.WriteFile(newFile, bz, 0644)
if err != nil {
return fmt.Errorf("failed to write state to %q: %w", s.currentFile, err)
}
@@ -160,7 +159,7 @@ func (s *State) Commit() (uint64, []byte, error) {
}
func (s *State) Rollback() error {
bz, err := ioutil.ReadFile(s.previousFile)
bz, err := os.ReadFile(s.previousFile)
if err != nil {
return fmt.Errorf("failed to read state from %q: %w", s.previousFile, err)
}
test/e2e/docker/Dockerfile
+1 -2
View File
@@ -1,7 +1,7 @@
# We need to build in a Linux environment to support C libraries, e.g. RocksDB.
# We use Debian instead of Alpine, so that we can use binary database packages
# instead of spending time compiling them.
FROM golang:1.15
FROM golang:1.18
RUN apt-get -qq update -y && apt-get -qq upgrade -y >/dev/null
RUN apt-get -qq install -y libleveldb-dev librocksdb-dev >/dev/null
@@ -18,7 +18,6 @@ RUN go mod download
COPY . .
RUN make build && cp build/tendermint /usr/bin/tendermint
COPY test/e2e/docker/entrypoint* /usr/bin/
RUN cd test/e2e && make maverick && cp build/maverick /usr/bin/maverick
RUN cd test/e2e && make node && cp build/node /usr/bin/app
# Set up runtime directory. We don't use a separate runtime image since we need
test/e2e/docker/entrypoint-maverick
-10
View File
@@ -1,10 +0,0 @@
#!/usr/bin/env bash
# Forcibly remove any stray UNIX sockets left behind from previous runs
rm -rf /var/run/privval.sock /var/run/app.sock
/usr/bin/app /tendermint/config/app.toml &
sleep 1
/usr/bin/maverick "$@"
test/e2e/generator/generate.go
+4 -37
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"math/rand"
"sort"
"strconv"
"strings"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
@@ -27,10 +26,9 @@ var (
nodeDatabases = uniformChoice{"goleveldb", "cleveldb", "rocksdb", "boltdb", "badgerdb"}
ipv6 = uniformChoice{false, true}
// FIXME: grpc disabled due to https://github.com/tendermint/tendermint/issues/5439
nodeABCIProtocols = uniformChoice{"unix", "tcp", "builtin"} // "grpc"
nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"}
// FIXME: v2 disabled due to flake
nodeFastSyncs = uniformChoice{"v0"} // "v2"
nodeABCIProtocols = uniformChoice{"unix", "tcp", "builtin"} // "grpc"
nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp"}
nodeFastSyncs = uniformChoice{false, true}
nodeStateSyncs = uniformChoice{false, true}
nodeMempools = uniformChoice{"v0", "v1"}
nodePersistIntervals = uniformChoice{0, 1, 5}
@@ -42,13 +40,6 @@ var (
"kill": 0.1,
"restart": 0.1,
}
nodeMisbehaviors = weightedChoice{
// FIXME: evidence disabled due to node panicing when not
// having sufficient block history to process evidence.
// https://github.com/tendermint/tendermint/issues/5617
// misbehaviorOption{"double-prevote"}: 1,
misbehaviorOption{}: 9,
}
)
// Generate generates random testnets using the given RNG.
@@ -210,7 +201,7 @@ func generateNode(
StartAt: startAt,
Database: nodeDatabases.Choose(r).(string),
PrivvalProtocol: nodePrivvalProtocols.Choose(r).(string),
FastSync: nodeFastSyncs.Choose(r).(string),
FastSync: nodeFastSyncs.Choose(r).(bool),
Mempool: nodeMempools.Choose(r).(string),
StateSync: nodeStateSyncs.Choose(r).(bool) && startAt > 0,
PersistInterval: ptrUint64(uint64(nodePersistIntervals.Choose(r).(int))),
@@ -226,17 +217,6 @@ func generateNode(
node.SnapshotInterval = 3
}
if node.Mode == string(e2e.ModeValidator) {
misbehaveAt := startAt + 5 + int64(r.Intn(10))
if startAt == 0 {
misbehaveAt += initialHeight - 1
}
node.Misbehaviors = nodeMisbehaviors.Choose(r).(misbehaviorOption).atHeight(misbehaveAt)
if len(node.Misbehaviors) != 0 {
node.PrivvalProtocol = "file"
}
}
// If a node which does not persist state also does not retain blocks, randomly
// choose to either persist state or retain all blocks.
if node.PersistInterval != nil && *node.PersistInterval == 0 && node.RetainBlocks > 0 {
@@ -274,16 +254,3 @@ func generateLightNode(r *rand.Rand, startAt int64, providers []string) *e2e.Man
func ptrUint64(i uint64) *uint64 {
return &i
}
type misbehaviorOption struct {
misbehavior string
}
func (m misbehaviorOption) atHeight(height int64) map[string]string {
misbehaviorMap := make(map[string]string)
if m.misbehavior == "" {
return misbehaviorMap
}
misbehaviorMap[strconv.Itoa(int(height))] = m.misbehavior
return misbehaviorMap
}
test/e2e/generator/random.go
-22
View File
@@ -56,28 +56,6 @@ func (uc uniformChoice) Choose(r *rand.Rand) interface{} {
return uc[r.Intn(len(uc))]
}
// weightedChoice chooses a single random key from a map of keys and weights.
type weightedChoice map[interface{}]uint
func (wc weightedChoice) Choose(r *rand.Rand) interface{} {
total := 0
choices := make([]interface{}, 0, len(wc))
for choice, weight := range wc {
total += int(weight)
choices = append(choices, choice)
}
rem := r.Intn(total)
for _, choice := range choices {
rem -= int(wc[choice])
if rem <= 0 {
return choice
}
}
return nil
}
// probSetChoice picks a set of strings based on each string's probability (0-1).
type probSetChoice map[string]float64
test/e2e/networks/ci.toml
+3 -6
View File
@@ -37,7 +37,6 @@ seeds = ["seed01"]
seeds = ["seed01"]
snapshot_interval = 5
perturb = ["disconnect"]
misbehaviors = { 1018 = "double-prevote" }
[node.validator02]
seeds = ["seed02"]
@@ -67,7 +66,7 @@ perturb = ["pause"]
start_at = 1005 # Becomes part of the validator set at 1010
seeds = ["seed02"]
database = "cleveldb"
fast_sync = "v0"
fast_sync = true
mempool_version = "v1"
# FIXME: should be grpc, disabled due to https://github.com/tendermint/tendermint/issues/5439
#abci_protocol = "grpc"
@@ -77,8 +76,7 @@ perturb = ["kill", "pause", "disconnect", "restart"]
[node.full01]
start_at = 1010
mode = "full"
# FIXME: should be v2, disabled due to flake
fast_sync = "v0"
fast_sync = true
persistent_peers = ["validator01", "validator02", "validator03", "validator04", "validator05"]
retain_blocks = 1
perturb = ["restart"]
@@ -86,8 +84,7 @@ perturb = ["restart"]
[node.full02]
start_at = 1015
mode = "full"
# FIXME: should be v2, disabled due to flake
fast_sync = "v0"
fast_sync = true
state_sync = true
seeds = ["seed01"]
perturb = ["restart"]
test/e2e/node/main.go
+3 -47
View File
@@ -7,7 +7,6 @@ import (
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"time"
@@ -30,8 +29,6 @@ import (
rpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"
"github.com/tendermint/tendermint/test/e2e/app"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
mcs "github.com/tendermint/tendermint/test/maverick/consensus"
maverick "github.com/tendermint/tendermint/test/maverick/node"
)
var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
@@ -75,14 +72,10 @@ func run(configFile string) error {
case "socket", "grpc":
err = startApp(cfg)
case "builtin":
if len(cfg.Misbehaviors) == 0 {
if cfg.Mode == string(e2e.ModeLight) {
err = startLightClient(cfg)
} else {
err = startNode(cfg)
}
if cfg.Mode == string(e2e.ModeLight) {
err = startLightClient(cfg)
} else {
err = startMaverick(cfg)
err = startNode(cfg)
}
default:
err = fmt.Errorf("invalid protocol %q", cfg.Protocol)
@@ -202,43 +195,6 @@ func startLightClient(cfg *Config) error {
return nil
}
// FIXME: Temporarily disconnected maverick until it is redesigned
// startMaverick starts a Maverick node that runs the application directly. It assumes the Tendermint
// configuration is in $TMHOME/config/tendermint.toml.
func startMaverick(cfg *Config) error {
app, err := app.NewApplication(cfg.App())
if err != nil {
return err
}
tmcfg, logger, nodeKey, err := setupNode()
if err != nil {
return fmt.Errorf("failed to setup config: %w", err)
}
misbehaviors := make(map[int64]mcs.Misbehavior, len(cfg.Misbehaviors))
for heightString, misbehaviorString := range cfg.Misbehaviors {
height, _ := strconv.ParseInt(heightString, 10, 64)
misbehaviors[height] = mcs.MisbehaviorList[misbehaviorString]
}
n, err := maverick.NewNode(tmcfg,
maverick.LoadOrGenFilePV(tmcfg.PrivValidatorKeyFile(), tmcfg.PrivValidatorStateFile()),
nodeKey,
proxy.NewLocalClientCreator(app),
maverick.DefaultGenesisDocProviderFunc(tmcfg),
maverick.DefaultDBProvider,
maverick.DefaultMetricsProvider(tmcfg.Instrumentation),
logger,
misbehaviors,
)
if err != nil {
return err
}
return n.Start()
}
// startSigner starts a signer server connecting to the given endpoint.
func startSigner(cfg *Config) error {
filePV := privval.LoadFilePV(cfg.PrivValKey, cfg.PrivValState)
test/e2e/pkg/manifest.go
+2 -13
View File
@@ -88,9 +88,8 @@ type ManifestNode struct {
// runner will wait for the network to reach at least this block height.
StartAt int64 `toml:"start_at"`
// FastSync specifies the fast sync mode: "" (disable), "v0", "v1", or "v2".
// Defaults to disabled.
FastSync string `toml:"fast_sync"`
// FastSync specifies whether to enable the fast sync protocol.
FastSync bool `toml:"fast_sync"`
// Mempool specifies which version of mempool to use. Either "v0" or "v1"
// This defaults to v0.
@@ -124,16 +123,6 @@ type ManifestNode struct {
// pause: temporarily pauses (freezes) the node
// restart: restarts the node, shutting it down with SIGTERM
Perturb []string `toml:"perturb"`
// Misbehaviors sets how a validator behaves during consensus at a
// certain height. Multiple misbehaviors at different heights can be used
//
// An example of misbehaviors
// { 10 = "double-prevote", 20 = "double-prevote"}
//
// For more information, look at the readme in the maverick folder.
// A list of all behaviors can be found in ../maverick/consensus/behavior.go
Misbehaviors map[string]string `toml:"misbehaviors"`
}
// Save saves the testnet manifest to a file.
test/e2e/pkg/testnet.go
+1 -54
View File
@@ -16,7 +16,6 @@ import (
"github.com/tendermint/tendermint/crypto/ed25519"
"github.com/tendermint/tendermint/crypto/secp256k1"
rpchttp "github.com/tendermint/tendermint/rpc/client/http"
mcs "github.com/tendermint/tendermint/test/maverick/consensus"
)
const (
@@ -73,7 +72,7 @@ type Node struct {
IP net.IP
ProxyPort uint32
StartAt int64
FastSync string
FastSync bool
StateSync bool
Mempool string
Database string
@@ -85,7 +84,6 @@ type Node struct {
Seeds []*Node
PersistentPeers []*Node
Perturbations []Perturbation
Misbehaviors map[int64]string
}
// LoadTestnet loads a testnet from a manifest file, using the filename to
@@ -164,7 +162,6 @@ func LoadTestnet(file string) (*Testnet, error) {
SnapshotInterval: nodeManifest.SnapshotInterval,
RetainBlocks: nodeManifest.RetainBlocks,
Perturbations: []Perturbation{},
Misbehaviors: make(map[int64]string),
}
if node.StartAt == testnet.InitialHeight {
node.StartAt = 0 // normalize to 0 for initial nodes, since code expects this
@@ -187,13 +184,6 @@ func LoadTestnet(file string) (*Testnet, error) {
for _, p := range nodeManifest.Perturb {
node.Perturbations = append(node.Perturbations, Perturbation(p))
}
for heightString, misbehavior := range nodeManifest.Misbehaviors {
height, err := strconv.ParseInt(heightString, 10, 64)
if err != nil {
return nil, fmt.Errorf("unable to parse height %s to int64: %w", heightString, err)
}
node.Misbehaviors[height] = misbehavior
}
testnet.Nodes = append(testnet.Nodes, node)
}
@@ -307,12 +297,6 @@ func (n Node) Validate(testnet Testnet) error {
}
}
}
switch n.FastSync {
case "", "v0", "v1", "v2":
default:
return fmt.Errorf("invalid fast sync setting %q", n.FastSync)
}
switch n.Mempool {
case "", "v0", "v1":
default:
@@ -362,30 +346,6 @@ func (n Node) Validate(testnet Testnet) error {
}
}
if (n.PrivvalProtocol != "file" || n.Mode != "validator") && len(n.Misbehaviors) != 0 {
return errors.New("must be using \"file\" privval protocol to implement misbehaviors")
}
for height, misbehavior := range n.Misbehaviors {
if height < n.StartAt {
return fmt.Errorf("misbehavior height %d is below node start height %d",
height, n.StartAt)
}
if height < testnet.InitialHeight {
return fmt.Errorf("misbehavior height %d is below network initial height %d",
height, testnet.InitialHeight)
}
exists := false
for possibleBehaviors := range mcs.MisbehaviorList {
if possibleBehaviors == misbehavior {
exists = true
}
}
if !exists {
return fmt.Errorf("misbehavior %s does not exist", misbehavior)
}
}
return nil
}
@@ -437,19 +397,6 @@ func (t Testnet) HasPerturbations() bool {
return false
}
// LastMisbehaviorHeight returns the height of the last misbehavior.
func (t Testnet) LastMisbehaviorHeight() int64 {
lastHeight := int64(0)
for _, node := range t.Nodes {
for height := range node.Misbehaviors {
if height > lastHeight {
lastHeight = height
}
}
}
return lastHeight
}
// Address returns a P2P endpoint address for the node.
func (n Node) AddressP2P(withID bool) string {
ip := n.IP.String()
test/e2e/runner/main.go
-8
View File
@@ -71,14 +71,6 @@ func NewCLI() *CLI {
return err
}
if lastMisbehavior := cli.testnet.LastMisbehaviorHeight(); lastMisbehavior > 0 {
// wait for misbehaviors before starting perturbations. We do a separate
// wait for another 5 blocks, since the last misbehavior height may be
// in the past depending on network startup ordering.
if err := WaitUntil(cli.testnet, lastMisbehavior); err != nil {
return err
}
}
if err := Wait(cli.testnet, 5); err != nil { // allow some txs to go through
return err
}
test/e2e/runner/setup.go
+5 -19
View File
@@ -7,7 +7,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
@@ -53,7 +52,7 @@ func Setup(testnet *e2e.Testnet) error {
if err != nil {
return err
}
err = ioutil.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
err = os.WriteFile(filepath.Join(testnet.Dir, "docker-compose.yml"), compose, 0644)
if err != nil {
return err
}
@@ -92,7 +91,7 @@ func Setup(testnet *e2e.Testnet) error {
if err != nil {
return err
}
err = ioutil.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644)
err = os.WriteFile(filepath.Join(nodeDir, "config", "app.toml"), appCfg, 0644)
if err != nil {
return err
}
@@ -168,9 +167,6 @@ services:
image: tendermint/e2e-node
{{- if eq .ABCIProtocol "builtin" }}
entrypoint: /usr/bin/entrypoint-builtin
{{- else if .Misbehaviors }}
entrypoint: /usr/bin/entrypoint-maverick
command: ["node", "--misbehaviors", "{{ misbehaviorsToString .Misbehaviors }}"]
{{- end }}
init: true
ports:
@@ -288,11 +284,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
cfg.Mempool.Version = node.Mempool
}
if node.FastSync == "" {
cfg.FastSyncMode = false
} else {
cfg.FastSync.Version = node.FastSync
}
cfg.FastSyncMode = node.FastSync
if node.StateSync {
cfg.StateSync.Enable = true
@@ -369,12 +361,6 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) {
}
}
misbehaviors := make(map[string]string)
for height, misbehavior := range node.Misbehaviors {
misbehaviors[strconv.Itoa(int(height))] = misbehavior
}
cfg["misbehaviors"] = misbehaviors
if len(node.Testnet.ValidatorUpdates) > 0 {
validatorUpdates := map[string]map[string]int64{}
for height, validators := range node.Testnet.ValidatorUpdates {
@@ -401,11 +387,11 @@ func UpdateConfigStateSync(node *e2e.Node, height int64, hash []byte) error {
// FIXME Apparently there's no function to simply load a config file without
// involving the entire Viper apparatus, so we'll just resort to regexps.
bz, err := ioutil.ReadFile(cfgPath)
bz, err := os.ReadFile(cfgPath)
if err != nil {
return err
}
bz = regexp.MustCompile(`(?m)^trust_height =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust_height = %v`, height)))
bz = regexp.MustCompile(`(?m)^trust_hash =.*`).ReplaceAll(bz, []byte(fmt.Sprintf(`trust_hash = "%X"`, hash)))
return ioutil.WriteFile(cfgPath, bz, 0644)
return os.WriteFile(cfgPath, bz, 0644)
}
test/e2e/tests/evidence_test.go
-57
View File
@@ -1,57 +0,0 @@
package e2e_test
import (
"bytes"
"testing"
"github.com/stretchr/testify/require"
e2e "github.com/tendermint/tendermint/test/e2e/pkg"
"github.com/tendermint/tendermint/types"
)
// assert that all nodes that have blocks at the height of a misbehavior has evidence
// for that misbehavior
func TestEvidence_Misbehavior(t *testing.T) {
blocks := fetchBlockChain(t)
testNode(t, func(t *testing.T, node e2e.Node) {
seenEvidence := make(map[int64]struct{})
for _, block := range blocks {
// Find any evidence blaming this node in this block
var nodeEvidence types.Evidence
for _, evidence := range block.Evidence.Evidence {
switch evidence := evidence.(type) {
case *types.DuplicateVoteEvidence:
if bytes.Equal(evidence.VoteA.ValidatorAddress, node.PrivvalKey.PubKey().Address()) {
nodeEvidence = evidence
}
default:
t.Fatalf("unexpected evidence type %T", evidence)
}
}
if nodeEvidence == nil {
continue // no evidence for the node at this height
}
// Check that evidence was as expected
misbehavior, ok := node.Misbehaviors[nodeEvidence.Height()]
require.True(t, ok, "found unexpected evidence %v in height %v",
nodeEvidence, block.Height)
switch misbehavior {
case "double-prevote":
require.IsType(t, &types.DuplicateVoteEvidence{}, nodeEvidence, "unexpected evidence type")
default:
t.Fatalf("unknown misbehavior %v", misbehavior)
}
seenEvidence[nodeEvidence.Height()] = struct{}{}
}
// see if there is any evidence that we were expecting but didn't see
for height, misbehavior := range node.Misbehaviors {
_, ok := seenEvidence[height]
require.True(t, ok, "expected evidence for %v misbehavior at height %v by node but was never found",
misbehavior, height)
}
})
}