diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index 203a60f6a..788f296b7 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/tendermint/tendermint/abci/example/code" abci "github.com/tendermint/tendermint/abci/types" @@ -80,6 +81,14 @@ type Config struct { // // height <-> pubkey <-> voting power ValidatorUpdates map[string]map[string]uint8 `toml:"validator_update"` + + // Add artificial delays to each of the main ABCI calls to mimic computation time + // of the application + PrepareProposalDelayMS uint64 `toml:"prepare_proposal_delay_ms"` + ProcessProposalDelayMS uint64 `toml:"process_proposal_delay_ms"` + CheckTxDelayMS uint64 `toml:"check_tx_delay_ms"` + VoteExtensionDelayMS uint64 `toml:"vote_extension_delay_ms"` + FinalizeBlockDelayMS uint64 `toml:"finalize_block_delay_ms"` } func DefaultConfig(dir string) *Config { @@ -164,6 +173,11 @@ func (app *Application) CheckTx(_ context.Context, req *abci.RequestCheckTx) (*a Code: code.CodeTypeEncodingError, }, nil } + + if app.cfg.CheckTxDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.CheckTxDelayMS) * time.Millisecond) + } + return &abci.ResponseCheckTx{Code: code.CodeTypeOK, GasWanted: 1}, nil } @@ -189,6 +203,10 @@ func (app *Application) FinalizeBlock(_ context.Context, req *abci.RequestFinali panic(err) } + if app.cfg.FinalizeBlockDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.FinalizeBlockDelayMS) * time.Millisecond) + } + return &abci.ResponseFinalizeBlock{ TxResults: txs, ValidatorUpdates: valUpdates, @@ -394,6 +412,11 @@ func (app *Application) PrepareProposal(_ context.Context, req *abci.RequestPrep Tx: tx, }) } + + if app.cfg.PrepareProposalDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.PrepareProposalDelayMS) * time.Millisecond) + } + return &abci.ResponsePrepareProposal{TxRecords: trs}, nil } @@ -415,6 +438,11 @@ func (app *Application) ProcessProposal(_ context.Context, req *abci.RequestProc } } } + + if app.cfg.ProcessProposalDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.ProcessProposalDelayMS) * time.Millisecond) + } + return &abci.ResponseProcessProposal{Status: abci.ResponseProcessProposal_ACCEPT}, nil } @@ -442,6 +470,11 @@ func (app *Application) ExtendVote(_ context.Context, req *abci.RequestExtendVot // nolint:gosec // G404: Use of weak random number generator num := rand.Int63n(voteExtensionMaxVal) extLen := binary.PutVarint(ext, num) + + if app.cfg.VoteExtensionDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.VoteExtensionDelayMS) * time.Millisecond) + } + app.logger.Info("generated vote extension", "num", num, "ext", fmt.Sprintf("%x", ext[:extLen]), "state.Height", app.state.Height) return &abci.ResponseExtendVote{ VoteExtension: ext[:extLen], @@ -476,6 +509,11 @@ func (app *Application) VerifyVoteExtension(_ context.Context, req *abci.Request Status: abci.ResponseVerifyVoteExtension_REJECT, }, nil } + + if app.cfg.VoteExtensionDelayMS != 0 { + time.Sleep(time.Duration(app.cfg.VoteExtensionDelayMS) * time.Millisecond) + } + app.logger.Info("verified vote extension value", "req", req, "num", num) return &abci.ResponseVerifyVoteExtension{ Status: abci.ResponseVerifyVoteExtension_ACCEPT, diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 5f917d746..0e8470111 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -15,13 +15,13 @@ var ( // separate testnet for each combination (Cartesian product) of options. testnetCombinations = map[string][]interface{}{ "topology": {"single", "quad", "large"}, - "queueType": {"priority"}, // "fifo" "initialHeight": {0, 1000}, "initialState": { map[string]string{}, map[string]string{"initial01": "a", "initial02": "b", "initial03": "c"}, }, "validators": {"genesis", "initchain"}, + "abci": {"builtin", "outofprocess"}, } // The following specify randomly chosen values for testnet nodes. @@ -32,11 +32,10 @@ var ( "rocksdb": 10, "cleveldb": 5, } - nodeABCIProtocols = weightedChoice{ - "builtin": 50, - "tcp": 20, - "grpc": 20, - "unix": 10, + ABCIProtocols = weightedChoice{ + "tcp": 20, + "grpc": 20, + "unix": 10, } nodePrivvalProtocols = weightedChoice{ "file": 50, @@ -62,10 +61,13 @@ var ( "kill": 0.1, "restart": 0.1, } - evidence = uniformChoice{0, 1, 10} - txSize = uniformChoice{1024, 4096} // either 1kb or 4kb - ipv6 = uniformChoice{false, true} - keyType = uniformChoice{types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1} + + // the following specify random chosen values for the entire testnet + evidence = uniformChoice{0, 1, 10} + txSize = uniformChoice{1024, 4096} // either 1kb or 4kb + ipv6 = uniformChoice{false, true} + keyType = uniformChoice{types.ABCIPubKeyTypeEd25519, types.ABCIPubKeyTypeSecp256k1} + abciDelays = uniformChoice{"none", "small", "large"} voteExtensionEnableHeightOffset = uniformChoice{int64(0), int64(10), int64(100)} voteExtensionEnabled = uniformChoice{true, false} @@ -107,7 +109,6 @@ type Options struct { func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, error) { manifest := e2e.Manifest{ IPv6: ipv6.Choose(r).(bool), - ABCIProtocol: nodeABCIProtocols.Choose(r), InitialHeight: int64(opt["initialHeight"].(int)), InitialState: opt["initialState"].(map[string]string), Validators: &map[string]int64{}, @@ -115,7 +116,7 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er Nodes: map[string]*e2e.ManifestNode{}, KeyType: keyType.Choose(r).(string), Evidence: evidence.Choose(r).(int), - QueueType: opt["queueType"].(string), + QueueType: "priority", TxSize: txSize.Choose(r).(int), } @@ -123,6 +124,27 @@ func generateTestnet(r *rand.Rand, opt map[string]interface{}) (e2e.Manifest, er manifest.VoteExtensionsEnableHeight = manifest.InitialHeight + voteExtensionEnableHeightOffset.Choose(r).(int64) } + if opt["abci"] == "builtin" { + manifest.ABCIProtocol = string(e2e.ProtocolBuiltin) + } else { + manifest.ABCIProtocol = ABCIProtocols.Choose(r) + } + + switch abciDelays.Choose(r).(string) { + case "none": + case "small": + manifest.PrepareProposalDelayMS = 100 + manifest.ProcessProposalDelayMS = 100 + manifest.VoteExtensionDelayMS = 20 + manifest.FinalizeBlockDelayMS = 200 + case "large": + manifest.PrepareProposalDelayMS = 200 + manifest.ProcessProposalDelayMS = 200 + manifest.CheckTxDelayMS = 20 + manifest.VoteExtensionDelayMS = 100 + manifest.FinalizeBlockDelayMS = 500 + } + var numSeeds, numValidators, numFulls, numLightClients int switch opt["topology"].(string) { case "single": diff --git a/test/e2e/networks/ci.toml b/test/e2e/networks/ci.toml index e2f7376a2..eb74dd111 100644 --- a/test/e2e/networks/ci.toml +++ b/test/e2e/networks/ci.toml @@ -5,6 +5,7 @@ evidence = 5 initial_height = 1000 initial_state = {initial01 = "a", initial02 = "b", initial03 = "c"} queue_type = "priority" +abci_protocol = "builtin" [validators] validator01 = 100 @@ -37,7 +38,6 @@ snapshot_interval = 5 block_sync = "v0" [node.validator02] -abci_protocol = "tcp" database = "cleveldb" persist_interval = 0 perturb = ["restart"] @@ -48,7 +48,6 @@ block_sync = "v0" [node.validator03] database = "badgerdb" seeds = ["seed01"] -abci_protocol = "grpc" persist_interval = 3 perturb = ["kill"] privval_protocol = "grpc" @@ -56,7 +55,6 @@ block_sync = "v0" retain_blocks = 10 [node.validator04] -abci_protocol = "builtin" snapshot_interval = 5 database = "rocksdb" persistent_peers = ["validator01"] @@ -69,7 +67,6 @@ block_sync = "v0" database = "badgerdb" state_sync = "p2p" start_at = 1005 # Becomes part of the validator set at 1010 -abci_protocol = "builtin" perturb = ["pause", "disconnect", "restart"] [node.full01] diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index dd2ad02ba..68ea8ca1d 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -64,7 +64,7 @@ type Manifest struct { QueueType string `toml:"queue_type"` // Number of bytes per tx. Default is 1kb (1024) - TxSize int + TxSize int `toml:"tx_size"` // VoteExtensionsEnableHeight configures the first height during which // the chain will use and require vote extension data to be present @@ -76,6 +76,14 @@ type Manifest struct { // builtin will build a complete Tendermint node into the application and // launch it instead of launching a separate Tendermint process. ABCIProtocol string `toml:"abci_protocol"` + + // Add artificial delays to each of the main ABCI calls to mimic computation time + // of the application + PrepareProposalDelayMS uint64 `toml:"prepare_proposal_delay_ms"` + ProcessProposalDelayMS uint64 `toml:"process_proposal_delay_ms"` + CheckTxDelayMS uint64 `toml:"check_tx_delay_ms"` + VoteExtensionDelayMS uint64 `toml:"vote_extension_delay_ms"` + FinalizeBlockDelayMS uint64 `toml:"finalize_block_delay_ms"` } // ManifestNode represents a node in a testnet manifest. diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index ad79c99c6..0e87466b0 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -72,7 +72,12 @@ type Testnet struct { VoteExtensionsEnableHeight int64 LogLevel string TxSize int - ABCIProtocol string + ABCIProtocol Protocol + PrepareProposalDelayMS int + ProcessProposalDelayMS int + CheckTxDelayMS int + VoteExtensionDelayMS int + FinalizeBlockDelayMS int } // Node represents a Tendermint node in a testnet. @@ -88,7 +93,6 @@ type Node struct { Mempool string StateSync string Database string - ABCIProtocol Protocol PrivvalProtocol Protocol PersistInterval uint64 SnapshotInterval uint64 @@ -128,20 +132,25 @@ func LoadTestnet(file string) (*Testnet, error) { proxyPortGen := newPortGenerator(proxyPortFirst) testnet := &Testnet{ - Name: filepath.Base(dir), - File: file, - Dir: dir, - IP: ipGen.Network(), - InitialHeight: 1, - InitialState: manifest.InitialState, - Validators: map[*Node]int64{}, - ValidatorUpdates: map[int64]map[*Node]int64{}, - Nodes: []*Node{}, - Evidence: manifest.Evidence, - KeyType: "ed25519", - LogLevel: manifest.LogLevel, - TxSize: manifest.TxSize, - ABCIProtocol: manifest.ABCIProtocol, + Name: filepath.Base(dir), + File: file, + Dir: dir, + IP: ipGen.Network(), + InitialHeight: 1, + InitialState: manifest.InitialState, + Validators: map[*Node]int64{}, + ValidatorUpdates: map[int64]map[*Node]int64{}, + Nodes: []*Node{}, + Evidence: manifest.Evidence, + KeyType: "ed25519", + LogLevel: manifest.LogLevel, + TxSize: manifest.TxSize, + ABCIProtocol: Protocol(manifest.ABCIProtocol), + PrepareProposalDelayMS: int(manifest.PrepareProposalDelayMS), + ProcessProposalDelayMS: int(manifest.ProcessProposalDelayMS), + CheckTxDelayMS: int(manifest.CheckTxDelayMS), + VoteExtensionDelayMS: int(manifest.VoteExtensionDelayMS), + FinalizeBlockDelayMS: int(manifest.FinalizeBlockDelayMS), } if len(manifest.KeyType) != 0 { testnet.KeyType = manifest.KeyType @@ -153,7 +162,7 @@ func LoadTestnet(file string) (*Testnet, error) { testnet.InitialHeight = manifest.InitialHeight } if testnet.ABCIProtocol == "" { - testnet.ABCIProtocol = string(ProtocolBuiltin) + testnet.ABCIProtocol = ProtocolBuiltin } // Set up nodes, in alphabetical order (IPs and ports get same order). @@ -174,7 +183,6 @@ func LoadTestnet(file string) (*Testnet, error) { ProxyPort: proxyPortGen.Next(), Mode: ModeValidator, Database: "goleveldb", - ABCIProtocol: Protocol(testnet.ABCIProtocol), PrivvalProtocol: ProtocolFile, StartAt: nodeManifest.StartAt, Mempool: nodeManifest.Mempool, @@ -192,9 +200,6 @@ func LoadTestnet(file string) (*Testnet, error) { if nodeManifest.Mode != "" { node.Mode = Mode(nodeManifest.Mode) } - if node.Mode == ModeLight { - node.ABCIProtocol = ProtocolBuiltin - } if nodeManifest.Database != "" { node.Database = nodeManifest.Database } @@ -354,14 +359,6 @@ func (n Node) Validate(testnet Testnet) error { default: return fmt.Errorf("invalid database setting %q", n.Database) } - switch n.ABCIProtocol { - case ProtocolBuiltin, ProtocolUNIX, ProtocolTCP, ProtocolGRPC: - default: - return fmt.Errorf("invalid ABCI protocol setting %q", n.ABCIProtocol) - } - if n.Mode == ModeLight && n.ABCIProtocol != ProtocolBuiltin { - return errors.New("light client must use builtin protocol") - } switch n.PrivvalProtocol { case ProtocolFile, ProtocolTCP, ProtocolGRPC, ProtocolUNIX: default: diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index 5f78a5b35..5887f13ef 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -141,6 +141,9 @@ func MakeDockerCompose(testnet *e2e.Testnet) ([]byte, error) { "addUint32": func(x, y uint32) uint32 { return x + y }, + "isBuiltin": func(protocol e2e.Protocol, mode e2e.Mode) bool { + return mode == e2e.ModeLight || protocol == e2e.ProtocolBuiltin + }, }).Parse(`version: '2.4' networks: @@ -163,7 +166,7 @@ services: e2e: true container_name: {{ .Name }} image: tendermint/e2e-node -{{- if eq .ABCIProtocol "builtin" }} +{{- if isBuiltin $.ABCIProtocol .Mode }} entrypoint: /usr/bin/entrypoint-builtin {{- else if .LogLevel }} command: start --log-level {{ .LogLevel }} @@ -254,7 +257,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.Mode = string(node.Mode) } - switch node.ABCIProtocol { + switch node.Testnet.ABCIProtocol { case e2e.ProtocolUNIX: cfg.ProxyApp = AppAddressUNIX case e2e.ProtocolTCP: @@ -266,7 +269,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { cfg.ProxyApp = "" cfg.ABCI = "" default: - return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol) + return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.Testnet.ABCIProtocol) } // Tendermint errors if it does not have a privval key set up, regardless of whether @@ -343,18 +346,23 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { // MakeAppConfig generates an ABCI application config for a node. func MakeAppConfig(node *e2e.Node) ([]byte, error) { cfg := map[string]interface{}{ - "chain_id": node.Testnet.Name, - "dir": "data/app", - "listen": AppAddressUNIX, - "mode": node.Mode, - "proxy_port": node.ProxyPort, - "protocol": "socket", - "persist_interval": node.PersistInterval, - "snapshot_interval": node.SnapshotInterval, - "retain_blocks": node.RetainBlocks, - "key_type": node.PrivvalKey.Type(), + "chain_id": node.Testnet.Name, + "dir": "data/app", + "listen": AppAddressUNIX, + "mode": node.Mode, + "proxy_port": node.ProxyPort, + "protocol": "socket", + "persist_interval": node.PersistInterval, + "snapshot_interval": node.SnapshotInterval, + "retain_blocks": node.RetainBlocks, + "key_type": node.PrivvalKey.Type(), + "prepare_proposal_delay_ms": node.Testnet.PrepareProposalDelayMS, + "process_proposal_delay_ms": node.Testnet.ProcessProposalDelayMS, + "check_tx_delay_ms": node.Testnet.CheckTxDelayMS, + "vote_extension_delay_ms": node.Testnet.VoteExtensionDelayMS, + "finalize_block_delay_ms": node.Testnet.FinalizeBlockDelayMS, } - switch node.ABCIProtocol { + switch node.Testnet.ABCIProtocol { case e2e.ProtocolUNIX: cfg["listen"] = AppAddressUNIX case e2e.ProtocolTCP: @@ -366,7 +374,7 @@ func MakeAppConfig(node *e2e.Node) ([]byte, error) { delete(cfg, "listen") cfg["protocol"] = "builtin" default: - return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.ABCIProtocol) + return nil, fmt.Errorf("unexpected ABCI protocol setting %q", node.Testnet.ABCIProtocol) } if node.Mode == e2e.ModeValidator { switch node.PrivvalProtocol {