mirrors/tendermint
1
0
Fork 0
mirror of https://github.com/tendermint/tendermint.git synced 2026-07-19 22:42:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into wb/issue-7950

Browse Source
This commit is contained in:
William Banfield
2022-04-06 22:20:44 -04:00
parent 6ef49b9315 7678ab8850
commit 32098b4353
75 changed files with 1366 additions and 1190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/lint.yml
+11 -5
View File
@@ -1,7 +1,11 @@
name: Golang Linter
# Lint runs golangci-lint over the entire Tendermint repository
# This workflow is run on every pull request and push to master
# The `golangci` job will pass without running if no *.{go, mod, sum} files have been modified.
# Lint runs golangci-lint over the entire Tendermint repository.
#
# This workflow is run on every pull request and push to master.
#
# The `golangci` job will pass without running if no *.{go, mod, sum}
# files have been modified.
on:
pull_request:
push:
@@ -25,8 +29,10 @@ jobs:
go.sum
- uses: golangci/golangci-lint-action@v3.1.0
with:
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
version: v1.44
# Required: the version of golangci-lint is required and
# must be specified without patch version: we always use the
# latest patch version.
version: v1.45
args: --timeout 10m
github-token: ${{ secrets.github_token }}
if: env.GIT_DIFF
.github/workflows/proto-lint.yml
+1 -1
View File
@@ -15,7 +15,7 @@ jobs:
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
- uses: bufbuild/buf-setup-action@v1.3.0
- uses: bufbuild/buf-setup-action@v1.3.1
- uses: bufbuild/buf-lint-action@v1
with:
input: 'proto'
CHANGELOG.md
+26 -6
View File
@@ -230,6 +230,26 @@ Special thanks to external contributors on this release: @JayT106,
- [cmd/tendermint/commands] [\#6623](https://github.com/tendermint/tendermint/pull/6623) replace `$HOME/.some/test/dir` with `t.TempDir` (@tanyabouman)
- [statesync] \6807 Implement P2P state provider as an alternative to RPC (@cmwaters)
## v0.34.18
### BREAKING CHANGES
- CLI/RPC/Config
- [cli] [\#8258](https://github.com/tendermint/tendermint/pull/8258) Fix a bug in the cli that caused `unsafe-reset-all` to panic
## v0.34.17
### BREAKING CHANGES
- CLI/RPC/Config
- [cli] [\#8081](https://github.com/tendermint/tendermint/issues/8081) make the reset command safe to use (@marbar3778).
### BUG FIXES
- [consensus] [\#8079](https://github.com/tendermint/tendermint/issues/8079) start the timeout ticker before relay (backport #7844) (@creachadair).
- [consensus] [\#7992](https://github.com/tendermint/tendermint/issues/7992) [\#7994](https://github.com/tendermint/tendermint/issues/7994) change lock handling in handleMsg and reactor to alleviate issues gossiping during long ABCI calls (@williambanfield).
## v0.34.16
Special thanks to external contributors on this release: @yihuang
@@ -1953,7 +1973,7 @@ more details.
- [rpc] [\#3269](https://github.com/tendermint/tendermint/issues/2826) Limit number of unique clientIDs with open subscriptions. Configurable via `rpc.max_subscription_clients`
- [rpc] [\#3269](https://github.com/tendermint/tendermint/issues/2826) Limit number of unique queries a given client can subscribe to at once. Configurable via `rpc.max_subscriptions_per_client`.
- [rpc] [\#3435](https://github.com/tendermint/tendermint/issues/3435) Default ReadTimeout and WriteTimeout changed to 10s. WriteTimeout can increased by setting `rpc.timeout_broadcast_tx_commit` in the config.
- [rpc/client] [\#3269](https://github.com/tendermint/tendermint/issues/3269) Update `EventsClient` interface to reflect new pubsub/eventBus API [ADR-33](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-033-pubsub.md). This includes `Subscribe`, `Unsubscribe`, and `UnsubscribeAll` methods.
- [rpc/client] [\#3269](https://github.com/tendermint/tendermint/issues/3269) Update `EventsClient` interface to reflect new pubsub/eventBus API [ADR-33](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-033-pubsub.md). This includes `Subscribe`, `Unsubscribe`, and `UnsubscribeAll` methods.
* Apps
- [abci] [\#3403](https://github.com/tendermint/tendermint/issues/3403) Remove `time_iota_ms` from BlockParams. This is a
@@ -2006,7 +2026,7 @@ more details.
- [blockchain] [\#3358](https://github.com/tendermint/tendermint/pull/3358) Fix timer leak in `BlockPool` (@guagualvcha)
- [cmd] [\#3408](https://github.com/tendermint/tendermint/issues/3408) Fix `testnet` command's panic when creating non-validator configs (using `--n` flag) (@srmo)
- [libs/db/remotedb/grpcdb] [\#3402](https://github.com/tendermint/tendermint/issues/3402) Close Iterator/ReverseIterator after use
- [libs/pubsub] [\#951](https://github.com/tendermint/tendermint/issues/951), [\#1880](https://github.com/tendermint/tendermint/issues/1880) Use non-blocking send when dispatching messages [ADR-33](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-033-pubsub.md)
- [libs/pubsub] [\#951](https://github.com/tendermint/tendermint/issues/951), [\#1880](https://github.com/tendermint/tendermint/issues/1880) Use non-blocking send when dispatching messages [ADR-33](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-033-pubsub.md)
- [lite] [\#3364](https://github.com/tendermint/tendermint/issues/3364) Fix `/validators` and `/abci_query` proxy endpoints
(@guagualvcha)
- [p2p/conn] [\#3347](https://github.com/tendermint/tendermint/issues/3347) Reject all-zero shared secrets in the Diffie-Hellman step of secret-connection
@@ -2710,7 +2730,7 @@ Special thanks to external contributors on this release:
This release is mostly about the ConsensusParams - removing fields and enforcing MaxGas.
It also addresses some issues found via security audit, removes various unused
functions from `libs/common`, and implements
[ADR-012](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-012-peer-transport.md).
[ADR-012](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-012-peer-transport.md).
BREAKING CHANGES:
@@ -2791,7 +2811,7 @@ BREAKING CHANGES:
- [abci] Added address of the original proposer of the block to Header
- [abci] Change ABCI Header to match Tendermint exactly
- [abci] [\#2159](https://github.com/tendermint/tendermint/issues/2159) Update use of `Validator` (see
[ADR-018](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-018-ABCI-Validators.md)):
[ADR-018](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-018-ABCI-Validators.md)):
- Remove PubKey from `Validator` (so it's just Address and Power)
- Introduce `ValidatorUpdate` (with just PubKey and Power)
- InitChain and EndBlock use ValidatorUpdate
@@ -2813,7 +2833,7 @@ BREAKING CHANGES:
- [state] [\#1815](https://github.com/tendermint/tendermint/issues/1815) Validator set changes are now delayed by one block (!)
- Add NextValidatorSet to State, changes on-disk representation of state
- [state] [\#2184](https://github.com/tendermint/tendermint/issues/2184) Enforce ConsensusParams.BlockSize.MaxBytes (See
[ADR-020](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-020-block-size.md)).
[ADR-020](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-020-block-size.md)).
- Remove ConsensusParams.BlockSize.MaxTxs
- Introduce maximum sizes for all components of a block, including ChainID
- [types] Updates to the block Header:
@@ -2824,7 +2844,7 @@ BREAKING CHANGES:
- [consensus] [\#2203](https://github.com/tendermint/tendermint/issues/2203) Implement BFT time
- Timestamp in block must be monotonic and equal the median of timestamps in block's LastCommit
- [crypto] [\#2239](https://github.com/tendermint/tendermint/issues/2239) Secp256k1 signature changes (See
[ADR-014](https://github.com/tendermint/tendermint/blob/develop/docs/architecture/adr-014-secp-malleability.md)):
[ADR-014](https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-014-secp-malleability.md)):
- format changed from DER to `r || s`, both little endian encoded as 32 bytes.
- malleability removed by requiring `s` to be in canonical form.
CHANGELOG_PENDING.md
+1
View File
@@ -20,6 +20,7 @@ Special thanks to external contributors on this release:
- [config] \#7930 Add new event subscription options and defaults. (@creachadair)
- [rpc] \#7982 Add new Events interface and deprecate Subscribe. (@creachadair)
- [cli] \#8081 make the reset command safe to use. (@marbar3778)
- [config] \#8222 default indexer configuration to null. (@creachadair)
- Apps
UPGRADING.md
+45
View File
@@ -26,6 +26,13 @@ application concern so be very sure to test the application thoroughly
using realistic workloads and the race detector to ensure your
applications remains correct.
### Config Changes
The default configuration for a newly-created node now disables indexing for
ABCI event metadata. Existing node configurations that already have indexing
turned on are not affected. Operators who wish to enable indexing for a new
node, however, must now edit the `config.toml` explicitly.
### RPC Changes
Tendermint v0.36 adds a new RPC event subscription API. The existing event
@@ -63,6 +70,44 @@ callback.
For more detailed information, see [ADR 075](https://tinyurl.com/adr075) which
defines and describes the new API in detail.
### Timeout Parameter Changes
Tendermint v0.36 updates how the Tendermint consensus timing parameters are
configured. These parameters, `timeout-propose`, `timeout-propose-delta`,
`timeout-prevote`, `timeout-prevote-delta`, `timeout-precommit`,
`timeout-precommit-delta`, `timeout-commit`, and `skip-timeout-commit`, were
previously configured in `config.toml`. These timing parameters have moved and
are no longer configured in the `config.toml` file. These parameters have been
migrated into the `ConsensusParameters`. Nodes with these parameters set in the
local configuration file will see a warning logged on startup indicating that
these parameters are no longer used.
These parameters have also been pared-down. There are no longer separate
parameters for both the `prevote` and `precommit` phases of Tendermint. The
separate `timeout-prevote` and `timeout-precommit` parameters have been merged
into a single `timeout-vote` parameter that configures both of these similar
phases of the consensus protocol.
A set of reasonable defaults have been put in place for these new parameters
that will take effect when the node starts up in version v0.36. New chains
created using v0.36 and beyond will be able to configure these parameters in the
chain's `genesis.json` file. Chains that upgrade to v0.36 from a previous
compatible version of Tendermint will begin running with the default values.
Upgrading applications that wish to use different values from the defaults for
these parameters may do so by setting the `ConsensusParams.Timeout` field of the
`FinalizeBlock` `ABCI` response.
As a safety measure in case of unusual timing issues during the upgrade to
v0.36, an operator may override the consensus timeout values for a single node.
Note, however, that these overrides will be removed in Tendermint v0.37. See
[configuration](https://github.com/tendermint/tendermint/blob/wb/issue-8182/docs/nodes/configuration.md)
for more information about these overrides.
For more discussion of this, see [ADR 074](https://tinyurl.com/adr074), which
lays out the reasoning for the changes as well as [RFC
009](https://tinyurl.com/rfc009) for a discussion of the complexities of
upgrading consensus parameters.
## v0.35
### ABCI Changes
abci/example/kvstore/kvstore.go
+17 -14
View File
@@ -285,8 +285,7 @@ func (app *Application) PrepareProposal(req types.RequestPrepareProposal) types.
defer app.mu.Unlock()
return types.ResponsePrepareProposal{
ModifiedTxStatus: types.ResponsePrepareProposal_MODIFIED,
TxRecords: app.substPrepareTx(req.Txs),
TxRecords: app.substPrepareTx(req.Txs, req.MaxTxBytes),
}
}
@@ -434,28 +433,32 @@ func (app *Application) execPrepareTx(tx []byte) *types.ExecTxResult {
}
// substPrepareTx substitutes all the transactions prefixed with 'prepare' in the
// proposal for transactions with the prefix strips.
// proposal for transactions with the prefix stripped.
// It marks all of the original transactions as 'REMOVED' so that
// Tendermint will remove them from its mempool.
func (app *Application) substPrepareTx(blockData [][]byte) []*types.TxRecord {
trs := make([]*types.TxRecord, len(blockData))
func (app *Application) substPrepareTx(blockData [][]byte, maxTxBytes int64) []*types.TxRecord {
trs := make([]*types.TxRecord, 0, len(blockData))
var removed []*types.TxRecord
for i, tx := range blockData {
var totalBytes int64
for _, tx := range blockData {
txMod := tx
action := types.TxRecord_UNMODIFIED
if isPrepareTx(tx) {
removed = append(removed, &types.TxRecord{
Tx: tx,
Action: types.TxRecord_REMOVED,
})
trs[i] = &types.TxRecord{
Tx: bytes.TrimPrefix(tx, []byte(PreparePrefix)),
Action: types.TxRecord_ADDED,
}
continue
txMod = bytes.TrimPrefix(tx, []byte(PreparePrefix))
action = types.TxRecord_ADDED
}
trs[i] = &types.TxRecord{
Tx: tx,
Action: types.TxRecord_UNMODIFIED,
totalBytes += int64(len(txMod))
if totalBytes > maxTxBytes {
break
}
trs = append(trs, &types.TxRecord{
Tx: txMod,
Action: action,
})
}
return append(trs, removed...)
abci/types/application.go
+13 -1
View File
@@ -95,7 +95,19 @@ func (BaseApplication) ApplySnapshotChunk(req RequestApplySnapshotChunk) Respons
}
func (BaseApplication) PrepareProposal(req RequestPrepareProposal) ResponsePrepareProposal {
return ResponsePrepareProposal{ModifiedTxStatus: ResponsePrepareProposal_UNMODIFIED}
trs := make([]*TxRecord, 0, len(req.Txs))
var totalBytes int64
for _, tx := range req.Txs {
totalBytes += int64(len(tx))
if totalBytes > req.MaxTxBytes {
break
}
trs = append(trs, &TxRecord{
Action: TxRecord_UNMODIFIED,
Tx: tx,
})
}
return ResponsePrepareProposal{TxRecords: trs}
}
func (BaseApplication) ProcessProposal(req RequestProcessProposal) ResponseProcessProposal {
abci/types/types.go
-8
View File
@@ -53,14 +53,6 @@ func (r ResponseQuery) IsErr() bool {
return r.Code != CodeTypeOK
}
func (r ResponsePrepareProposal) IsTxStatusUnknown() bool {
return r.ModifiedTxStatus == ResponsePrepareProposal_UNKNOWN
}
func (r ResponsePrepareProposal) IsTxStatusModified() bool {
return r.ModifiedTxStatus == ResponsePrepareProposal_MODIFIED
}
func (r ResponseProcessProposal) IsAccepted() bool {
return r.Status == ResponseProcessProposal_ACCEPT
}
cmd/tendermint/commands/debug/debug.go
+1
View File
@@ -2,6 +2,7 @@ package debug
import (
"github.com/spf13/cobra"
"github.com/tendermint/tendermint/libs/log"
)
cmd/tendermint/commands/root.go
+3 -1
View File
@@ -51,10 +51,12 @@ func RootCommand(conf *config.Config, logger log.Logger) *cobra.Command {
}
*conf = *pconf
config.EnsureRoot(conf.RootDir)
if err := log.OverrideWithNewLogger(logger, conf.LogFormat, conf.LogLevel); err != nil {
return err
}
if warning := pconf.DeprecatedFieldWarning(); warning != nil {
logger.Info("WARNING", "deprecated field warning", warning)
}
return nil
},
config/config.go
+61 -7
View File
@@ -8,6 +8,7 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/tendermint/tendermint/libs/log"
@@ -145,6 +146,10 @@ func (cfg *Config) ValidateBasic() error {
return nil
}
func (cfg *Config) DeprecatedFieldWarning() error {
return cfg.Consensus.DeprecatedFieldWarning()
}
//-----------------------------------------------------------------------------
// BaseConfig
@@ -998,6 +1003,20 @@ type ConsensusConfig struct {
// If it is set to true, the consensus engine will proceed to the next height
// as soon as the node has gathered votes from all of the validators on the network.
UnsafeBypassCommitTimeoutOverride *bool `mapstructure:"unsafe-bypass-commit-timeout-override"`
// Deprecated timeout parameters. These parameters are present in this struct
// so that they can be parsed so that validation can check if they have erroneously
// been included and provide a helpful error message.
// These fields should be completely removed in v0.37.
// See: https://github.com/tendermint/tendermint/issues/8188
DeprecatedTimeoutPropose *interface{} `mapstructure:"timeout-propose"`
DeprecatedTimeoutProposeDelta *interface{} `mapstructure:"timeout-propose-delta"`
DeprecatedTimeoutPrevote *interface{} `mapstructure:"timeout-prevote"`
DeprecatedTimeoutPrevoteDelta *interface{} `mapstructure:"timeout-prevote-delta"`
DeprecatedTimeoutPrecommit *interface{} `mapstructure:"timeout-precommit"`
DeprecatedTimeoutPrecommitDelta *interface{} `mapstructure:"timeout-precommit-delta"`
DeprecatedTimeoutCommit *interface{} `mapstructure:"timeout-commit"`
DeprecatedSkipTimeoutCommit *interface{} `mapstructure:"skip-timeout-commit"`
}
// DefaultConsensusConfig returns a default configuration for the consensus service
@@ -1072,6 +1091,44 @@ func (cfg *ConsensusConfig) ValidateBasic() error {
return nil
}
func (cfg *ConsensusConfig) DeprecatedFieldWarning() error {
var fields []string
if cfg.DeprecatedSkipTimeoutCommit != nil {
fields = append(fields, "skip-timeout-commit")
}
if cfg.DeprecatedTimeoutPropose != nil {
fields = append(fields, "timeout-propose")
}
if cfg.DeprecatedTimeoutProposeDelta != nil {
fields = append(fields, "timeout-propose-delta")
}
if cfg.DeprecatedTimeoutPrevote != nil {
fields = append(fields, "timeout-prevote")
}
if cfg.DeprecatedTimeoutPrevoteDelta != nil {
fields = append(fields, "timeout-prevote-delta")
}
if cfg.DeprecatedTimeoutPrecommit != nil {
fields = append(fields, "timeout-precommit")
}
if cfg.DeprecatedTimeoutPrecommitDelta != nil {
fields = append(fields, "timeout-precommit-delta")
}
if cfg.DeprecatedTimeoutCommit != nil {
fields = append(fields, "timeout-commit")
}
if cfg.DeprecatedSkipTimeoutCommit != nil {
fields = append(fields, "skip-timeout-commit")
}
if len(fields) != 0 {
return fmt.Errorf("the following deprecated fields were set in the "+
"configuration file: %s. These fields were removed in v0.36. Timeout "+
"configuration has been moved to the ConsensusParams. For more information see "+
"https://tinyurl.com/adr074", strings.Join(fields, ", "))
}
return nil
}
//-----------------------------------------------------------------------------
// TxIndexConfig
// Remember that Event has the following structure:
@@ -1088,9 +1145,8 @@ type TxIndexConfig struct {
// If list contains `null`, meaning no indexer service will be used.
//
// Options:
// 1) "null" - no indexer services.
// 2) "kv" (default) - the simplest possible indexer,
// backed by key-value storage (defaults to levelDB; see DBBackend).
// 1) "null" (default) - no indexer services.
// 2) "kv" - a simple indexer backed by key-value storage (see DBBackend)
// 3) "psql" - the indexer services backed by PostgreSQL.
Indexer []string `mapstructure:"indexer"`
@@ -1101,14 +1157,12 @@ type TxIndexConfig struct {
// DefaultTxIndexConfig returns a default configuration for the transaction indexer.
func DefaultTxIndexConfig() *TxIndexConfig {
return &TxIndexConfig{
Indexer: []string{"kv"},
}
return &TxIndexConfig{Indexer: []string{"null"}}
}
// TestTxIndexConfig returns a default configuration for the transaction indexer.
func TestTxIndexConfig() *TxIndexConfig {
return DefaultTxIndexConfig()
return &TxIndexConfig{Indexer: []string{"kv"}}
}
//-----------------------------------------------------------------------------
config/toml.go
+3 -3
View File
@@ -486,7 +486,7 @@ peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"
# This field provides an unsafe override of the Vote timeout consensus parameter.
# This field configures how long the consensus engine will wait after
# receiving +2/3 votes in a around.
# receiving +2/3 votes in a round.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-vote-timeout-override = {{ .Consensus.UnsafeVoteTimeoutOverride }}
@@ -520,8 +520,8 @@ peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"
# to decide which txs to index based on configuration set in the application.
#
# Options:
# 1) "null"
# 2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend).
# 1) "null" (default) - no indexer services.
# 2) "kv" - a simple indexer backed by key-value storage (see DBBackend)
# 3) "psql" - the indexer services backed by PostgreSQL.
# When "kv" or "psql" is chosen "tx.height" and "tx.hash" will always be indexed.
indexer = [{{ range $i, $e := .TxIndex.Indexer }}{{if $i}}, {{end}}{{ printf "%q" $e}}{{end}}]
docs/nodes/configuration.md
+137 -169
View File
@@ -16,7 +16,8 @@ the parameters set with their default values. It will look something
like the file below, however, double check by inspecting the
`config.toml` created with your version of `tendermint` installed:
```toml# This is a TOML config file.
```toml
# This is a TOML config file.
# For more information, see https://github.com/toml-lang/toml
# NOTE: Any path below can be absolute (e.g. "/var/myawesomeapp/data") or
@@ -33,11 +34,10 @@ like the file below, however, double check by inspecting the
proxy-app = "tcp://127.0.0.1:26658"
# A custom human readable name for this node
moniker = "ape"
moniker = "sidewinder"
# Mode of Node: full | validator | seed (default: "validator")
# * validator node (default)
# Mode of Node: full | validator | seed
# * validator node
# - all reactors
# - with priv_validator_key.json, priv_validator_state.json
# * full node
@@ -48,11 +48,6 @@ moniker = "ape"
# - No priv_validator_key.json, priv_validator_state.json
mode = "validator"
# If this node is many blocks behind the tip of the chain, FastSync
# allows them to catchup quickly by downloading blocks in parallel
# and verifying their commits
fast-sync = true
# Database backend: goleveldb | cleveldb | boltdb | rocksdb | badgerdb
# * goleveldb (github.com/syndtr/goleveldb - most popular implementation)
# - pure go
@@ -120,10 +115,10 @@ laddr = ""
client-certificate-file = ""
# Client key generated while creating certificates for secure connection
validator-client-key-file = ""
client-key-file = ""
# Path to the Root Certificate Authority used to sign both client and server certificates
certificate-authority = ""
root-ca-file = ""
#######################################################################
@@ -149,26 +144,10 @@ cors-allowed-methods = ["HEAD", "GET", "POST", ]
# A list of non simple headers the client is allowed to use with cross-domain requests
cors-allowed-headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time", ]
# TCP or UNIX socket address for the gRPC server to listen on
# NOTE: This server only supports /broadcast_tx_commit
# Deprecated gRPC in the RPC layer of Tendermint will be deprecated in 0.36.
grpc-laddr = ""
# Maximum number of simultaneous connections.
# Does not include RPC (HTTP&WebSocket) connections. See max-open-connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
# Deprecated gRPC in the RPC layer of Tendermint will be deprecated in 0.36.
grpc-max-open-connections = 900
# Activate unsafe RPC commands like /dial-seeds and /unsafe-flush-mempool
unsafe = false
# Maximum number of simultaneous connections (including WebSocket).
# Does not include gRPC connections. See grpc-max-open-connections
# If you want to accept a larger number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
@@ -182,10 +161,37 @@ max-open-connections = 900
max-subscription-clients = 100
# Maximum number of unique queries a given client can /subscribe to
# If you're using GRPC (or Local RPC client) and /broadcast_tx_commit, set to
# the estimated # maximum number of broadcast_tx_commit calls per block.
# If you're using a Local RPC client and /broadcast_tx_commit, set this
# to the estimated maximum number of broadcast_tx_commit calls per block.
max-subscriptions-per-client = 5
# If true, disable the websocket interface to the RPC service. This has
# the effect of disabling the /subscribe, /unsubscribe, and /unsubscribe_all
# methods for event subscription.
#
# EXPERIMENTAL: This setting will be removed in Tendermint v0.37.
experimental-disable-websocket = false
# The time window size for the event log. All events up to this long before
# the latest (up to EventLogMaxItems) will be available for subscribers to
# fetch via the /events method. If 0 (the default) the event log and the
# /events RPC method are disabled.
event-log-window-size = "0s"
# The maxiumum number of events that may be retained by the event log. If
# this value is 0, no upper limit is set. Otherwise, items in excess of
# this number will be discarded from the event log.
#
# Warning: This setting is a safety valve. Setting it too low may cause
# subscribers to miss events. Try to choose a value higher than the
# maximum worst-case expected event load within the chosen window size in
# ordinary operation.
#
# For example, if the window size is 10 minutes and the node typically
# averages 1000 events per ten minutes, but with occasional known spikes of
# up to 2000, choose a value > 2000.
event-log-max-items = 0
# How long to wait for a tx to be committed during /broadcast_tx_commit.
# WARNING: Using a value larger than 10s will result in increasing the
# global HTTP write timeout, which applies to all connections and endpoints.
@@ -252,63 +258,12 @@ persistent-peers = ""
# UPNP port forwarding
upnp = false
# Path to address book
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
addr-book-file = "config/addrbook.json"
# Set true for strict address routability rules
# Set false for private or local networks
addr-book-strict = true
# Maximum number of inbound peers
#
# TODO: Remove once p2p refactor is complete in favor of MaxConnections.
# ref: https://github.com/tendermint/tendermint/issues/5670
max-num-inbound-peers = 40
# Maximum number of outbound peers to connect to, excluding persistent peers
#
# TODO: Remove once p2p refactor is complete in favor of MaxConnections.
# ref: https://github.com/tendermint/tendermint/issues/5670
max-num-outbound-peers = 10
# Maximum number of connections (inbound and outbound).
max-connections = 64
# Rate limits the number of incoming connection attempts per IP address.
max-incoming-connection-attempts = 100
# List of node IDs, to which a connection will be (re)established ignoring any existing limits
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
unconditional-peer-ids = ""
# Maximum pause when redialing a persistent peer (if zero, exponential backoff is used)
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
persistent-peers-max-dial-period = "0s"
# Time to wait before flushing messages out on the connection
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
flush-throttle-timeout = "100ms"
# Maximum size of a message packet payload, in bytes
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
max-packet-msg-payload-size = 1400
# Rate at which packets can be sent, in bytes/second
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
send-rate = 5120000
# Rate at which packets can be received, in bytes/second
# TODO: Remove once p2p refactor is complete
# ref: https:#github.com/tendermint/tendermint/issues/5670
recv-rate = 5120000
# Set true to enable the peer-exchange reactor
pex = true
@@ -323,16 +278,28 @@ allow-duplicate-ip = false
handshake-timeout = "20s"
dial-timeout = "3s"
# Time to wait before flushing messages out on the connection
# TODO: Remove once MConnConnection is removed.
flush-throttle-timeout = "100ms"
# Maximum size of a message packet payload, in bytes
# TODO: Remove once MConnConnection is removed.
max-packet-msg-payload-size = 1400
# Rate at which packets can be sent, in bytes/second
# TODO: Remove once MConnConnection is removed.
send-rate = 5120000
# Rate at which packets can be received, in bytes/second
# TODO: Remove once MConnConnection is removed.
recv-rate = 5120000
#######################################################
### Mempool Configuration Option ###
#######################################################
[mempool]
# Mempool version to use:
# 1) "v0" - The legacy non-prioritized mempool reactor.
# 2) "v1" (default) - The prioritized mempool reactor.
version = "v1"
recheck = true
broadcast = true
@@ -388,22 +355,30 @@ ttl-num-blocks = 0
# starting from the height of the snapshot.
enable = false
# RPC servers (comma-separated) for light client verification of the synced state machine and
# retrieval of state data for node bootstrapping. Also needs a trusted height and corresponding
# header hash obtained from a trusted source, and a period during which validators can be trusted.
#
# For Cosmos SDK-based chains, trust-period should usually be about 2/3 of the unbonding time (~2
# weeks) during which they can be financially punished (slashed) for misbehavior.
# State sync uses light client verification to verify state. This can be done either through the
# P2P layer or RPC layer. Set this to true to use the P2P layer. If false (default), RPC layer
# will be used.
use-p2p = false
# If using RPC, at least two addresses need to be provided. They should be compatible with net.Dial,
# for example: "host.example.com:2125"
rpc-servers = ""
# The hash and height of a trusted block. Must be within the trust-period.
trust-height = 0
trust-hash = ""
# The trust period should be set so that Tendermint can detect and gossip misbehavior before
# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding
# period should suffice.
trust-period = "168h0m0s"
# Time to spend discovering snapshots before initiating a restore.
discovery-time = "15s"
# Temporary directory for state sync snapshot chunks, defaults to the OS tempdir (typically /tmp).
# Will create a new, randomly named directory within, and remove it when done.
# Temporary directory for state sync snapshot chunks, defaults to os.TempDir().
# The synchronizer will create a new, randomly named directory within this directory
# and remove it when the sync is complete.
temp-dir = ""
# The timeout duration before re-requesting a chunk, possibly from a different
@@ -413,21 +388,6 @@ chunk-request-timeout = "15s"
# The number of concurrent chunk and block fetchers to run (default: 4).
fetchers = "4"
#######################################################
### Block Sync Configuration Connections ###
#######################################################
[blocksync]
# If this node is many blocks behind the tip of the chain, BlockSync
# allows them to catchup quickly by downloading blocks in parallel
# and verifying their commits
enable = true
# Block Sync version to use:
# 1) "v0" (default) - the standard block sync implementation
# 2) "v2" - DEPRECATED, please use v0
version = "v0"
#######################################################
### Consensus Configuration Options ###
#######################################################
@@ -435,32 +395,12 @@ version = "v0"
wal-file = "data/cs.wal/wal"
# How long we wait for a proposal block before prevoting nil
timeout-propose = "3s"
# How much timeout-propose increases with each round
timeout-propose-delta = "500ms"
# How long we wait after receiving +2/3 prevotes for “anything” (ie. not a single block or nil)
timeout-prevote = "1s"
# How much the timeout-prevote increases with each round
timeout-prevote-delta = "500ms"
# How long we wait after receiving +2/3 precommits for “anything” (ie. not a single block or nil)
timeout-precommit = "1s"
# How much the timeout-precommit increases with each round
timeout-precommit-delta = "500ms"
# How long we wait after committing a block, before starting on the new
# height (this gives us a chance to receive some more precommits, even
# though we already have +2/3).
timeout-commit = "1s"
# How many blocks to look back to check existence of the node's consensus votes before joining consensus
# When non-zero, the node will panic upon restart
# if the same consensus key was used to sign {double-sign-check-height} last blocks.
# So, validators should stop the state machine, wait for some blocks, and then restart the state machine to avoid panic.
double-sign-check-height = 0
# Make progress as soon as we have all the precommits (as if TimeoutCommit = 0)
skip-timeout-commit = false
# EmptyBlocks mode and possible interval between empty blocks
create-empty-blocks = true
create-empty-blocks-interval = "0s"
@@ -469,6 +409,50 @@ create-empty-blocks-interval = "0s"
peer-gossip-sleep-duration = "100ms"
peer-query-maj23-sleep-duration = "2s"
### Unsafe Timeout Overrides ###
# These fields provide temporary overrides for the Timeout consensus parameters.
# Use of these parameters is strongly discouraged. Using these parameters may have serious
# liveness implications for the validator and for the chain.
#
# These fields will be removed from the configuration file in the v0.37 release of Tendermint.
# For additional information, see ADR-74:
# https://github.com/tendermint/tendermint/blob/master/docs/architecture/adr-074-timeout-params.md
# This field provides an unsafe override of the Propose timeout consensus parameter.
# This field configures how long the consensus engine will wait for a proposal block before prevoting nil.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-propose-timeout-override = 0s
# This field provides an unsafe override of the ProposeDelta timeout consensus parameter.
# This field configures how much the propose timeout increases with each round.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-propose-timeout-delta-override = 0s
# This field provides an unsafe override of the Vote timeout consensus parameter.
# This field configures how long the consensus engine will wait after
# receiving +2/3 votes in a around.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-vote-timeout-override = 0s
# This field provides an unsafe override of the VoteDelta timeout consensus parameter.
# This field configures how much the vote timeout increases with each round.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-vote-timeout-delta-override = 0s
# This field provides an unsafe override of the Commit timeout consensus parameter.
# This field configures how long the consensus engine will wait after receiving
# +2/3 precommits before beginning the next height.
# If this field is set to a value greater than 0, it will take effect.
# unsafe-commit-timeout-override = 0s
# This field provides an unsafe override of the BypassCommitTimeout consensus parameter.
# This field configures if the consensus engine will wait for the full Commit timeout
# before proceeding to the next height.
# If this field is set to true, the consensus engine will proceed to the next height
# as soon as the node has gathered votes from all of the validators on the network.
# unsafe-bypass-commit-timeout-override =
#######################################################
### Transaction Indexer Configuration Options ###
#######################################################
@@ -543,46 +527,6 @@ transactions every `create-empty-blocks-interval`. For instance, with
Tendermint will only create blocks if there are transactions, or after waiting
30 seconds without receiving any transactions.
## Consensus timeouts explained
There's a variety of information about timeouts in [Running in
production](../tendermint-core/running-in-production.md)
You can also find more detailed technical explanation in the spec: [The latest
gossip on BFT consensus](https://arxiv.org/abs/1807.04938).
```toml
[consensus]
...
timeout-propose = "3s"
timeout-propose-delta = "500ms"
timeout-prevote = "1s"
timeout-prevote-delta = "500ms"
timeout-precommit = "1s"
timeout-precommit-delta = "500ms"
timeout-commit = "1s"
```
Note that in a successful round, the only timeout that we absolutely wait no
matter what is `timeout-commit`.
Here's a brief summary of the timeouts:
- `timeout-propose` = how long we wait for a proposal block before prevoting
nil
- `timeout-propose-delta` = how much timeout-propose increases with each round
- `timeout-prevote` = how long we wait after receiving +2/3 prevotes for
anything (ie. not a single block or nil)
- `timeout-prevote-delta` = how much the timeout-prevote increases with each
round
- `timeout-precommit` = how long we wait after receiving +2/3 precommits for
anything (ie. not a single block or nil)
- `timeout-precommit-delta` = how much the timeout-precommit increases with
each round
- `timeout-commit` = how long we wait after committing a block, before starting
on the new height (this gives us a chance to receive some more precommits,
even though we already have +2/3)
## P2P settings
@@ -648,3 +592,27 @@ Example:
```shell
$ psql ... -f state/indexer/sink/psql/schema.sql
```
## Unsafe Consensus Timeout Overrides
Tendermint version v0.36 provides a set of unsafe overrides for the consensus
timing parameters. These parameters are provided as a safety measure in case of
unusual timing issues during the upgrade to v0.36 so that an operator may
override the timings for a single node. These overrides will completely be
removed in Tendermint v0.37.
- `unsafe-propose-override`: How long the Tendermint consensus engine will wait
for a proposal block before prevoting nil.
- `unsafe-propose-delta-override`: How much the propose timeout increase with
each round.
- `unsafe-vote-override`: How long the consensus engine will wait after
receiving +2/3 votes in a round.
- `unsafe-vote-delta-override`: How much the vote timeout increases with each
round.
- `unsafe-commit-override`: How long the consensus engine will wait after
receiving +2/3 precommits before beginning the next height.
- `unsafe-bypass-commit-timeout-override`: Configures if the consensus engine
will wait for the full commit timeout before proceeding to the next height. If
this field is set to true, the consensus engine will proceed to the next
height as soon as the node has gathered votes from all of the validators on
the network.
go.mod
+2 -2
View File
@@ -3,7 +3,7 @@ module github.com/tendermint/tendermint
go 1.17
require (
github.com/BurntSushi/toml v1.0.0
github.com/BurntSushi/toml v1.1.0
github.com/adlio/schema v1.3.0
github.com/btcsuite/btcd v0.22.0-beta
github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
@@ -41,7 +41,7 @@ require (
github.com/creachadair/atomicfile v0.2.4
github.com/golangci/golangci-lint v1.45.2
github.com/google/go-cmp v0.5.7
github.com/vektra/mockery/v2 v2.10.0
github.com/vektra/mockery/v2 v2.10.4
gotest.tools v2.2.0+incompatible
)
go.sum
+4 -3
View File
@@ -68,8 +68,9 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I=
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
@@ -1035,8 +1036,8 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/VPSJnLYn+LmLk8=
github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
github.com/vektra/mockery/v2 v2.10.0 h1:MiiQWxwdq7/ET6dCXLaJzSGEN17k758H7JHS9kOdiks=
github.com/vektra/mockery/v2 v2.10.0/go.mod h1:m/WO2UzWzqgVX3nvqpRQq70I4Z7jbSCRhdmkgtp+Ab4=
github.com/vektra/mockery/v2 v2.10.4 h1:nMdsCKIS7ZdNTRNS/77Bx6Q/UbasGcfc3Nx7JO7HGTg=
github.com/vektra/mockery/v2 v2.10.4/go.mod h1:m/WO2UzWzqgVX3nvqpRQq70I4Z7jbSCRhdmkgtp+Ab4=
github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE=
github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
internal/blocksync/reactor.go
+82 -82
View File
@@ -11,6 +11,7 @@ import (
"github.com/tendermint/tendermint/internal/consensus"
"github.com/tendermint/tendermint/internal/eventbus"
"github.com/tendermint/tendermint/internal/p2p"
"github.com/tendermint/tendermint/internal/p2p/conn"
sm "github.com/tendermint/tendermint/internal/state"
"github.com/tendermint/tendermint/internal/store"
"github.com/tendermint/tendermint/libs/log"
@@ -45,6 +46,7 @@ func GetChannelDescriptor() *p2p.ChannelDescriptor {
SendQueueCapacity: 1000,
RecvBufferCapacity: 1024,
RecvMessageCapacity: MaxMsgSize,
Name: "blockSync",
}
}
@@ -79,8 +81,8 @@ type Reactor struct {
consReactor consensusReactor
blockSync *atomicBool
blockSyncCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
chCreator p2p.ChannelCreator
peerEvents p2p.PeerEventSubscriber
requestsCh <-chan BlockRequest
errorsCh <-chan peerError
@@ -93,23 +95,17 @@ type Reactor struct {
// NewReactor returns new reactor instance.
func NewReactor(
ctx context.Context,
logger log.Logger,
stateStore sm.Store,
blockExec *sm.BlockExecutor,
store *store.BlockStore,
consReactor consensusReactor,
channelCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
peerEvents p2p.PeerEventSubscriber,
blockSync bool,
metrics *consensus.Metrics,
eventBus *eventbus.EventBus,
) (*Reactor, error) {
blockSyncCh, err := channelCreator(ctx, GetChannelDescriptor())
if err != nil {
return nil, err
}
) *Reactor {
r := &Reactor{
logger: logger,
stateStore: stateStore,
@@ -117,14 +113,14 @@ func NewReactor(
store: store,
consReactor: consReactor,
blockSync: newAtomicBool(blockSync),
blockSyncCh: blockSyncCh,
peerUpdates: peerUpdates,
chCreator: channelCreator,
peerEvents: peerEvents,
metrics: metrics,
eventBus: eventBus,
}
r.BaseService = *service.NewBaseService(logger, "BlockSync", r)
return r, nil
return r
}
// OnStart starts separate go routines for each p2p Channel and listens for
@@ -135,6 +131,12 @@ func NewReactor(
// If blockSync is enabled, we also start the pool and the pool processing
// goroutine. If the pool fails to start, an error is returned.
func (r *Reactor) OnStart(ctx context.Context) error {
blockSyncCh, err := r.chCreator(ctx, GetChannelDescriptor())
if err != nil {
return err
}
r.chCreator = func(context.Context, *conn.ChannelDescriptor) (*p2p.Channel, error) { return blockSyncCh, nil }
state, err := r.stateStore.Load()
if err != nil {
return err
@@ -160,13 +162,13 @@ func (r *Reactor) OnStart(ctx context.Context) error {
if err := r.pool.Start(ctx); err != nil {
return err
}
go r.requestRoutine(ctx)
go r.requestRoutine(ctx, blockSyncCh)
go r.poolRoutine(ctx, false)
go r.poolRoutine(ctx, false, blockSyncCh)
}
go r.processBlockSyncCh(ctx)
go r.processPeerUpdates(ctx)
go r.processBlockSyncCh(ctx, blockSyncCh)
go r.processPeerUpdates(ctx, r.peerEvents(ctx), blockSyncCh)
return nil
}
@@ -181,7 +183,7 @@ func (r *Reactor) OnStop() {
// respondToPeer loads a block and sends it to the requesting peer, if we have it.
// Otherwise, we'll respond saying we do not have it.
func (r *Reactor) respondToPeer(ctx context.Context, msg *bcproto.BlockRequest, peerID types.NodeID) error {
func (r *Reactor) respondToPeer(ctx context.Context, msg *bcproto.BlockRequest, peerID types.NodeID, blockSyncCh *p2p.Channel) error {
block := r.store.LoadBlock(msg.Height)
if block != nil {
blockProto, err := block.ToProto()
@@ -190,7 +192,7 @@ func (r *Reactor) respondToPeer(ctx context.Context, msg *bcproto.BlockRequest,
return err
}
return r.blockSyncCh.Send(ctx, p2p.Envelope{
return blockSyncCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &bcproto.BlockResponse{Block: blockProto},
})
@@ -198,55 +200,16 @@ func (r *Reactor) respondToPeer(ctx context.Context, msg *bcproto.BlockRequest,
r.logger.Info("peer requesting a block we do not have", "peer", peerID, "height", msg.Height)
return r.blockSyncCh.Send(ctx, p2p.Envelope{
return blockSyncCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &bcproto.NoBlockResponse{Height: msg.Height},
})
}
// handleBlockSyncMessage handles envelopes sent from peers on the
// BlockSyncChannel. It returns an error only if the Envelope.Message is unknown
// for this channel. This should never be called outside of handleMessage.
func (r *Reactor) handleBlockSyncMessage(ctx context.Context, envelope *p2p.Envelope) error {
logger := r.logger.With("peer", envelope.From)
switch msg := envelope.Message.(type) {
case *bcproto.BlockRequest:
return r.respondToPeer(ctx, msg, envelope.From)
case *bcproto.BlockResponse:
block, err := types.BlockFromProto(msg.Block)
if err != nil {
logger.Error("failed to convert block from proto", "err", err)
return err
}
r.pool.AddBlock(envelope.From, block, block.Size())
case *bcproto.StatusRequest:
return r.blockSyncCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &bcproto.StatusResponse{
Height: r.store.Height(),
Base: r.store.Base(),
},
})
case *bcproto.StatusResponse:
r.pool.SetPeerRange(envelope.From, msg.Base, msg.Height)
case *bcproto.NoBlockResponse:
logger.Debug("peer does not have the requested block", "height", msg.Height)
default:
return fmt.Errorf("received unknown message: %T", msg)
}
return nil
}
// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
// It will handle errors and any possible panics gracefully. A caller can handle
// any error returned by sending a PeerError on the respective channel.
func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelope *p2p.Envelope) (err error) {
func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelope *p2p.Envelope, blockSyncCh *p2p.Channel) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("panic in processing message: %v", e)
@@ -262,7 +225,39 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
switch chID {
case BlockSyncChannel:
err = r.handleBlockSyncMessage(ctx, envelope)
switch msg := envelope.Message.(type) {
case *bcproto.BlockRequest:
return r.respondToPeer(ctx, msg, envelope.From, blockSyncCh)
case *bcproto.BlockResponse:
block, err := types.BlockFromProto(msg.Block)
if err != nil {
r.logger.Error("failed to convert block from proto",
"peer", envelope.From,
"err", err)
return err
}
r.pool.AddBlock(envelope.From, block, block.Size())
case *bcproto.StatusRequest:
return blockSyncCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &bcproto.StatusResponse{
Height: r.store.Height(),
Base: r.store.Base(),
},
})
case *bcproto.StatusResponse:
r.pool.SetPeerRange(envelope.From, msg.Base, msg.Height)
case *bcproto.NoBlockResponse:
r.logger.Debug("peer does not have the requested block",
"peer", envelope.From,
"height", msg.Height)
default:
return fmt.Errorf("received unknown message: %T", msg)
}
default:
err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
@@ -276,17 +271,17 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
// message execution will result in a PeerError being sent on the BlockSyncChannel.
// When the reactor is stopped, we will catch the signal and close the p2p Channel
// gracefully.
func (r *Reactor) processBlockSyncCh(ctx context.Context) {
iter := r.blockSyncCh.Receive(ctx)
func (r *Reactor) processBlockSyncCh(ctx context.Context, blockSyncCh *p2p.Channel) {
iter := blockSyncCh.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.blockSyncCh.ID, envelope); err != nil {
if err := r.handleMessage(ctx, blockSyncCh.ID, envelope, blockSyncCh); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
r.logger.Error("failed to process message", "ch_id", r.blockSyncCh.ID, "envelope", envelope, "err", err)
if serr := r.blockSyncCh.SendError(ctx, p2p.PeerError{
r.logger.Error("failed to process message", "ch_id", blockSyncCh.ID, "envelope", envelope, "err", err)
if serr := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -297,7 +292,7 @@ func (r *Reactor) processBlockSyncCh(ctx context.Context) {
}
// processPeerUpdate processes a PeerUpdate.
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate) {
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate, blockSyncCh *p2p.Channel) {
r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
// XXX: Pool#RedoRequest can sometimes give us an empty peer.
@@ -308,7 +303,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
switch peerUpdate.Status {
case p2p.PeerStatusUp:
// send a status update the newly added peer
if err := r.blockSyncCh.Send(ctx, p2p.Envelope{
if err := blockSyncCh.Send(ctx, p2p.Envelope{
To: peerUpdate.NodeID,
Message: &bcproto.StatusResponse{
Base: r.store.Base(),
@@ -316,7 +311,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
},
}); err != nil {
r.pool.RemovePeer(peerUpdate.NodeID)
if err := r.blockSyncCh.SendError(ctx, p2p.PeerError{
if err := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: peerUpdate.NodeID,
Err: err,
}); err != nil {
@@ -332,13 +327,13 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates, blockSyncCh *p2p.Channel) {
for {
select {
case <-ctx.Done():
return
case peerUpdate := <-r.peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate)
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate, blockSyncCh)
}
}
}
@@ -356,13 +351,18 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error {
r.syncStartTime = time.Now()
go r.requestRoutine(ctx)
go r.poolRoutine(ctx, true)
bsCh, err := r.chCreator(ctx, GetChannelDescriptor())
if err != nil {
return err
}
go r.requestRoutine(ctx, bsCh)
go r.poolRoutine(ctx, true, bsCh)
return nil
}
func (r *Reactor) requestRoutine(ctx context.Context) {
func (r *Reactor) requestRoutine(ctx context.Context, blockSyncCh *p2p.Channel) {
statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second)
defer statusUpdateTicker.Stop()
@@ -371,11 +371,11 @@ func (r *Reactor) requestRoutine(ctx context.Context) {
case <-ctx.Done():
return
case request := <-r.requestsCh:
if err := r.blockSyncCh.Send(ctx, p2p.Envelope{
if err := blockSyncCh.Send(ctx, p2p.Envelope{
To: request.PeerID,
Message: &bcproto.BlockRequest{Height: request.Height},
}); err != nil {
if err := r.blockSyncCh.SendError(ctx, p2p.PeerError{
if err := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: request.PeerID,
Err: err,
}); err != nil {
@@ -383,14 +383,14 @@ func (r *Reactor) requestRoutine(ctx context.Context) {
}
}
case pErr := <-r.errorsCh:
if err := r.blockSyncCh.SendError(ctx, p2p.PeerError{
if err := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: pErr.peerID,
Err: pErr.err,
}); err != nil {
return
}
case <-statusUpdateTicker.C:
if err := r.blockSyncCh.Send(ctx, p2p.Envelope{
if err := blockSyncCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: &bcproto.StatusRequest{},
}); err != nil {
@@ -404,7 +404,7 @@ func (r *Reactor) requestRoutine(ctx context.Context) {
// do.
//
// NOTE: Don't sleep in the FOR_LOOP or otherwise slow it down!
func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) {
func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool, blockSyncCh *p2p.Channel) {
var (
trySyncTicker = time.NewTicker(trySyncIntervalMS * time.Millisecond)
switchToConsensusTicker = time.NewTicker(switchToConsensusIntervalSeconds * time.Second)
@@ -522,7 +522,7 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) {
// NOTE: We've already removed the peer's request, but we still need
// to clean up the rest.
peerID := r.pool.RedoRequest(first.Height)
if serr := r.blockSyncCh.SendError(ctx, p2p.PeerError{
if serr := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: peerID,
Err: err,
}); serr != nil {
@@ -531,7 +531,7 @@ func (r *Reactor) poolRoutine(ctx context.Context, stateSynced bool) {
peerID2 := r.pool.RedoRequest(second.Height)
if peerID2 != peerID {
if serr := r.blockSyncCh.SendError(ctx, p2p.PeerError{
if serr := blockSyncCh.SendError(ctx, p2p.PeerError{
NodeID: peerID2,
Err: err,
}); serr != nil {
internal/blocksync/reactor_test.go
+3 -4
View File
@@ -145,6 +145,7 @@ func (rts *reactorTestSuite) addNode(
sm.EmptyEvidencePool{},
blockStore,
eventbus,
sm.NopMetrics(),
)
for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
@@ -189,20 +190,18 @@ func (rts *reactorTestSuite) addNode(
chCreator := func(ctx context.Context, chdesc *p2p.ChannelDescriptor) (*p2p.Channel, error) {
return rts.blockSyncChannels[nodeID], nil
}
rts.reactors[nodeID], err = NewReactor(
ctx,
rts.reactors[nodeID] = NewReactor(
rts.logger.With("nodeID", nodeID),
stateStore,
blockExec,
blockStore,
nil,
chCreator,
rts.peerUpdates[nodeID],
func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
rts.blockSync,
consensus.NopMetrics(),
nil, // eventbus, can be nil
)
require.NoError(t, err)
require.NoError(t, rts.reactors[nodeID].Start(ctx))
require.True(t, rts.reactors[nodeID].IsRunning())
internal/consensus/byzantine_test.go
+5 -3
View File
@@ -95,7 +95,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
evpool := evidence.NewPool(logger.With("module", "evidence"), evidenceDB, stateStore, blockStore, evidence.NopMetrics(), eventBus)
// Make State
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyAppConnCon, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyAppConnCon, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
cs, err := NewState(ctx, logger, thisConfig.Consensus, stateStore, blockExec, blockStore, mempool, evpool, eventBus)
require.NoError(t, err)
// set private validator
@@ -141,8 +141,10 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
// send two votes to all peers (1st to one half, 2nd to another half)
i := 0
for _, ps := range bzReactor.peers {
voteCh := rts.voteChannels[bzNodeID]
if i < len(bzReactor.peers)/2 {
require.NoError(t, bzReactor.voteCh.Send(ctx,
require.NoError(t, voteCh.Send(ctx,
p2p.Envelope{
To: ps.peerID,
Message: &tmcons.Vote{
@@ -150,7 +152,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) {
},
}))
} else {
require.NoError(t, bzReactor.voteCh.Send(ctx,
require.NoError(t, voteCh.Send(ctx,
p2p.Envelope{
To: ps.peerID,
Message: &tmcons.Vote{
internal/consensus/common_test.go
+1 -1
View File
@@ -490,7 +490,7 @@ func newStateWithConfigAndBlockStore(
eventBus := eventbus.NewDefault(logger.With("module", "events"))
require.NoError(t, eventBus.Start(ctx))
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyAppConnCon, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyAppConnCon, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
cs, err := NewState(ctx,
logger.With("module", "consensus"),
thisConfig.Consensus,
internal/consensus/invalid_test.go
+3 -2
View File
@@ -57,7 +57,7 @@ func TestReactorInvalidPrecommit(t *testing.T) {
privVal := byzState.privValidator
byzState.doPrevote = func(ctx context.Context, height int64, round int32) {
defer close(signal)
invalidDoPrevoteFunc(ctx, t, height, round, byzState, byzReactor, privVal)
invalidDoPrevoteFunc(ctx, t, height, round, byzState, byzReactor, rts.voteChannels[node.NodeID], privVal)
}
byzState.mtx.Unlock()
@@ -107,6 +107,7 @@ func invalidDoPrevoteFunc(
round int32,
cs *State,
r *Reactor,
voteCh *p2p.Channel,
pv types.PrivValidator,
) {
// routine to:
@@ -155,7 +156,7 @@ func invalidDoPrevoteFunc(
count := 0
for _, peerID := range ids {
count++
err := r.voteCh.Send(ctx, p2p.Envelope{
err := voteCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &tmcons.Vote{
Vote: precommit.ToProto(),
internal/consensus/mempool_test.go
+13 -2
View File
@@ -317,9 +317,20 @@ func (app *CounterApplication) Commit() abci.ResponseCommit {
func (app *CounterApplication) PrepareProposal(
req abci.RequestPrepareProposal) abci.ResponsePrepareProposal {
return abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_UNMODIFIED,
trs := make([]*abci.TxRecord, 0, len(req.Txs))
var totalBytes int64
for _, tx := range req.Txs {
totalBytes += int64(len(tx))
if totalBytes > req.MaxTxBytes {
break
}
trs = append(trs, &abci.TxRecord{
Action: abci.TxRecord_UNMODIFIED,
Tx: tx,
})
}
return abci.ResponsePrepareProposal{TxRecords: trs}
}
func (app *CounterApplication) ProcessProposal(
internal/consensus/peer_state_test.go
+1
View File
@@ -4,6 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/tendermint/tendermint/libs/log"
tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
"github.com/tendermint/tendermint/types"
internal/consensus/reactor.go
+131 -140
View File
@@ -38,6 +38,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 64,
RecvMessageCapacity: maxMsgSize,
RecvBufferCapacity: 128,
Name: "state",
},
DataChannel: {
// TODO: Consider a split between gossiping current block and catchup
@@ -49,6 +50,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 64,
RecvBufferCapacity: 512,
RecvMessageCapacity: maxMsgSize,
Name: "data",
},
VoteChannel: {
ID: VoteChannel,
@@ -57,6 +59,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 64,
RecvBufferCapacity: 128,
RecvMessageCapacity: maxMsgSize,
Name: "vote",
},
VoteSetBitsChannel: {
ID: VoteSetBitsChannel,
@@ -65,6 +68,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 8,
RecvBufferCapacity: 128,
RecvMessageCapacity: maxMsgSize,
Name: "voteSet",
},
}
}
@@ -122,11 +126,8 @@ type Reactor struct {
rs *cstypes.RoundState
readySignal chan struct{} // closed when the node is ready to start consensus
stateCh *p2p.Channel
dataCh *p2p.Channel
voteCh *p2p.Channel
voteSetBitsCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
peerEvents p2p.PeerEventSubscriber
chCreator p2p.ChannelCreator
}
// NewReactor returns a reference to a new consensus reactor, which implements
@@ -134,49 +135,25 @@ type Reactor struct {
// to relevant p2p Channels and a channel to listen for peer updates on. The
// reactor will close all p2p Channels when stopping.
func NewReactor(
ctx context.Context,
logger log.Logger,
cs *State,
channelCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
peerEvents p2p.PeerEventSubscriber,
eventBus *eventbus.EventBus,
waitSync bool,
metrics *Metrics,
) (*Reactor, error) {
chans := getChannelDescriptors()
stateCh, err := channelCreator(ctx, chans[StateChannel])
if err != nil {
return nil, err
}
dataCh, err := channelCreator(ctx, chans[DataChannel])
if err != nil {
return nil, err
}
voteCh, err := channelCreator(ctx, chans[VoteChannel])
if err != nil {
return nil, err
}
voteSetBitsCh, err := channelCreator(ctx, chans[VoteSetBitsChannel])
if err != nil {
return nil, err
}
) *Reactor {
r := &Reactor{
logger: logger,
state: cs,
waitSync: waitSync,
rs: cs.GetRoundState(),
peers: make(map[types.NodeID]*PeerState),
eventBus: eventBus,
Metrics: metrics,
stateCh: stateCh,
dataCh: dataCh,
voteCh: voteCh,
voteSetBitsCh: voteSetBitsCh,
peerUpdates: peerUpdates,
readySignal: make(chan struct{}),
logger: logger,
state: cs,
waitSync: waitSync,
rs: cs.GetRoundState(),
peers: make(map[types.NodeID]*PeerState),
eventBus: eventBus,
Metrics: metrics,
peerEvents: peerEvents,
chCreator: channelCreator,
readySignal: make(chan struct{}),
}
r.BaseService = *service.NewBaseService(logger, "Consensus", r)
@@ -184,7 +161,14 @@ func NewReactor(
close(r.readySignal)
}
return r, nil
return r
}
type channelBundle struct {
state *p2p.Channel
data *p2p.Channel
vote *p2p.Channel
votSet *p2p.Channel
}
// OnStart starts separate go routines for each p2p Channel and listens for
@@ -194,13 +178,39 @@ func NewReactor(
func (r *Reactor) OnStart(ctx context.Context) error {
r.logger.Debug("consensus wait sync", "wait_sync", r.WaitSync())
peerUpdates := r.peerEvents(ctx)
var chBundle channelBundle
var err error
chans := getChannelDescriptors()
chBundle.state, err = r.chCreator(ctx, chans[StateChannel])
if err != nil {
return err
}
chBundle.data, err = r.chCreator(ctx, chans[DataChannel])
if err != nil {
return err
}
chBundle.vote, err = r.chCreator(ctx, chans[VoteChannel])
if err != nil {
return err
}
chBundle.votSet, err = r.chCreator(ctx, chans[VoteSetBitsChannel])
if err != nil {
return err
}
// start routine that computes peer statistics for evaluating peer quality
//
// TODO: Evaluate if we need this to be synchronized via WaitGroup as to not
// leak the goroutine when stopping the reactor.
go r.peerStatsRoutine(ctx)
go r.peerStatsRoutine(ctx, peerUpdates)
r.subscribeToBroadcastEvents()
r.subscribeToBroadcastEvents(chBundle.state)
go r.updateRoundStateRoutine()
if !r.WaitSync() {
@@ -209,11 +219,11 @@ func (r *Reactor) OnStart(ctx context.Context) error {
}
}
go r.processStateCh(ctx)
go r.processDataCh(ctx)
go r.processVoteCh(ctx)
go r.processVoteSetBitsCh(ctx)
go r.processPeerUpdates(ctx)
go r.processStateCh(ctx, chBundle)
go r.processDataCh(ctx, chBundle)
go r.processVoteCh(ctx, chBundle)
go r.processVoteSetBitsCh(ctx, chBundle)
go r.processPeerUpdates(ctx, peerUpdates, chBundle)
return nil
}
@@ -289,22 +299,6 @@ func (r *Reactor) String() string {
return "ConsensusReactor"
}
// StringIndented returns an indented string representation of the Reactor.
func (r *Reactor) StringIndented(indent string) string {
r.mtx.RLock()
defer r.mtx.RUnlock()
s := "ConsensusReactor{\n"
s += indent + " " + r.state.StringIndented(indent+" ") + "\n"
for _, ps := range r.peers {
s += indent + " " + ps.StringIndented(indent+" ") + "\n"
}
s += indent + "}"
return s
}
// GetPeerState returns PeerState for a given NodeID.
func (r *Reactor) GetPeerState(peerID types.NodeID) (*PeerState, bool) {
r.mtx.RLock()
@@ -314,16 +308,16 @@ func (r *Reactor) GetPeerState(peerID types.NodeID) (*PeerState, bool) {
return ps, ok
}
func (r *Reactor) broadcastNewRoundStepMessage(ctx context.Context, rs *cstypes.RoundState) error {
return r.stateCh.Send(ctx, p2p.Envelope{
func (r *Reactor) broadcastNewRoundStepMessage(ctx context.Context, rs *cstypes.RoundState, stateCh *p2p.Channel) error {
return stateCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: makeRoundStepMessage(rs),
})
}
func (r *Reactor) broadcastNewValidBlockMessage(ctx context.Context, rs *cstypes.RoundState) error {
func (r *Reactor) broadcastNewValidBlockMessage(ctx context.Context, rs *cstypes.RoundState, stateCh *p2p.Channel) error {
psHeader := rs.ProposalBlockParts.Header()
return r.stateCh.Send(ctx, p2p.Envelope{
return stateCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: &tmcons.NewValidBlock{
Height: rs.Height,
@@ -335,8 +329,8 @@ func (r *Reactor) broadcastNewValidBlockMessage(ctx context.Context, rs *cstypes
})
}
func (r *Reactor) broadcastHasVoteMessage(ctx context.Context, vote *types.Vote) error {
return r.stateCh.Send(ctx, p2p.Envelope{
func (r *Reactor) broadcastHasVoteMessage(ctx context.Context, vote *types.Vote, stateCh *p2p.Channel) error {
return stateCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: &tmcons.HasVote{
Height: vote.Height,
@@ -350,14 +344,14 @@ func (r *Reactor) broadcastHasVoteMessage(ctx context.Context, vote *types.Vote)
// subscribeToBroadcastEvents subscribes for new round steps and votes using the
// internal pubsub defined in the consensus state to broadcast them to peers
// upon receiving.
func (r *Reactor) subscribeToBroadcastEvents() {
func (r *Reactor) subscribeToBroadcastEvents(stateCh *p2p.Channel) {
onStopCh := r.state.getOnStopCh()
err := r.state.evsw.AddListenerForEvent(
listenerIDConsensus,
types.EventNewRoundStepValue,
func(ctx context.Context, data tmevents.EventData) error {
if err := r.broadcastNewRoundStepMessage(ctx, data.(*cstypes.RoundState)); err != nil {
if err := r.broadcastNewRoundStepMessage(ctx, data.(*cstypes.RoundState), stateCh); err != nil {
return err
}
select {
@@ -378,7 +372,7 @@ func (r *Reactor) subscribeToBroadcastEvents() {
listenerIDConsensus,
types.EventValidBlockValue,
func(ctx context.Context, data tmevents.EventData) error {
return r.broadcastNewValidBlockMessage(ctx, data.(*cstypes.RoundState))
return r.broadcastNewValidBlockMessage(ctx, data.(*cstypes.RoundState), stateCh)
},
)
if err != nil {
@@ -389,7 +383,7 @@ func (r *Reactor) subscribeToBroadcastEvents() {
listenerIDConsensus,
types.EventVoteValue,
func(ctx context.Context, data tmevents.EventData) error {
return r.broadcastHasVoteMessage(ctx, data.(*types.Vote))
return r.broadcastHasVoteMessage(ctx, data.(*types.Vote), stateCh)
},
)
if err != nil {
@@ -407,8 +401,8 @@ func makeRoundStepMessage(rs *cstypes.RoundState) *tmcons.NewRoundStep {
}
}
func (r *Reactor) sendNewRoundStepMessage(ctx context.Context, peerID types.NodeID) error {
return r.stateCh.Send(ctx, p2p.Envelope{
func (r *Reactor) sendNewRoundStepMessage(ctx context.Context, peerID types.NodeID, stateCh *p2p.Channel) error {
return stateCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: makeRoundStepMessage(r.getRoundState()),
})
@@ -434,7 +428,7 @@ func (r *Reactor) getRoundState() *cstypes.RoundState {
return r.rs
}
func (r *Reactor) gossipDataForCatchup(ctx context.Context, rs *cstypes.RoundState, prs *cstypes.PeerRoundState, ps *PeerState) {
func (r *Reactor) gossipDataForCatchup(ctx context.Context, rs *cstypes.RoundState, prs *cstypes.PeerRoundState, ps *PeerState, dataCh *p2p.Channel) {
logger := r.logger.With("height", prs.Height).With("peer", ps.peerID)
if index, ok := prs.ProposalBlockParts.Not().PickRandom(); ok {
@@ -483,7 +477,7 @@ func (r *Reactor) gossipDataForCatchup(ctx context.Context, rs *cstypes.RoundSta
}
logger.Debug("sending block part for catchup", "round", prs.Round, "index", index)
_ = r.dataCh.Send(ctx, p2p.Envelope{
_ = dataCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.BlockPart{
Height: prs.Height, // not our height, so it does not matter.
@@ -498,7 +492,7 @@ func (r *Reactor) gossipDataForCatchup(ctx context.Context, rs *cstypes.RoundSta
time.Sleep(r.state.config.PeerGossipSleepDuration)
}
func (r *Reactor) gossipDataRoutine(ctx context.Context, ps *PeerState) {
func (r *Reactor) gossipDataRoutine(ctx context.Context, ps *PeerState, dataCh *p2p.Channel) {
logger := r.logger.With("peer", ps.peerID)
timer := time.NewTimer(0)
@@ -530,7 +524,7 @@ OUTER_LOOP:
}
logger.Debug("sending block part", "height", prs.Height, "round", prs.Round)
if err := r.dataCh.Send(ctx, p2p.Envelope{
if err := dataCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.BlockPart{
Height: rs.Height, // this tells peer that this part applies to us
@@ -577,7 +571,7 @@ OUTER_LOOP:
continue OUTER_LOOP
}
r.gossipDataForCatchup(ctx, rs, prs, ps)
r.gossipDataForCatchup(ctx, rs, prs, ps, dataCh)
continue OUTER_LOOP
}
@@ -604,7 +598,7 @@ OUTER_LOOP:
propProto := rs.Proposal.ToProto()
logger.Debug("sending proposal", "height", prs.Height, "round", prs.Round)
if err := r.dataCh.Send(ctx, p2p.Envelope{
if err := dataCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.Proposal{
Proposal: *propProto,
@@ -627,7 +621,7 @@ OUTER_LOOP:
pPolProto := pPol.ToProto()
logger.Debug("sending POL", "height", prs.Height, "round", prs.Round)
if err := r.dataCh.Send(ctx, p2p.Envelope{
if err := dataCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.ProposalPOL{
Height: rs.Height,
@@ -655,14 +649,14 @@ OUTER_LOOP:
// pickSendVote picks a vote and sends it to the peer. It will return true if
// there is a vote to send and false otherwise.
func (r *Reactor) pickSendVote(ctx context.Context, ps *PeerState, votes types.VoteSetReader) (bool, error) {
func (r *Reactor) pickSendVote(ctx context.Context, ps *PeerState, votes types.VoteSetReader, voteCh *p2p.Channel) (bool, error) {
vote, ok := ps.PickVoteToSend(votes)
if !ok {
return false, nil
}
r.logger.Debug("sending vote message", "ps", ps, "vote", vote)
if err := r.voteCh.Send(ctx, p2p.Envelope{
if err := voteCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.Vote{
Vote: vote.ToProto(),
@@ -683,12 +677,13 @@ func (r *Reactor) gossipVotesForHeight(
rs *cstypes.RoundState,
prs *cstypes.PeerRoundState,
ps *PeerState,
voteCh *p2p.Channel,
) (bool, error) {
logger := r.logger.With("height", prs.Height).With("peer", ps.peerID)
// if there are lastCommits to send...
if prs.Step == cstypes.RoundStepNewHeight {
if ok, err := r.pickSendVote(ctx, ps, rs.LastCommit); err != nil {
if ok, err := r.pickSendVote(ctx, ps, rs.LastCommit, voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.LastCommit to send")
@@ -700,7 +695,7 @@ func (r *Reactor) gossipVotesForHeight(
// if there are POL prevotes to send...
if prs.Step <= cstypes.RoundStepPropose && prs.Round != -1 && prs.Round <= rs.Round && prs.ProposalPOLRound != -1 {
if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil {
if ok, err := r.pickSendVote(ctx, ps, polPrevotes); err != nil {
if ok, err := r.pickSendVote(ctx, ps, polPrevotes, voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.Prevotes(prs.ProposalPOLRound) to send", "round", prs.ProposalPOLRound)
@@ -711,7 +706,7 @@ func (r *Reactor) gossipVotesForHeight(
// if there are prevotes to send...
if prs.Step <= cstypes.RoundStepPrevoteWait && prs.Round != -1 && prs.Round <= rs.Round {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Prevotes(prs.Round)); err != nil {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Prevotes(prs.Round), voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
@@ -721,7 +716,7 @@ func (r *Reactor) gossipVotesForHeight(
// if there are precommits to send...
if prs.Step <= cstypes.RoundStepPrecommitWait && prs.Round != -1 && prs.Round <= rs.Round {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Precommits(prs.Round)); err != nil {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Precommits(prs.Round), voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.Precommits(prs.Round) to send", "round", prs.Round)
@@ -731,7 +726,7 @@ func (r *Reactor) gossipVotesForHeight(
// if there are prevotes to send...(which are needed because of validBlock mechanism)
if prs.Round != -1 && prs.Round <= rs.Round {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Prevotes(prs.Round)); err != nil {
if ok, err := r.pickSendVote(ctx, ps, rs.Votes.Prevotes(prs.Round), voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.Prevotes(prs.Round) to send", "round", prs.Round)
@@ -742,7 +737,7 @@ func (r *Reactor) gossipVotesForHeight(
// if there are POLPrevotes to send...
if prs.ProposalPOLRound != -1 {
if polPrevotes := rs.Votes.Prevotes(prs.ProposalPOLRound); polPrevotes != nil {
if ok, err := r.pickSendVote(ctx, ps, polPrevotes); err != nil {
if ok, err := r.pickSendVote(ctx, ps, polPrevotes, voteCh); err != nil {
return false, err
} else if ok {
logger.Debug("picked rs.Prevotes(prs.ProposalPOLRound) to send", "round", prs.ProposalPOLRound)
@@ -754,7 +749,7 @@ func (r *Reactor) gossipVotesForHeight(
return false, nil
}
func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState) {
func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState, voteCh *p2p.Channel) {
logger := r.logger.With("peer", ps.peerID)
// XXX: simple hack to throttle logs upon sleep
@@ -786,7 +781,7 @@ func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState) {
// if height matches, then send LastCommit, Prevotes, and Precommits
if rs.Height == prs.Height {
if ok, err := r.gossipVotesForHeight(ctx, rs, prs, ps); err != nil {
if ok, err := r.gossipVotesForHeight(ctx, rs, prs, ps, voteCh); err != nil {
return
} else if ok {
continue
@@ -795,7 +790,7 @@ func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState) {
// special catchup logic -- if peer is lagging by height 1, send LastCommit
if prs.Height != 0 && rs.Height == prs.Height+1 {
if ok, err := r.pickSendVote(ctx, ps, rs.LastCommit); err != nil {
if ok, err := r.pickSendVote(ctx, ps, rs.LastCommit, voteCh); err != nil {
return
} else if ok {
logger.Debug("picked rs.LastCommit to send", "height", prs.Height)
@@ -809,7 +804,7 @@ func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState) {
// Load the block commit for prs.Height, which contains precommit
// signatures for prs.Height.
if commit := r.state.blockStore.LoadBlockCommit(prs.Height); commit != nil {
if ok, err := r.pickSendVote(ctx, ps, commit); err != nil {
if ok, err := r.pickSendVote(ctx, ps, commit, voteCh); err != nil {
return
} else if ok {
logger.Debug("picked Catchup commit to send", "height", prs.Height)
@@ -843,7 +838,7 @@ func (r *Reactor) gossipVotesRoutine(ctx context.Context, ps *PeerState) {
// NOTE: `queryMaj23Routine` has a simple crude design since it only comes
// into play for liveness when there's a signature DDoS attack happening.
func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState, stateCh *p2p.Channel) {
timer := time.NewTimer(0)
defer timer.Stop()
@@ -879,7 +874,7 @@ func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
// maybe send Height/Round/Prevotes
if maj23, ok := rs.Votes.Prevotes(prs.Round).TwoThirdsMajority(); ok {
if err := r.stateCh.Send(ctx, p2p.Envelope{
if err := stateCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.VoteSetMaj23{
Height: prs.Height,
@@ -900,7 +895,7 @@ func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
// maybe send Height/Round/ProposalPOL
if maj23, ok := rs.Votes.Prevotes(prs.ProposalPOLRound).TwoThirdsMajority(); ok {
if err := r.stateCh.Send(ctx, p2p.Envelope{
if err := stateCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.VoteSetMaj23{
Height: prs.Height,
@@ -921,7 +916,7 @@ func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
// maybe send Height/Round/Precommits
if maj23, ok := rs.Votes.Precommits(prs.Round).TwoThirdsMajority(); ok {
if err := r.stateCh.Send(ctx, p2p.Envelope{
if err := stateCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.VoteSetMaj23{
Height: prs.Height,
@@ -946,7 +941,7 @@ func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
if prs.Height <= r.state.blockStore.Height() && prs.Height >= r.state.blockStore.Base() {
// maybe send Height/CatchupCommitRound/CatchupCommit
if commit := r.state.LoadCommit(prs.Height); commit != nil {
if err := r.stateCh.Send(ctx, p2p.Envelope{
if err := stateCh.Send(ctx, p2p.Envelope{
To: ps.peerID,
Message: &tmcons.VoteSetMaj23{
Height: prs.Height,
@@ -979,7 +974,7 @@ func (r *Reactor) queryMaj23Routine(ctx context.Context, ps *PeerState) {
// be the case, and we spawn all the relevant goroutine to broadcast messages to
// the peer. During peer removal, we remove the peer for our set of peers and
// signal to all spawned goroutines to gracefully exit in a non-blocking manner.
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate) {
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate, chans channelBundle) {
r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
r.mtx.Lock()
@@ -1020,14 +1015,14 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
return
}
// start goroutines for this peer
go r.gossipDataRoutine(ctx, ps)
go r.gossipVotesRoutine(ctx, ps)
go r.queryMaj23Routine(ctx, ps)
go r.gossipDataRoutine(ctx, ps, chans.data)
go r.gossipVotesRoutine(ctx, ps, chans.vote)
go r.queryMaj23Routine(ctx, ps, chans.state)
// Send our state to the peer. If we're block-syncing, broadcast a
// RoundStepMessage later upon SwitchToConsensus().
if !r.WaitSync() {
go func() { _ = r.sendNewRoundStepMessage(ctx, ps.peerID) }()
go func() { _ = r.sendNewRoundStepMessage(ctx, ps.peerID, chans.state) }()
}
}()
@@ -1054,7 +1049,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
// If we fail to find the peer state for the envelope sender, we perform a no-op
// and return. This can happen when we process the envelope after the peer is
// removed.
func (r *Reactor) handleStateMessage(ctx context.Context, envelope *p2p.Envelope, msgI Message) error {
func (r *Reactor) handleStateMessage(ctx context.Context, envelope *p2p.Envelope, msgI Message, voteSetCh *p2p.Channel) error {
ps, ok := r.GetPeerState(envelope.From)
if !ok || ps == nil {
r.logger.Debug("failed to find peer state", "peer", envelope.From, "ch_id", "StateChannel")
@@ -1124,7 +1119,7 @@ func (r *Reactor) handleStateMessage(ctx context.Context, envelope *p2p.Envelope
eMsg.Votes = *votesProto
}
if err := r.voteSetBitsCh.Send(ctx, p2p.Envelope{
if err := voteSetCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: eMsg,
}); err != nil {
@@ -1292,7 +1287,7 @@ func (r *Reactor) handleVoteSetBitsMessage(ctx context.Context, envelope *p2p.En
// the p2p channel.
//
// NOTE: We block on consensus state for proposals, block parts, and votes.
func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelope *p2p.Envelope) (err error) {
func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelope *p2p.Envelope, chans channelBundle) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("panic in processing message: %v", e)
@@ -1323,17 +1318,13 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
switch chID {
case StateChannel:
err = r.handleStateMessage(ctx, envelope, msgI)
err = r.handleStateMessage(ctx, envelope, msgI, chans.votSet)
case DataChannel:
err = r.handleDataMessage(ctx, envelope, msgI)
case VoteChannel:
err = r.handleVoteMessage(ctx, envelope, msgI)
case VoteSetBitsChannel:
err = r.handleVoteSetBitsMessage(ctx, envelope, msgI)
default:
err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
}
@@ -1346,13 +1337,13 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
// execution will result in a PeerError being sent on the StateChannel. When
// the reactor is stopped, we will catch the signal and close the p2p Channel
// gracefully.
func (r *Reactor) processStateCh(ctx context.Context) {
iter := r.stateCh.Receive(ctx)
func (r *Reactor) processStateCh(ctx context.Context, chans channelBundle) {
iter := chans.state.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.stateCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", r.stateCh.ID, "envelope", envelope, "err", err)
if serr := r.stateCh.SendError(ctx, p2p.PeerError{
if err := r.handleMessage(ctx, chans.state.ID, envelope, chans); err != nil {
r.logger.Error("failed to process message", "ch_id", chans.state.ID, "envelope", envelope, "err", err)
if serr := chans.state.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -1367,13 +1358,13 @@ func (r *Reactor) processStateCh(ctx context.Context) {
// execution will result in a PeerError being sent on the DataChannel. When
// the reactor is stopped, we will catch the signal and close the p2p Channel
// gracefully.
func (r *Reactor) processDataCh(ctx context.Context) {
iter := r.dataCh.Receive(ctx)
func (r *Reactor) processDataCh(ctx context.Context, chans channelBundle) {
iter := chans.data.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.dataCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", r.dataCh.ID, "envelope", envelope, "err", err)
if serr := r.dataCh.SendError(ctx, p2p.PeerError{
if err := r.handleMessage(ctx, chans.data.ID, envelope, chans); err != nil {
r.logger.Error("failed to process message", "ch_id", chans.data.ID, "envelope", envelope, "err", err)
if serr := chans.data.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -1388,13 +1379,13 @@ func (r *Reactor) processDataCh(ctx context.Context) {
// execution will result in a PeerError being sent on the VoteChannel. When
// the reactor is stopped, we will catch the signal and close the p2p Channel
// gracefully.
func (r *Reactor) processVoteCh(ctx context.Context) {
iter := r.voteCh.Receive(ctx)
func (r *Reactor) processVoteCh(ctx context.Context, chans channelBundle) {
iter := chans.vote.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.voteCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", r.voteCh.ID, "envelope", envelope, "err", err)
if serr := r.voteCh.SendError(ctx, p2p.PeerError{
if err := r.handleMessage(ctx, chans.vote.ID, envelope, chans); err != nil {
r.logger.Error("failed to process message", "ch_id", chans.vote.ID, "envelope", envelope, "err", err)
if serr := chans.vote.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -1409,18 +1400,18 @@ func (r *Reactor) processVoteCh(ctx context.Context) {
// execution will result in a PeerError being sent on the VoteSetBitsChannel.
// When the reactor is stopped, we will catch the signal and close the p2p
// Channel gracefully.
func (r *Reactor) processVoteSetBitsCh(ctx context.Context) {
iter := r.voteSetBitsCh.Receive(ctx)
func (r *Reactor) processVoteSetBitsCh(ctx context.Context, chans channelBundle) {
iter := chans.votSet.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.voteSetBitsCh.ID, envelope); err != nil {
if err := r.handleMessage(ctx, chans.votSet.ID, envelope, chans); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
r.logger.Error("failed to process message", "ch_id", r.voteSetBitsCh.ID, "envelope", envelope, "err", err)
if serr := r.voteSetBitsCh.SendError(ctx, p2p.PeerError{
r.logger.Error("failed to process message", "ch_id", chans.votSet.ID, "envelope", envelope, "err", err)
if serr := chans.votSet.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -1433,18 +1424,18 @@ func (r *Reactor) processVoteSetBitsCh(ctx context.Context) {
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates, chans channelBundle) {
for {
select {
case <-ctx.Done():
return
case peerUpdate := <-r.peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate)
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate, chans)
}
}
}
func (r *Reactor) peerStatsRoutine(ctx context.Context) {
func (r *Reactor) peerStatsRoutine(ctx context.Context, peerUpdates *p2p.PeerUpdates) {
for {
if !r.IsRunning() {
r.logger.Info("stopping peerStatsRoutine")
@@ -1462,7 +1453,7 @@ func (r *Reactor) peerStatsRoutine(ctx context.Context) {
switch msg.Msg.(type) {
case *VoteMessage:
if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 {
r.peerUpdates.SendUpdate(ctx, p2p.PeerUpdate{
peerUpdates.SendUpdate(ctx, p2p.PeerUpdate{
NodeID: msg.PeerID,
Status: p2p.PeerStatusGood,
})
@@ -1470,7 +1461,7 @@ func (r *Reactor) peerStatsRoutine(ctx context.Context) {
case *BlockPartMessage:
if numParts := ps.RecordBlockPart(); numParts%blocksToContributeToBecomeGoodPeer == 0 {
r.peerUpdates.SendUpdate(ctx, p2p.PeerUpdate{
peerUpdates.SendUpdate(ctx, p2p.PeerUpdate{
NodeID: msg.PeerID,
Status: p2p.PeerStatusGood,
})
internal/consensus/reactor_test.go
+3 -4
View File
@@ -104,16 +104,15 @@ func setup(
for nodeID, node := range rts.network.Nodes {
state := states[i]
reactor, err := NewReactor(ctx,
reactor := NewReactor(
state.logger.With("node", nodeID),
state,
chCreator(nodeID),
node.MakePeerUpdates(ctx, t),
func(ctx context.Context) *p2p.PeerUpdates { return node.MakePeerUpdates(ctx, t) },
state.eventBus,
true,
NopMetrics(),
)
require.NoError(t, err)
blocksSub, err := state.eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{
ClientID: testSubscriber,
@@ -504,7 +503,7 @@ func TestReactorWithEvidence(t *testing.T) {
eventBus := eventbus.NewDefault(log.NewNopLogger().With("module", "events"))
require.NoError(t, eventBus.Start(ctx))
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyAppConnCon, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyAppConnCon, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
cs, err := NewState(ctx, logger.With("validator", i, "module", "consensus"),
thisConfig.Consensus, stateStore, blockExec, blockStore, mempool, evpool2, eventBus)
internal/consensus/replay.go
+2 -2
View File
@@ -484,7 +484,7 @@ func (h *Handshaker) replayBlocks(
if i == finalBlock && !mutateState {
// We emit events for the index services at the final block due to the sync issue when
// the node shutdown during the block committing status.
blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, appClient, emptyMempool{}, sm.EmptyEvidencePool{}, h.store, h.eventBus)
blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, appClient, emptyMempool{}, sm.EmptyEvidencePool{}, h.store, h.eventBus, sm.NopMetrics())
appHash, err = sm.ExecCommitBlock(ctx,
blockExec, appClient, block, h.logger, h.stateStore, h.genDoc.InitialHeight, state)
if err != nil {
@@ -526,7 +526,7 @@ func (h *Handshaker) replayBlock(
// Use stubs for both mempool and evidence pool since no transactions nor
// evidence are needed here - block already exists.
blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, appClient, emptyMempool{}, sm.EmptyEvidencePool{}, h.store, h.eventBus)
blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, appClient, emptyMempool{}, sm.EmptyEvidencePool{}, h.store, h.eventBus, sm.NopMetrics())
var err error
state, err = blockExec.ApplyBlock(ctx, state, meta.BlockID, block)
internal/consensus/replay_file.go
+1 -1
View File
@@ -348,7 +348,7 @@ func newConsensusStateForReplay(
}
mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{}
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyApp, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyApp, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
consensusState, err := NewState(ctx, logger, csConfig, stateStore, blockExec,
blockStore, mempool, evpool, eventBus)
internal/consensus/replay_test.go
+1 -1
View File
@@ -826,7 +826,7 @@ func applyBlock(
eventBus *eventbus.EventBus,
) sm.State {
testPartSize := types.BlockPartSizeBytes
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), appClient, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), appClient, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
bps, err := blk.MakePartSet(testPartSize)
require.NoError(t, err)
internal/consensus/state.go
+5 -1
View File
@@ -1415,7 +1415,11 @@ func (cs *State) createProposalBlock(ctx context.Context) (*types.Block, error)
proposerAddr := cs.privValidatorPubKey.Address()
return cs.blockExec.CreateProposalBlock(ctx, cs.Height, cs.state, commit, proposerAddr, cs.LastCommit.GetVotes())
ret, err := cs.blockExec.CreateProposalBlock(ctx, cs.Height, cs.state, commit, proposerAddr, cs.LastCommit.GetVotes())
if err != nil {
panic(err)
}
return ret, nil
}
// Enter: `timeoutPropose` after entering Propose.
internal/consensus/wal_generator.go
+1 -1
View File
@@ -80,7 +80,7 @@ func WALGenerateNBlocks(ctx context.Context, t *testing.T, logger log.Logger, wr
mempool := emptyMempool{}
evpool := sm.EmptyEvidencePool{}
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyApp, mempool, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyApp, mempool, evpool, blockStore, eventBus, sm.NopMetrics())
consensusState, err := NewState(ctx, logger, cfg.Consensus, stateStore, blockExec, blockStore, mempool, evpool, eventBus)
if err != nil {
t.Fatal(err)
internal/evidence/reactor.go
+29 -32
View File
@@ -38,6 +38,7 @@ func GetChannelDescriptor() *p2p.ChannelDescriptor {
Priority: 6,
RecvMessageCapacity: maxMsgSize,
RecvBufferCapacity: 32,
Name: "evidence",
}
}
@@ -46,9 +47,9 @@ type Reactor struct {
service.BaseService
logger log.Logger
evpool *Pool
evidenceCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
evpool *Pool
chCreator p2p.ChannelCreator
peerEvents p2p.PeerEventSubscriber
mtx sync.Mutex
@@ -59,28 +60,22 @@ type Reactor struct {
// service.Service interface. It accepts a p2p Channel dedicated for handling
// envelopes with EvidenceList messages.
func NewReactor(
ctx context.Context,
logger log.Logger,
chCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
peerEvents p2p.PeerEventSubscriber,
evpool *Pool,
) (*Reactor, error) {
evidenceCh, err := chCreator(ctx, GetChannelDescriptor())
if err != nil {
return nil, err
}
) *Reactor {
r := &Reactor{
logger: logger,
evpool: evpool,
evidenceCh: evidenceCh,
peerUpdates: peerUpdates,
chCreator: chCreator,
peerEvents: peerEvents,
peerRoutines: make(map[types.NodeID]context.CancelFunc),
}
r.BaseService = *service.NewBaseService(logger, "Evidence", r)
return r, err
return r
}
// OnStart starts separate go routines for each p2p Channel and listens for
@@ -88,18 +83,20 @@ func NewReactor(
// messages on that p2p channel accordingly. The caller must be sure to execute
// OnStop to ensure the outbound p2p Channels are closed. No error is returned.
func (r *Reactor) OnStart(ctx context.Context) error {
go r.processEvidenceCh(ctx)
go r.processPeerUpdates(ctx)
ch, err := r.chCreator(ctx, GetChannelDescriptor())
if err != nil {
return err
}
go r.processEvidenceCh(ctx, ch)
go r.processPeerUpdates(ctx, r.peerEvents(ctx), ch)
return nil
}
// OnStop stops the reactor by signaling to all spawned goroutines to exit and
// blocking until they all exit.
func (r *Reactor) OnStop() {
// Close the evidence db
r.evpool.Close()
}
func (r *Reactor) OnStop() { r.evpool.Close() }
// handleEvidenceMessage handles envelopes sent from peers on the EvidenceChannel.
// It returns an error only if the Envelope.Message is unknown for this channel
@@ -163,13 +160,13 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
// processEvidenceCh implements a blocking event loop where we listen for p2p
// Envelope messages from the evidenceCh.
func (r *Reactor) processEvidenceCh(ctx context.Context) {
iter := r.evidenceCh.Receive(ctx)
func (r *Reactor) processEvidenceCh(ctx context.Context, evidenceCh *p2p.Channel) {
iter := evidenceCh.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.evidenceCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", r.evidenceCh.ID, "envelope", envelope, "err", err)
if serr := r.evidenceCh.SendError(ctx, p2p.PeerError{
if err := r.handleMessage(ctx, evidenceCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", evidenceCh.ID, "envelope", envelope, "err", err)
if serr := evidenceCh.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -190,7 +187,7 @@ func (r *Reactor) processEvidenceCh(ctx context.Context) {
// connects/disconnects frequently from the broadcasting peer(s).
//
// REF: https://github.com/tendermint/tendermint/issues/4727
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate) {
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate, evidenceCh *p2p.Channel) {
r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
r.mtx.Lock()
@@ -213,7 +210,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
if !ok {
pctx, pcancel := context.WithCancel(ctx)
r.peerRoutines[peerUpdate.NodeID] = pcancel
go r.broadcastEvidenceLoop(pctx, peerUpdate.NodeID)
go r.broadcastEvidenceLoop(pctx, peerUpdate.NodeID, evidenceCh)
}
case p2p.PeerStatusDown:
@@ -231,11 +228,11 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates, evidenceCh *p2p.Channel) {
for {
select {
case peerUpdate := <-r.peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate)
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate, evidenceCh)
case <-ctx.Done():
return
}
@@ -253,7 +250,7 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) {
// that the peer has already received or may not be ready for.
//
// REF: https://github.com/tendermint/tendermint/issues/4727
func (r *Reactor) broadcastEvidenceLoop(ctx context.Context, peerID types.NodeID) {
func (r *Reactor) broadcastEvidenceLoop(ctx context.Context, peerID types.NodeID, evidenceCh *p2p.Channel) {
var next *clist.CElement
defer func() {
@@ -300,7 +297,7 @@ func (r *Reactor) broadcastEvidenceLoop(ctx context.Context, peerID types.NodeID
// peer may receive this piece of evidence multiple times if it added and
// removed frequently from the broadcasting peer.
if err := r.evidenceCh.Send(ctx, p2p.Envelope{
if err := evidenceCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: evProto,
}); err != nil {
internal/evidence/reactor_test.go
+5 -6
View File
@@ -92,21 +92,20 @@ func setup(ctx context.Context, t *testing.T, stateStores []sm.Store, chBuf uint
require.NoError(t, err)
rts.peerChans[nodeID] = make(chan p2p.PeerUpdate)
rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
rts.network.Nodes[nodeID].PeerManager.Register(ctx, rts.peerUpdates[nodeID])
pu := p2p.NewPeerUpdates(rts.peerChans[nodeID], 1)
rts.peerUpdates[nodeID] = pu
rts.network.Nodes[nodeID].PeerManager.Register(ctx, pu)
rts.nodes = append(rts.nodes, rts.network.Nodes[nodeID])
chCreator := func(ctx context.Context, chdesc *p2p.ChannelDescriptor) (*p2p.Channel, error) {
return rts.evidenceChannels[nodeID], nil
}
rts.reactors[nodeID], err = evidence.NewReactor(
ctx,
rts.reactors[nodeID] = evidence.NewReactor(
logger,
chCreator,
rts.peerUpdates[nodeID],
func(ctx context.Context) *p2p.PeerUpdates { return pu },
rts.pools[nodeID])
require.NoError(t, err)
require.NoError(t, rts.reactors[nodeID].Start(ctx))
require.True(t, rts.reactors[nodeID].IsRunning())
internal/mempool/mempool_bench_test.go
+1
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/stretchr/testify/require"
abciclient "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/example/kvstore"
"github.com/tendermint/tendermint/libs/log"
internal/mempool/reactor.go
+38 -43
View File
@@ -40,13 +40,9 @@ type Reactor struct {
mempool *TxMempool
ids *IDs
// XXX: Currently, this is the only way to get information about a peer. Ideally,
// we rely on message-oriented communication to get necessary peer data.
// ref: https://github.com/tendermint/tendermint/issues/5670
peerMgr PeerManager
mempoolCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
getPeerHeight func(types.NodeID) int64
peerEvents p2p.PeerEventSubscriber
chCreator p2p.ChannelCreator
// observePanic is a function for observing panics that were recovered in methods on
// Reactor. observePanic is called with the recovered value.
@@ -58,34 +54,27 @@ type Reactor struct {
// NewReactor returns a reference to a new reactor.
func NewReactor(
ctx context.Context,
logger log.Logger,
cfg *config.MempoolConfig,
peerMgr PeerManager,
txmp *TxMempool,
chCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
) (*Reactor, error) {
ch, err := chCreator(ctx, getChannelDescriptor(cfg))
if err != nil {
return nil, err
}
peerEvents p2p.PeerEventSubscriber,
getPeerHeight func(types.NodeID) int64,
) *Reactor {
r := &Reactor{
logger: logger,
cfg: cfg,
peerMgr: peerMgr,
mempool: txmp,
ids: NewMempoolIDs(),
mempoolCh: ch,
peerUpdates: peerUpdates,
peerRoutines: make(map[types.NodeID]context.CancelFunc),
observePanic: defaultObservePanic,
logger: logger,
cfg: cfg,
mempool: txmp,
ids: NewMempoolIDs(),
chCreator: chCreator,
peerEvents: peerEvents,
getPeerHeight: getPeerHeight,
peerRoutines: make(map[types.NodeID]context.CancelFunc),
observePanic: defaultObservePanic,
}
r.BaseService = *service.NewBaseService(logger, "Mempool", r)
return r, nil
return r
}
func defaultObservePanic(r interface{}) {}
@@ -106,6 +95,7 @@ func getChannelDescriptor(cfg *config.MempoolConfig) *p2p.ChannelDescriptor {
Priority: 5,
RecvMessageCapacity: batchMsg.Size(),
RecvBufferCapacity: 128,
Name: "mempool",
}
}
@@ -118,8 +108,13 @@ func (r *Reactor) OnStart(ctx context.Context) error {
r.logger.Info("tx broadcasting is disabled")
}
go r.processMempoolCh(ctx)
go r.processPeerUpdates(ctx)
ch, err := r.chCreator(ctx, getChannelDescriptor(r.cfg))
if err != nil {
return err
}
go r.processMempoolCh(ctx, ch)
go r.processPeerUpdates(ctx, r.peerEvents(ctx), ch)
return nil
}
@@ -202,13 +197,13 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
// processMempoolCh implements a blocking event loop where we listen for p2p
// Envelope messages from the mempoolCh.
func (r *Reactor) processMempoolCh(ctx context.Context) {
iter := r.mempoolCh.Receive(ctx)
func (r *Reactor) processMempoolCh(ctx context.Context, mempoolCh *p2p.Channel) {
iter := mempoolCh.Receive(ctx)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, r.mempoolCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", r.mempoolCh.ID, "envelope", envelope, "err", err)
if serr := r.mempoolCh.SendError(ctx, p2p.PeerError{
if err := r.handleMessage(ctx, mempoolCh.ID, envelope); err != nil {
r.logger.Error("failed to process message", "ch_id", mempoolCh.ID, "envelope", envelope, "err", err)
if serr := mempoolCh.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -223,7 +218,7 @@ func (r *Reactor) processMempoolCh(ctx context.Context) {
// goroutine or not. If not, we start one for the newly added peer. For down or
// removed peers, we remove the peer from the mempool peer ID set and signal to
// stop the tx broadcasting goroutine.
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate) {
func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate, mempoolCh *p2p.Channel) {
r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
r.mtx.Lock()
@@ -251,7 +246,7 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
r.ids.ReserveForPeer(peerUpdate.NodeID)
// start a broadcast routine ensuring all txs are forwarded to the peer
go r.broadcastTxRoutine(pctx, peerUpdate.NodeID)
go r.broadcastTxRoutine(pctx, peerUpdate.NodeID, mempoolCh)
}
}
@@ -272,18 +267,18 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates, mempoolCh *p2p.Channel) {
for {
select {
case <-ctx.Done():
return
case peerUpdate := <-r.peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate)
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate, mempoolCh)
}
}
}
func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) {
func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID, mempoolCh *p2p.Channel) {
peerMempoolID := r.ids.GetForPeer(peerID)
var nextGossipTx *clist.CElement
@@ -324,8 +319,8 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) {
memTx := nextGossipTx.Value.(*WrappedTx)
if r.peerMgr != nil {
height := r.peerMgr.GetHeight(peerID)
if r.getPeerHeight != nil {
height := r.getPeerHeight(peerID)
if height > 0 && height < memTx.height-1 {
// allow for a lag of one block
time.Sleep(PeerCatchupSleepIntervalMS * time.Millisecond)
@@ -338,7 +333,7 @@ func (r *Reactor) broadcastTxRoutine(ctx context.Context, peerID types.NodeID) {
if ok := r.mempool.txStore.TxHasPeer(memTx.hash, peerMempoolID); !ok {
// Send the mempool tx to the corresponding peer. Note, the peer may be
// behind and thus would not be able to process the mempool tx correctly.
if err := r.mempoolCh.Send(ctx, p2p.Envelope{
if err := mempoolCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &protomem.Txs{
Txs: [][]byte{memTx.tx},
internal/mempool/reactor_test.go
+4 -7
View File
@@ -79,17 +79,14 @@ func setupReactors(ctx context.Context, t *testing.T, logger log.Logger, numNode
return rts.mempoolChannels[nodeID], nil
}
rts.reactors[nodeID], err = NewReactor(
ctx,
rts.reactors[nodeID] = NewReactor(
rts.logger.With("nodeID", nodeID),
cfg.Mempool,
rts.network.Nodes[nodeID].PeerManager,
mempool,
chCreator,
rts.peerUpdates[nodeID],
func(ctx context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
rts.network.Nodes[nodeID].PeerManager.GetHeight,
)
require.NoError(t, err)
rts.nodes = append(rts.nodes, nodeID)
require.NoError(t, rts.reactors[nodeID].Start(ctx))
@@ -179,7 +176,7 @@ func TestReactorBroadcastDoesNotPanic(t *testing.T) {
// run the router
rts.start(ctx, t)
go primaryReactor.broadcastTxRoutine(ctx, secondary)
go primaryReactor.broadcastTxRoutine(ctx, secondary, rts.mempoolChannels[primary])
wg := &sync.WaitGroup{}
for i := 0; i < 50; i++ {
internal/p2p/channel.go
+3
View File
@@ -60,6 +60,7 @@ type Channel struct {
errCh chan<- PeerError // peer error reporting
messageType proto.Message // the channel's message type, used for unmarshaling
name string
}
// NewChannel creates a new channel. It is primarily for internal and test
@@ -102,6 +103,8 @@ func (ch *Channel) SendError(ctx context.Context, pe PeerError) error {
}
}
func (ch *Channel) String() string { return fmt.Sprintf("p2p.Channel<%d:%s>", ch.ID, ch.name) }
// Receive returns a new unbuffered iterator to receive messages from ch.
// The iterator runs until ctx ends.
func (ch *Channel) Receive(ctx context.Context) *ChannelIterator {
internal/p2p/conn/connection.go
+4
View File
@@ -616,6 +616,10 @@ type ChannelDescriptor struct {
// RecvBufferCapacity defines the max buffer size of inbound messages for a
// given p2p Channel queue.
RecvBufferCapacity int
// Human readable name of the channel, used in logging and
// diagnostics.
Name string
}
func (chDesc ChannelDescriptor) FillDefaults() (filled ChannelDescriptor) {
internal/p2p/p2ptest/network.go
+1 -2
View File
@@ -259,12 +259,11 @@ func (n *Network) MakeNode(ctx context.Context, t *testing.T, opts NodeOptions)
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
n.logger,
p2p.NopMetrics(),
nodeInfo,
privKey,
peerManager,
func() *types.NodeInfo { return &nodeInfo },
[]p2p.Transport{transport},
transport.Endpoints(),
p2p.RouterOptions{DialSleep: func(_ context.Context) {}},
internal/p2p/peermanager.go
+5
View File
@@ -828,6 +828,11 @@ func (m *PeerManager) Advertise(peerID types.NodeID, limit uint16) []NodeAddress
return addresses
}
// PeerEventSubscriber describes the type of the subscription method, to assist
// in isolating reactors specific construction and lifecycle from the
// peer manager.
type PeerEventSubscriber func(context.Context) *PeerUpdates
// Subscribe subscribes to peer updates. The caller must consume the peer
// updates in a timely fashion and close the subscription when done, otherwise
// the PeerManager will halt.
internal/p2p/pex/reactor.go
+28 -29
View File
@@ -63,6 +63,7 @@ func ChannelDescriptor() *conn.ChannelDescriptor {
SendQueueCapacity: 10,
RecvMessageCapacity: maxMsgSize,
RecvBufferCapacity: 128,
Name: "pex",
}
}
@@ -79,9 +80,8 @@ type Reactor struct {
logger log.Logger
peerManager *p2p.PeerManager
pexCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
chCreator p2p.ChannelCreator
peerEvents p2p.PeerEventSubscriber
// list of available peers to loop through and send peer requests to
availablePeers map[types.NodeID]struct{}
@@ -104,30 +104,23 @@ type Reactor struct {
// NewReactor returns a reference to a new reactor.
func NewReactor(
ctx context.Context,
logger log.Logger,
peerManager *p2p.PeerManager,
channelCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
) (*Reactor, error) {
channel, err := channelCreator(ctx, ChannelDescriptor())
if err != nil {
return nil, err
}
peerEvents p2p.PeerEventSubscriber,
) *Reactor {
r := &Reactor{
logger: logger,
peerManager: peerManager,
pexCh: channel,
peerUpdates: peerUpdates,
chCreator: channelCreator,
peerEvents: peerEvents,
availablePeers: make(map[types.NodeID]struct{}),
requestsSent: make(map[types.NodeID]struct{}),
lastReceivedRequests: make(map[types.NodeID]time.Time),
}
r.BaseService = *service.NewBaseService(logger, "PEX", r)
return r, nil
return r
}
// OnStart starts separate go routines for each p2p Channel and listens for
@@ -135,8 +128,14 @@ func NewReactor(
// messages on that p2p channel accordingly. The caller must be sure to execute
// OnStop to ensure the outbound p2p Channels are closed.
func (r *Reactor) OnStart(ctx context.Context) error {
go r.processPexCh(ctx)
go r.processPeerUpdates(ctx)
channel, err := r.chCreator(ctx, ChannelDescriptor())
if err != nil {
return err
}
peerUpdates := r.peerEvents(ctx)
go r.processPexCh(ctx, channel)
go r.processPeerUpdates(ctx, peerUpdates)
return nil
}
@@ -146,11 +145,11 @@ func (r *Reactor) OnStop() {}
// processPexCh implements a blocking event loop where we listen for p2p
// Envelope messages from the pexCh.
func (r *Reactor) processPexCh(ctx context.Context) {
func (r *Reactor) processPexCh(ctx context.Context, pexCh *p2p.Channel) {
incoming := make(chan *p2p.Envelope)
go func() {
defer close(incoming)
iter := r.pexCh.Receive(ctx)
iter := pexCh.Receive(ctx)
for iter.Next(ctx) {
select {
case <-ctx.Done():
@@ -176,7 +175,7 @@ func (r *Reactor) processPexCh(ctx context.Context) {
case <-timer.C:
// Send a request for more peer addresses.
if err := r.sendRequestForPeers(ctx); err != nil {
if err := r.sendRequestForPeers(ctx, pexCh); err != nil {
return
// TODO(creachadair): Do we really want to stop processing the PEX
// channel just because of an error here?
@@ -191,11 +190,11 @@ func (r *Reactor) processPexCh(ctx context.Context) {
}
// A request from another peer, or a response to one of our requests.
dur, err := r.handlePexMessage(ctx, envelope)
dur, err := r.handlePexMessage(ctx, envelope, pexCh)
if err != nil {
r.logger.Error("failed to process message",
"ch_id", r.pexCh.ID, "envelope", envelope, "err", err)
if serr := r.pexCh.SendError(ctx, p2p.PeerError{
"ch_id", pexCh.ID, "envelope", envelope, "err", err)
if serr := pexCh.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
Err: err,
}); serr != nil {
@@ -212,12 +211,12 @@ func (r *Reactor) processPexCh(ctx context.Context) {
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates) {
for {
select {
case <-ctx.Done():
return
case peerUpdate := <-r.peerUpdates.Updates():
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(peerUpdate)
}
}
@@ -226,7 +225,7 @@ func (r *Reactor) processPeerUpdates(ctx context.Context) {
// handlePexMessage handles envelopes sent from peers on the PexChannel.
// If an update was received, a new polling interval is returned; otherwise the
// duration is 0.
func (r *Reactor) handlePexMessage(ctx context.Context, envelope *p2p.Envelope) (time.Duration, error) {
func (r *Reactor) handlePexMessage(ctx context.Context, envelope *p2p.Envelope, pexCh *p2p.Channel) (time.Duration, error) {
logger := r.logger.With("peer", envelope.From)
switch msg := envelope.Message.(type) {
@@ -245,7 +244,7 @@ func (r *Reactor) handlePexMessage(ctx context.Context, envelope *p2p.Envelope)
URL: addr.String(),
}
}
return 0, r.pexCh.Send(ctx, p2p.Envelope{
return 0, pexCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &protop2p.PexResponse{Addresses: pexAddresses},
})
@@ -309,7 +308,7 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) {
// that peer a request for more peer addresses. The chosen peer is moved into
// the requestsSent bucket so that we will not attempt to contact them again
// until they've replied or updated.
func (r *Reactor) sendRequestForPeers(ctx context.Context) error {
func (r *Reactor) sendRequestForPeers(ctx context.Context, pexCh *p2p.Channel) error {
r.mtx.Lock()
defer r.mtx.Unlock()
if len(r.availablePeers) == 0 {
@@ -324,7 +323,7 @@ func (r *Reactor) sendRequestForPeers(ctx context.Context) error {
break
}
if err := r.pexCh.Send(ctx, p2p.Envelope{
if err := pexCh.Send(ctx, p2p.Envelope{
To: peerID,
Message: &protop2p.PexRequest{},
}); err != nil {
internal/p2p/pex/reactor_test.go
+15 -16
View File
@@ -23,8 +23,8 @@ import (
const (
checkFrequency = 500 * time.Millisecond
defaultBufferSize = 2
shortWait = 10 * time.Second
longWait = 60 * time.Second
shortWait = 5 * time.Second
longWait = 20 * time.Second
firstNode = 0
secondNode = 1
@@ -211,7 +211,8 @@ func TestReactorSmallPeerStoreInALargeNetwork(t *testing.T) {
require.Eventually(t, func() bool {
// nolint:scopelint
return testNet.network.Nodes[nodeID].PeerManager.PeerRatio() >= 0.9
}, longWait, checkFrequency)
}, longWait, checkFrequency,
"peer ratio is: %f", testNet.network.Nodes[nodeID].PeerManager.PeerRatio())
}
}
@@ -303,8 +304,7 @@ func setupSingle(ctx context.Context, t *testing.T) *singleTestReactor {
return pexCh, nil
}
reactor, err := pex.NewReactor(ctx, log.NewNopLogger(), peerManager, chCreator, peerUpdates)
require.NoError(t, err)
reactor := pex.NewReactor(log.NewNopLogger(), peerManager, chCreator, func(_ context.Context) *p2p.PeerUpdates { return peerUpdates })
require.NoError(t, reactor.Start(ctx))
t.Cleanup(reactor.Wait)
@@ -381,6 +381,10 @@ func setupNetwork(ctx context.Context, t *testing.T, opts testOptions) *reactorT
idx := 0
for nodeID := range rts.network.Nodes {
// make a copy to avoid getting hit by the range ref
// confusion:
nodeID := nodeID
rts.peerChans[nodeID] = make(chan p2p.PeerUpdate, chBuf)
rts.peerUpdates[nodeID] = p2p.NewPeerUpdates(rts.peerChans[nodeID], chBuf)
rts.network.Nodes[nodeID].PeerManager.Register(ctx, rts.peerUpdates[nodeID])
@@ -393,15 +397,12 @@ func setupNetwork(ctx context.Context, t *testing.T, opts testOptions) *reactorT
if idx < opts.MockNodes {
rts.mocks = append(rts.mocks, nodeID)
} else {
var err error
rts.reactors[nodeID], err = pex.NewReactor(
ctx,
rts.reactors[nodeID] = pex.NewReactor(
rts.logger.With("nodeID", nodeID),
rts.network.Nodes[nodeID].PeerManager,
chCreator,
rts.peerUpdates[nodeID],
func(_ context.Context) *p2p.PeerUpdates { return rts.peerUpdates[nodeID] },
)
require.NoError(t, err)
}
rts.nodes = append(rts.nodes, nodeID)
@@ -426,9 +427,10 @@ func setupNetwork(ctx context.Context, t *testing.T, opts testOptions) *reactorT
func (r *reactorTestSuite) start(ctx context.Context, t *testing.T) {
t.Helper()
for _, reactor := range r.reactors {
for name, reactor := range r.reactors {
require.NoError(t, reactor.Start(ctx))
require.True(t, reactor.IsRunning())
t.Log("started", name)
}
}
@@ -451,15 +453,12 @@ func (r *reactorTestSuite) addNodes(ctx context.Context, t *testing.T, nodes int
return r.pexChannels[nodeID], nil
}
var err error
r.reactors[nodeID], err = pex.NewReactor(
ctx,
r.reactors[nodeID] = pex.NewReactor(
r.logger.With("nodeID", nodeID),
r.network.Nodes[nodeID].PeerManager,
chCreator,
r.peerUpdates[nodeID],
func(_ context.Context) *p2p.PeerUpdates { return r.peerUpdates[nodeID] },
)
require.NoError(t, err)
r.nodes = append(r.nodes, nodeID)
r.total++
}
internal/p2p/router.go
+32 -23
View File
@@ -150,7 +150,6 @@ type Router struct {
metrics *Metrics
options RouterOptions
nodeInfo types.NodeInfo
privKey crypto.PrivKey
peerManager *PeerManager
chDescs []*ChannelDescriptor
@@ -162,8 +161,9 @@ type Router struct {
peerMtx sync.RWMutex
peerQueues map[types.NodeID]queue // outbound messages per peer for all channels
// the channels that the peer queue has open
peerChannels map[types.NodeID]ChannelIDSet
queueFactory func(int) queue
peerChannels map[types.NodeID]ChannelIDSet
queueFactory func(int) queue
nodeInfoProducer func() *types.NodeInfo
// FIXME: We don't strictly need to use a mutex for this if we seal the
// channels on router start. This depends on whether we want to allow
@@ -177,12 +177,11 @@ type Router struct {
// listening on appropriate interfaces, and will be closed by the Router when it
// stops.
func NewRouter(
ctx context.Context,
logger log.Logger,
metrics *Metrics,
nodeInfo types.NodeInfo,
privKey crypto.PrivKey,
peerManager *PeerManager,
nodeInfoProducer func() *types.NodeInfo,
transports []Transport,
endpoints []Endpoint,
options RouterOptions,
@@ -193,10 +192,10 @@ func NewRouter(
}
router := &Router{
logger: logger,
metrics: metrics,
nodeInfo: nodeInfo,
privKey: privKey,
logger: logger,
metrics: metrics,
privKey: privKey,
nodeInfoProducer: nodeInfoProducer,
connTracker: newConnTracker(
options.MaxIncomingConnectionAttempts,
options.IncomingConnectionWindow,
@@ -215,13 +214,6 @@ func NewRouter(
router.BaseService = service.NewBaseService(logger, "router", router)
qf, err := router.createQueueFactory(ctx)
if err != nil {
return nil, err
}
router.queueFactory = qf
for _, transport := range transports {
for _, protocol := range transport.Protocols() {
if _, ok := router.protocolTransports[protocol]; !ok {
@@ -281,6 +273,7 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (*C
outCh := make(chan Envelope, chDesc.RecvBufferCapacity)
errCh := make(chan PeerError, chDesc.RecvBufferCapacity)
channel := NewChannel(id, messageType, queue.dequeue(), outCh, errCh)
channel.name = chDesc.Name
var wrapper Wrapper
if w, ok := messageType.(Wrapper); ok {
@@ -291,7 +284,7 @@ func (r *Router) OpenChannel(ctx context.Context, chDesc *ChannelDescriptor) (*C
r.channelMessages[id] = messageType
// add the channel to the nodeInfo if it's not already there.
r.nodeInfo.AddChannel(uint16(chDesc.ID))
r.nodeInfoProducer().AddChannel(uint16(chDesc.ID))
for _, t := range r.transports {
t.AddChannelDescriptors([]*ChannelDescriptor{chDesc})
@@ -722,7 +715,8 @@ func (r *Router) handshakePeer(
defer cancel()
}
peerInfo, peerKey, err := conn.Handshake(ctx, r.nodeInfo, r.privKey)
nodeInfo := r.nodeInfoProducer()
peerInfo, peerKey, err := conn.Handshake(ctx, *nodeInfo, r.privKey)
if err != nil {
return peerInfo, err
}
@@ -737,7 +731,7 @@ func (r *Router) handshakePeer(
return peerInfo, fmt.Errorf("expected to connect with peer %q, got %q",
expectID, peerInfo.NodeID)
}
if err := r.nodeInfo.CompatibleWith(peerInfo); err != nil {
if err := r.nodeInfoProducer().CompatibleWith(peerInfo); err != nil {
return peerInfo, ErrRejected{
err: err,
id: peerInfo.ID(),
@@ -937,11 +931,25 @@ func (r *Router) evictPeers(ctx context.Context) {
// NodeInfo returns a copy of the current NodeInfo. Used for testing.
func (r *Router) NodeInfo() types.NodeInfo {
return r.nodeInfo.Copy()
return r.nodeInfoProducer().Copy()
}
func (r *Router) setupQueueFactory(ctx context.Context) error {
qf, err := r.createQueueFactory(ctx)
if err != nil {
return err
}
r.queueFactory = qf
return nil
}
// OnStart implements service.Service.
func (r *Router) OnStart(ctx context.Context) error {
if err := r.setupQueueFactory(ctx); err != nil {
return err
}
for _, transport := range r.transports {
for _, endpoint := range r.endpoints {
if err := transport.Listen(endpoint); err != nil {
@@ -950,11 +958,12 @@ func (r *Router) OnStart(ctx context.Context) error {
}
}
nodeInfo := r.nodeInfoProducer()
r.logger.Info(
"starting router",
"node_id", r.nodeInfo.NodeID,
"channels", r.nodeInfo.Channels,
"listen_addr", r.nodeInfo.ListenAddr,
"node_id", nodeInfo.NodeID,
"channels", nodeInfo.Channels,
"listen_addr", nodeInfo.ListenAddr,
"transports", len(r.transports),
)
internal/p2p/router_init_test.go
+10 -4
View File
@@ -23,29 +23,35 @@ func TestRouter_ConstructQueueFactory(t *testing.T) {
t.Run("Default", func(t *testing.T) {
require.Zero(t, os.Getenv("TM_P2P_QUEUE"))
opts := RouterOptions{}
r, err := NewRouter(ctx, log.NewNopLogger(), nil, types.NodeInfo{}, nil, nil, nil, nil, opts)
r, err := NewRouter(log.NewNopLogger(), nil, nil, nil, func() *types.NodeInfo { return &types.NodeInfo{} }, nil, nil, opts)
require.NoError(t, err)
require.NoError(t, r.setupQueueFactory(ctx))
_, ok := r.queueFactory(1).(*fifoQueue)
require.True(t, ok)
})
t.Run("Fifo", func(t *testing.T) {
opts := RouterOptions{QueueType: queueTypeFifo}
r, err := NewRouter(ctx, log.NewNopLogger(), nil, types.NodeInfo{}, nil, nil, nil, nil, opts)
r, err := NewRouter(log.NewNopLogger(), nil, nil, nil, func() *types.NodeInfo { return &types.NodeInfo{} }, nil, nil, opts)
require.NoError(t, err)
require.NoError(t, r.setupQueueFactory(ctx))
_, ok := r.queueFactory(1).(*fifoQueue)
require.True(t, ok)
})
t.Run("Priority", func(t *testing.T) {
opts := RouterOptions{QueueType: queueTypePriority}
r, err := NewRouter(ctx, log.NewNopLogger(), nil, types.NodeInfo{}, nil, nil, nil, nil, opts)
r, err := NewRouter(log.NewNopLogger(), nil, nil, nil, func() *types.NodeInfo { return &types.NodeInfo{} }, nil, nil, opts)
require.NoError(t, err)
require.NoError(t, r.setupQueueFactory(ctx))
q, ok := r.queueFactory(1).(*pqScheduler)
require.True(t, ok)
defer q.close()
})
t.Run("NonExistant", func(t *testing.T) {
opts := RouterOptions{QueueType: "fast"}
_, err := NewRouter(ctx, log.NewNopLogger(), nil, types.NodeInfo{}, nil, nil, nil, nil, opts)
_, err := NewRouter(log.NewNopLogger(), nil, nil, nil, func() *types.NodeInfo { return &types.NodeInfo{} }, nil, nil, opts)
require.Error(t, err)
require.Contains(t, err.Error(), "fast")
})
internal/p2p/router_test.go
+10 -20
View File
@@ -106,12 +106,11 @@ func TestRouter_Channel_Basic(t *testing.T) {
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
nil,
nil,
p2p.RouterOptions{},
@@ -409,12 +408,11 @@ func TestRouter_AcceptPeers(t *testing.T) {
sub := peerManager.Subscribe(ctx)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -464,12 +462,11 @@ func TestRouter_AcceptPeers_Error(t *testing.T) {
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -502,12 +499,11 @@ func TestRouter_AcceptPeers_ErrorEOF(t *testing.T) {
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -554,12 +550,11 @@ func TestRouter_AcceptPeers_HeadOfLineBlocking(t *testing.T) {
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -658,12 +653,11 @@ func TestRouter_DialPeers(t *testing.T) {
sub := peerManager.Subscribe(ctx)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -744,12 +738,11 @@ func TestRouter_DialPeers_Parallel(t *testing.T) {
require.True(t, added)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{
@@ -819,12 +812,11 @@ func TestRouter_EvictPeers(t *testing.T) {
sub := peerManager.Subscribe(ctx)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -882,12 +874,11 @@ func TestRouter_ChannelCompatability(t *testing.T) {
require.NoError(t, err)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
@@ -938,12 +929,11 @@ func TestRouter_DontSendOnInvalidChannel(t *testing.T) {
sub := peerManager.Subscribe(ctx)
router, err := p2p.NewRouter(
ctx,
log.NewNopLogger(),
p2p.NopMetrics(),
selfInfo,
selfKey,
peerManager,
func() *types.NodeInfo { return &selfInfo },
[]p2p.Transport{mockTransport},
nil,
p2p.RouterOptions{},
internal/proxy/client.go
+1
View File
@@ -8,6 +8,7 @@ import (
"time"
"github.com/go-kit/kit/metrics"
abciclient "github.com/tendermint/tendermint/abci/client"
"github.com/tendermint/tendermint/abci/example/kvstore"
"github.com/tendermint/tendermint/abci/types"
internal/proxy/client_test.go
+2 -1
View File
@@ -14,6 +14,8 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"gotest.tools/assert"
abciclient "github.com/tendermint/tendermint/abci/client"
abcimocks "github.com/tendermint/tendermint/abci/client/mocks"
"github.com/tendermint/tendermint/abci/example/kvstore"
@@ -21,7 +23,6 @@ import (
"github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
tmrand "github.com/tendermint/tendermint/libs/rand"
"gotest.tools/assert"
)
//----------------------------------------
internal/state/execution.go
+8 -29
View File
@@ -48,14 +48,6 @@ type BlockExecutor struct {
cache map[string]struct{}
}
type BlockExecutorOption func(executor *BlockExecutor)
func BlockExecutorWithMetrics(metrics *Metrics) BlockExecutorOption {
return func(blockExec *BlockExecutor) {
blockExec.metrics = metrics
}
}
// NewBlockExecutor returns a new BlockExecutor with a NopEventBus.
// Call SetEventBus to provide one.
func NewBlockExecutor(
@@ -66,25 +58,19 @@ func NewBlockExecutor(
evpool EvidencePool,
blockStore BlockStore,
eventBus *eventbus.EventBus,
options ...BlockExecutorOption,
metrics *Metrics,
) *BlockExecutor {
res := &BlockExecutor{
return &BlockExecutor{
eventBus: eventBus,
store: stateStore,
appClient: appClient,
mempool: pool,
evpool: evpool,
logger: logger,
metrics: NopMetrics(),
metrics: metrics,
cache: make(map[string]struct{}),
blockStore: blockStore,
}
for _, option := range options {
option(res)
}
return res
}
func (blockExec *BlockExecutor) Store() Store {
@@ -138,22 +124,15 @@ func (blockExec *BlockExecutor) CreateProposalBlock(
// transaction causing an error.
//
// Also, the App can simply skip any transaction that could cause any kind of trouble.
// Either way, we can not recover in a meaningful way, unless we skip proposing
// this block, repair what caused the error and try again. Hence, we panic on
// purpose for now.
panic(err)
}
if rpp.IsTxStatusUnknown() {
panic(fmt.Sprintf("PrepareProposal responded with ModifiedTxStatus %s", rpp.ModifiedTxStatus.String()))
}
if !rpp.IsTxStatusModified() {
return block, nil
// Either way, we cannot recover in a meaningful way, unless we skip proposing
// this block, repair what caused the error and try again. Hence, we return an
// error for now (the production code calling this function is expected to panic).
return nil, err
}
txrSet := types.NewTxRecordSet(rpp.TxRecords)
if err := txrSet.Validate(maxDataBytes, block.Txs); err != nil {
panic(fmt.Errorf("ResponsePrepareProposal validation: %w", err))
return nil, err
}
for _, rtx := range txrSet.RemovedTxs() {
internal/state/execution_test.go
+105 -54
View File
@@ -2,6 +2,7 @@ package state_test
import (
"context"
"errors"
"testing"
"time"
@@ -11,6 +12,7 @@ import (
dbm "github.com/tendermint/tm-db"
abciclient "github.com/tendermint/tendermint/abci/client"
abciclientmocks "github.com/tendermint/tendermint/abci/client/mocks"
abci "github.com/tendermint/tendermint/abci/types"
abcimocks "github.com/tendermint/tendermint/abci/types/mocks"
"github.com/tendermint/tendermint/crypto"
@@ -64,7 +66,7 @@ func TestApplyBlock(t *testing.T) {
mock.Anything,
mock.Anything,
mock.Anything).Return(nil)
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyApp, mp, sm.EmptyEvidencePool{}, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, logger, proxyApp, mp, sm.EmptyEvidencePool{}, blockStore, eventBus, sm.NopMetrics())
block := sf.MakeBlock(state, 1, new(types.Commit))
bps, err := block.MakePartSet(testPartSize)
@@ -128,7 +130,7 @@ func TestFinalizeBlockDecidedLastCommit(t *testing.T) {
eventBus := eventbus.NewDefault(logger)
require.NoError(t, eventBus.Start(ctx))
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), appClient, mp, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), appClient, mp, evpool, blockStore, eventBus, sm.NopMetrics())
state, _, lastCommit := makeAndCommitGoodBlock(ctx, t, state, 1, new(types.Commit), state.NextValidators.Validators[0].Address, blockExec, privVals, nil)
for idx, isAbsent := range tc.absentCommitSigs {
@@ -252,8 +254,7 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) {
blockStore := store.NewBlockStore(dbm.NewMemDB())
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyApp,
mp, evpool, blockStore, eventBus)
blockExec := sm.NewBlockExecutor(stateStore, log.NewNopLogger(), proxyApp, mp, evpool, blockStore, eventBus, sm.NopMetrics())
block := sf.MakeBlock(state, 1, new(types.Commit))
block.Evidence = ev
@@ -272,7 +273,7 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) {
func TestProcessProposal(t *testing.T) {
const height = 2
txs := factory.MakeTenTxs(height)
txs := factory.MakeNTxs(height, 10)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -298,6 +299,7 @@ func TestProcessProposal(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
block0 := sf.MakeBlock(state, height-1, new(types.Commit))
@@ -516,6 +518,7 @@ func TestFinalizeBlockValidatorUpdates(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
updatesSub, err := eventBus.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{
@@ -586,6 +589,7 @@ func TestFinalizeBlockValidatorUpdatesResultingInEmptySet(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
block := sf.MakeBlock(state, 1, new(types.Commit))
@@ -616,9 +620,7 @@ func TestEmptyPrepareProposal(t *testing.T) {
require.NoError(t, eventBus.Start(ctx))
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_UNMODIFIED,
}, nil)
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{})
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
err := proxyApp.Start(ctx)
@@ -647,6 +649,7 @@ func TestEmptyPrepareProposal(t *testing.T) {
sm.EmptyEvidencePool{},
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
@@ -654,9 +657,10 @@ func TestEmptyPrepareProposal(t *testing.T) {
require.NoError(t, err)
}
// TestPrepareProposalPanicOnInvalid tests that the block creation logic panics
// if the ResponsePrepareProposal returned from the application is invalid.
func TestPrepareProposalPanicOnInvalid(t *testing.T) {
// TestPrepareProposalErrorOnNonExistingRemoved tests that the block creation logic returns
// an error if the ResponsePrepareProposal returned from the application marks
// a transaction as REMOVED that was not present in the original proposal.
func TestPrepareProposalErrorOnNonExistingRemoved(t *testing.T) {
const height = 2
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -678,7 +682,6 @@ func TestPrepareProposalPanicOnInvalid(t *testing.T) {
// create an invalid ResponsePrepareProposal
rpp := abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_MODIFIED,
TxRecords: []*abci.TxRecord{
{
Action: abci.TxRecord_REMOVED,
@@ -686,7 +689,7 @@ func TestPrepareProposalPanicOnInvalid(t *testing.T) {
},
},
}
app.On("PrepareProposal", mock.Anything).Return(rpp, nil)
app.On("PrepareProposal", mock.Anything).Return(rpp)
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
@@ -701,13 +704,13 @@ func TestPrepareProposalPanicOnInvalid(t *testing.T) {
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
require.Panics(t,
func() {
blockExec.CreateProposalBlock(ctx, height, state, commit, pa, nil) //nolint:errcheck
})
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, nil)
require.Nil(t, block)
require.ErrorContains(t, err, "new transaction incorrectly marked as removed")
mp.AssertExpectations(t)
}
@@ -730,7 +733,7 @@ func TestPrepareProposalRemoveTxs(t *testing.T) {
evpool := &mocks.EvidencePool{}
evpool.On("PendingEvidence", mock.Anything).Return([]types.Evidence{}, int64(0))
txs := factory.MakeTenTxs(height)
txs := factory.MakeNTxs(height, 10)
mp := &mpmocks.Mempool{}
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs(txs))
@@ -741,9 +744,8 @@ func TestPrepareProposalRemoveTxs(t *testing.T) {
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_MODIFIED,
TxRecords: trs,
}, nil)
TxRecords: trs,
})
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
@@ -758,6 +760,7 @@ func TestPrepareProposalRemoveTxs(t *testing.T) {
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
@@ -790,7 +793,7 @@ func TestPrepareProposalAddedTxsIncluded(t *testing.T) {
evpool := &mocks.EvidencePool{}
evpool.On("PendingEvidence", mock.Anything).Return([]types.Evidence{}, int64(0))
txs := factory.MakeTenTxs(height)
txs := factory.MakeNTxs(height, 10)
mp := &mpmocks.Mempool{}
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs(txs[2:]))
@@ -800,9 +803,8 @@ func TestPrepareProposalAddedTxsIncluded(t *testing.T) {
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_MODIFIED,
TxRecords: trs,
}, nil)
TxRecords: trs,
})
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
@@ -817,6 +819,7 @@ func TestPrepareProposalAddedTxsIncluded(t *testing.T) {
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
@@ -846,7 +849,7 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
evpool := &mocks.EvidencePool{}
evpool.On("PendingEvidence", mock.Anything).Return([]types.Evidence{}, int64(0))
txs := factory.MakeTenTxs(height)
txs := factory.MakeNTxs(height, 10)
mp := &mpmocks.Mempool{}
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs(txs))
@@ -856,9 +859,8 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_MODIFIED,
TxRecords: trs,
}, nil)
TxRecords: trs,
})
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
@@ -873,6 +875,7 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
@@ -886,10 +889,67 @@ func TestPrepareProposalReorderTxs(t *testing.T) {
}
// TestPrepareProposalModifiedTxStatusFalse tests that CreateBlock correctly ignores
// the ResponsePrepareProposal TxRecords if ResponsePrepareProposal does not
// set ModifiedTxStatus to true.
func TestPrepareProposalModifiedTxStatusFalse(t *testing.T) {
// TestPrepareProposalErrorOnTooManyTxs tests that the block creation logic returns
// an error if the ResponsePrepareProposal returned from the application is invalid.
func TestPrepareProposalErrorOnTooManyTxs(t *testing.T) {
const height = 2
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logger := log.NewNopLogger()
eventBus := eventbus.NewDefault(logger)
require.NoError(t, eventBus.Start(ctx))
state, stateDB, privVals := makeState(t, 1, height)
// limit max block size
state.ConsensusParams.Block.MaxBytes = 60 * 1024
stateStore := sm.NewStore(stateDB)
evpool := &mocks.EvidencePool{}
evpool.On("PendingEvidence", mock.Anything).Return([]types.Evidence{}, int64(0))
const nValidators = 1
var bytesPerTx int64 = 3
maxDataBytes := types.MaxDataBytes(state.ConsensusParams.Block.MaxBytes, 0, nValidators)
txs := factory.MakeNTxs(height, maxDataBytes/bytesPerTx+2) // +2 so that tx don't fit
mp := &mpmocks.Mempool{}
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs(txs))
trs := txsToTxRecords(types.Txs(txs))
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
TxRecords: trs,
})
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
err := proxyApp.Start(ctx)
require.NoError(t, err)
blockExec := sm.NewBlockExecutor(
stateStore,
logger,
proxyApp,
mp,
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, nil)
require.Nil(t, block)
require.ErrorContains(t, err, "transaction data size exceeds maximum")
mp.AssertExpectations(t)
}
// TestPrepareProposalErrorOnPrepareProposalError tests when the client returns an error
// upon calling PrepareProposal on it.
func TestPrepareProposalErrorOnPrepareProposalError(t *testing.T) {
const height = 2
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -904,27 +964,18 @@ func TestPrepareProposalModifiedTxStatusFalse(t *testing.T) {
evpool := &mocks.EvidencePool{}
evpool.On("PendingEvidence", mock.Anything).Return([]types.Evidence{}, int64(0))
txs := factory.MakeTenTxs(height)
txs := factory.MakeNTxs(height, 10)
mp := &mpmocks.Mempool{}
mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs(txs))
trs := txsToTxRecords(types.Txs(txs))
trs = append(trs[len(trs)/2:], trs[:len(trs)/2]...)
trs = trs[1:]
trs[0].Action = abci.TxRecord_REMOVED
trs[1] = &abci.TxRecord{
Tx: []byte("new"),
Action: abci.TxRecord_ADDED,
}
cm := &abciclientmocks.Client{}
cm.On("IsRunning").Return(true)
cm.On("Error").Return(nil)
cm.On("Start", mock.Anything).Return(nil).Once()
cm.On("Wait").Return(nil).Once()
cm.On("PrepareProposal", mock.Anything, mock.Anything).Return(nil, errors.New("an injected error")).Once()
app := abcimocks.NewBaseMock()
app.On("PrepareProposal", mock.Anything).Return(abci.ResponsePrepareProposal{
ModifiedTxStatus: abci.ResponsePrepareProposal_UNMODIFIED,
TxRecords: trs,
}, nil)
cc := abciclient.NewLocalClient(logger, app)
proxyApp := proxy.New(cc, logger, proxy.NopMetrics())
proxyApp := proxy.New(cm, logger, proxy.NopMetrics())
err := proxyApp.Start(ctx)
require.NoError(t, err)
@@ -936,14 +987,14 @@ func TestPrepareProposalModifiedTxStatusFalse(t *testing.T) {
evpool,
nil,
eventBus,
sm.NopMetrics(),
)
pa, _ := state.Validators.GetByIndex(0)
commit := makeValidCommit(ctx, t, height, types.BlockID{}, state.Validators, privVals)
block, err := blockExec.CreateProposalBlock(ctx, height, state, commit, pa, nil)
require.NoError(t, err)
for i, tx := range block.Data.Txs {
require.Equal(t, txs[i], tx)
}
require.Nil(t, block)
require.ErrorContains(t, err, "an injected error")
mp.AssertExpectations(t)
}
internal/state/helpers_test.go
+1 -1
View File
@@ -62,7 +62,7 @@ func makeAndApplyGoodBlock(
evidence []types.Evidence,
) (sm.State, types.BlockID) {
t.Helper()
block := state.MakeBlock(height, factory.MakeTenTxs(height), lastCommit, evidence, proposerAddr)
block := state.MakeBlock(height, factory.MakeNTxs(height, 10), lastCommit, evidence, proposerAddr)
partSet, err := block.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
internal/state/test/factory/block.go
+1 -1
View File
@@ -45,7 +45,7 @@ func MakeBlocks(ctx context.Context, t *testing.T, n int, state *sm.State, privV
func MakeBlock(state sm.State, height int64, c *types.Commit) *types.Block {
return state.MakeBlock(
height,
factory.MakeTenTxs(state.LastBlockHeight),
factory.MakeNTxs(state.LastBlockHeight, 10),
c,
nil,
state.Validators.GetProposer().Address,
internal/state/validation_test.go
+4 -1
View File
@@ -63,6 +63,7 @@ func TestValidateBlockHeader(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
@@ -166,6 +167,7 @@ func TestValidateBlockCommit(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
wrongSigsCommit := types.NewCommit(1, 0, types.BlockID{}, nil)
@@ -315,6 +317,7 @@ func TestValidateBlockEvidence(t *testing.T) {
evpool,
blockStore,
eventBus,
sm.NopMetrics(),
)
lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil)
@@ -335,7 +338,7 @@ func TestValidateBlockEvidence(t *testing.T) {
evidence = append(evidence, newEv)
currentBytes += int64(len(newEv.Bytes()))
}
block := state.MakeBlock(height, testfactory.MakeTenTxs(height), lastCommit, evidence, proposerAddr)
block := state.MakeBlock(height, testfactory.MakeNTxs(height, 10), lastCommit, evidence, proposerAddr)
err := blockExec.ValidateBlock(ctx, state, block)
if assert.Error(t, err) {
internal/statesync/reactor.go
+159 -126
View File
@@ -75,13 +75,13 @@ const (
func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
return map[p2p.ChannelID]*p2p.ChannelDescriptor{
SnapshotChannel: {
ID: SnapshotChannel,
MessageType: new(ssproto.Message),
Priority: 6,
SendQueueCapacity: 10,
RecvMessageCapacity: snapshotMsgSize,
RecvBufferCapacity: 128,
Name: "snapshot",
},
ChunkChannel: {
ID: ChunkChannel,
@@ -90,6 +90,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 4,
RecvMessageCapacity: chunkMsgSize,
RecvBufferCapacity: 128,
Name: "chunk",
},
LightBlockChannel: {
ID: LightBlockChannel,
@@ -98,6 +99,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 10,
RecvMessageCapacity: lightBlockMsgSize,
RecvBufferCapacity: 128,
Name: "light-block",
},
ParamsChannel: {
ID: ParamsChannel,
@@ -106,6 +108,7 @@ func getChannelDescriptors() map[p2p.ChannelID]*p2p.ChannelDescriptor {
SendQueueCapacity: 10,
RecvMessageCapacity: paramMsgSize,
RecvBufferCapacity: 128,
Name: "params",
},
}
@@ -135,13 +138,11 @@ type Reactor struct {
stateStore sm.Store
blockStore *store.BlockStore
conn abciclient.Client
tempDir string
snapshotCh *p2p.Channel
chunkCh *p2p.Channel
blockCh *p2p.Channel
paramsCh *p2p.Channel
peerUpdates *p2p.PeerUpdates
conn abciclient.Client
tempDir string
peerEvents p2p.PeerEventSubscriber
chCreator p2p.ChannelCreator
sendBlockError func(context.Context, p2p.PeerError) error
// Dispatcher is used to multiplex light block requests and responses over multiple
// peers used by the p2p state provider and in reverse sync.
@@ -151,10 +152,13 @@ type Reactor struct {
// These will only be set when a state sync is in progress. It is used to feed
// received snapshots and chunks into the syncer and manage incoming and outgoing
// providers.
mtx sync.RWMutex
syncer *syncer
providers map[types.NodeID]*BlockProvider
stateProvider StateProvider
mtx sync.RWMutex
initSyncer func() *syncer
requestSnaphot func() error
syncer *syncer
providers map[types.NodeID]*BlockProvider
initStateProvider func(ctx context.Context, chainID string, initialHeight int64) error
stateProvider StateProvider
eventBus *eventbus.EventBus
metrics *Metrics
@@ -174,32 +178,13 @@ func NewReactor(
logger log.Logger,
conn abciclient.Client,
channelCreator p2p.ChannelCreator,
peerUpdates *p2p.PeerUpdates,
peerEvents p2p.PeerEventSubscriber,
stateStore sm.Store,
blockStore *store.BlockStore,
tempDir string,
ssMetrics *Metrics,
eventBus *eventbus.EventBus,
) (*Reactor, error) {
chDesc := getChannelDescriptors()
snapshotCh, err := channelCreator(ctx, chDesc[SnapshotChannel])
if err != nil {
return nil, err
}
chunkCh, err := channelCreator(ctx, chDesc[ChunkChannel])
if err != nil {
return nil, err
}
blockCh, err := channelCreator(ctx, chDesc[LightBlockChannel])
if err != nil {
return nil, err
}
paramsCh, err := channelCreator(ctx, chDesc[ParamsChannel])
if err != nil {
return nil, err
}
) *Reactor {
r := &Reactor{
logger: logger,
@@ -207,23 +192,19 @@ func NewReactor(
initialHeight: initialHeight,
cfg: cfg,
conn: conn,
snapshotCh: snapshotCh,
chunkCh: chunkCh,
blockCh: blockCh,
paramsCh: paramsCh,
peerUpdates: peerUpdates,
chCreator: channelCreator,
peerEvents: peerEvents,
tempDir: tempDir,
stateStore: stateStore,
blockStore: blockStore,
peers: newPeerList(),
dispatcher: NewDispatcher(blockCh),
providers: make(map[types.NodeID]*BlockProvider),
metrics: ssMetrics,
eventBus: eventBus,
}
r.BaseService = *service.NewBaseService(logger, "StateSync", r)
return r, nil
return r
}
// OnStart starts separate go routines for each p2p Channel and listens for
@@ -233,11 +214,91 @@ func NewReactor(
// The caller must be sure to execute OnStop to ensure the outbound p2p Channels are
// closed. No error is returned.
func (r *Reactor) OnStart(ctx context.Context) error {
go r.processCh(ctx, r.snapshotCh, "snapshot")
go r.processCh(ctx, r.chunkCh, "chunk")
go r.processCh(ctx, r.blockCh, "light block")
go r.processCh(ctx, r.paramsCh, "consensus params")
go r.processPeerUpdates(ctx)
// construct channels
chDesc := getChannelDescriptors()
snapshotCh, err := r.chCreator(ctx, chDesc[SnapshotChannel])
if err != nil {
return err
}
chunkCh, err := r.chCreator(ctx, chDesc[ChunkChannel])
if err != nil {
return err
}
blockCh, err := r.chCreator(ctx, chDesc[LightBlockChannel])
if err != nil {
return err
}
paramsCh, err := r.chCreator(ctx, chDesc[ParamsChannel])
if err != nil {
return err
}
// define constructor and helper functions, that hold
// references to these channels for use later. This is not
// ideal.
r.initSyncer = func() *syncer {
return &syncer{
logger: r.logger,
stateProvider: r.stateProvider,
conn: r.conn,
snapshots: newSnapshotPool(),
snapshotCh: snapshotCh,
chunkCh: chunkCh,
tempDir: r.tempDir,
fetchers: r.cfg.Fetchers,
retryTimeout: r.cfg.ChunkRequestTimeout,
metrics: r.metrics,
}
}
r.dispatcher = NewDispatcher(blockCh)
r.requestSnaphot = func() error {
// request snapshots from all currently connected peers
return snapshotCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: &ssproto.SnapshotsRequest{},
})
}
r.sendBlockError = blockCh.SendError
r.initStateProvider = func(ctx context.Context, chainID string, initialHeight int64) error {
to := light.TrustOptions{
Period: r.cfg.TrustPeriod,
Height: r.cfg.TrustHeight,
Hash: r.cfg.TrustHashBytes(),
}
spLogger := r.logger.With("module", "stateprovider")
spLogger.Info("initializing state provider", "trustPeriod", to.Period,
"trustHeight", to.Height, "useP2P", r.cfg.UseP2P)
if r.cfg.UseP2P {
if err := r.waitForEnoughPeers(ctx, 2); err != nil {
return err
}
peers := r.peers.All()
providers := make([]provider.Provider, len(peers))
for idx, p := range peers {
providers[idx] = NewBlockProvider(p, chainID, r.dispatcher)
}
stateProvider, err := NewP2PStateProvider(ctx, chainID, initialHeight, providers, to, paramsCh, r.logger.With("module", "stateprovider"))
if err != nil {
return fmt.Errorf("failed to initialize P2P state provider: %w", err)
}
r.stateProvider = stateProvider
return nil
}
stateProvider, err := NewRPCStateProvider(ctx, chainID, initialHeight, r.cfg.RPCServers, to, spLogger)
if err != nil {
return fmt.Errorf("failed to initialize RPC state provider: %w", err)
}
r.stateProvider = stateProvider
return nil
}
go r.processChannels(ctx, snapshotCh, chunkCh, blockCh, paramsCh)
go r.processPeerUpdates(ctx, r.peerEvents(ctx))
return nil
}
@@ -280,17 +341,9 @@ func (r *Reactor) Sync(ctx context.Context) (sm.State, error) {
return sm.State{}, err
}
r.syncer = newSyncer(
r.cfg,
r.logger,
r.conn,
r.stateProvider,
r.snapshotCh,
r.chunkCh,
r.tempDir,
r.metrics,
)
r.syncer = r.initSyncer()
r.mtx.Unlock()
defer func() {
r.mtx.Lock()
// reset syncing objects at the close of Sync
@@ -299,15 +352,7 @@ func (r *Reactor) Sync(ctx context.Context) (sm.State, error) {
r.mtx.Unlock()
}()
requestSnapshotsHook := func() error {
// request snapshots from all currently connected peers
return r.snapshotCh.Send(ctx, p2p.Envelope{
Broadcast: true,
Message: &ssproto.SnapshotsRequest{},
})
}
state, commit, err := r.syncer.SyncAny(ctx, r.cfg.DiscoveryTime, requestSnapshotsHook)
state, commit, err := r.syncer.SyncAny(ctx, r.cfg.DiscoveryTime, r.requestSnaphot)
if err != nil {
return sm.State{}, err
}
@@ -434,7 +479,7 @@ func (r *Reactor) backfill(
r.logger.Info("backfill: fetched light block failed validate basic, removing peer...",
"err", err, "height", height)
queue.retry(height)
if serr := r.blockCh.SendError(ctx, p2p.PeerError{
if serr := r.sendBlockError(ctx, p2p.PeerError{
NodeID: peer,
Err: fmt.Errorf("received invalid light block: %w", err),
}); serr != nil {
@@ -471,7 +516,7 @@ func (r *Reactor) backfill(
if w, g := trustedBlockID.Hash, resp.block.Hash(); !bytes.Equal(w, g) {
r.logger.Info("received invalid light block. header hash doesn't match trusted LastBlockID",
"trustedHash", w, "receivedHash", g, "height", resp.block.Height)
if err := r.blockCh.SendError(ctx, p2p.PeerError{
if err := r.sendBlockError(ctx, p2p.PeerError{
NodeID: resp.peer,
Err: fmt.Errorf("received invalid light block. Expected hash %v, got: %v", w, g),
}); err != nil {
@@ -532,7 +577,7 @@ func (r *Reactor) backfill(
// handleSnapshotMessage handles envelopes sent from peers on the
// SnapshotChannel. It returns an error only if the Envelope.Message is unknown
// for this channel. This should never be called outside of handleMessage.
func (r *Reactor) handleSnapshotMessage(ctx context.Context, envelope *p2p.Envelope) error {
func (r *Reactor) handleSnapshotMessage(ctx context.Context, envelope *p2p.Envelope, snapshotCh *p2p.Channel) error {
logger := r.logger.With("peer", envelope.From)
switch msg := envelope.Message.(type) {
@@ -551,7 +596,7 @@ func (r *Reactor) handleSnapshotMessage(ctx context.Context, envelope *p2p.Envel
"peer", envelope.From,
)
if err := r.snapshotCh.Send(ctx, p2p.Envelope{
if err := snapshotCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &ssproto.SnapshotsResponse{
Height: snapshot.Height,
@@ -587,8 +632,8 @@ func (r *Reactor) handleSnapshotMessage(ctx context.Context, envelope *p2p.Envel
"failed to add snapshot",
"height", msg.Height,
"format", msg.Format,
"channel", snapshotCh.ID,
"err", err,
"channel", r.snapshotCh.ID,
)
return nil
}
@@ -604,7 +649,7 @@ func (r *Reactor) handleSnapshotMessage(ctx context.Context, envelope *p2p.Envel
// handleChunkMessage handles envelopes sent from peers on the ChunkChannel.
// It returns an error only if the Envelope.Message is unknown for this channel.
// This should never be called outside of handleMessage.
func (r *Reactor) handleChunkMessage(ctx context.Context, envelope *p2p.Envelope) error {
func (r *Reactor) handleChunkMessage(ctx context.Context, envelope *p2p.Envelope, chunkCh *p2p.Channel) error {
switch msg := envelope.Message.(type) {
case *ssproto.ChunkRequest:
r.logger.Debug(
@@ -638,7 +683,7 @@ func (r *Reactor) handleChunkMessage(ctx context.Context, envelope *p2p.Envelope
"chunk", msg.Index,
"peer", envelope.From,
)
if err := r.chunkCh.Send(ctx, p2p.Envelope{
if err := chunkCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &ssproto.ChunkResponse{
Height: msg.Height,
@@ -693,7 +738,7 @@ func (r *Reactor) handleChunkMessage(ctx context.Context, envelope *p2p.Envelope
return nil
}
func (r *Reactor) handleLightBlockMessage(ctx context.Context, envelope *p2p.Envelope) error {
func (r *Reactor) handleLightBlockMessage(ctx context.Context, envelope *p2p.Envelope, blockCh *p2p.Channel) error {
switch msg := envelope.Message.(type) {
case *ssproto.LightBlockRequest:
r.logger.Info("received light block request", "height", msg.Height)
@@ -703,7 +748,7 @@ func (r *Reactor) handleLightBlockMessage(ctx context.Context, envelope *p2p.Env
return err
}
if lb == nil {
if err := r.blockCh.Send(ctx, p2p.Envelope{
if err := blockCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &ssproto.LightBlockResponse{
LightBlock: nil,
@@ -722,7 +767,7 @@ func (r *Reactor) handleLightBlockMessage(ctx context.Context, envelope *p2p.Env
// NOTE: If we don't have the light block we will send a nil light block
// back to the requested node, indicating that we don't have it.
if err := r.blockCh.Send(ctx, p2p.Envelope{
if err := blockCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &ssproto.LightBlockResponse{
LightBlock: lbproto,
@@ -750,7 +795,7 @@ func (r *Reactor) handleLightBlockMessage(ctx context.Context, envelope *p2p.Env
return nil
}
func (r *Reactor) handleParamsMessage(ctx context.Context, envelope *p2p.Envelope) error {
func (r *Reactor) handleParamsMessage(ctx context.Context, envelope *p2p.Envelope, paramsCh *p2p.Channel) error {
switch msg := envelope.Message.(type) {
case *ssproto.ParamsRequest:
r.logger.Debug("received consensus params request", "height", msg.Height)
@@ -761,7 +806,7 @@ func (r *Reactor) handleParamsMessage(ctx context.Context, envelope *p2p.Envelop
}
cpproto := cp.ToProto()
if err := r.paramsCh.Send(ctx, p2p.Envelope{
if err := paramsCh.Send(ctx, p2p.Envelope{
To: envelope.From,
Message: &ssproto.ParamsResponse{
Height: msg.Height,
@@ -780,6 +825,8 @@ func (r *Reactor) handleParamsMessage(ctx context.Context, envelope *p2p.Envelop
if sp, ok := r.stateProvider.(*stateProviderP2P); ok {
select {
case sp.paramsRecvCh <- cp:
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Second):
return errors.New("failed to send consensus params, stateprovider not ready for response")
}
@@ -797,7 +844,7 @@ func (r *Reactor) handleParamsMessage(ctx context.Context, envelope *p2p.Envelop
// handleMessage handles an Envelope sent from a peer on a specific p2p Channel.
// It will handle errors and any possible panics gracefully. A caller can handle
// any error returned by sending a PeerError on the respective channel.
func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelope *p2p.Envelope) (err error) {
func (r *Reactor) handleMessage(ctx context.Context, envelope *p2p.Envelope, chans map[p2p.ChannelID]*p2p.Channel) (err error) {
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("panic in processing message: %v", e)
@@ -811,17 +858,17 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
r.logger.Debug("received message", "message", reflect.TypeOf(envelope.Message), "peer", envelope.From)
switch chID {
switch envelope.ChannelID {
case SnapshotChannel:
err = r.handleSnapshotMessage(ctx, envelope)
err = r.handleSnapshotMessage(ctx, envelope, chans[SnapshotChannel])
case ChunkChannel:
err = r.handleChunkMessage(ctx, envelope)
err = r.handleChunkMessage(ctx, envelope, chans[ChunkChannel])
case LightBlockChannel:
err = r.handleLightBlockMessage(ctx, envelope)
err = r.handleLightBlockMessage(ctx, envelope, chans[LightBlockChannel])
case ParamsChannel:
err = r.handleParamsMessage(ctx, envelope)
err = r.handleParamsMessage(ctx, envelope, chans[ParamsChannel])
default:
err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", chID, envelope)
err = fmt.Errorf("unknown channel ID (%d) for envelope (%v)", envelope.ChannelID, envelope)
}
return err
@@ -831,15 +878,35 @@ func (r *Reactor) handleMessage(ctx context.Context, chID p2p.ChannelID, envelop
// encountered during message execution will result in a PeerError being sent on
// the respective channel. When the reactor is stopped, we will catch the signal
// and close the p2p Channel gracefully.
func (r *Reactor) processCh(ctx context.Context, ch *p2p.Channel, chName string) {
iter := ch.Receive(ctx)
func (r *Reactor) processChannels(ctx context.Context, chs ...*p2p.Channel) {
// make sure that the iterator gets cleaned up in case of error
ctx, cancel := context.WithCancel(ctx)
defer cancel()
chanTable := make(map[p2p.ChannelID]*p2p.Channel, len(chs))
for idx := range chs {
ch := chs[idx]
chanTable[ch.ID] = ch
}
iter := p2p.MergedChannelIterator(ctx, chs...)
for iter.Next(ctx) {
envelope := iter.Envelope()
if err := r.handleMessage(ctx, ch.ID, envelope); err != nil {
if err := r.handleMessage(ctx, envelope, chanTable); err != nil {
ch, ok := chanTable[envelope.ChannelID]
if !ok {
r.logger.Error("received impossible message",
"envelope_from", envelope.From,
"envelope_ch", envelope.ChannelID,
"num_chs", len(chanTable),
"err", err,
)
return
}
r.logger.Error("failed to process message",
"err", err,
"channel", chName,
"ch_id", ch.ID,
"channel", ch.String(),
"ch_id", envelope.ChannelID,
"envelope", envelope)
if serr := ch.SendError(ctx, p2p.PeerError{
NodeID: envelope.From,
@@ -875,14 +942,15 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
r.mtx.Lock()
defer r.mtx.Unlock()
if r.syncer == nil {
return
}
switch peerUpdate.Status {
case p2p.PeerStatusUp:
newProvider := NewBlockProvider(peerUpdate.NodeID, r.chainID, r.dispatcher)
r.providers[peerUpdate.NodeID] = newProvider
err := r.syncer.AddPeer(ctx, peerUpdate.NodeID)
if err != nil {
@@ -905,12 +973,12 @@ func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpda
// processPeerUpdates initiates a blocking process where we listen for and handle
// PeerUpdate messages. When the reactor is stopped, we will catch the signal and
// close the p2p PeerUpdatesCh gracefully.
func (r *Reactor) processPeerUpdates(ctx context.Context) {
func (r *Reactor) processPeerUpdates(ctx context.Context, peerUpdates *p2p.PeerUpdates) {
for {
select {
case <-ctx.Done():
return
case peerUpdate := <-r.peerUpdates.Updates():
case peerUpdate := <-peerUpdates.Updates():
r.processPeerUpdate(ctx, peerUpdate)
}
}
@@ -1015,41 +1083,6 @@ func (r *Reactor) waitForEnoughPeers(ctx context.Context, numPeers int) error {
return nil
}
func (r *Reactor) initStateProvider(ctx context.Context, chainID string, initialHeight int64) error {
var err error
to := light.TrustOptions{
Period: r.cfg.TrustPeriod,
Height: r.cfg.TrustHeight,
Hash: r.cfg.TrustHashBytes(),
}
spLogger := r.logger.With("module", "stateprovider")
spLogger.Info("initializing state provider", "trustPeriod", to.Period,
"trustHeight", to.Height, "useP2P", r.cfg.UseP2P)
if r.cfg.UseP2P {
if err := r.waitForEnoughPeers(ctx, 2); err != nil {
return err
}
peers := r.peers.All()
providers := make([]provider.Provider, len(peers))
for idx, p := range peers {
providers[idx] = NewBlockProvider(p, chainID, r.dispatcher)
}
r.stateProvider, err = NewP2PStateProvider(ctx, chainID, initialHeight, providers, to, r.paramsCh, spLogger)
if err != nil {
return fmt.Errorf("failed to initialize P2P state provider: %w", err)
}
} else {
r.stateProvider, err = NewRPCStateProvider(ctx, chainID, initialHeight, r.cfg.RPCServers, to, spLogger)
if err != nil {
return fmt.Errorf("failed to initialize RPC state provider: %w", err)
}
}
return nil
}
func (r *Reactor) TotalSnapshots() int64 {
r.mtx.RLock()
defer r.mtx.RUnlock()
internal/statesync/reactor_test.go
+43 -33
View File
@@ -154,8 +154,7 @@ func setup(
logger := log.NewNopLogger()
var err error
rts.reactor, err = NewReactor(
rts.reactor = NewReactor(
ctx,
factory.DefaultTestChainID,
1,
@@ -163,25 +162,26 @@ func setup(
logger.With("component", "reactor"),
conn,
chCreator,
rts.peerUpdates,
func(context.Context) *p2p.PeerUpdates { return rts.peerUpdates },
rts.stateStore,
rts.blockStore,
"",
m,
nil, // eventbus can be nil
)
require.NoError(t, err)
rts.syncer = newSyncer(
*cfg,
logger.With("component", "syncer"),
conn,
stateProvider,
rts.snapshotChannel,
rts.chunkChannel,
"",
rts.reactor.metrics,
)
rts.syncer = &syncer{
logger: logger,
stateProvider: stateProvider,
conn: conn,
snapshots: newSnapshotPool(),
snapshotCh: rts.snapshotChannel,
chunkCh: rts.chunkChannel,
tempDir: t.TempDir(),
fetchers: cfg.Fetchers,
retryTimeout: cfg.ChunkRequestTimeout,
metrics: rts.reactor.metrics,
}
ctx, cancel := context.WithCancel(ctx)
@@ -196,11 +196,11 @@ func setup(
}
func TestReactor_Sync(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
const snapshotHeight = 7
rts := setup(ctx, t, nil, nil, 2)
rts := setup(ctx, t, nil, nil, 100)
chain := buildLightBlockChain(ctx, t, 1, 10, time.Now())
// app accepts any snapshot
rts.conn.On("OfferSnapshot", ctx, mock.AnythingOfType("types.RequestOfferSnapshot")).
@@ -224,8 +224,7 @@ func TestReactor_Sync(t *testing.T) {
closeCh := make(chan struct{})
defer close(closeCh)
go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh,
rts.blockInCh, closeCh, 0)
go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0)
go graduallyAddPeers(ctx, t, rts.peerUpdateCh, closeCh, 1*time.Second)
go handleSnapshotRequests(ctx, t, rts.snapshotOutCh, rts.snapshotInCh, closeCh, []snapshot{
{
@@ -257,8 +256,9 @@ func TestReactor_ChunkRequest_InvalidRequest(t *testing.T) {
rts := setup(ctx, t, nil, nil, 2)
rts.chunkInCh <- p2p.Envelope{
From: types.NodeID("aa"),
Message: &ssproto.SnapshotsRequest{},
From: types.NodeID("aa"),
ChannelID: ChunkChannel,
Message: &ssproto.SnapshotsRequest{},
}
response := <-rts.chunkPeerErrCh
@@ -315,8 +315,9 @@ func TestReactor_ChunkRequest(t *testing.T) {
rts := setup(ctx, t, conn, nil, 2)
rts.chunkInCh <- p2p.Envelope{
From: types.NodeID("aa"),
Message: tc.request,
From: types.NodeID("aa"),
ChannelID: ChunkChannel,
Message: tc.request,
}
response := <-rts.chunkOutCh
@@ -335,8 +336,9 @@ func TestReactor_SnapshotsRequest_InvalidRequest(t *testing.T) {
rts := setup(ctx, t, nil, nil, 2)
rts.snapshotInCh <- p2p.Envelope{
From: types.NodeID("aa"),
Message: &ssproto.ChunkRequest{},
From: types.NodeID("aa"),
ChannelID: SnapshotChannel,
Message: &ssproto.ChunkRequest{},
}
response := <-rts.snapshotPeerErrCh
@@ -400,8 +402,9 @@ func TestReactor_SnapshotsRequest(t *testing.T) {
rts := setup(ctx, t, conn, nil, 100)
rts.snapshotInCh <- p2p.Envelope{
From: types.NodeID("aa"),
Message: &ssproto.SnapshotsRequest{},
From: types.NodeID("aa"),
ChannelID: SnapshotChannel,
Message: &ssproto.SnapshotsRequest{},
}
if len(tc.expectResponses) > 0 {
@@ -457,7 +460,8 @@ func TestReactor_LightBlockResponse(t *testing.T) {
rts.stateStore.On("LoadValidators", height).Return(vals, nil)
rts.blockInCh <- p2p.Envelope{
From: types.NodeID("aa"),
From: types.NodeID("aa"),
ChannelID: LightBlockChannel,
Message: &ssproto.LightBlockRequest{
Height: 10,
},
@@ -733,7 +737,8 @@ func handleLightBlockRequests(
require.NoError(t, err)
select {
case sending <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: LightBlockChannel,
Message: &ssproto.LightBlockResponse{
LightBlock: lb,
},
@@ -750,7 +755,8 @@ func handleLightBlockRequests(
require.NoError(t, err)
select {
case sending <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: LightBlockChannel,
Message: &ssproto.LightBlockResponse{
LightBlock: differntLB,
},
@@ -761,7 +767,8 @@ func handleLightBlockRequests(
case 1: // send nil block i.e. pretend we don't have it
select {
case sending <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: LightBlockChannel,
Message: &ssproto.LightBlockResponse{
LightBlock: nil,
},
@@ -802,7 +809,8 @@ func handleConsensusParamsRequest(
}
select {
case sending <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: ParamsChannel,
Message: &ssproto.ParamsResponse{
Height: msg.Height,
ConsensusParams: paramsProto,
@@ -913,7 +921,8 @@ func handleSnapshotRequests(
require.True(t, ok)
for _, snapshot := range snapshots {
sendingCh <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: SnapshotChannel,
Message: &ssproto.SnapshotsResponse{
Height: snapshot.Height,
Format: snapshot.Format,
@@ -946,7 +955,8 @@ func handleChunkRequests(
msg, ok := envelope.Message.(*ssproto.ChunkRequest)
require.True(t, ok)
sendingCh <- p2p.Envelope{
From: envelope.To,
From: envelope.To,
ChannelID: ChunkChannel,
Message: &ssproto.ChunkResponse{
Height: msg.Height,
Format: msg.Format,
internal/statesync/syncer.go
-26
View File
@@ -10,7 +10,6 @@ import (
abciclient "github.com/tendermint/tendermint/abci/client"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/config"
"github.com/tendermint/tendermint/internal/p2p"
"github.com/tendermint/tendermint/internal/proxy"
sm "github.com/tendermint/tendermint/internal/state"
@@ -72,31 +71,6 @@ type syncer struct {
processingSnapshot *snapshot
}
// newSyncer creates a new syncer.
func newSyncer(
cfg config.StateSyncConfig,
logger log.Logger,
conn abciclient.Client,
stateProvider StateProvider,
snapshotCh *p2p.Channel,
chunkCh *p2p.Channel,
tempDir string,
metrics *Metrics,
) *syncer {
return &syncer{
logger: logger,
stateProvider: stateProvider,
conn: conn,
snapshots: newSnapshotPool(),
snapshotCh: snapshotCh,
chunkCh: chunkCh,
tempDir: tempDir,
fetchers: cfg.Fetchers,
retryTimeout: cfg.ChunkRequestTimeout,
metrics: metrics,
}
}
// AddChunk adds a chunk to the chunk queue, if any. It returns false if the chunk has already
// been added to the queue, or an error if there's no sync in progress.
func (s *syncer) AddChunk(chunk *chunk) (bool, error) {
internal/test/factory/tx.go
+3 -3
View File
@@ -2,10 +2,10 @@ package factory
import "github.com/tendermint/tendermint/types"
func MakeTenTxs(height int64) []types.Tx {
txs := make([]types.Tx, 10)
func MakeNTxs(height, n int64) []types.Tx {
txs := make([]types.Tx, n)
for i := range txs {
txs[i] = types.Tx([]byte{byte(height), byte(i)})
txs[i] = types.Tx([]byte{byte(height), byte(i / 256), byte(i % 256)})
}
return txs
}
light/rpc/client.go
+3 -9
View File
@@ -51,7 +51,6 @@ type Client struct {
keyPathFn KeyPathFunc
closers []func()
quitCh chan struct{}
}
var _ rpcclient.Client = (*Client)(nil)
@@ -92,10 +91,9 @@ func DefaultMerkleKeyPathFn() KeyPathFunc {
// NewClient returns a new client.
func NewClient(logger log.Logger, next rpcclient.Client, lc LightClient, opts ...Option) *Client {
c := &Client{
next: next,
lc: lc,
prt: merkle.DefaultProofRuntime(),
quitCh: make(chan struct{}),
next: next,
lc: lc,
prt: merkle.DefaultProofRuntime(),
}
c.BaseService = *service.NewBaseService(logger, "Client", c)
for _, o := range opts {
@@ -111,10 +109,6 @@ func (c *Client) OnStart(ctx context.Context) error {
return err
}
c.closers = append(c.closers, ncancel)
go func() {
defer close(c.quitCh)
c.Wait()
}()
return nil
}
node/node.go
+81 -96
View File
@@ -88,12 +88,11 @@ func newDefaultNode(
}
if cfg.Mode == config.ModeSeed {
return makeSeedNode(
ctx,
logger,
cfg,
config.DefaultDBProvider,
nodeKey,
defaultGenesisDocProviderFunc(cfg),
logger,
)
}
pval, err := makeDefaultPrivval(cfg)
@@ -214,13 +213,6 @@ func makeNode(
}
}
// Determine whether we should attempt state sync.
stateSync := cfg.StateSync.Enable && !onlyValidatorIsUs(state, pubKey)
if stateSync && state.LastBlockHeight > 0 {
logger.Info("Found local state with non-zero height, skipping state sync")
stateSync = false
}
// Create the handshaker, which calls RequestInfo, sets the AppVersion on the state,
// and replays any blocks as necessary to sync tendermint with the app.
if err := consensus.NewHandshaker(
@@ -244,7 +236,7 @@ func makeNode(
// TODO: Fetch and provide real options and do proper p2p bootstrapping.
// TODO: Use a persistent peer database.
nodeInfo, err := makeNodeInfo(cfg, nodeKey, eventSinks, genDoc, state)
nodeInfo, err := makeNodeInfo(cfg, nodeKey, eventSinks, genDoc, state.Version.Consensus)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
@@ -257,28 +249,67 @@ func makeNode(
makeCloser(closers))
}
router, err := createRouter(ctx, logger, nodeMetrics.p2p, nodeInfo, nodeKey,
peerManager, cfg, proxyApp)
// TODO construct node here:
node := &nodeImpl{
config: cfg,
logger: logger,
genesisDoc: genDoc,
privValidator: privValidator,
peerManager: peerManager,
nodeInfo: nodeInfo,
nodeKey: nodeKey,
eventSinks: eventSinks,
services: []service.Service{eventBus},
stateStore: stateStore,
blockStore: blockStore,
shutdownOps: makeCloser(closers),
rpcEnv: &rpccore.Environment{
ProxyApp: proxyApp,
StateStore: stateStore,
BlockStore: blockStore,
PeerManager: peerManager,
GenDoc: genDoc,
EventSinks: eventSinks,
EventBus: eventBus,
EventLog: eventLog,
Logger: logger.With("module", "rpc"),
Config: *cfg.RPC,
},
}
node.router, err = createRouter(logger, nodeMetrics.p2p, node.NodeInfo, nodeKey, peerManager, cfg, proxyApp)
if err != nil {
return nil, combineCloseError(
fmt.Errorf("failed to create router: %w", err),
makeCloser(closers))
}
mpReactor, mp, err := createMempoolReactor(ctx,
cfg, proxyApp, stateStore, nodeMetrics.mempool, peerManager, router, logger,
)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
evReactor, evPool, edbCloser, err := createEvidenceReactor(ctx,
cfg, dbProvider, stateStore, blockStore, peerManager, router, logger, nodeMetrics.evidence, eventBus,
)
evReactor, evPool, edbCloser, err := createEvidenceReactor(logger, cfg, dbProvider,
stateStore, blockStore, peerManager.Subscribe, node.router.OpenChannel, nodeMetrics.evidence, eventBus)
closers = append(closers, edbCloser)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
node.services = append(node.services, evReactor)
node.rpcEnv.EvidencePool = evPool
node.evPool = evPool
mpReactor, mp, err := createMempoolReactor(logger, cfg, proxyApp, stateStore, nodeMetrics.mempool,
peerManager.Subscribe, node.router.OpenChannel, peerManager.GetHeight)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
node.rpcEnv.Mempool = mp
node.services = append(node.services, mpReactor)
// make block executor for consensus and blockchain reactors to execute blocks
blockExec := sm.NewBlockExecutor(
@@ -289,41 +320,49 @@ func makeNode(
evPool,
blockStore,
eventBus,
sm.BlockExecutorWithMetrics(nodeMetrics.state),
nodeMetrics.state,
)
// Determine whether we should attempt state sync.
stateSync := cfg.StateSync.Enable && !onlyValidatorIsUs(state, pubKey)
if stateSync && state.LastBlockHeight > 0 {
logger.Info("Found local state with non-zero height, skipping state sync")
stateSync = false
}
// Determine whether we should do block sync. This must happen after the handshake, since the
// app may modify the validator set, specifying ourself as the only validator.
blockSync := !onlyValidatorIsUs(state, pubKey)
waitSync := stateSync || blockSync
csReactor, csState, err := createConsensusReactor(ctx,
cfg, stateStore, blockExec, blockStore, mp, evPool,
privValidator, nodeMetrics.consensus, stateSync || blockSync, eventBus,
peerManager, router, logger,
privValidator, nodeMetrics.consensus, waitSync, eventBus,
peerManager, node.router.OpenChannel, logger,
)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
node.services = append(node.services, csReactor)
node.rpcEnv.ConsensusState = csState
node.rpcEnv.ConsensusReactor = csReactor
// Create the blockchain reactor. Note, we do not start block sync if we're
// doing a state sync first.
bcReactor, err := blocksync.NewReactor(ctx,
bcReactor := blocksync.NewReactor(
logger.With("module", "blockchain"),
stateStore,
blockExec,
blockStore,
csReactor,
router.OpenChannel,
peerManager.Subscribe(ctx),
node.router.OpenChannel,
peerManager.Subscribe,
blockSync && !stateSync,
nodeMetrics.consensus,
eventBus,
)
if err != nil {
return nil, combineCloseError(
fmt.Errorf("could not create blocksync reactor: %w", err),
makeCloser(closers))
}
node.services = append(node.services, bcReactor)
node.rpcEnv.BlockSyncReactor = bcReactor
// Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first.
// FIXME We need to update metrics here, since other reactors don't have access to them.
@@ -337,83 +376,25 @@ func makeNode(
// FIXME The way we do phased startups (e.g. replay -> block sync -> consensus) is very messy,
// we should clean this whole thing up. See:
// https://github.com/tendermint/tendermint/issues/4644
stateSyncReactor, err := statesync.NewReactor(
node.stateSync = stateSync
node.stateSyncReactor = statesync.NewReactor(
ctx,
genDoc.ChainID,
genDoc.InitialHeight,
*cfg.StateSync,
logger.With("module", "statesync"),
proxyApp,
router.OpenChannel,
peerManager.Subscribe(ctx),
node.router.OpenChannel,
peerManager.Subscribe,
stateStore,
blockStore,
cfg.StateSync.TempDir,
nodeMetrics.statesync,
eventBus,
)
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
var pexReactor service.Service = service.NopService{}
if cfg.P2P.PexReactor {
pexReactor, err = pex.NewReactor(ctx, logger, peerManager, router.OpenChannel, peerManager.Subscribe(ctx))
if err != nil {
return nil, combineCloseError(err, makeCloser(closers))
}
}
node := &nodeImpl{
config: cfg,
logger: logger,
genesisDoc: genDoc,
privValidator: privValidator,
peerManager: peerManager,
router: router,
nodeInfo: nodeInfo,
nodeKey: nodeKey,
eventSinks: eventSinks,
services: []service.Service{
eventBus,
evReactor,
mpReactor,
csReactor,
bcReactor,
pexReactor,
},
stateStore: stateStore,
blockStore: blockStore,
stateSyncReactor: stateSyncReactor,
stateSync: stateSync,
evPool: evPool,
shutdownOps: makeCloser(closers),
rpcEnv: &rpccore.Environment{
ProxyApp: proxyApp,
EvidencePool: evPool,
ConsensusState: csState,
StateStore: stateStore,
BlockStore: blockStore,
ConsensusReactor: csReactor,
BlockSyncReactor: bcReactor,
PeerManager: peerManager,
GenDoc: genDoc,
EventSinks: eventSinks,
EventBus: eventBus,
EventLog: eventLog,
Mempool: mp,
Logger: logger.With("module", "rpc"),
Config: *cfg.RPC,
},
node.services = append(node.services, pex.NewReactor(logger, peerManager, node.router.OpenChannel, peerManager.Subscribe))
}
if cfg.Mode == config.ModeValidator {
@@ -658,6 +639,10 @@ func (n *nodeImpl) startPrometheusServer(ctx context.Context, addr string) *http
return srv
}
func (n *nodeImpl) NodeInfo() *types.NodeInfo {
return &n.nodeInfo
}
// EventBus returns the Node's EventBus.
func (n *nodeImpl) EventBus() *eventbus.EventBus {
return n.rpcEnv.EventBus
node/node_test.go
+5 -2
View File
@@ -333,6 +333,7 @@ func TestCreateProposalBlock(t *testing.T) {
evidencePool,
blockStore,
eventBus,
sm.NopMetrics(),
)
commit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
@@ -412,6 +413,7 @@ func TestMaxTxsProposalBlockSize(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
commit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
@@ -487,6 +489,7 @@ func TestMaxProposalBlockSize(t *testing.T) {
sm.EmptyEvidencePool{},
blockStore,
eventBus,
sm.NopMetrics(),
)
blockID := types.BlockID{
@@ -578,12 +581,12 @@ func TestNodeNewSeedNode(t *testing.T) {
logger := log.NewNopLogger()
ns, err := makeSeedNode(ctx,
ns, err := makeSeedNode(
logger,
cfg,
config.DefaultDBProvider,
nodeKey,
defaultGenesisDocProviderFunc(cfg),
logger,
)
t.Cleanup(ns.Wait)
t.Cleanup(leaktest.CheckTimeout(t, time.Second))
node/public.go
+1 -1
View File
@@ -68,7 +68,7 @@ func New(
config.DefaultDBProvider,
logger)
case config.ModeSeed:
return makeSeedNode(ctx, conf, config.DefaultDBProvider, nodeKey, genProvider, logger)
return makeSeedNode(logger, conf, config.DefaultDBProvider, nodeKey, genProvider)
default:
return nil, fmt.Errorf("%q is not a valid mode", conf.Mode)
}
node/seed.go
+3 -12
View File
@@ -29,7 +29,6 @@ type seedNodeImpl struct {
// network
peerManager *p2p.PeerManager
router *p2p.Router
nodeInfo types.NodeInfo
nodeKey types.NodeKey // our node privkey
isListening bool
@@ -40,12 +39,11 @@ type seedNodeImpl struct {
// makeSeedNode returns a new seed node, containing only p2p, pex reactor
func makeSeedNode(
ctx context.Context,
logger log.Logger,
cfg *config.Config,
dbProvider config.DBProvider,
nodeKey types.NodeKey,
genesisDocProvider genesisDocProvider,
logger log.Logger,
) (service.Service, error) {
if !cfg.P2P.PexReactor {
return nil, errors.New("cannot run seed nodes with PEX disabled")
@@ -76,32 +74,25 @@ func makeSeedNode(
closer)
}
router, err := createRouter(ctx, logger, p2pMetrics, nodeInfo, nodeKey,
peerManager, cfg, nil)
router, err := createRouter(logger, p2pMetrics, func() *types.NodeInfo { return &nodeInfo }, nodeKey, peerManager, cfg, nil)
if err != nil {
return nil, combineCloseError(
fmt.Errorf("failed to create router: %w", err),
closer)
}
pexReactor, err := pex.NewReactor(ctx, logger, peerManager, router.OpenChannel, peerManager.Subscribe(ctx))
if err != nil {
return nil, combineCloseError(err, closer)
}
node := &seedNodeImpl{
config: cfg,
logger: logger,
genesisDoc: genDoc,
nodeInfo: nodeInfo,
nodeKey: nodeKey,
peerManager: peerManager,
router: router,
shutdownOps: closer,
pexReactor: pexReactor,
pexReactor: pex.NewReactor(logger, peerManager, router.OpenChannel, peerManager.Subscribe),
}
node.BaseService = *service.NewBaseService(logger, "SeedNode", node)
node/setup.go
+23 -39
View File
@@ -169,14 +169,14 @@ func onlyValidatorIsUs(state sm.State, pubKey crypto.PubKey) bool {
}
func createMempoolReactor(
ctx context.Context,
logger log.Logger,
cfg *config.Config,
appClient abciclient.Client,
store sm.Store,
memplMetrics *mempool.Metrics,
peerManager *p2p.PeerManager,
router *p2p.Router,
logger log.Logger,
peerEvents p2p.PeerEventSubscriber,
chCreator p2p.ChannelCreator,
peerHeight func(types.NodeID) int64,
) (service.Service, mempool.Mempool, error) {
logger = logger.With("module", "mempool")
@@ -189,18 +189,14 @@ func createMempoolReactor(
mempool.WithPostCheck(sm.TxPostCheckFromStore(store)),
)
reactor, err := mempool.NewReactor(
ctx,
reactor := mempool.NewReactor(
logger,
cfg.Mempool,
peerManager,
mp,
router.OpenChannel,
peerManager.Subscribe(ctx),
chCreator,
peerEvents,
peerHeight,
)
if err != nil {
return nil, nil, err
}
if cfg.Consensus.WaitForTxs() {
mp.EnableTxsAvailable()
@@ -210,14 +206,13 @@ func createMempoolReactor(
}
func createEvidenceReactor(
ctx context.Context,
logger log.Logger,
cfg *config.Config,
dbProvider config.DBProvider,
store sm.Store,
blockStore *store.BlockStore,
peerManager *p2p.PeerManager,
router *p2p.Router,
logger log.Logger,
peerEvents p2p.PeerEventSubscriber,
chCreator p2p.ChannelCreator,
metrics *evidence.Metrics,
eventBus *eventbus.EventBus,
) (*evidence.Reactor, *evidence.Pool, closer, error) {
@@ -231,16 +226,12 @@ func createEvidenceReactor(
evidencePool := evidence.NewPool(logger, evidenceDB, store, blockStore, metrics, eventBus)
evidenceReactor, err := evidence.NewReactor(
ctx,
evidenceReactor := evidence.NewReactor(
logger,
router.OpenChannel,
peerManager.Subscribe(ctx),
chCreator,
peerEvents,
evidencePool,
)
if err != nil {
return nil, nil, dbCloser, fmt.Errorf("creating evidence reactor: %w", err)
}
return evidenceReactor, evidencePool, dbCloser, nil
}
@@ -258,7 +249,7 @@ func createConsensusReactor(
waitSync bool,
eventBus *eventbus.EventBus,
peerManager *p2p.PeerManager,
router *p2p.Router,
chCreator p2p.ChannelCreator,
logger log.Logger,
) (*consensus.Reactor, *consensus.State, error) {
logger = logger.With("module", "consensus")
@@ -282,20 +273,15 @@ func createConsensusReactor(
consensusState.SetPrivValidator(ctx, privValidator)
}
reactor, err := consensus.NewReactor(
ctx,
reactor := consensus.NewReactor(
logger,
consensusState,
router.OpenChannel,
peerManager.Subscribe(ctx),
chCreator,
peerManager.Subscribe,
eventBus,
waitSync,
csMetrics,
)
if err != nil {
return nil, nil, err
}
return reactor, consensusState, nil
}
@@ -375,10 +361,9 @@ func createPeerManager(
}
func createRouter(
ctx context.Context,
logger log.Logger,
p2pMetrics *p2p.Metrics,
nodeInfo types.NodeInfo,
nodeInfoProducer func() *types.NodeInfo,
nodeKey types.NodeKey,
peerManager *p2p.PeerManager,
cfg *config.Config,
@@ -405,12 +390,11 @@ func createRouter(
}
return p2p.NewRouter(
ctx,
p2pLogger,
p2pMetrics,
nodeInfo,
nodeKey.PrivKey,
peerManager,
nodeInfoProducer,
[]p2p.Transport{transport},
[]p2p.Endpoint{ep},
getRouterConfig(cfg, appClient),
@@ -422,7 +406,7 @@ func makeNodeInfo(
nodeKey types.NodeKey,
eventSinks []indexer.EventSink,
genDoc *types.GenesisDoc,
state sm.State,
versionInfo version.Consensus,
) (types.NodeInfo, error) {
txIndexerStatus := "off"
@@ -434,8 +418,8 @@ func makeNodeInfo(
nodeInfo := types.NodeInfo{
ProtocolVersion: types.ProtocolVersion{
P2P: version.P2PProtocol, // global
Block: state.Version.Consensus.Block,
App: state.Version.Consensus.App,
Block: versionInfo.Block,
App: versionInfo.App,
},
NodeID: nodeKey.ID,
Network: genDoc.ChainID,
proto/tendermint/abci/types.proto.intermediate
+5 -12
View File
@@ -326,18 +326,11 @@ message ResponseApplySnapshotChunk {
}
message ResponsePrepareProposal {
ModifiedTxStatus modified_tx_status = 1;
repeated TxRecord tx_records = 2;
bytes app_hash = 3;
repeated ExecTxResult tx_results = 4;
repeated ValidatorUpdate validator_updates = 5;
tendermint.types.ConsensusParams consensus_param_updates = 6;
enum ModifiedTxStatus {
UNKNOWN = 0;
UNMODIFIED = 1;
MODIFIED = 2;
}
repeated TxRecord tx_records = 1;
bytes app_hash = 2;
repeated ExecTxResult tx_results = 3;
repeated ValidatorUpdate validator_updates = 4;
tendermint.types.ConsensusParams consensus_param_updates = 5;
}
message ResponseProcessProposal {
proto/tendermint/types/params.proto
+1 -1
View File
@@ -75,7 +75,7 @@ message HashedParams {
// see the specification of proposer-based timestamps:
// https://github.com/tendermint/tendermint/tree/master/spec/consensus/proposer-based-timestamp
message SynchronyParams {
// message_delay bounds how long a proposal message may take to reach all validators on a newtork
// message_delay bounds how long a proposal message may take to reach all validators on a network
// and still be considered valid.
google.protobuf.Duration message_delay = 1 [(gogoproto.stdduration) = true];
// precision bounds how skewed a proposer's clock may be from any validator
rpc/client/mocks/client.go
+23
View File
@@ -411,6 +411,29 @@ func (_m *Client) DumpConsensusState(_a0 context.Context) (*coretypes.ResultDump
return r0, r1
}
// Events provides a mock function with given fields: ctx, req
func (_m *Client) Events(ctx context.Context, req *coretypes.RequestEvents) (*coretypes.ResultEvents, error) {
ret := _m.Called(ctx, req)
var r0 *coretypes.ResultEvents
if rf, ok := ret.Get(0).(func(context.Context, *coretypes.RequestEvents) *coretypes.ResultEvents); ok {
r0 = rf(ctx, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*coretypes.ResultEvents)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(context.Context, *coretypes.RequestEvents) error); ok {
r1 = rf(ctx, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Genesis provides a mock function with given fields: _a0
func (_m *Client) Genesis(_a0 context.Context) (*coretypes.ResultGenesis, error) {
ret := _m.Called(_a0)
spec/abci++/abci++_methods_002_draft.md
+14 -34
View File
@@ -299,7 +299,6 @@ title: Methods
| Name | Type | Description | Field Number |
|-------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------|--------------|
| modified_tx_status | [TxModifiedStatus](#TxModifiedStatus) | `enum` signaling if the application has made changes to the list of transactions. | 1 |
| tx_records | repeated [TxRecord](#txrecord) | Possibly modified list of transactions that have been picked as part of the proposed block. | 2 |
| app_hash | bytes | The Merkle root hash of the application state. | 3 |
| tx_results | repeated [ExecTxResult](#txresult) | List of structures containing the data resulting from executing the transactions | 4 |
@@ -311,19 +310,15 @@ title: Methods
and `RequestFinalizeBlock`.
* The height and timestamp values match the values from the header of the proposed block.
* `RequestPrepareProposal` contains a preliminary set of transactions `txs` that Tendermint considers to be a good block proposal, called _raw proposal_. The Application can modify this set via `ResponsePrepareProposal.tx_records` (see [TxRecord](#txrecord)).
* In this case, the Application should set `ResponsePrepareProposal.modified_tx_status` to `MODIFIED`.
* The Application _can_ reorder, remove or add transactions to the raw proposal. Let `tx` be a transaction in `txs`:
* If the Application considers that `tx` should not be proposed in this block, e.g., there are other transactions with higher priority, then it should not include it in `tx_records`. In this case, Tendermint won't remove `tx` from the mempool. The Application should be extra-careful, as abusing this feature may cause transactions to stay forever in the mempool.
* If the Application considers that a `tx` should not be included in the proposal and removed from the mempool, then the Application should include it in `tx_records` and _mark_ it as `REMOVED`. In this case, Tendermint will remove `tx` from the mempool.
* If the Application wants to add a new transaction, then the Application should include it in `tx_records` and _mark_ it as `ADD`. In this case, Tendermint will add it to the mempool.
* The Application should be aware that removing and adding transactions may compromise _traceability_.
> Consider the following example: the Application transforms a client-submitted transaction `t1` into a second transaction `t2`, i.e., the Application asks Tendermint to remove `t1` and add `t2` to the mempool. If a client wants to eventually check what happened to `t1`, it will discover that `t_1` is not in the mempool or in a committed block, getting the wrong idea that `t_1` did not make it into a block. Note that `t_2` _will be_ in a committed block, but unless the Application tracks this information, no component will be aware of it. Thus, if the Application wants traceability, it is its responsability to support it. For instance, the Application could attach to a transformed transaction a list with the hashes of the transactions it derives from.
* If the Application does not modify the preliminary set of transactions `txs`, then it sets `ResponsePrepareProposal.modified_tx_status` to `UNMODIFIED`. In this case, Tendermint will ignore the contents of `ResponsePrepareProposal.tx_records`.
* Tendermint MAY include a list of transactions in `RequestPrepareProposal.txs` whose total size in bytes exceeds `RequestPrepareProposal.max_tx_bytes`.
Therefore, if the size of `RequestPrepareProposal.txs` is greater than `RequestPrepareProposal.max_tx_bytes`:
* the Application MUST make sure that the `RequestPrepareProposal.max_tx_bytes` limit is respected by those
transaction records returned in `ResponsePrepareProposal.tx_records` that are marked as `UNMODIFIED` or `ADDED`.
* the Application MUST set `ResponsePrepareProposal.modified_tx_status` to `MODIFIED`. Tendermint will panic otherwise.
Therefore, if the size of `RequestPrepareProposal.txs` is greater than `RequestPrepareProposal.max_tx_bytes`, the Application MUST make sure that the
`RequestPrepareProposal.max_tx_bytes` limit is respected by those transaction records returned in `ResponsePrepareProposal.tx_records` that are marked as `UNMODIFIED` or `ADDED`.
* In same-block execution mode, the Application must provide values for `ResponsePrepareProposal.app_hash`,
`ResponsePrepareProposal.tx_results`, `ResponsePrepareProposal.validator_updates`, and
`ResponsePrepareProposal.consensus_param_updates`, as a result of fully executing the block.
@@ -414,7 +409,7 @@ Note that, if _p_ has a non-`nil` _validValue_, Tendermint will use it as propos
| Name | Type | Description | Field Number |
|-------------------------|--------------------------------------------------|-----------------------------------------------------------------------------------|--------------|
| status | [ProposalStatus](#ProposalStatus) | `enum` that signals if the application finds the proposal valid. | 1 |
| status | [ProposalStatus](#proposalstatus) | `enum` that signals if the application finds the proposal valid. | 1 |
| app_hash | bytes | The Merkle root hash of the application state. | 2 |
| tx_results | repeated [ExecTxResult](#txresult) | List of structures containing the data resulting from executing the transactions. | 3 |
| validator_updates | repeated [ValidatorUpdate](#validatorupdate) | Changes to validator set (set voting power to 0 to remove). | 4 |
@@ -533,7 +528,7 @@ a [CanonicalVoteExtension](#canonicalvoteextension) field in the `precommit nil`
| Name | Type | Description | Field Number |
|--------|-------------------------------|----------------------------------------------------------------|--------------|
| status | [VerifyStatus](#VerifyStatus) | `enum` signaling if the application accepts the vote extension | 1 |
| status | [VerifyStatus](#verifystatus) | `enum` signaling if the application accepts the vote extension | 1 |
* **Usage**:
* `RequestVerifyVoteExtension.vote_extension` can be an empty byte array. The Application's interpretation of it should be
@@ -827,7 +822,7 @@ Most of the data structures used in ABCI are shared [common data structures](../
| info | string | Additional information. **May be non-deterministic.** | 4 |
| gas_wanted | int64 | Amount of gas requested for transaction. | 5 |
| gas_used | int64 | Amount of gas consumed by transaction. | 6 |
| events | repeated [Event](abci++_basic_concepts_002_draft.md#events) | Type & Key-Value events for indexing transactions (e.g. by account). | 7 |
| events | repeated [Event](abci++_basic_concepts_002_draft.md#events) | Type & Key-Value events for indexing transactions (e.g. by account). | 7 |
| codespace | string | Namespace for the `code`. | 8 |
### TxAction
@@ -847,6 +842,7 @@ enum TxAction {
* If `Action` is `ADDED`, Tendermint includes the transaction in the proposal. The transaction is _not_ added to the mempool.
* If `Action` is `REMOVED`, Tendermint excludes the transaction from the proposal. The transaction is also removed from the mempool if it exists,
similar to `CheckTx` returning _false_.
### TxRecord
* **Fields**:
@@ -867,26 +863,10 @@ enum ProposalStatus {
```
* **Usage**:
* Used within the [ProcessProposal](#ProcessProposal) response.
* If `Status` is `UNKNOWN`, a problem happened in the Application. Tendermint will assume the application is faulty and crash.
* If `Status` is `ACCEPT`, Tendermint accepts the proposal and will issue a Prevote message for it.
* If `Status` is `REJECT`, Tendermint rejects the proposal and will issue a Prevote for `nil` instead.
### TxModifiedStatus
```proto
enum ModifiedTxStatus {
UNKNOWN = 0; // Unknown status. Returning this from the application is always an error.
UNMODIFIED = 1; // Status that signals the application has modified the returned list of transactions.
MODIFIED = 2; // Status that signals that the application has not modified the list of transactions.
}
```
* **Usage**:
* Used within the [PrepareProposal](#PrepareProposal) response.
* If `TxModifiedStatus` is `UNKNOWN`, a problem happened in the Application. Tendermint will assume the application is faulty and crash.
* If `TxModifiedStatus` is `UNMODIFIED`, Tendermint will ignore the contents of the `PrepareProposal` response and use the transactions originally passed to the application during `PrepareProposal`.
* If `TxModifiedStatus` is `MODIFIED`, Tendermint will update the block proposal using the contents of the `PrepareProposal` response returned by the application.
* Used within the [ProcessProposal](#processproposal) response.
* If `Status` is `UNKNOWN`, a problem happened in the Application. Tendermint will assume the application is faulty and crash.
* If `Status` is `ACCEPT`, Tendermint accepts the proposal and will issue a Prevote message for it.
* If `Status` is `REJECT`, Tendermint rejects the proposal and will issue a Prevote for `nil` instead.
### VerifyStatus
@@ -899,10 +879,10 @@ enum VerifyStatus {
```
* **Usage**:
* Used within the [VerifyVoteExtension](#VerifyVoteExtension) response.
* If `Status` is `UNKNOWN`, a problem happened in the Application. Tendermint will assume the application is faulty and crash.
* If `Status` is `ACCEPT`, Tendermint will accept the vote as valid.
* If `Status` is `REJECT`, Tendermint will reject the vote as invalid.
* Used within the [VerifyVoteExtension](#verifyvoteextension) response.
* If `Status` is `UNKNOWN`, a problem happened in the Application. Tendermint will assume the application is faulty and crash.
* If `Status` is `ACCEPT`, Tendermint will accept the vote as valid.
* If `Status` is `REJECT`, Tendermint will reject the vote as invalid.
### CanonicalVoteExtension
spec/abci++/abci++_tmint_expected_behavior_002_draft.md
+13 -10
View File
@@ -202,14 +202,17 @@ to undergo any changes in their implementation.
As for the new methods:
* `PrepareProposal` should check whether the size of transactions exceeds the byte limit.
* If it does: remove transactions at the end of the list until the total byte size conforms to the limit,
then set `ResponsePrepareProposal.modified_tx_status` to `MODIFIED` and return.
* Else, set `ResponsePrepareProposal.modified_tx_status` to `UNMODIFIED` and return.
* `ProcessProposal` should set `ResponseProcessProposal.accept` to _true_ and return.
* `ExtendVote` should set `ResponseExtendVote.extension` to an empty byte array and return.
* `VerifyVoteExtension` should set `ResponseVerifyVoteExtension.accept` to _true_ if the extension is an empty byte array
* `PrepareProposal` must create a list of [TxRecord](./abci++_methods_002_draft.md#txrecord) each containing a
transaction passed in `RequestPrepareProposal.txs`, in the same other. The field `action` must be set to `UNMODIFIED`
for all [TxRecord](./abci++_methods_002_draft.md#txrecord) elements in the list.
The Application must check whether the size of all transactions exceeds the byte limit
(`RequestPrepareProposal.max_tx_bytes`). If so, the Application must remove transactions at the end of the list
until the total byte size is at or below the limit.
* `ProcessProposal` must set `ResponseProcessProposal.accept` to _true_ and return.
* `ExtendVote` is to set `ResponseExtendVote.extension` to an empty byte array and return.
* `VerifyVoteExtension` must set `ResponseVerifyVoteExtension.accept` to _true_ if the extension is an empty byte array
and _false_ otherwise, then return.
* `FinalizeBlock` should coalesce the implementation of methods `BeginBlock`, `DeliverTx`, `EndBlock`, and `Commit`.
The logic extracted from `DeliverTx` should be wrappped by a loop that will execute as many times as
transactions exist in `RequestFinalizeBlock.tx`.
* `FinalizeBlock` is to coalesce the implementation of methods `BeginBlock`, `DeliverTx`, `EndBlock`, and `Commit`.
Legacy applications looking to reuse old code that implemented `DeliverTx` should wrap the legacy
`DeliverTx` logic in a loop that executes one transaction iteration per
transaction in `RequestFinalizeBlock.tx`.
spec/core/genesis.md
+21 -20
View File
@@ -4,31 +4,32 @@ The genesis file is the starting point of a chain. An application will populate
## Genesis Fields
- `genesis_time`: The genesis time is the time the blockchain started or will start. If nodes are started before this time they will sit idle until the time specified.
- `chain_id`: The chainid is the chain identifier. Every chain should have a unique identifier. When conducting a fork based upgrade, we recommend changing the chainid to avoid network or consensus errors.
- `initial_height`: This field is the starting height of the blockchain. When conducting a chain restart to avoid restarting at height 1, the network is able to start at a specified height.
- `genesis_time`: The time the blockchain started or will start. If nodes are started before this time they will sit idle until the time specified.
- `chain_id`: The chain identifier. Every chain should have a unique identifier. When conducting a fork based upgrade, we recommend changing the chainid to avoid network or consensus errors.
- `initial_height`: The starting height of the blockchain. When conducting a chain restart to avoid restarting at height 1, the network is able to start at a specified height.
- `consensus_params`
- `block`
- `max_bytes`: The max amount of bytes a block can be.
- `max_gas`: The maximum amount of gas that a block can have.
- `time_iota_ms`: This parameter has no value anymore in Tendermint-core.
- `evidence`
- `max_age_num_blocks`: After this preset amount of blocks has passed a single piece of evidence is considered invalid
- `max_age_duration`: After this preset amount of time has passed a single piece of evidence is considered invalid.
- `max_bytes`: The max amount of bytes of all evidence included in a block.
> Note: For evidence to be considered invalid, evidence must be older than both `max_age_num_blocks` and `max_age_duration`
- `validator`
- `pub_key_types`: Defines which curves are to be accepted as a valid validator consensus key. Tendermint supports ed25519, sr25519 and secp256k1.
- `version`
- `app_version`: The version of the application. This is set by the application and is used to identify which version of the app a user should be using in order to operate a node.
- `evidence`
- `max_age_num_blocks`: After this preset amount of blocks has passed a single piece of evidence is considered invalid.
- `max_age_duration`: After this preset amount of time has passed a single piece of evidence is considered invalid.
- `max_bytes`: The max amount of bytes of all evidence included in a block.
- `validator`
- `pub_key_types`: Defines which curves are to be accepted as a valid validator consensus key. Tendermint supports ed25519, sr25519 and secp256k1.
- `version`
- `app_version`: The version of the application. This is set by the application and is used to identify which version of the app a user should be using in order to operate a node.
- `synchrony`
- `message_delay`: A bound on how long a proposal message may take to reach all validators on a network and still be considered valid.
- `precision`: A bound on how skewed the proposer's clock may be from any validator on the network while still producing valid proposals.
- `timeout`
- `propose`: How long the Tendermint consensus engine will wait for a proposal block before prevoting nil.
- `propose_delta`: How much the propose timeout increase with each round.
- `vote`: How long the consensus engine will wait after receiving +2/3 votes in a round.
- `vote_delta`: How much the vote timeout increases with each round.
- `commit`: How long the consensus engine will wait after receiving +2/3 precommits before beginning the next height.
- `bypass_commit_timeout`: Configures if the consensus engine will wait for the full commit timeout before proceeding to the next height. If this field is set to true, the conesnsus engine will proceed to the next height as soon as the node has gathered votes from all of the validators on the network.
- `validators`
- This is an array of validators. This validator set is used as the starting validator set of the chain. This field can be empty, if the application sets the validator set in `InitChain`.
- `app_hash`: The applications state root hash. This field does not need to be populated at the start of the chain, the application may provide the needed information via `Initchain`.
- `app_state`: This section is filled in by the application and is unknown to Tendermint.
test/app/test.sh
+17 -8
View File
@@ -1,5 +1,5 @@
#! /bin/bash
set -ex
#!/bin/bash
set -exo pipefail
#- kvstore over socket, curl
@@ -8,9 +8,19 @@ set -ex
export PATH="$GOBIN:$PATH"
export TMHOME=$HOME/.tendermint_app
function kvstore_over_socket(){
rm -rf $TMHOME
function init_validator() {
rm -rf -- "$TMHOME"
tendermint init validator
# The default configuration sets a null indexer, but these tests require
# indexing to be enabled. Rewrite the config file to set the "kv" indexer
# before starting up the node.
sed -i'' -e '/indexer = \["null"\]/c\
indexer = ["kv"]' "$TMHOME/config/config.toml"
}
function kvstore_over_socket() {
init_validator
echo "Starting kvstore_over_socket"
abci-cli kvstore > /dev/null &
pid_kvstore=$!
@@ -25,9 +35,8 @@ function kvstore_over_socket(){
}
# start tendermint first
function kvstore_over_socket_reorder(){
rm -rf $TMHOME
tendermint init validator
function kvstore_over_socket_reorder() {
init_validator
echo "Starting kvstore_over_socket_reorder (ie. start tendermint first)"
tendermint start --mode validator > tendermint.log &
pid_tendermint=$!
@@ -42,7 +51,7 @@ function kvstore_over_socket_reorder(){
kill -9 $pid_kvstore $pid_tendermint
}
case "$1" in
case "$1" in
"kvstore_over_socket")
kvstore_over_socket
;;
test/docker/config-template.toml
+3
View File
@@ -1,2 +1,5 @@
[rpc]
laddr = "tcp://0.0.0.0:26657"
[tx-index]
indexer = ["kv"]
test/e2e/app/app.go
+13 -1
View File
@@ -306,7 +306,19 @@ func (app *Application) ApplySnapshotChunk(req abci.RequestApplySnapshotChunk) a
func (app *Application) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePrepareProposal {
// None of the transactions are modified by this application.
return abci.ResponsePrepareProposal{ModifiedTxStatus: abci.ResponsePrepareProposal_UNMODIFIED}
trs := make([]*abci.TxRecord, 0, len(req.Txs))
var totalBytes int64
for _, tx := range req.Txs {
totalBytes += int64(len(tx))
if totalBytes > req.MaxTxBytes {
break
}
trs = append(trs, &abci.TxRecord{
Action: abci.TxRecord_UNMODIFIED,
Tx: tx,
})
}
return abci.ResponsePrepareProposal{TxRecords: trs}
}
// ProcessProposal implements part of the Application interface.
test/e2e/app/snapshots.go
+1 -1
View File
@@ -92,7 +92,7 @@ func (s *SnapshotStore) Create(state *State) (abci.Snapshot, error) {
snapshot := abci.Snapshot{
Height: state.Height,
Format: 1,
Hash: hashItems(state.Values),
Hash: hashItems(state.Values, state.Height),
Chunks: byteChunks(bz),
}
err = os.WriteFile(filepath.Join(s.dir, fmt.Sprintf("%v.json", state.Height)), bz, 0644)
test/e2e/app/state.go
+8 -4
View File
@@ -3,6 +3,7 @@ package app
import (
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
@@ -38,7 +39,7 @@ func NewState(dir string, persistInterval uint64) (*State, error) {
previousFile: filepath.Join(dir, prevStateFileName),
persistInterval: persistInterval,
}
state.Hash = hashItems(state.Values)
state.Hash = hashItems(state.Values, state.Height)
err := state.load()
switch {
case errors.Is(err, os.ErrNotExist):
@@ -114,7 +115,7 @@ func (s *State) Import(height uint64, jsonBytes []byte) error {
}
s.Height = height
s.Values = values
s.Hash = hashItems(values)
s.Hash = hashItems(values, height)
return s.save()
}
@@ -140,7 +141,6 @@ func (s *State) Set(key, value string) {
func (s *State) Commit() (uint64, []byte, error) {
s.Lock()
defer s.Unlock()
s.Hash = hashItems(s.Values)
switch {
case s.Height > 0:
s.Height++
@@ -149,6 +149,7 @@ func (s *State) Commit() (uint64, []byte, error) {
default:
s.Height = 1
}
s.Hash = hashItems(s.Values, s.Height)
if s.persistInterval > 0 && s.Height%s.persistInterval == 0 {
err := s.save()
if err != nil {
@@ -171,7 +172,7 @@ func (s *State) Rollback() error {
}
// hashItems hashes a set of key/value items.
func hashItems(items map[string]string) []byte {
func hashItems(items map[string]string, height uint64) []byte {
keys := make([]string, 0, len(items))
for key := range items {
keys = append(keys, key)
@@ -179,6 +180,9 @@ func hashItems(items map[string]string) []byte {
sort.Strings(keys)
hasher := sha256.New()
var b [8]byte
binary.BigEndian.PutUint64(b[:], height)
_, _ = hasher.Write(b[:])
for _, key := range keys {
_, _ = hasher.Write([]byte(key))
_, _ = hasher.Write([]byte{0})
test/e2e/runner/setup.go
+1
View File
@@ -237,6 +237,7 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) {
cfg := config.DefaultConfig()
cfg.Moniker = node.Name
cfg.ProxyApp = AppAddressTCP
cfg.TxIndex = config.TestTxIndexConfig()
if node.LogLevel != "" {
cfg.LogLevel = node.LogLevel
test/e2e/tests/app_test.go
+14 -13
View File
@@ -48,23 +48,24 @@ func TestApp_Hash(t *testing.T) {
info, err := client.ABCIInfo(ctx)
require.NoError(t, err)
require.NotEmpty(t, info.Response.LastBlockAppHash, "expected app to return app hash")
// In next-block execution, the app hash is stored in the next block
blockHeight := info.Response.LastBlockHeight + 1
status, err := client.Status(ctx)
require.NoError(t, err)
require.NotZero(t, status.SyncInfo.LatestBlockHeight)
require.Eventually(t, func() bool {
status, err := client.Status(ctx)
require.NoError(t, err)
require.NotZero(t, status.SyncInfo.LatestBlockHeight)
return status.SyncInfo.LatestBlockHeight >= blockHeight
}, 60*time.Second, 500*time.Millisecond)
block, err := client.Block(ctx, &info.Response.LastBlockHeight)
block, err := client.Block(ctx, &blockHeight)
require.NoError(t, err)
if info.Response.LastBlockHeight == block.Block.Height {
require.Equal(t,
fmt.Sprintf("%x", info.Response.LastBlockAppHash),
fmt.Sprintf("%x", block.Block.AppHash.Bytes()),
"app hash does not match last block's app hash")
}
require.True(t, status.SyncInfo.LatestBlockHeight >= info.Response.LastBlockHeight,
"status out of sync with application")
require.Equal(t, blockHeight, block.Block.Height)
require.Equal(t,
fmt.Sprintf("%x", info.Response.LastBlockAppHash),
fmt.Sprintf("%x", block.Block.AppHash.Bytes()),
"app hash does not match last block's app hash")
})
}
types/tx.go
+8 -6
View File
@@ -188,13 +188,7 @@ func (t TxRecordSet) Validate(maxSizeBytes int64, otxs Txs) error {
// Only the slices are copied, the transaction contents are shared.
allCopy := sortedCopy(t.all)
var size int64
for i, cur := range allCopy {
size += int64(len(cur))
if size > maxSizeBytes {
return fmt.Errorf("transaction data size %d exceeds maximum %d", size, maxSizeBytes)
}
// allCopy is sorted, so any duplicated data will be adjacent.
if i+1 < len(allCopy) && bytes.Equal(cur, allCopy[i+1]) {
return fmt.Errorf("found duplicate transaction with hash: %x", cur.Hash())
@@ -207,6 +201,14 @@ func (t TxRecordSet) Validate(maxSizeBytes int64, otxs Txs) error {
removedCopy := sortedCopy(t.removed)
unmodifiedCopy := sortedCopy(t.unmodified)
var size int64
for _, cur := range append(unmodifiedCopy, addedCopy...) {
size += int64(len(cur))
if size > maxSizeBytes {
return fmt.Errorf("transaction data size exceeds maximum %d", maxSizeBytes)
}
}
// make a defensive copy of otxs so that the order of
// the caller's data is not altered.
otxsCopy := sortedCopy(otxs)
types/tx_test.go
+19
View File
@@ -64,6 +64,25 @@ func TestValidateTxRecordSet(t *testing.T) {
err := txrSet.Validate(9, []Tx{})
require.Error(t, err)
})
t.Run("should not error on removed transaction size exceeding max data size", func(t *testing.T) {
trs := []*abci.TxRecord{
{
Action: abci.TxRecord_ADDED,
Tx: Tx([]byte{1, 2, 3, 4, 5}),
},
{
Action: abci.TxRecord_ADDED,
Tx: Tx([]byte{6, 7, 8, 9}),
},
{
Action: abci.TxRecord_REMOVED,
Tx: Tx([]byte{10}),
},
}
txrSet := NewTxRecordSet(trs)
err := txrSet.Validate(9, []Tx{[]byte{10}})
require.NoError(t, err)
})
t.Run("should error on duplicate transactions with the same action", func(t *testing.T) {
trs := []*abci.TxRecord{
{