From 7638235d33aae14efbdb7c7bccbc43a7c3edbca8 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 20 Jul 2021 15:57:48 -0400 Subject: [PATCH 01/14] e2e: longer test harness timeouts (#6728) --- .github/workflows/e2e-nightly-master.yml | 2 +- test/e2e/runner/evidence.go | 14 +++++++++++++- test/e2e/runner/main.go | 2 +- test/e2e/runner/perturb.go | 6 +++--- test/e2e/runner/start.go | 2 +- test/e2e/runner/wait.go | 2 +- 6 files changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-nightly-master.yml b/.github/workflows/e2e-nightly-master.yml index 0f8b6602b..d6d459abc 100644 --- a/.github/workflows/e2e-nightly-master.yml +++ b/.github/workflows/e2e-nightly-master.yml @@ -35,7 +35,7 @@ jobs: - name: Generate testnets working-directory: test/e2e # When changing -g, also change the matrix groups above - run: ./build/generator -g 2 -d networks/nightly/${{ matrix.p2p }} -p ${{ matrix.p2p }} + run: ./build/generator -g 2 -d networks/nightly/${{ matrix.p2p }} -p ${{ matrix.p2p }} - name: Run ${{ matrix.p2p }} p2p testnets in group ${{ matrix.group }} working-directory: test/e2e diff --git a/test/e2e/runner/evidence.go b/test/e2e/runner/evidence.go index 7868d9174..6a246dcb5 100644 --- a/test/e2e/runner/evidence.go +++ b/test/e2e/runner/evidence.go @@ -91,6 +91,13 @@ func InjectEvidence(testnet *e2e.Testnet, amount int) error { } } + // wait for the node to reach the height above the forged height so that + // it is able to validate the evidence + _, err = waitForNode(targetNode, blockRes.Block.Height+2, 10*time.Second) + if err != nil { + return err + } + logger.Info(fmt.Sprintf("Finished sending evidence (height %d)", blockRes.Block.Height+2)) return nil @@ -186,7 +193,12 @@ func generateDuplicateVoteEvidence( if err != nil { return nil, err } - return types.NewDuplicateVoteEvidence(voteA, voteB, time, vals), nil + ev := types.NewDuplicateVoteEvidence(voteA, voteB, time, vals) + if ev == nil { + return nil, fmt.Errorf("could not generate evidence a=%v b=%v vals=%v", voteA, voteB, vals) + } + + return ev, nil } func readPrivKey(keyFilePath string) (crypto.PrivKey, error) { diff --git a/test/e2e/runner/main.go b/test/e2e/runner/main.go index 3f7d878e0..cb3d7d6bc 100644 --- a/test/e2e/runner/main.go +++ b/test/e2e/runner/main.go @@ -85,7 +85,7 @@ func NewCLI() *CLI { if err := InjectEvidence(cli.testnet, cli.testnet.Evidence); err != nil { return err } - if err := Wait(cli.testnet, 1); err != nil { // ensure chain progress + if err := Wait(cli.testnet, 5); err != nil { // ensure chain progress return err } } diff --git a/test/e2e/runner/perturb.go b/test/e2e/runner/perturb.go index 070153720..6909c665a 100644 --- a/test/e2e/runner/perturb.go +++ b/test/e2e/runner/perturb.go @@ -16,7 +16,7 @@ func Perturb(testnet *e2e.Testnet) error { if err != nil { return err } - time.Sleep(15 * time.Second) // give network some time to recover between each + time.Sleep(20 * time.Second) // give network some time to recover between each } } return nil @@ -42,7 +42,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul if err := execCompose(testnet.Dir, "kill", "-s", "SIGKILL", node.Name); err != nil { return nil, err } - time.Sleep(2 * time.Second) + time.Sleep(10 * time.Second) if err := execCompose(testnet.Dir, "start", node.Name); err != nil { return nil, err } @@ -73,7 +73,7 @@ func PerturbNode(node *e2e.Node, perturbation e2e.Perturbation) (*rpctypes.Resul return nil, nil } - status, err := waitForNode(node, 0, 2*time.Minute) + status, err := waitForNode(node, 0, 3*time.Minute) if err != nil { return nil, err } diff --git a/test/e2e/runner/start.go b/test/e2e/runner/start.go index 2eee9f756..70e496af3 100644 --- a/test/e2e/runner/start.go +++ b/test/e2e/runner/start.go @@ -75,7 +75,7 @@ func Start(testnet *e2e.Testnet) error { if err := execCompose(testnet.Dir, "up", "-d", node.Name); err != nil { return err } - status, err := waitForNode(node, node.StartAt, 5*time.Minute) + status, err := waitForNode(node, node.StartAt, 8*time.Minute) if err != nil { return err } diff --git a/test/e2e/runner/wait.go b/test/e2e/runner/wait.go index eb006227e..4c16fb808 100644 --- a/test/e2e/runner/wait.go +++ b/test/e2e/runner/wait.go @@ -30,5 +30,5 @@ func WaitUntil(testnet *e2e.Testnet, height int64) error { // waitingTime estimates how long it should take for a node to reach the height. // More nodes in a network implies we may expect a slower network and may have to wait longer. func waitingTime(nodes int) time.Duration { - return time.Minute + (time.Duration(nodes) * (15 * time.Second)) + return time.Minute + (time.Duration(nodes) * (30 * time.Second)) } From 7d63e991c5596ee3b07fe4c19658e8431881db05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Jul 2021 17:38:25 -0400 Subject: [PATCH 02/14] build(deps): Bump codecov/codecov-action from 1.5.2 to 2.0.1 (#6739) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 1.5.2 to 2.0.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v1.5.2...v2.0.1) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sam Kleinman --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d38da66c4..ee6b0068f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -121,7 +121,7 @@ jobs: - run: | cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt if: env.GIT_DIFF - - uses: codecov/codecov-action@v1.5.2 + - uses: codecov/codecov-action@v2.0.1 with: file: ./coverage.txt if: env.GIT_DIFF From 08e4e2ed3dae15094f89b3bf8b9f6832b519924c Mon Sep 17 00:00:00 2001 From: Marko Date: Wed, 21 Jul 2021 10:39:16 +0000 Subject: [PATCH 03/14] ignore issues for stale bot (#6747) --- .github/workflows/stale.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c7de0c550..929f0075e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,6 +13,7 @@ jobs: stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions." - days-before-stale: 10 - days-before-close: 4 + days-before-stale: -1 + days-before-pr-stale: 10 + days-before-pr-close: 4 exempt-pr-labels: "S:wip" From 478f5321ad74feab86933742ea087263cb138c38 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Wed, 21 Jul 2021 09:54:14 -0400 Subject: [PATCH 04/14] light: run examples as integration tests (#6745) --- light/example_test.go | 2 - light/light_test.go | 138 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 light/light_test.go diff --git a/light/example_test.go b/light/example_test.go index a542b8d3d..0fd6640c4 100644 --- a/light/example_test.go +++ b/light/example_test.go @@ -93,7 +93,6 @@ func ExampleClient_Update() { } else { fmt.Println("update failed") } - // Output: successful update } // Manually getting light blocks and verifying them. @@ -169,5 +168,4 @@ func ExampleClient_VerifyLightBlockAtHeight() { } fmt.Println("got header", h.Height) - // Output: got header 3 } diff --git a/light/light_test.go b/light/light_test.go new file mode 100644 index 000000000..a224d71b2 --- /dev/null +++ b/light/light_test.go @@ -0,0 +1,138 @@ +package light_test + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/light" + "github.com/tendermint/tendermint/light/provider" + httpp "github.com/tendermint/tendermint/light/provider/http" + dbs "github.com/tendermint/tendermint/light/store/db" + rpctest "github.com/tendermint/tendermint/rpc/test" +) + +// NOTE: these are ports of the tests from example_test.go but +// rewritten as more conventional tests. + +// Automatically getting new headers and verifying them. +func TestClientIntegration_Update(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conf := rpctest.CreateConfig(t.Name()) + + // Start a test application + app := kvstore.NewApplication() + _, closer, err := rpctest.StartTendermint(ctx, conf, app, rpctest.SuppressStdout) + require.NoError(t, err) + defer func() { require.NoError(t, closer(ctx)) }() + + // give Tendermint time to generate some blocks + time.Sleep(5 * time.Second) + + dbDir, err := ioutil.TempDir("", "light-client-test-update-example") + require.NoError(t, err) + defer os.RemoveAll(dbDir) + + chainID := conf.ChainID() + + primary, err := httpp.New(chainID, conf.RPC.ListenAddress) + require.NoError(t, err) + + block, err := primary.LightBlock(ctx, 2) + require.NoError(t, err) + + db, err := dbm.NewGoLevelDB("light-client-db", dbDir) + require.NoError(t, err) + + c, err := light.NewClient( + ctx, + chainID, + light.TrustOptions{ + Period: 504 * time.Hour, // 21 days + Height: 2, + Hash: block.Hash(), + }, + primary, + []provider.Provider{primary}, // NOTE: primary should not be used here + dbs.New(db), + light.Logger(log.TestingLogger()), + ) + require.NoError(t, err) + + defer func() { require.NoError(t, c.Cleanup()) }() + + time.Sleep(2 * time.Second) + + h, err := c.Update(ctx, time.Now()) + require.NoError(t, err) + require.NotNil(t, h) + + require.True(t, h.Height > 2) +} + +// Manually getting light blocks and verifying them. +func TestClientIntegration_VerifyLightBlockAtHeight(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + conf := rpctest.CreateConfig(t.Name()) + + // Start a test application + app := kvstore.NewApplication() + + _, closer, err := rpctest.StartTendermint(ctx, conf, app, rpctest.SuppressStdout) + require.NoError(t, err) + defer func() { require.NoError(t, closer(ctx)) }() + + // give Tendermint time to generate some blocks + time.Sleep(5 * time.Second) + + dbDir, err := ioutil.TempDir("", "light-client-test-verify-example") + require.NoError(t, err) + defer os.RemoveAll(dbDir) + + chainID := conf.ChainID() + + primary, err := httpp.New(chainID, conf.RPC.ListenAddress) + require.NoError(t, err) + + block, err := primary.LightBlock(ctx, 2) + require.NoError(t, err) + + db, err := dbm.NewGoLevelDB("light-client-db", dbDir) + require.NoError(t, err) + + c, err := light.NewClient(ctx, + chainID, + light.TrustOptions{ + Period: 504 * time.Hour, // 21 days + Height: 2, + Hash: block.Hash(), + }, + primary, + []provider.Provider{primary}, // NOTE: primary should not be used here + dbs.New(db), + light.Logger(log.TestingLogger()), + ) + require.NoError(t, err) + + defer func() { require.NoError(t, c.Cleanup()) }() + + _, err = c.VerifyLightBlockAtHeight(ctx, 3, time.Now()) + require.NoError(t, err) + + h, err := c.TrustedLightBlock(3) + require.NoError(t, err) + + require.EqualValues(t, 3, h.Height) +} From e70445f9424d08f5edfc5bfcbd0ae823b47e70bf Mon Sep 17 00:00:00 2001 From: JayT106 Date: Thu, 22 Jul 2021 02:16:50 -0400 Subject: [PATCH 05/14] statesync/event: emit statesync start/end event (#6700) --- CHANGELOG_PENDING.md | 1 + docs/tendermint-core/state-sync.md | 7 + internal/consensus/mocks/cons_sync_reactor.go | 28 ++++ internal/consensus/mocks/fast_sync_reactor.go | 71 +++++++++ internal/consensus/reactor.go | 16 +++ internal/statesync/mock_sync_reactor.go | 50 +++++++ internal/statesync/reactor.go | 11 ++ node/node.go | 135 +++++++++++------- node/node_test.go | 75 ++++++++++ types/event_bus.go | 8 ++ types/event_bus_test.go | 3 + types/events.go | 32 +++-- 12 files changed, 373 insertions(+), 64 deletions(-) create mode 100644 internal/consensus/mocks/cons_sync_reactor.go create mode 100644 internal/consensus/mocks/fast_sync_reactor.go create mode 100644 internal/statesync/mock_sync_reactor.go diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 5a7b024cf..9c3add9b8 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -99,6 +99,7 @@ Friendly reminder: We have a [bug bounty program](https://hackerone.com/tendermi - Transactions are gossiped in FIFO order as they are in `v0`. - [config/indexer] \#6411 Introduce support for custom event indexing data sources, specifically PostgreSQL. (@JayT106) - [fastsync/event] \#6619 Emit fastsync status event when switching consensus/fastsync (@JayT106) +- [statesync/event] \#6700 Emit statesync status start/end event (@JayT106) ### IMPROVEMENTS - [libs/log] Console log formatting changes as a result of \#6534 and \#6589. (@tychoish) diff --git a/docs/tendermint-core/state-sync.md b/docs/tendermint-core/state-sync.md index 38bf8bf33..623de4953 100644 --- a/docs/tendermint-core/state-sync.md +++ b/docs/tendermint-core/state-sync.md @@ -9,3 +9,10 @@ With state sync your node will download data related to the head or near the hea This leads to drastically shorter times for joining a network. Information on how to configure state sync is located in the [nodes section](../nodes/state-sync.md) + +## Events + +When a node starts with the statesync flag enabled in the config file, it will emit two events: one upon starting statesync and the other upon completion. + +The user can query the events by subscribing `EventQueryStateSyncStatus` +Please check [types](https://pkg.go.dev/github.com/tendermint/tendermint/types?utm_source=godoc#pkg-constants) for the details. \ No newline at end of file diff --git a/internal/consensus/mocks/cons_sync_reactor.go b/internal/consensus/mocks/cons_sync_reactor.go new file mode 100644 index 000000000..fcaa5696d --- /dev/null +++ b/internal/consensus/mocks/cons_sync_reactor.go @@ -0,0 +1,28 @@ +// Code generated by mockery 2.7.5. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + state "github.com/tendermint/tendermint/state" +) + +// ConsSyncReactor is an autogenerated mock type for the ConsSyncReactor type +type ConsSyncReactor struct { + mock.Mock +} + +// SetFastSyncingMetrics provides a mock function with given fields: _a0 +func (_m *ConsSyncReactor) SetFastSyncingMetrics(_a0 float64) { + _m.Called(_a0) +} + +// SetStateSyncingMetrics provides a mock function with given fields: _a0 +func (_m *ConsSyncReactor) SetStateSyncingMetrics(_a0 float64) { + _m.Called(_a0) +} + +// SwitchToConsensus provides a mock function with given fields: _a0, _a1 +func (_m *ConsSyncReactor) SwitchToConsensus(_a0 state.State, _a1 bool) { + _m.Called(_a0, _a1) +} diff --git a/internal/consensus/mocks/fast_sync_reactor.go b/internal/consensus/mocks/fast_sync_reactor.go new file mode 100644 index 000000000..f6a7fabbc --- /dev/null +++ b/internal/consensus/mocks/fast_sync_reactor.go @@ -0,0 +1,71 @@ +// Code generated by mockery 2.7.5. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + state "github.com/tendermint/tendermint/state" + + time "time" +) + +// FastSyncReactor is an autogenerated mock type for the FastSyncReactor type +type FastSyncReactor struct { + mock.Mock +} + +// GetMaxPeerBlockHeight provides a mock function with given fields: +func (_m *FastSyncReactor) GetMaxPeerBlockHeight() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// GetRemainingSyncTime provides a mock function with given fields: +func (_m *FastSyncReactor) GetRemainingSyncTime() time.Duration { + ret := _m.Called() + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// GetTotalSyncedTime provides a mock function with given fields: +func (_m *FastSyncReactor) GetTotalSyncedTime() time.Duration { + ret := _m.Called() + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// SwitchToFastSync provides a mock function with given fields: _a0 +func (_m *FastSyncReactor) SwitchToFastSync(_a0 state.State) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(state.State) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 1e5399f52..e01a6f329 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -112,6 +112,14 @@ type FastSyncReactor interface { GetRemainingSyncTime() time.Duration } +//go:generate mockery --case underscore --name ConsSyncReactor +// ConsSyncReactor defines an interface used for testing abilities of node.startStateSync. +type ConsSyncReactor interface { + SwitchToConsensus(sm.State, bool) + SetStateSyncingMetrics(float64) + SetFastSyncingMetrics(float64) +} + // Reactor defines a reactor for the consensus service. type Reactor struct { service.BaseService @@ -1429,3 +1437,11 @@ func (r *Reactor) peerStatsRoutine() { func (r *Reactor) GetConsensusState() *State { return r.state } + +func (r *Reactor) SetStateSyncingMetrics(v float64) { + r.Metrics.StateSyncing.Set(v) +} + +func (r *Reactor) SetFastSyncingMetrics(v float64) { + r.Metrics.FastSyncing.Set(v) +} diff --git a/internal/statesync/mock_sync_reactor.go b/internal/statesync/mock_sync_reactor.go new file mode 100644 index 000000000..6688ce4d2 --- /dev/null +++ b/internal/statesync/mock_sync_reactor.go @@ -0,0 +1,50 @@ +package statesync + +import ( + "context" + "time" + + mock "github.com/stretchr/testify/mock" + state "github.com/tendermint/tendermint/state" +) + +// MockSyncReactor is an autogenerated mock type for the SyncReactor type. +// Because of the stateprovider uses in Sync(), we use package statesync instead of mocks. +type MockSyncReactor struct { + mock.Mock +} + +// Backfill provides a mock function with given fields: _a0 +func (_m *MockSyncReactor) Backfill(_a0 state.State) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(state.State) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// Sync provides a mock function with given fields: _a0, _a1, _a2 +func (_m *MockSyncReactor) Sync(_a0 context.Context, _a1 StateProvider, _a2 time.Duration) (state.State, error) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 state.State + if rf, ok := ret.Get(0).(func(context.Context, StateProvider, time.Duration) state.State); ok { + r0 = rf(_a0, _a1, _a2) + } else { + r0 = ret.Get(0).(state.State) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, StateProvider, time.Duration) error); ok { + r1 = rf(_a0, _a1, _a2) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/internal/statesync/reactor.go b/internal/statesync/reactor.go index 3921e477e..59cbabd14 100644 --- a/internal/statesync/reactor.go +++ b/internal/statesync/reactor.go @@ -102,6 +102,12 @@ const ( maxLightBlockRequestRetries = 20 ) +// SyncReactor defines an interface used for testing abilities of node.startStateSync. +type SyncReactor interface { + Sync(context.Context, StateProvider, time.Duration) (sm.State, error) + Backfill(sm.State) error +} + // Reactor handles state sync, both restoring snapshots for the local node and // serving snapshots for other nodes. type Reactor struct { @@ -222,6 +228,11 @@ func (r *Reactor) Sync( return sm.State{}, errors.New("a state sync is already in progress") } + if stateProvider == nil { + r.mtx.Unlock() + return sm.State{}, errors.New("the stateProvider should not be nil when doing the state sync") + } + r.syncer = newSyncer( r.cfg, r.Logger, diff --git a/node/node.go b/node/node.go index 2d994b189..cb32c5ce6 100644 --- a/node/node.go +++ b/node/node.go @@ -63,26 +63,25 @@ type nodeImpl struct { isListening bool // services - eventBus *types.EventBus // pub/sub for services - stateStore sm.Store - blockStore *store.BlockStore // store the blockchain to disk - bcReactor service.Service // for fast-syncing - mempoolReactor service.Service // for gossipping transactions - mempool mempool.Mempool - stateSync bool // whether the node should state sync on startup - stateSyncReactor *statesync.Reactor // for hosting and restoring state sync snapshots - stateSyncProvider statesync.StateProvider // provides state data for bootstrapping a node - consensusState *cs.State // latest consensus state - consensusReactor *cs.Reactor // for participating in the consensus - pexReactor *pex.Reactor // for exchanging peer addresses - pexReactorV2 *pex.ReactorV2 // for exchanging peer addresses - evidenceReactor *evidence.Reactor - evidencePool *evidence.Pool // tracking evidence - proxyApp proxy.AppConns // connection to the application - rpcListeners []net.Listener // rpc servers - eventSinks []indexer.EventSink - indexerService *indexer.Service - prometheusSrv *http.Server + eventBus *types.EventBus // pub/sub for services + stateStore sm.Store + blockStore *store.BlockStore // store the blockchain to disk + bcReactor service.Service // for fast-syncing + mempoolReactor service.Service // for gossipping transactions + mempool mempool.Mempool + stateSync bool // whether the node should state sync on startup + stateSyncReactor *statesync.Reactor // for hosting and restoring state sync snapshots + consensusState *cs.State // latest consensus state + consensusReactor *cs.Reactor // for participating in the consensus + pexReactor *pex.Reactor // for exchanging peer addresses + pexReactorV2 *pex.ReactorV2 // for exchanging peer addresses + evidenceReactor *evidence.Reactor + evidencePool *evidence.Pool // tracking evidence + proxyApp proxy.AppConns // connection to the application + rpcListeners []net.Listener // rpc servers + eventSinks []indexer.EventSink + indexerService *indexer.Service + prometheusSrv *http.Server } // newDefaultNode returns a Tendermint node with default settings for the @@ -663,9 +662,15 @@ func (n *nodeImpl) OnStart() error { return fmt.Errorf("unable to derive state: %w", err) } - err = startStateSync(n.stateSyncReactor, bcR, n.consensusReactor, n.stateSyncProvider, - n.config.StateSync, n.config.FastSyncMode, n.stateStore, n.blockStore, state, n.eventBus) + ssc := n.config.StateSync + sp, err := constructStateProvider(ssc, state, n.Logger.With("module", "light")) + if err != nil { + return fmt.Errorf("failed to set up light client state provider: %w", err) + } + + if err := startStateSync(n.stateSyncReactor, bcR, n.consensusReactor, sp, + ssc, n.config.FastSyncMode, state.InitialHeight, n.eventBus); err != nil { return fmt.Errorf("failed to start state sync: %w", err) } } @@ -1027,54 +1032,57 @@ func (n *nodeImpl) NodeInfo() types.NodeInfo { } // startStateSync starts an asynchronous state sync process, then switches to fast sync mode. -func startStateSync(ssR *statesync.Reactor, bcR cs.FastSyncReactor, conR *cs.Reactor, - stateProvider statesync.StateProvider, config *cfg.StateSyncConfig, fastSync bool, - stateStore sm.Store, blockStore *store.BlockStore, state sm.State, eventbus *types.EventBus) error { - ssR.Logger.Info("starting state sync...") +func startStateSync( + ssR statesync.SyncReactor, + bcR cs.FastSyncReactor, + conR cs.ConsSyncReactor, + sp statesync.StateProvider, + config *cfg.StateSyncConfig, + fastSync bool, + stateInitHeight int64, + eb *types.EventBus, +) error { + stateSyncLogger := eb.Logger.With("module", "statesync") - if stateProvider == nil { - var err error - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - stateProvider, err = statesync.NewLightClientStateProvider( - ctx, - state.ChainID, state.Version, state.InitialHeight, - config.RPCServers, light.TrustOptions{ - Period: config.TrustPeriod, - Height: config.TrustHeight, - Hash: config.TrustHashBytes(), - }, ssR.Logger.With("module", "light")) - if err != nil { - return fmt.Errorf("failed to set up light client state provider: %w", err) - } + stateSyncLogger.Info("starting state sync...") + + // at the beginning of the statesync start, we use the initialHeight as the event height + // because of the statesync doesn't have the concreate state height before fetched the snapshot. + d := types.EventDataStateSyncStatus{Complete: false, Height: stateInitHeight} + if err := eb.PublishEventStateSyncStatus(d); err != nil { + stateSyncLogger.Error("failed to emit the statesync start event", "err", err) } go func() { - state, err := ssR.Sync(context.TODO(), stateProvider, config.DiscoveryTime) + state, err := ssR.Sync(context.TODO(), sp, config.DiscoveryTime) if err != nil { - ssR.Logger.Error("state sync failed", "err", err) + stateSyncLogger.Error("state sync failed", "err", err) return } - err = ssR.Backfill(state) - if err != nil { - ssR.Logger.Error("backfill failed; node has insufficient history to verify all evidence;"+ + if err := ssR.Backfill(state); err != nil { + stateSyncLogger.Error("backfill failed; node has insufficient history to verify all evidence;"+ " proceeding optimistically...", "err", err) } - conR.Metrics.StateSyncing.Set(0) + conR.SetStateSyncingMetrics(0) + + d := types.EventDataStateSyncStatus{Complete: true, Height: state.LastBlockHeight} + if err := eb.PublishEventStateSyncStatus(d); err != nil { + stateSyncLogger.Error("failed to emit the statesync start event", "err", err) + } + if fastSync { // FIXME Very ugly to have these metrics bleed through here. - conR.Metrics.FastSyncing.Set(1) - err = bcR.SwitchToFastSync(state) - if err != nil { - ssR.Logger.Error("failed to switch to fast sync", "err", err) + conR.SetFastSyncingMetrics(1) + if err := bcR.SwitchToFastSync(state); err != nil { + stateSyncLogger.Error("failed to switch to fast sync", "err", err) return } d := types.EventDataFastSyncStatus{Complete: false, Height: state.LastBlockHeight} - if err := eventbus.PublishEventFastSyncStatus(d); err != nil { - ssR.Logger.Error("failed to emit the fastsync starting event", "err", err) + if err := eb.PublishEventFastSyncStatus(d); err != nil { + stateSyncLogger.Error("failed to emit the fastsync starting event", "err", err) } } else { @@ -1266,3 +1274,24 @@ func getChannelsFromShim(reactorShim *p2p.ReactorShim) map[p2p.ChannelID]*p2p.Ch return channels } + +func constructStateProvider( + ssc *cfg.StateSyncConfig, + state sm.State, + logger log.Logger, +) (statesync.StateProvider, error) { + ctx, cancel := context.WithTimeout(context.TODO(), 10*time.Second) + defer cancel() + + to := light.TrustOptions{ + Period: ssc.TrustPeriod, + Height: ssc.TrustHeight, + Hash: ssc.TrustHashBytes(), + } + + return statesync.NewLightClientStateProvider( + ctx, + state.ChainID, state.Version, state.InitialHeight, + ssc.RPCServers, to, logger, + ) +} diff --git a/node/node_test.go b/node/node_test.go index d619d22d9..eca622bd3 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -21,11 +21,16 @@ import ( "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/tmhash" + consmocks "github.com/tendermint/tendermint/internal/consensus/mocks" + ssmocks "github.com/tendermint/tendermint/internal/statesync/mocks" + "github.com/tendermint/tendermint/internal/evidence" "github.com/tendermint/tendermint/internal/mempool" mempoolv0 "github.com/tendermint/tendermint/internal/mempool/v0" + statesync "github.com/tendermint/tendermint/internal/statesync" "github.com/tendermint/tendermint/internal/test/factory" "github.com/tendermint/tendermint/libs/log" + tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmrand "github.com/tendermint/tendermint/libs/rand" tmtime "github.com/tendermint/tendermint/libs/time" "github.com/tendermint/tendermint/privval" @@ -626,6 +631,12 @@ func state(nVals int, height int64) (sm.State, dbm.DB, []types.PrivValidator) { } func TestLoadStateFromGenesis(t *testing.T) { + _ = loadStatefromGenesis(t) +} + +func loadStatefromGenesis(t *testing.T) sm.State { + t.Helper() + stateDB := dbm.NewMemDB() stateStore := sm.NewStore(stateDB) config := cfg.ResetTestRoot("load_state_from_genesis") @@ -642,4 +653,68 @@ func TestLoadStateFromGenesis(t *testing.T) { ) require.NoError(t, err) require.NotNil(t, state) + + return state +} + +func TestNodeStartStateSync(t *testing.T) { + mockSSR := &statesync.MockSyncReactor{} + mockFSR := &consmocks.FastSyncReactor{} + mockCSR := &consmocks.ConsSyncReactor{} + mockSP := &ssmocks.StateProvider{} + state := loadStatefromGenesis(t) + config := cfg.ResetTestRoot("load_state_from_genesis") + + eventBus, err := createAndStartEventBus(log.TestingLogger()) + defer func() { + err := eventBus.Stop() + require.NoError(t, err) + }() + + require.NoError(t, err) + require.NotNil(t, eventBus) + + sub, err := eventBus.Subscribe(context.Background(), "test-client", types.EventQueryStateSyncStatus, 10) + require.NoError(t, err) + require.NotNil(t, sub) + + cfgSS := config.StateSync + + mockSSR.On("Sync", context.TODO(), mockSP, cfgSS.DiscoveryTime).Return(state, nil). + On("Backfill", state).Return(nil) + mockCSR.On("SetStateSyncingMetrics", float64(0)).Return(). + On("SwitchToConsensus", state, true).Return() + + require.NoError(t, + startStateSync(mockSSR, mockFSR, mockCSR, mockSP, config.StateSync, false, state.InitialHeight, eventBus)) + + for cnt := 0; cnt < 2; { + select { + case <-time.After(3 * time.Second): + t.Errorf("StateSyncStatus timeout") + case msg := <-sub.Out(): + if cnt == 0 { + ensureStateSyncStatus(t, msg, false, state.InitialHeight) + cnt++ + } else { + // the state height = 0 because we are not actually update the state in this test + ensureStateSyncStatus(t, msg, true, 0) + cnt++ + } + } + } + + mockSSR.AssertNumberOfCalls(t, "Sync", 1) + mockSSR.AssertNumberOfCalls(t, "Backfill", 1) + mockCSR.AssertNumberOfCalls(t, "SetStateSyncingMetrics", 1) + mockCSR.AssertNumberOfCalls(t, "SwitchToConsensus", 1) +} + +func ensureStateSyncStatus(t *testing.T, msg tmpubsub.Message, complete bool, height int64) { + t.Helper() + status, ok := msg.Data().(types.EventDataStateSyncStatus) + + require.True(t, ok) + require.Equal(t, complete, status.Complete) + require.Equal(t, height, status.Height) } diff --git a/types/event_bus.go b/types/event_bus.go index e3a530d2e..5290181ee 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -157,6 +157,10 @@ func (b *EventBus) PublishEventFastSyncStatus(data EventDataFastSyncStatus) erro return b.Publish(EventFastSyncStatusValue, data) } +func (b *EventBus) PublishEventStateSyncStatus(data EventDataStateSyncStatus) error { + return b.Publish(EventStateSyncStatusValue, data) +} + // PublishEventTx publishes tx event with events from Result. Note it will add // predefined keys (EventTypeKey, TxHashKey). Existing events with the same keys // will be overwritten. @@ -316,3 +320,7 @@ func (NopEventBus) PublishEventValidatorSetUpdates(data EventDataValidatorSetUpd func (NopEventBus) PublishEventFastSyncStatus(data EventDataFastSyncStatus) error { return nil } + +func (NopEventBus) PublishEventStateSyncStatus(data EventDataStateSyncStatus) error { + return nil +} diff --git a/types/event_bus_test.go b/types/event_bus_test.go index 0ea51cdfe..987a10eab 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -372,6 +372,8 @@ func TestEventBusPublish(t *testing.T) { require.NoError(t, err) err = eventBus.PublishEventFastSyncStatus(EventDataFastSyncStatus{}) require.NoError(t, err) + err = eventBus.PublishEventStateSyncStatus(EventDataStateSyncStatus{}) + require.NoError(t, err) select { case <-done: @@ -479,6 +481,7 @@ var events = []string{ EventTimeoutWaitValue, EventVoteValue, EventFastSyncStatusValue, + EventStateSyncStatusValue, } func randEventValue() string { diff --git a/types/events.go b/types/events.go index 326a1610d..2e234d3b7 100644 --- a/types/events.go +++ b/types/events.go @@ -29,17 +29,18 @@ const ( EventCompleteProposalValue = "CompleteProposal" // The FastSyncStatus event will be emitted when the node switching // state sync mechanism between the consensus reactor and the fastsync reactor. - EventFastSyncStatusValue = "FastSyncStatus" - EventLockValue = "Lock" - EventNewRoundValue = "NewRound" - EventNewRoundStepValue = "NewRoundStep" - EventPolkaValue = "Polka" - EventRelockValue = "Relock" - EventTimeoutProposeValue = "TimeoutPropose" - EventTimeoutWaitValue = "TimeoutWait" - EventUnlockValue = "Unlock" - EventValidBlockValue = "ValidBlock" - EventVoteValue = "Vote" + EventFastSyncStatusValue = "FastSyncStatus" + EventLockValue = "Lock" + EventNewRoundValue = "NewRound" + EventNewRoundStepValue = "NewRoundStep" + EventPolkaValue = "Polka" + EventRelockValue = "Relock" + EventStateSyncStatusValue = "StateSyncStatus" + EventTimeoutProposeValue = "TimeoutPropose" + EventTimeoutWaitValue = "TimeoutWait" + EventUnlockValue = "Unlock" + EventValidBlockValue = "ValidBlock" + EventVoteValue = "Vote" ) // Pre-populated ABCI Tendermint-reserved events @@ -104,6 +105,7 @@ func init() { tmjson.RegisterType(EventDataValidatorSetUpdates{}, "tendermint/event/ValidatorSetUpdates") tmjson.RegisterType(EventDataString(""), "tendermint/event/ProposalString") tmjson.RegisterType(EventDataFastSyncStatus{}, "tendermint/event/FastSyncStatus") + tmjson.RegisterType(EventDataStateSyncStatus{}, "tendermint/event/StateSyncStatus") } // Most event messages are basic types (a block, a transaction) @@ -181,6 +183,13 @@ type EventDataFastSyncStatus struct { Height int64 `json:"height"` } +// EventDataStateSyncStatus shows the statesync status and the +// height when the node state sync mechanism changes. +type EventDataStateSyncStatus struct { + Complete bool `json:"complete"` + Height int64 `json:"height"` +} + // PUBSUB const ( @@ -219,6 +228,7 @@ var ( EventQueryValidBlock = QueryForEvent(EventValidBlockValue) EventQueryVote = QueryForEvent(EventVoteValue) EventQueryFastSyncStatus = QueryForEvent(EventFastSyncStatusValue) + EventQueryStateSyncStatus = QueryForEvent(EventStateSyncStatusValue) ) func EventQueryTxFor(tx Tx) tmpubsub.Query { From 84c15857e41f036e99399c5f9167f1d72cf45a03 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Thu, 22 Jul 2021 10:52:29 -0400 Subject: [PATCH 06/14] mempool: return mempool errors to the abci client (#6740) This changes adds an `MempoolError` field to the `ResponseCheckTx`. This will allow clients to understand that their transaction was rejected from the mempool despite passing the ABCI check. This change also updates the code to make use of early returns to prevent highly nested code blocks. Namely, it returns when the type assertion fails at the beginning of the method, instead of wrapping the entire method in a large if statement. This has a somewhat large effect on the diff as rendered by github. addresses: #3546 --- abci/types/types.pb.go | 401 ++++++++++++++++------------ internal/mempool/v1/mempool.go | 188 ++++++------- internal/mempool/v1/mempool_test.go | 56 +++- proto/tendermint/abci/types.proto | 21 +- proto/tendermint/p2p/types.pb.go | 13 +- rpc/core/mempool.go | 11 +- rpc/core/types/responses.go | 9 +- 7 files changed, 404 insertions(+), 295 deletions(-) diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 566bb29ac..f8781c0c4 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -1827,16 +1827,17 @@ func (m *ResponseBeginBlock) GetEvents() []Event { } type ResponseCheckTx struct { - Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` - Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` - Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` - Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` - GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,proto3" json:"gas_wanted,omitempty"` - GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,proto3" json:"gas_used,omitempty"` - Events []Event `protobuf:"bytes,7,rep,name=events,proto3" json:"events,omitempty"` - Codespace string `protobuf:"bytes,8,opt,name=codespace,proto3" json:"codespace,omitempty"` - Sender string `protobuf:"bytes,9,opt,name=sender,proto3" json:"sender,omitempty"` - Priority int64 `protobuf:"varint,10,opt,name=priority,proto3" json:"priority,omitempty"` + Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` + Log string `protobuf:"bytes,3,opt,name=log,proto3" json:"log,omitempty"` + Info string `protobuf:"bytes,4,opt,name=info,proto3" json:"info,omitempty"` + GasWanted int64 `protobuf:"varint,5,opt,name=gas_wanted,proto3" json:"gas_wanted,omitempty"` + GasUsed int64 `protobuf:"varint,6,opt,name=gas_used,proto3" json:"gas_used,omitempty"` + Events []Event `protobuf:"bytes,7,rep,name=events,proto3" json:"events,omitempty"` + Codespace string `protobuf:"bytes,8,opt,name=codespace,proto3" json:"codespace,omitempty"` + Sender string `protobuf:"bytes,9,opt,name=sender,proto3" json:"sender,omitempty"` + Priority int64 `protobuf:"varint,10,opt,name=priority,proto3" json:"priority,omitempty"` + MempoolError string `protobuf:"bytes,11,opt,name=mempool_error,json=mempoolError,proto3" json:"mempool_error,omitempty"` } func (m *ResponseCheckTx) Reset() { *m = ResponseCheckTx{} } @@ -1942,6 +1943,13 @@ func (m *ResponseCheckTx) GetPriority() int64 { return 0 } +func (m *ResponseCheckTx) GetMempoolError() string { + if m != nil { + return m.MempoolError + } + return "" +} + type ResponseDeliverTx struct { Code uint32 `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"` Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` @@ -2954,170 +2962,172 @@ func init() { func init() { proto.RegisterFile("tendermint/abci/types.proto", fileDescriptor_252557cfdd89a31a) } var fileDescriptor_252557cfdd89a31a = []byte{ - // 2608 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5a, 0xbd, 0x73, 0x1b, 0xc7, - 0x15, 0xc7, 0x37, 0x70, 0x8f, 0x04, 0x08, 0xae, 0x68, 0x19, 0x86, 0x65, 0x52, 0x3e, 0x8f, 0x1d, - 0x4b, 0xb6, 0xc9, 0x98, 0x1a, 0x29, 0xf2, 0x38, 0x1f, 0x26, 0x20, 0x28, 0xa0, 0xc5, 0x90, 0xcc, - 0x12, 0x92, 0xc7, 0x49, 0xac, 0xf3, 0x01, 0xb7, 0x04, 0xce, 0x02, 0xee, 0xce, 0x77, 0x0b, 0x8a, - 0x74, 0x99, 0x49, 0x1a, 0x4d, 0x0a, 0x95, 0x69, 0x3c, 0x93, 0xff, 0x20, 0x65, 0x52, 0xa5, 0x4a, - 0xe1, 0x22, 0x99, 0x71, 0x99, 0xca, 0xc9, 0x48, 0x5d, 0xfe, 0x81, 0x54, 0x99, 0xc9, 0xec, 0xc7, - 0x7d, 0x01, 0x38, 0x02, 0x8c, 0xd3, 0xa5, 0xdb, 0xdd, 0x7b, 0xef, 0x61, 0xf7, 0xed, 0xbe, 0xdf, - 0xfb, 0xed, 0x5b, 0xc0, 0xcb, 0x94, 0x58, 0x06, 0x71, 0x47, 0xa6, 0x45, 0xb7, 0xf4, 0x6e, 0xcf, - 0xdc, 0xa2, 0x67, 0x0e, 0xf1, 0x36, 0x1d, 0xd7, 0xa6, 0x36, 0x5a, 0x09, 0x3f, 0x6e, 0xb2, 0x8f, - 0xf5, 0x57, 0x22, 0xd2, 0x3d, 0xf7, 0xcc, 0xa1, 0xf6, 0x96, 0xe3, 0xda, 0xf6, 0xb1, 0x90, 0xaf, - 0x5f, 0x89, 0x7c, 0xe6, 0x76, 0xa2, 0xd6, 0x62, 0x5f, 0xa5, 0xf2, 0x23, 0x72, 0xe6, 0x7f, 0x7d, - 0x65, 0x4a, 0xd7, 0xd1, 0x5d, 0x7d, 0xe4, 0x7f, 0xde, 0xe8, 0xdb, 0x76, 0x7f, 0x48, 0xb6, 0x78, - 0xaf, 0x3b, 0x3e, 0xde, 0xa2, 0xe6, 0x88, 0x78, 0x54, 0x1f, 0x39, 0x52, 0x60, 0xad, 0x6f, 0xf7, - 0x6d, 0xde, 0xdc, 0x62, 0x2d, 0x31, 0xaa, 0xfe, 0xb5, 0x08, 0x45, 0x4c, 0x3e, 0x1f, 0x13, 0x8f, - 0xa2, 0x6d, 0xc8, 0x91, 0xde, 0xc0, 0xae, 0xa5, 0xaf, 0xa6, 0xdf, 0x5c, 0xda, 0xbe, 0xb2, 0x39, - 0xb1, 0xb8, 0x4d, 0x29, 0xd7, 0xea, 0x0d, 0xec, 0x76, 0x0a, 0x73, 0x59, 0x74, 0x13, 0xf2, 0xc7, - 0xc3, 0xb1, 0x37, 0xa8, 0x65, 0xb8, 0xd2, 0x2b, 0x49, 0x4a, 0x77, 0x99, 0x50, 0x3b, 0x85, 0x85, - 0x34, 0xfb, 0x29, 0xd3, 0x3a, 0xb6, 0x6b, 0xd9, 0xf3, 0x7f, 0x6a, 0xd7, 0x3a, 0xe6, 0x3f, 0xc5, - 0x64, 0x51, 0x03, 0xc0, 0xb4, 0x4c, 0xaa, 0xf5, 0x06, 0xba, 0x69, 0xd5, 0x72, 0x5c, 0xf3, 0xd5, - 0x64, 0x4d, 0x93, 0x36, 0x99, 0x60, 0x3b, 0x85, 0x15, 0xd3, 0xef, 0xb0, 0xe9, 0x7e, 0x3e, 0x26, - 0xee, 0x59, 0x2d, 0x7f, 0xfe, 0x74, 0x7f, 0xca, 0x84, 0xd8, 0x74, 0xb9, 0x34, 0x6a, 0xc1, 0x52, - 0x97, 0xf4, 0x4d, 0x4b, 0xeb, 0x0e, 0xed, 0xde, 0xa3, 0x5a, 0x81, 0x2b, 0xab, 0x49, 0xca, 0x0d, - 0x26, 0xda, 0x60, 0x92, 0xed, 0x14, 0x86, 0x6e, 0xd0, 0x43, 0xdf, 0x87, 0x52, 0x6f, 0x40, 0x7a, - 0x8f, 0x34, 0x7a, 0x5a, 0x2b, 0x72, 0x1b, 0x1b, 0x49, 0x36, 0x9a, 0x4c, 0xae, 0x73, 0xda, 0x4e, - 0xe1, 0x62, 0x4f, 0x34, 0xd9, 0xfa, 0x0d, 0x32, 0x34, 0x4f, 0x88, 0xcb, 0xf4, 0x4b, 0xe7, 0xaf, - 0xff, 0x8e, 0x90, 0xe4, 0x16, 0x14, 0xc3, 0xef, 0xa0, 0x1f, 0x81, 0x42, 0x2c, 0x43, 0x2e, 0x43, - 0xe1, 0x26, 0xae, 0x26, 0xee, 0xb3, 0x65, 0xf8, 0x8b, 0x28, 0x11, 0xd9, 0x46, 0xb7, 0xa1, 0xd0, - 0xb3, 0x47, 0x23, 0x93, 0xd6, 0x80, 0x6b, 0xaf, 0x27, 0x2e, 0x80, 0x4b, 0xb5, 0x53, 0x58, 0xca, - 0xa3, 0x7d, 0xa8, 0x0c, 0x4d, 0x8f, 0x6a, 0x9e, 0xa5, 0x3b, 0xde, 0xc0, 0xa6, 0x5e, 0x6d, 0x89, - 0x5b, 0x78, 0x3d, 0xc9, 0xc2, 0x9e, 0xe9, 0xd1, 0x23, 0x5f, 0xb8, 0x9d, 0xc2, 0xe5, 0x61, 0x74, - 0x80, 0xd9, 0xb3, 0x8f, 0x8f, 0x89, 0x1b, 0x18, 0xac, 0x2d, 0x9f, 0x6f, 0xef, 0x80, 0x49, 0xfb, - 0xfa, 0xcc, 0x9e, 0x1d, 0x1d, 0x40, 0x3f, 0x87, 0x4b, 0x43, 0x5b, 0x37, 0x02, 0x73, 0x5a, 0x6f, - 0x30, 0xb6, 0x1e, 0xd5, 0xca, 0xdc, 0xe8, 0xb5, 0xc4, 0x49, 0xda, 0xba, 0xe1, 0x9b, 0x68, 0x32, - 0x85, 0x76, 0x0a, 0xaf, 0x0e, 0x27, 0x07, 0xd1, 0x43, 0x58, 0xd3, 0x1d, 0x67, 0x78, 0x36, 0x69, - 0xbd, 0xc2, 0xad, 0x5f, 0x4f, 0xb2, 0xbe, 0xc3, 0x74, 0x26, 0xcd, 0x23, 0x7d, 0x6a, 0xb4, 0x51, - 0x84, 0xfc, 0x89, 0x3e, 0x1c, 0x13, 0xf5, 0x3b, 0xb0, 0x14, 0x09, 0x53, 0x54, 0x83, 0xe2, 0x88, - 0x78, 0x9e, 0xde, 0x27, 0x3c, 0xaa, 0x15, 0xec, 0x77, 0xd5, 0x0a, 0x2c, 0x47, 0x43, 0x53, 0x7d, - 0x9a, 0x0e, 0x34, 0x59, 0xd4, 0x31, 0xcd, 0x13, 0xe2, 0x7a, 0xa6, 0x6d, 0xf9, 0x9a, 0xb2, 0x8b, - 0x5e, 0x83, 0x32, 0x3f, 0x3f, 0x9a, 0xff, 0x9d, 0x85, 0x7e, 0x0e, 0x2f, 0xf3, 0xc1, 0x07, 0x52, - 0x68, 0x03, 0x96, 0x9c, 0x6d, 0x27, 0x10, 0xc9, 0x72, 0x11, 0x70, 0xb6, 0x1d, 0x5f, 0xe0, 0x55, - 0x58, 0x66, 0x2b, 0x0d, 0x24, 0x72, 0xfc, 0x47, 0x96, 0xd8, 0x98, 0x14, 0x51, 0xff, 0x92, 0x81, - 0xea, 0x64, 0x38, 0xa3, 0xdb, 0x90, 0x63, 0xc8, 0x26, 0x41, 0xaa, 0xbe, 0x29, 0x60, 0x6f, 0xd3, - 0x87, 0xbd, 0xcd, 0x8e, 0x0f, 0x7b, 0x8d, 0xd2, 0x57, 0xdf, 0x6c, 0xa4, 0x9e, 0xfe, 0x7d, 0x23, - 0x8d, 0xb9, 0x06, 0x7a, 0x89, 0x45, 0x9f, 0x6e, 0x5a, 0x9a, 0x69, 0xf0, 0x29, 0x2b, 0x2c, 0xb4, - 0x74, 0xd3, 0xda, 0x35, 0xd0, 0x1e, 0x54, 0x7b, 0xb6, 0xe5, 0x11, 0xcb, 0x1b, 0x7b, 0x9a, 0x80, - 0x55, 0x09, 0x4d, 0xb1, 0x00, 0x13, 0x60, 0xdd, 0xf4, 0x25, 0x0f, 0xb9, 0x20, 0x5e, 0xe9, 0xc5, - 0x07, 0xd0, 0x5d, 0x80, 0x13, 0x7d, 0x68, 0x1a, 0x3a, 0xb5, 0x5d, 0xaf, 0x96, 0xbb, 0x9a, 0x9d, - 0x19, 0x65, 0x0f, 0x7c, 0x91, 0xfb, 0x8e, 0xa1, 0x53, 0xd2, 0xc8, 0xb1, 0xe9, 0xe2, 0x88, 0x26, - 0x7a, 0x03, 0x56, 0x74, 0xc7, 0xd1, 0x3c, 0xaa, 0x53, 0xa2, 0x75, 0xcf, 0x28, 0xf1, 0x38, 0x6c, - 0x2d, 0xe3, 0xb2, 0xee, 0x38, 0x47, 0x6c, 0xb4, 0xc1, 0x06, 0xd1, 0xeb, 0x50, 0x61, 0x08, 0x67, - 0xea, 0x43, 0x6d, 0x40, 0xcc, 0xfe, 0x80, 0x72, 0x80, 0xca, 0xe2, 0xb2, 0x1c, 0x6d, 0xf3, 0x41, - 0xd5, 0x08, 0x76, 0x9c, 0xa3, 0x1b, 0x42, 0x90, 0x33, 0x74, 0xaa, 0x73, 0x4f, 0x2e, 0x63, 0xde, - 0x66, 0x63, 0x8e, 0x4e, 0x07, 0xd2, 0x3f, 0xbc, 0x8d, 0x2e, 0x43, 0x41, 0x9a, 0xcd, 0x72, 0xb3, - 0xb2, 0x87, 0xd6, 0x20, 0xef, 0xb8, 0xf6, 0x09, 0xe1, 0x5b, 0x57, 0xc2, 0xa2, 0xa3, 0xfe, 0x2a, - 0x03, 0xab, 0x53, 0x38, 0xc8, 0xec, 0x0e, 0x74, 0x6f, 0xe0, 0xff, 0x16, 0x6b, 0xa3, 0x5b, 0xcc, - 0xae, 0x6e, 0x10, 0x57, 0xe6, 0x8e, 0xda, 0xb4, 0xab, 0xdb, 0xfc, 0xbb, 0x74, 0x8d, 0x94, 0x46, - 0x07, 0x50, 0x1d, 0xea, 0x1e, 0xd5, 0x04, 0xae, 0x68, 0x91, 0x3c, 0x32, 0x8d, 0xa6, 0x7b, 0xba, - 0x8f, 0x44, 0xec, 0x50, 0x4b, 0x43, 0x95, 0x61, 0x6c, 0x14, 0x61, 0x58, 0xeb, 0x9e, 0x7d, 0xa1, - 0x5b, 0xd4, 0xb4, 0x88, 0x36, 0xb5, 0x73, 0x2f, 0x4d, 0x19, 0x6d, 0x9d, 0x98, 0x06, 0xb1, 0x7a, - 0xfe, 0x96, 0x5d, 0x0a, 0x94, 0x83, 0x2d, 0xf5, 0x54, 0x0c, 0x95, 0x38, 0x92, 0xa3, 0x0a, 0x64, - 0xe8, 0xa9, 0x74, 0x40, 0x86, 0x9e, 0xa2, 0xef, 0x42, 0x8e, 0x2d, 0x92, 0x2f, 0xbe, 0x32, 0x23, - 0x05, 0x4a, 0xbd, 0xce, 0x99, 0x43, 0x30, 0x97, 0x54, 0xd5, 0x20, 0x1c, 0x02, 0x74, 0x9f, 0xb4, - 0xaa, 0x5e, 0x83, 0x95, 0x09, 0xf8, 0x8e, 0xec, 0x5f, 0x3a, 0xba, 0x7f, 0xea, 0x0a, 0x94, 0x63, - 0x58, 0xad, 0x5e, 0x86, 0xb5, 0x59, 0xd0, 0xab, 0x0e, 0x82, 0xf1, 0x18, 0x84, 0xa2, 0x9b, 0x50, - 0x0a, 0xb0, 0x57, 0x84, 0xe3, 0xb4, 0xaf, 0x7c, 0x61, 0x1c, 0x88, 0xb2, 0x38, 0x64, 0xc7, 0x9a, - 0x9f, 0x87, 0x0c, 0x9f, 0x78, 0x51, 0x77, 0x9c, 0xb6, 0xee, 0x0d, 0xd4, 0x4f, 0xa1, 0x96, 0x84, - 0xab, 0x13, 0xcb, 0xc8, 0x05, 0xc7, 0xf0, 0x32, 0x14, 0x8e, 0x6d, 0x77, 0xa4, 0x53, 0x6e, 0xac, - 0x8c, 0x65, 0x8f, 0x1d, 0x4f, 0x81, 0xb1, 0x59, 0x3e, 0x2c, 0x3a, 0xaa, 0x06, 0x2f, 0x25, 0x62, - 0x2b, 0x53, 0x31, 0x2d, 0x83, 0x08, 0x7f, 0x96, 0xb1, 0xe8, 0x84, 0x86, 0xc4, 0x64, 0x45, 0x87, - 0xfd, 0xac, 0xc7, 0xd7, 0xca, 0xed, 0x2b, 0x58, 0xf6, 0xd4, 0xdf, 0x95, 0xa0, 0x84, 0x89, 0xe7, - 0x30, 0x4c, 0x40, 0x0d, 0x50, 0xc8, 0x69, 0x8f, 0x38, 0xd4, 0x87, 0xd1, 0xd9, 0xac, 0x41, 0x48, - 0xb7, 0x7c, 0x49, 0x96, 0xb2, 0x03, 0x35, 0x74, 0x43, 0xb2, 0xb2, 0x64, 0x82, 0x25, 0xd5, 0xa3, - 0xb4, 0xec, 0x96, 0x4f, 0xcb, 0xb2, 0x89, 0x59, 0x5a, 0x68, 0x4d, 0xf0, 0xb2, 0x1b, 0x92, 0x97, - 0xe5, 0xe6, 0xfc, 0x58, 0x8c, 0x98, 0x35, 0x63, 0xc4, 0x2c, 0x3f, 0x67, 0x99, 0x09, 0xcc, 0xec, - 0x96, 0xcf, 0xcc, 0x0a, 0x73, 0x66, 0x3c, 0x41, 0xcd, 0xee, 0xc6, 0xa9, 0x99, 0xa0, 0x55, 0xaf, - 0x25, 0x6a, 0x27, 0x72, 0xb3, 0x1f, 0x44, 0xb8, 0x59, 0x29, 0x91, 0x18, 0x09, 0x23, 0x33, 0xc8, - 0x59, 0x33, 0x46, 0xce, 0x94, 0x39, 0x3e, 0x48, 0x60, 0x67, 0x1f, 0x44, 0xd9, 0x19, 0x24, 0x12, - 0x3c, 0xb9, 0xdf, 0xb3, 0xe8, 0xd9, 0x7b, 0x01, 0x3d, 0x5b, 0x4a, 0xe4, 0x97, 0x72, 0x0d, 0x93, - 0xfc, 0xec, 0x60, 0x8a, 0x9f, 0x09, 0x3e, 0xf5, 0x46, 0xa2, 0x89, 0x39, 0x04, 0xed, 0x60, 0x8a, - 0xa0, 0x95, 0xe7, 0x18, 0x9c, 0xc3, 0xd0, 0x7e, 0x31, 0x9b, 0xa1, 0x25, 0x73, 0x28, 0x39, 0xcd, - 0xc5, 0x28, 0x9a, 0x96, 0x40, 0xd1, 0x56, 0xb8, 0xf9, 0xb7, 0x12, 0xcd, 0x5f, 0x9c, 0xa3, 0x5d, - 0x63, 0x19, 0x72, 0x22, 0xe6, 0x19, 0xca, 0x10, 0xd7, 0xb5, 0x5d, 0xc9, 0xb6, 0x44, 0x47, 0x7d, - 0x93, 0xe5, 0xec, 0x30, 0xbe, 0xcf, 0xe1, 0x73, 0x1c, 0xcd, 0x23, 0x31, 0xad, 0xfe, 0x31, 0x1d, - 0xea, 0xf2, 0x34, 0x17, 0xcd, 0xf7, 0x8a, 0xcc, 0xf7, 0x11, 0x96, 0x97, 0x89, 0xb3, 0xbc, 0x0d, - 0x58, 0x62, 0x28, 0x3d, 0x41, 0xe0, 0x74, 0x27, 0x20, 0x70, 0xd7, 0x61, 0x95, 0xa7, 0x61, 0xc1, - 0x05, 0x25, 0x34, 0xe7, 0x78, 0x86, 0x59, 0x61, 0x1f, 0xc4, 0xe1, 0x14, 0x18, 0xfd, 0x0e, 0x5c, - 0x8a, 0xc8, 0x06, 0xe8, 0x2f, 0xd8, 0x4c, 0x35, 0x90, 0xde, 0x91, 0x69, 0xe0, 0xcf, 0xe9, 0xd0, - 0x43, 0x21, 0xf3, 0x9b, 0x45, 0xd2, 0xd2, 0xff, 0x23, 0x92, 0x96, 0xf9, 0xaf, 0x49, 0x5a, 0x34, - 0x9b, 0x65, 0xe3, 0xd9, 0xec, 0x5f, 0xe9, 0x70, 0x4f, 0x02, 0xca, 0xd5, 0xb3, 0x0d, 0x22, 0xf3, - 0x0b, 0x6f, 0xa3, 0x2a, 0x64, 0x87, 0x76, 0x5f, 0x66, 0x11, 0xd6, 0x64, 0x52, 0x01, 0x08, 0x2b, - 0x12, 0x63, 0x83, 0xd4, 0x94, 0xe7, 0x1e, 0x96, 0xa9, 0xa9, 0x0a, 0xd9, 0x47, 0x44, 0x40, 0xe6, - 0x32, 0x66, 0x4d, 0x26, 0xc7, 0x0f, 0x19, 0x07, 0xc2, 0x65, 0x2c, 0x3a, 0xe8, 0x36, 0x28, 0xbc, - 0x0c, 0xa1, 0xd9, 0x8e, 0x27, 0xd1, 0xed, 0xe5, 0xe8, 0x5a, 0x45, 0xb5, 0x61, 0xf3, 0x90, 0xc9, - 0x1c, 0x38, 0x1e, 0x2e, 0x39, 0xb2, 0x15, 0xc9, 0xba, 0x4a, 0x8c, 0xfc, 0x5d, 0x01, 0x85, 0xcd, - 0xde, 0x73, 0xf4, 0x1e, 0xe1, 0x50, 0xa5, 0xe0, 0x70, 0x40, 0x7d, 0x08, 0x68, 0x1a, 0x70, 0x51, - 0x1b, 0x0a, 0xe4, 0x84, 0x58, 0x94, 0x6d, 0x1b, 0x73, 0xf7, 0xe5, 0x19, 0xcc, 0x8a, 0x58, 0xb4, - 0x51, 0x63, 0x4e, 0xfe, 0xe7, 0x37, 0x1b, 0x55, 0x21, 0xfd, 0xb6, 0x3d, 0x32, 0x29, 0x19, 0x39, - 0xf4, 0x0c, 0x4b, 0x7d, 0xf5, 0x0f, 0x19, 0x46, 0x73, 0x62, 0x60, 0x3c, 0xd3, 0xb7, 0xfe, 0x91, - 0xcf, 0x44, 0x28, 0xee, 0x62, 0xfe, 0x5e, 0x07, 0xe8, 0xeb, 0x9e, 0xf6, 0x58, 0xb7, 0x28, 0x31, - 0xa4, 0xd3, 0x23, 0x23, 0xa8, 0x0e, 0x25, 0xd6, 0x1b, 0x7b, 0xc4, 0x90, 0x6c, 0x3b, 0xe8, 0x47, - 0xd6, 0x59, 0xfc, 0x76, 0xeb, 0x8c, 0x7b, 0xb9, 0x34, 0xe1, 0xe5, 0x08, 0x05, 0x51, 0xa2, 0x14, - 0x84, 0xcd, 0xcd, 0x71, 0x4d, 0xdb, 0x35, 0xe9, 0x19, 0xdf, 0x9a, 0x2c, 0x0e, 0xfa, 0xea, 0xaf, - 0x33, 0x61, 0x68, 0x85, 0x2c, 0xf2, 0xff, 0xce, 0x77, 0xea, 0x6f, 0xf8, 0xdd, 0x32, 0x9e, 0x49, - 0xd1, 0x11, 0xac, 0x06, 0x91, 0xad, 0x8d, 0x79, 0xc4, 0xfb, 0x67, 0x75, 0x51, 0x68, 0xa8, 0x9e, - 0xc4, 0x87, 0x3d, 0xf4, 0x31, 0xbc, 0x38, 0x01, 0x5b, 0x81, 0xe9, 0xcc, 0xa2, 0xe8, 0xf5, 0x42, - 0x1c, 0xbd, 0x7c, 0xd3, 0xa1, 0xb3, 0xb2, 0xdf, 0x32, 0xa0, 0x76, 0xd9, 0x75, 0x25, 0x4a, 0x0c, - 0x66, 0x6e, 0xff, 0x6b, 0x50, 0x76, 0x09, 0x65, 0x57, 0xe8, 0xd8, 0x85, 0x70, 0x59, 0x0c, 0xca, - 0x6b, 0xe6, 0x21, 0xbc, 0x30, 0x93, 0x20, 0xa0, 0xef, 0x81, 0x12, 0x72, 0x8b, 0x74, 0xc2, 0xdd, - 0x2a, 0xb8, 0x2f, 0x84, 0xb2, 0xea, 0x9f, 0xd2, 0xa1, 0xc9, 0xf8, 0x0d, 0xa4, 0x05, 0x05, 0x97, - 0x78, 0xe3, 0xa1, 0xb8, 0x13, 0x54, 0xb6, 0xdf, 0x59, 0x8c, 0x5a, 0xb0, 0xd1, 0xf1, 0x90, 0x62, - 0xa9, 0xac, 0x3e, 0x84, 0x82, 0x18, 0x41, 0x4b, 0x50, 0xbc, 0xbf, 0x7f, 0x6f, 0xff, 0xe0, 0xa3, - 0xfd, 0x6a, 0x0a, 0x01, 0x14, 0x76, 0x9a, 0xcd, 0xd6, 0x61, 0xa7, 0x9a, 0x46, 0x0a, 0xe4, 0x77, - 0x1a, 0x07, 0xb8, 0x53, 0xcd, 0xb0, 0x61, 0xdc, 0xfa, 0xb0, 0xd5, 0xec, 0x54, 0xb3, 0x68, 0x15, - 0xca, 0xa2, 0xad, 0xdd, 0x3d, 0xc0, 0x3f, 0xd9, 0xe9, 0x54, 0x73, 0x91, 0xa1, 0xa3, 0xd6, 0xfe, - 0x9d, 0x16, 0xae, 0xe6, 0xd5, 0x77, 0xd9, 0xa5, 0x23, 0x81, 0x8c, 0x84, 0xd7, 0x8b, 0x74, 0xe4, - 0x7a, 0xa1, 0xfe, 0x36, 0x03, 0xf5, 0x64, 0x86, 0x81, 0x3e, 0x9c, 0x58, 0xf8, 0xf6, 0x05, 0xe8, - 0xc9, 0xc4, 0xea, 0xd1, 0xeb, 0x50, 0x71, 0xc9, 0x31, 0xa1, 0xbd, 0x81, 0x60, 0x3c, 0x22, 0x1b, - 0x96, 0x71, 0x59, 0x8e, 0x72, 0x25, 0x4f, 0x88, 0x7d, 0x46, 0x7a, 0x54, 0x13, 0x30, 0x23, 0x0e, - 0x9d, 0xc2, 0xc4, 0xd8, 0xe8, 0x91, 0x18, 0x54, 0x3f, 0xbd, 0x90, 0x2f, 0x15, 0xc8, 0xe3, 0x56, - 0x07, 0x7f, 0x5c, 0xcd, 0x22, 0x04, 0x15, 0xde, 0xd4, 0x8e, 0xf6, 0x77, 0x0e, 0x8f, 0xda, 0x07, - 0xcc, 0x97, 0x97, 0x60, 0xc5, 0xf7, 0xa5, 0x3f, 0x98, 0x57, 0x3f, 0x81, 0x4a, 0xfc, 0x5a, 0xcf, - 0x5c, 0xe8, 0xda, 0x63, 0xcb, 0xe0, 0xce, 0xc8, 0x63, 0xd1, 0x41, 0x37, 0x21, 0x7f, 0x62, 0x8b, - 0x30, 0x9b, 0x7d, 0xd6, 0x1e, 0xd8, 0x94, 0x44, 0xca, 0x02, 0x42, 0x5a, 0xfd, 0x02, 0xf2, 0x3c, - 0x6a, 0x58, 0x04, 0xf0, 0x0b, 0xba, 0xe4, 0x4b, 0xac, 0x8d, 0x3e, 0x01, 0xd0, 0x29, 0x75, 0xcd, - 0xee, 0x38, 0x34, 0xbc, 0x31, 0x3b, 0xea, 0x76, 0x7c, 0xb9, 0xc6, 0x15, 0x19, 0x7e, 0x6b, 0xa1, - 0x6a, 0x24, 0x04, 0x23, 0x06, 0xd5, 0x7d, 0xa8, 0xc4, 0x75, 0xfd, 0x0c, 0x2f, 0xe6, 0x10, 0xcf, - 0xf0, 0x82, 0xb0, 0xc9, 0x0c, 0x1f, 0xf0, 0x83, 0xac, 0x28, 0xc6, 0xf0, 0x8e, 0xfa, 0x24, 0x0d, - 0xa5, 0xce, 0xa9, 0xdc, 0x8f, 0x84, 0x3a, 0x40, 0xa8, 0x9a, 0x89, 0xde, 0x7a, 0x45, 0x61, 0x21, - 0x1b, 0x94, 0x2b, 0x3e, 0x08, 0x4e, 0x5c, 0x6e, 0xd1, 0xcb, 0x8d, 0x5f, 0xb7, 0x91, 0x51, 0xf6, - 0x3e, 0x28, 0x01, 0x66, 0x32, 0xe2, 0xa9, 0x1b, 0x86, 0x4b, 0x3c, 0x4f, 0x9e, 0x7b, 0xbf, 0xcb, - 0xcb, 0x4a, 0xf6, 0x63, 0x79, 0xaf, 0xce, 0x62, 0xd1, 0x51, 0x0d, 0x58, 0x99, 0x00, 0x5c, 0xf4, - 0x3e, 0x14, 0x9d, 0x71, 0x57, 0xf3, 0xdd, 0x33, 0xf1, 0x8c, 0xe0, 0x53, 0x9a, 0x71, 0x77, 0x68, - 0xf6, 0xee, 0x91, 0x33, 0x7f, 0x32, 0xce, 0xb8, 0x7b, 0x4f, 0x78, 0x51, 0xfc, 0x4a, 0x26, 0xfa, - 0x2b, 0x27, 0x50, 0xf2, 0x0f, 0x05, 0xfa, 0x21, 0x28, 0x01, 0x96, 0x07, 0xd5, 0xc6, 0xc4, 0x24, - 0x20, 0xcd, 0x87, 0x2a, 0x8c, 0x1f, 0x7b, 0x66, 0xdf, 0x22, 0x86, 0x16, 0x52, 0x5f, 0xfe, 0x6b, - 0x25, 0xbc, 0x22, 0x3e, 0xec, 0xf9, 0xbc, 0x57, 0xfd, 0x77, 0x1a, 0x4a, 0x7e, 0x55, 0x09, 0xbd, - 0x1b, 0x39, 0x77, 0x95, 0x19, 0x77, 0x70, 0x5f, 0x30, 0xac, 0x0c, 0xc5, 0xe7, 0x9a, 0xb9, 0xf8, - 0x5c, 0x93, 0x4a, 0x7c, 0x7e, 0xb1, 0x35, 0x77, 0xe1, 0x62, 0xeb, 0xdb, 0x80, 0xa8, 0x4d, 0xf5, - 0xa1, 0x76, 0x62, 0x53, 0xd3, 0xea, 0x6b, 0xc2, 0xd9, 0x82, 0x0b, 0x54, 0xf9, 0x97, 0x07, 0xfc, - 0xc3, 0x21, 0xf7, 0xfb, 0x2f, 0xd3, 0x50, 0x0a, 0x40, 0xfd, 0xa2, 0x85, 0x9e, 0xcb, 0x50, 0x90, - 0xb8, 0x25, 0x2a, 0x3d, 0xb2, 0x17, 0xd4, 0x1c, 0x73, 0x91, 0x9a, 0x63, 0x1d, 0x4a, 0x23, 0x42, - 0x75, 0x9e, 0xd9, 0xc4, 0xed, 0x23, 0xe8, 0x5f, 0x7f, 0x0f, 0x96, 0x22, 0x35, 0x37, 0x16, 0x79, - 0xfb, 0xad, 0x8f, 0xaa, 0xa9, 0x7a, 0xf1, 0xc9, 0x97, 0x57, 0xb3, 0xfb, 0xe4, 0x31, 0x3b, 0xb3, - 0xb8, 0xd5, 0x6c, 0xb7, 0x9a, 0xf7, 0xaa, 0xe9, 0xfa, 0xd2, 0x93, 0x2f, 0xaf, 0x16, 0x31, 0xe1, - 0xf7, 0xff, 0xeb, 0x6d, 0x58, 0x8e, 0xee, 0x4a, 0x1c, 0xfa, 0x10, 0x54, 0xee, 0xdc, 0x3f, 0xdc, - 0xdb, 0x6d, 0xee, 0x74, 0x5a, 0xda, 0x83, 0x83, 0x4e, 0xab, 0x9a, 0x46, 0x2f, 0xc2, 0xa5, 0xbd, - 0xdd, 0x1f, 0xb7, 0x3b, 0x5a, 0x73, 0x6f, 0xb7, 0xb5, 0xdf, 0xd1, 0x76, 0x3a, 0x9d, 0x9d, 0xe6, - 0xbd, 0x6a, 0x66, 0xfb, 0xf7, 0x0a, 0xac, 0xec, 0x34, 0x9a, 0xbb, 0x0c, 0xb6, 0xcd, 0x9e, 0xce, - 0xaf, 0x86, 0x4d, 0xc8, 0xf1, 0xcb, 0xdf, 0xb9, 0x2f, 0x72, 0xf5, 0xf3, 0x2b, 0x43, 0xe8, 0x2e, - 0xe4, 0xf9, 0xbd, 0x10, 0x9d, 0xff, 0x44, 0x57, 0x9f, 0x53, 0x2a, 0x62, 0x93, 0xe1, 0xe1, 0x71, - 0xee, 0x9b, 0x5d, 0xfd, 0xfc, 0xca, 0x11, 0xc2, 0xa0, 0x84, 0xe4, 0x73, 0xfe, 0x1b, 0x56, 0x7d, - 0x01, 0xb0, 0x41, 0x7b, 0x50, 0xf4, 0xaf, 0x02, 0xf3, 0x5e, 0xd5, 0xea, 0x73, 0x4b, 0x3b, 0xcc, - 0x5d, 0xe2, 0xca, 0x76, 0xfe, 0x13, 0x61, 0x7d, 0x4e, 0x9d, 0x0a, 0xed, 0x42, 0x41, 0x12, 0xaa, - 0x39, 0x2f, 0x65, 0xf5, 0x79, 0xa5, 0x1a, 0xe6, 0xb4, 0xf0, 0x32, 0x3c, 0xff, 0xe1, 0xb3, 0xbe, - 0x40, 0x09, 0x0e, 0xdd, 0x07, 0x88, 0x5c, 0xd0, 0x16, 0x78, 0xd1, 0xac, 0x2f, 0x52, 0x5a, 0x43, - 0x07, 0x50, 0x0a, 0x48, 0xf5, 0xdc, 0xf7, 0xc5, 0xfa, 0xfc, 0x1a, 0x17, 0x7a, 0x08, 0xe5, 0x38, - 0x99, 0x5c, 0xec, 0xd5, 0xb0, 0xbe, 0x60, 0xf1, 0x8a, 0xd9, 0x8f, 0x33, 0xcb, 0xc5, 0x5e, 0x11, - 0xeb, 0x0b, 0xd6, 0xb2, 0xd0, 0x67, 0xb0, 0x3a, 0xcd, 0xfc, 0x16, 0x7f, 0x54, 0xac, 0x5f, 0xa0, - 0xba, 0x85, 0x46, 0x80, 0x66, 0x30, 0xc6, 0x0b, 0xbc, 0x31, 0xd6, 0x2f, 0x52, 0xec, 0x6a, 0xb4, - 0xbe, 0x7a, 0xb6, 0x9e, 0xfe, 0xfa, 0xd9, 0x7a, 0xfa, 0x1f, 0xcf, 0xd6, 0xd3, 0x4f, 0x9f, 0xaf, - 0xa7, 0xbe, 0x7e, 0xbe, 0x9e, 0xfa, 0xdb, 0xf3, 0xf5, 0xd4, 0xcf, 0xde, 0xea, 0x9b, 0x74, 0x30, - 0xee, 0x6e, 0xf6, 0xec, 0xd1, 0x56, 0xf4, 0xcf, 0x0b, 0xb3, 0xfe, 0x50, 0xd1, 0x2d, 0xf0, 0xa4, - 0x72, 0xe3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xed, 0x8f, 0xef, 0x31, 0x70, 0x21, 0x00, 0x00, + // 2627 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5a, 0xcd, 0x73, 0xdb, 0xc6, + 0x15, 0xe7, 0x37, 0x89, 0x47, 0x91, 0xa2, 0xd6, 0x8a, 0x43, 0x33, 0xb6, 0xe4, 0xc0, 0xe3, 0x34, + 0x76, 0x12, 0xa9, 0x91, 0xc7, 0xae, 0x33, 0xe9, 0x47, 0x44, 0x9a, 0x2e, 0x15, 0xab, 0x92, 0xba, + 0xa2, 0x9d, 0x49, 0xdb, 0x18, 0x01, 0x89, 0x15, 0x89, 0x98, 0x04, 0x10, 0x60, 0x29, 0x4b, 0x39, + 0x76, 0xda, 0x8b, 0xa7, 0x07, 0x1f, 0x7b, 0xc9, 0x4c, 0xff, 0x83, 0x5e, 0x7b, 0xea, 0xa9, 0x87, + 0x1c, 0xda, 0x99, 0x1c, 0x7b, 0xe8, 0xa4, 0x1d, 0xfb, 0xd6, 0x7f, 0xa0, 0xa7, 0xce, 0x74, 0xf6, + 0x03, 0x20, 0x40, 0x12, 0x22, 0xd5, 0xf4, 0xd6, 0xdb, 0xee, 0xc3, 0x7b, 0x8f, 0xbb, 0x6f, 0xf7, + 0xfd, 0xf6, 0xb7, 0x6f, 0x09, 0xaf, 0x51, 0x62, 0x19, 0xc4, 0x1d, 0x9a, 0x16, 0xdd, 0xd4, 0x3b, + 0x5d, 0x73, 0x93, 0x9e, 0x3a, 0xc4, 0xdb, 0x70, 0x5c, 0x9b, 0xda, 0x68, 0x79, 0xfc, 0x71, 0x83, + 0x7d, 0xac, 0x5d, 0x09, 0x69, 0x77, 0xdd, 0x53, 0x87, 0xda, 0x9b, 0x8e, 0x6b, 0xdb, 0x47, 0x42, + 0xbf, 0x76, 0x39, 0xf4, 0x99, 0xfb, 0x09, 0x7b, 0x8b, 0x7c, 0x95, 0xc6, 0x4f, 0xc8, 0xa9, 0xff, + 0xf5, 0xca, 0x94, 0xad, 0xa3, 0xbb, 0xfa, 0xd0, 0xff, 0xbc, 0xde, 0xb3, 0xed, 0xde, 0x80, 0x6c, + 0xf2, 0x5e, 0x67, 0x74, 0xb4, 0x49, 0xcd, 0x21, 0xf1, 0xa8, 0x3e, 0x74, 0xa4, 0xc2, 0x6a, 0xcf, + 0xee, 0xd9, 0xbc, 0xb9, 0xc9, 0x5a, 0x42, 0xaa, 0xfe, 0x25, 0x0f, 0x79, 0x4c, 0x3e, 0x1f, 0x11, + 0x8f, 0xa2, 0x2d, 0xc8, 0x90, 0x6e, 0xdf, 0xae, 0x26, 0xaf, 0x26, 0xdf, 0x2c, 0x6e, 0x5d, 0xde, + 0x98, 0x98, 0xdc, 0x86, 0xd4, 0x6b, 0x76, 0xfb, 0x76, 0x2b, 0x81, 0xb9, 0x2e, 0xba, 0x0d, 0xd9, + 0xa3, 0xc1, 0xc8, 0xeb, 0x57, 0x53, 0xdc, 0xe8, 0x4a, 0x9c, 0xd1, 0x7d, 0xa6, 0xd4, 0x4a, 0x60, + 0xa1, 0xcd, 0x7e, 0xca, 0xb4, 0x8e, 0xec, 0x6a, 0xfa, 0xec, 0x9f, 0xda, 0xb1, 0x8e, 0xf8, 0x4f, + 0x31, 0x5d, 0x54, 0x07, 0x30, 0x2d, 0x93, 0x6a, 0xdd, 0xbe, 0x6e, 0x5a, 0xd5, 0x0c, 0xb7, 0x7c, + 0x3d, 0xde, 0xd2, 0xa4, 0x0d, 0xa6, 0xd8, 0x4a, 0x60, 0xc5, 0xf4, 0x3b, 0x6c, 0xb8, 0x9f, 0x8f, + 0x88, 0x7b, 0x5a, 0xcd, 0x9e, 0x3d, 0xdc, 0x9f, 0x32, 0x25, 0x36, 0x5c, 0xae, 0x8d, 0x9a, 0x50, + 0xec, 0x90, 0x9e, 0x69, 0x69, 0x9d, 0x81, 0xdd, 0x7d, 0x52, 0xcd, 0x71, 0x63, 0x35, 0xce, 0xb8, + 0xce, 0x54, 0xeb, 0x4c, 0xb3, 0x95, 0xc0, 0xd0, 0x09, 0x7a, 0xe8, 0xfb, 0x50, 0xe8, 0xf6, 0x49, + 0xf7, 0x89, 0x46, 0x4f, 0xaa, 0x79, 0xee, 0x63, 0x3d, 0xce, 0x47, 0x83, 0xe9, 0xb5, 0x4f, 0x5a, + 0x09, 0x9c, 0xef, 0x8a, 0x26, 0x9b, 0xbf, 0x41, 0x06, 0xe6, 0x31, 0x71, 0x99, 0x7d, 0xe1, 0xec, + 0xf9, 0xdf, 0x13, 0x9a, 0xdc, 0x83, 0x62, 0xf8, 0x1d, 0xf4, 0x23, 0x50, 0x88, 0x65, 0xc8, 0x69, + 0x28, 0xdc, 0xc5, 0xd5, 0xd8, 0x75, 0xb6, 0x0c, 0x7f, 0x12, 0x05, 0x22, 0xdb, 0xe8, 0x2e, 0xe4, + 0xba, 0xf6, 0x70, 0x68, 0xd2, 0x2a, 0x70, 0xeb, 0xb5, 0xd8, 0x09, 0x70, 0xad, 0x56, 0x02, 0x4b, + 0x7d, 0xb4, 0x07, 0xe5, 0x81, 0xe9, 0x51, 0xcd, 0xb3, 0x74, 0xc7, 0xeb, 0xdb, 0xd4, 0xab, 0x16, + 0xb9, 0x87, 0xeb, 0x71, 0x1e, 0x76, 0x4d, 0x8f, 0x1e, 0xfa, 0xca, 0xad, 0x04, 0x2e, 0x0d, 0xc2, + 0x02, 0xe6, 0xcf, 0x3e, 0x3a, 0x22, 0x6e, 0xe0, 0xb0, 0xba, 0x74, 0xb6, 0xbf, 0x7d, 0xa6, 0xed, + 0xdb, 0x33, 0x7f, 0x76, 0x58, 0x80, 0x7e, 0x0e, 0x17, 0x06, 0xb6, 0x6e, 0x04, 0xee, 0xb4, 0x6e, + 0x7f, 0x64, 0x3d, 0xa9, 0x96, 0xb8, 0xd3, 0x1b, 0xb1, 0x83, 0xb4, 0x75, 0xc3, 0x77, 0xd1, 0x60, + 0x06, 0xad, 0x04, 0x5e, 0x19, 0x4c, 0x0a, 0xd1, 0x63, 0x58, 0xd5, 0x1d, 0x67, 0x70, 0x3a, 0xe9, + 0xbd, 0xcc, 0xbd, 0xdf, 0x8c, 0xf3, 0xbe, 0xcd, 0x6c, 0x26, 0xdd, 0x23, 0x7d, 0x4a, 0x5a, 0xcf, + 0x43, 0xf6, 0x58, 0x1f, 0x8c, 0x88, 0xfa, 0x1d, 0x28, 0x86, 0xd2, 0x14, 0x55, 0x21, 0x3f, 0x24, + 0x9e, 0xa7, 0xf7, 0x08, 0xcf, 0x6a, 0x05, 0xfb, 0x5d, 0xb5, 0x0c, 0x4b, 0xe1, 0xd4, 0x54, 0x9f, + 0x27, 0x03, 0x4b, 0x96, 0x75, 0xcc, 0xf2, 0x98, 0xb8, 0x9e, 0x69, 0x5b, 0xbe, 0xa5, 0xec, 0xa2, + 0x6b, 0x50, 0xe2, 0xfb, 0x47, 0xf3, 0xbf, 0xb3, 0xd4, 0xcf, 0xe0, 0x25, 0x2e, 0x7c, 0x24, 0x95, + 0xd6, 0xa1, 0xe8, 0x6c, 0x39, 0x81, 0x4a, 0x9a, 0xab, 0x80, 0xb3, 0xe5, 0xf8, 0x0a, 0xaf, 0xc3, + 0x12, 0x9b, 0x69, 0xa0, 0x91, 0xe1, 0x3f, 0x52, 0x64, 0x32, 0xa9, 0xa2, 0xfe, 0x39, 0x05, 0x95, + 0xc9, 0x74, 0x46, 0x77, 0x21, 0xc3, 0x90, 0x4d, 0x82, 0x54, 0x6d, 0x43, 0xc0, 0xde, 0x86, 0x0f, + 0x7b, 0x1b, 0x6d, 0x1f, 0xf6, 0xea, 0x85, 0xaf, 0xbe, 0x59, 0x4f, 0x3c, 0xff, 0xfb, 0x7a, 0x12, + 0x73, 0x0b, 0x74, 0x89, 0x65, 0x9f, 0x6e, 0x5a, 0x9a, 0x69, 0xf0, 0x21, 0x2b, 0x2c, 0xb5, 0x74, + 0xd3, 0xda, 0x31, 0xd0, 0x2e, 0x54, 0xba, 0xb6, 0xe5, 0x11, 0xcb, 0x1b, 0x79, 0x9a, 0x80, 0x55, + 0x09, 0x4d, 0x91, 0x04, 0x13, 0x60, 0xdd, 0xf0, 0x35, 0x0f, 0xb8, 0x22, 0x5e, 0xee, 0x46, 0x05, + 0xe8, 0x3e, 0xc0, 0xb1, 0x3e, 0x30, 0x0d, 0x9d, 0xda, 0xae, 0x57, 0xcd, 0x5c, 0x4d, 0xcf, 0xcc, + 0xb2, 0x47, 0xbe, 0xca, 0x43, 0xc7, 0xd0, 0x29, 0xa9, 0x67, 0xd8, 0x70, 0x71, 0xc8, 0x12, 0xbd, + 0x01, 0xcb, 0xba, 0xe3, 0x68, 0x1e, 0xd5, 0x29, 0xd1, 0x3a, 0xa7, 0x94, 0x78, 0x1c, 0xb6, 0x96, + 0x70, 0x49, 0x77, 0x9c, 0x43, 0x26, 0xad, 0x33, 0x21, 0xba, 0x0e, 0x65, 0x86, 0x70, 0xa6, 0x3e, + 0xd0, 0xfa, 0xc4, 0xec, 0xf5, 0x29, 0x07, 0xa8, 0x34, 0x2e, 0x49, 0x69, 0x8b, 0x0b, 0x55, 0x23, + 0x58, 0x71, 0x8e, 0x6e, 0x08, 0x41, 0xc6, 0xd0, 0xa9, 0xce, 0x23, 0xb9, 0x84, 0x79, 0x9b, 0xc9, + 0x1c, 0x9d, 0xf6, 0x65, 0x7c, 0x78, 0x1b, 0x5d, 0x84, 0x9c, 0x74, 0x9b, 0xe6, 0x6e, 0x65, 0x0f, + 0xad, 0x42, 0xd6, 0x71, 0xed, 0x63, 0xc2, 0x97, 0xae, 0x80, 0x45, 0x47, 0xfd, 0x55, 0x0a, 0x56, + 0xa6, 0x70, 0x90, 0xf9, 0xed, 0xeb, 0x5e, 0xdf, 0xff, 0x2d, 0xd6, 0x46, 0x77, 0x98, 0x5f, 0xdd, + 0x20, 0xae, 0x3c, 0x3b, 0xaa, 0xd3, 0xa1, 0x6e, 0xf1, 0xef, 0x32, 0x34, 0x52, 0x1b, 0xed, 0x43, + 0x65, 0xa0, 0x7b, 0x54, 0x13, 0xb8, 0xa2, 0x85, 0xce, 0x91, 0x69, 0x34, 0xdd, 0xd5, 0x7d, 0x24, + 0x62, 0x9b, 0x5a, 0x3a, 0x2a, 0x0f, 0x22, 0x52, 0x84, 0x61, 0xb5, 0x73, 0xfa, 0x85, 0x6e, 0x51, + 0xd3, 0x22, 0xda, 0xd4, 0xca, 0x5d, 0x9a, 0x72, 0xda, 0x3c, 0x36, 0x0d, 0x62, 0x75, 0xfd, 0x25, + 0xbb, 0x10, 0x18, 0x07, 0x4b, 0xea, 0xa9, 0x18, 0xca, 0x51, 0x24, 0x47, 0x65, 0x48, 0xd1, 0x13, + 0x19, 0x80, 0x14, 0x3d, 0x41, 0xdf, 0x85, 0x0c, 0x9b, 0x24, 0x9f, 0x7c, 0x79, 0xc6, 0x11, 0x28, + 0xed, 0xda, 0xa7, 0x0e, 0xc1, 0x5c, 0x53, 0x55, 0x83, 0x74, 0x08, 0xd0, 0x7d, 0xd2, 0xab, 0x7a, + 0x03, 0x96, 0x27, 0xe0, 0x3b, 0xb4, 0x7e, 0xc9, 0xf0, 0xfa, 0xa9, 0xcb, 0x50, 0x8a, 0x60, 0xb5, + 0x7a, 0x11, 0x56, 0x67, 0x41, 0xaf, 0xda, 0x0f, 0xe4, 0x11, 0x08, 0x45, 0xb7, 0xa1, 0x10, 0x60, + 0xaf, 0x48, 0xc7, 0xe9, 0x58, 0xf9, 0xca, 0x38, 0x50, 0x65, 0x79, 0xc8, 0xb6, 0x35, 0xdf, 0x0f, + 0x29, 0x3e, 0xf0, 0xbc, 0xee, 0x38, 0x2d, 0xdd, 0xeb, 0xab, 0x9f, 0x42, 0x35, 0x0e, 0x57, 0x27, + 0xa6, 0x91, 0x09, 0xb6, 0xe1, 0x45, 0xc8, 0x1d, 0xd9, 0xee, 0x50, 0xa7, 0xdc, 0x59, 0x09, 0xcb, + 0x1e, 0xdb, 0x9e, 0x02, 0x63, 0xd3, 0x5c, 0x2c, 0x3a, 0xaa, 0x06, 0x97, 0x62, 0xb1, 0x95, 0x99, + 0x98, 0x96, 0x41, 0x44, 0x3c, 0x4b, 0x58, 0x74, 0xc6, 0x8e, 0xc4, 0x60, 0x45, 0x87, 0xfd, 0xac, + 0xc7, 0xe7, 0xca, 0xfd, 0x2b, 0x58, 0xf6, 0xd4, 0xdf, 0x15, 0xa0, 0x80, 0x89, 0xe7, 0x30, 0x4c, + 0x40, 0x75, 0x50, 0xc8, 0x49, 0x97, 0x38, 0xd4, 0x87, 0xd1, 0xd9, 0xac, 0x41, 0x68, 0x37, 0x7d, + 0x4d, 0x76, 0x64, 0x07, 0x66, 0xe8, 0x96, 0x64, 0x65, 0xf1, 0x04, 0x4b, 0x9a, 0x87, 0x69, 0xd9, + 0x1d, 0x9f, 0x96, 0xa5, 0x63, 0x4f, 0x69, 0x61, 0x35, 0xc1, 0xcb, 0x6e, 0x49, 0x5e, 0x96, 0x99, + 0xf3, 0x63, 0x11, 0x62, 0xd6, 0x88, 0x10, 0xb3, 0xec, 0x9c, 0x69, 0xc6, 0x30, 0xb3, 0x3b, 0x3e, + 0x33, 0xcb, 0xcd, 0x19, 0xf1, 0x04, 0x35, 0xbb, 0x1f, 0xa5, 0x66, 0x82, 0x56, 0x5d, 0x8b, 0xb5, + 0x8e, 0xe5, 0x66, 0x3f, 0x08, 0x71, 0xb3, 0x42, 0x2c, 0x31, 0x12, 0x4e, 0x66, 0x90, 0xb3, 0x46, + 0x84, 0x9c, 0x29, 0x73, 0x62, 0x10, 0xc3, 0xce, 0x3e, 0x08, 0xb3, 0x33, 0x88, 0x25, 0x78, 0x72, + 0xbd, 0x67, 0xd1, 0xb3, 0xf7, 0x02, 0x7a, 0x56, 0x8c, 0xe5, 0x97, 0x72, 0x0e, 0x93, 0xfc, 0x6c, + 0x7f, 0x8a, 0x9f, 0x09, 0x3e, 0xf5, 0x46, 0xac, 0x8b, 0x39, 0x04, 0x6d, 0x7f, 0x8a, 0xa0, 0x95, + 0xe6, 0x38, 0x9c, 0xc3, 0xd0, 0x7e, 0x31, 0x9b, 0xa1, 0xc5, 0x73, 0x28, 0x39, 0xcc, 0xc5, 0x28, + 0x9a, 0x16, 0x43, 0xd1, 0x96, 0xb9, 0xfb, 0xb7, 0x62, 0xdd, 0x9f, 0x9f, 0xa3, 0xdd, 0x60, 0x27, + 0xe4, 0x44, 0xce, 0x33, 0x94, 0x21, 0xae, 0x6b, 0xbb, 0x92, 0x6d, 0x89, 0x8e, 0xfa, 0x26, 0x3b, + 0xb3, 0xc7, 0xf9, 0x7d, 0x06, 0x9f, 0xe3, 0x68, 0x1e, 0xca, 0x69, 0xf5, 0x0f, 0xc9, 0xb1, 0x2d, + 0x3f, 0xe6, 0xc2, 0xe7, 0xbd, 0x22, 0xcf, 0xfb, 0x10, 0xcb, 0x4b, 0x45, 0x59, 0xde, 0x3a, 0x14, + 0x19, 0x4a, 0x4f, 0x10, 0x38, 0xdd, 0x09, 0x08, 0xdc, 0x4d, 0x58, 0xe1, 0xc7, 0xb0, 0xe0, 0x82, + 0x12, 0x9a, 0x33, 0xfc, 0x84, 0x59, 0x66, 0x1f, 0xc4, 0xe6, 0x14, 0x18, 0xfd, 0x0e, 0x5c, 0x08, + 0xe9, 0x06, 0xe8, 0x2f, 0xd8, 0x4c, 0x25, 0xd0, 0xde, 0x96, 0xc7, 0xc0, 0x9f, 0x92, 0xe3, 0x08, + 0x8d, 0x99, 0xdf, 0x2c, 0x92, 0x96, 0xfc, 0x1f, 0x91, 0xb4, 0xd4, 0x7f, 0x4d, 0xd2, 0xc2, 0xa7, + 0x59, 0x3a, 0x7a, 0x9a, 0xfd, 0x2b, 0x39, 0x5e, 0x93, 0x80, 0x72, 0x75, 0x6d, 0x83, 0xc8, 0xf3, + 0x85, 0xb7, 0x51, 0x05, 0xd2, 0x03, 0xbb, 0x27, 0x4f, 0x11, 0xd6, 0x64, 0x5a, 0x01, 0x08, 0x2b, + 0x12, 0x63, 0x83, 0xa3, 0x29, 0xcb, 0x23, 0x2c, 0x8f, 0xa6, 0x0a, 0xa4, 0x9f, 0x10, 0x01, 0x99, + 0x4b, 0x98, 0x35, 0x99, 0x1e, 0xdf, 0x64, 0x1c, 0x08, 0x97, 0xb0, 0xe8, 0xa0, 0xbb, 0xa0, 0xf0, + 0x32, 0x84, 0x66, 0x3b, 0x9e, 0x44, 0xb7, 0xd7, 0xc2, 0x73, 0x15, 0xd5, 0x86, 0x8d, 0x03, 0xa6, + 0xb3, 0xef, 0x78, 0xb8, 0xe0, 0xc8, 0x56, 0xe8, 0xd4, 0x55, 0x22, 0xe4, 0xef, 0x32, 0x28, 0x6c, + 0xf4, 0x9e, 0xa3, 0x77, 0x09, 0x87, 0x2a, 0x05, 0x8f, 0x05, 0xea, 0x63, 0x40, 0xd3, 0x80, 0x8b, + 0x5a, 0x90, 0x23, 0xc7, 0xc4, 0xa2, 0x6c, 0xd9, 0x58, 0xb8, 0x2f, 0xce, 0x60, 0x56, 0xc4, 0xa2, + 0xf5, 0x2a, 0x0b, 0xf2, 0x3f, 0xbf, 0x59, 0xaf, 0x08, 0xed, 0xb7, 0xed, 0xa1, 0x49, 0xc9, 0xd0, + 0xa1, 0xa7, 0x58, 0xda, 0xab, 0x7f, 0x4b, 0x31, 0x9a, 0x13, 0x01, 0xe3, 0x99, 0xb1, 0xf5, 0xb7, + 0x7c, 0x2a, 0x44, 0x71, 0x17, 0x8b, 0xf7, 0x1a, 0x40, 0x4f, 0xf7, 0xb4, 0xa7, 0xba, 0x45, 0x89, + 0x21, 0x83, 0x1e, 0x92, 0xa0, 0x1a, 0x14, 0x58, 0x6f, 0xe4, 0x11, 0x43, 0xb2, 0xed, 0xa0, 0x1f, + 0x9a, 0x67, 0xfe, 0xdb, 0xcd, 0x33, 0x1a, 0xe5, 0xc2, 0x44, 0x94, 0x43, 0x14, 0x44, 0x09, 0x53, + 0x10, 0x36, 0x36, 0xc7, 0x35, 0x6d, 0xd7, 0xa4, 0xa7, 0x7c, 0x69, 0xd2, 0x38, 0xe8, 0xb3, 0xcb, + 0xdb, 0x90, 0x0c, 0x1d, 0xdb, 0x1e, 0x68, 0x02, 0x6e, 0x8a, 0xdc, 0x74, 0x49, 0x0a, 0x9b, 0x1c, + 0x75, 0x7e, 0x9d, 0x1a, 0xe7, 0xdf, 0x98, 0x6a, 0xfe, 0xdf, 0x05, 0x58, 0xfd, 0x0d, 0xbf, 0x80, + 0x46, 0x8f, 0x5b, 0x74, 0x08, 0x2b, 0x41, 0xfa, 0x6b, 0x23, 0x0e, 0x0b, 0xfe, 0x86, 0x5e, 0x14, + 0x3f, 0x2a, 0xc7, 0x51, 0xb1, 0x87, 0x3e, 0x86, 0x57, 0x27, 0xb0, 0x2d, 0x70, 0x9d, 0x5a, 0x14, + 0xe2, 0x5e, 0x89, 0x42, 0x9c, 0xef, 0x7a, 0x1c, 0xac, 0xf4, 0xb7, 0xcc, 0xba, 0x1d, 0x76, 0xa7, + 0x09, 0xb3, 0x87, 0x99, 0xcb, 0x7f, 0x0d, 0x4a, 0x2e, 0xa1, 0xec, 0x9e, 0x1d, 0xb9, 0x35, 0x2e, + 0x09, 0xa1, 0xbc, 0x8b, 0x1e, 0xc0, 0x2b, 0x33, 0x59, 0x04, 0xfa, 0x1e, 0x28, 0x63, 0x02, 0x92, + 0x8c, 0xb9, 0x80, 0x05, 0x97, 0x8a, 0xb1, 0xae, 0xfa, 0xc7, 0xe4, 0xd8, 0x65, 0xf4, 0x9a, 0xd2, + 0x84, 0x9c, 0x4b, 0xbc, 0xd1, 0x40, 0x5c, 0x1c, 0xca, 0x5b, 0xef, 0x2c, 0xc6, 0x3f, 0x98, 0x74, + 0x34, 0xa0, 0x58, 0x1a, 0xab, 0x8f, 0x21, 0x27, 0x24, 0xa8, 0x08, 0xf9, 0x87, 0x7b, 0x0f, 0xf6, + 0xf6, 0x3f, 0xda, 0xab, 0x24, 0x10, 0x40, 0x6e, 0xbb, 0xd1, 0x68, 0x1e, 0xb4, 0x2b, 0x49, 0xa4, + 0x40, 0x76, 0xbb, 0xbe, 0x8f, 0xdb, 0x95, 0x14, 0x13, 0xe3, 0xe6, 0x87, 0xcd, 0x46, 0xbb, 0x92, + 0x46, 0x2b, 0x50, 0x12, 0x6d, 0xed, 0xfe, 0x3e, 0xfe, 0xc9, 0x76, 0xbb, 0x92, 0x09, 0x89, 0x0e, + 0x9b, 0x7b, 0xf7, 0x9a, 0xb8, 0x92, 0x55, 0xdf, 0x65, 0x37, 0x93, 0x18, 0xc6, 0x32, 0xbe, 0x83, + 0x24, 0x43, 0x77, 0x10, 0xf5, 0xb7, 0x29, 0xa8, 0xc5, 0xd3, 0x10, 0xf4, 0xe1, 0xc4, 0xc4, 0xb7, + 0xce, 0xc1, 0x61, 0x26, 0x66, 0x8f, 0xae, 0x43, 0xd9, 0x25, 0x47, 0x84, 0x76, 0xfb, 0x82, 0x16, + 0x89, 0x23, 0xb3, 0x84, 0x4b, 0x52, 0xca, 0x8d, 0x3c, 0xa1, 0xf6, 0x19, 0xe9, 0x52, 0x4d, 0x60, + 0x91, 0xd8, 0x74, 0x0a, 0x53, 0x63, 0xd2, 0x43, 0x21, 0x54, 0x3f, 0x3d, 0x57, 0x2c, 0x15, 0xc8, + 0xe2, 0x66, 0x1b, 0x7f, 0x5c, 0x49, 0x23, 0x04, 0x65, 0xde, 0xd4, 0x0e, 0xf7, 0xb6, 0x0f, 0x0e, + 0x5b, 0xfb, 0x2c, 0x96, 0x17, 0x60, 0xd9, 0x8f, 0xa5, 0x2f, 0xcc, 0xaa, 0x9f, 0x40, 0x39, 0x7a, + 0xf7, 0x67, 0x21, 0x74, 0xed, 0x91, 0x65, 0xf0, 0x60, 0x64, 0xb1, 0xe8, 0xa0, 0xdb, 0x90, 0x3d, + 0xb6, 0x45, 0x9a, 0xcd, 0xde, 0x6b, 0x8f, 0x6c, 0x4a, 0x42, 0xb5, 0x03, 0xa1, 0xad, 0x7e, 0x01, + 0x59, 0x9e, 0x35, 0x2c, 0x03, 0xf8, 0x2d, 0x5e, 0x92, 0x2a, 0xd6, 0x46, 0x9f, 0x00, 0xe8, 0x94, + 0xba, 0x66, 0x67, 0x34, 0x76, 0xbc, 0x3e, 0x3b, 0xeb, 0xb6, 0x7d, 0xbd, 0xfa, 0x65, 0x99, 0x7e, + 0xab, 0x63, 0xd3, 0x50, 0x0a, 0x86, 0x1c, 0xaa, 0x7b, 0x50, 0x8e, 0xda, 0xfa, 0x34, 0x40, 0x8c, + 0x21, 0x4a, 0x03, 0x04, 0xab, 0x93, 0x34, 0x20, 0x20, 0x11, 0x69, 0x51, 0xb1, 0xe1, 0x1d, 0xf5, + 0x59, 0x12, 0x0a, 0xed, 0x13, 0xb9, 0x1e, 0x31, 0xc5, 0x82, 0xb1, 0x69, 0x2a, 0x7c, 0x35, 0x16, + 0xd5, 0x87, 0x74, 0x50, 0xd3, 0xf8, 0x20, 0xd8, 0x71, 0x99, 0x45, 0x6f, 0x40, 0x7e, 0x71, 0x47, + 0x66, 0xd9, 0xfb, 0xa0, 0x04, 0x98, 0xc9, 0xd8, 0xa9, 0x6e, 0x18, 0x2e, 0xf1, 0x3c, 0xb9, 0xef, + 0xfd, 0x2e, 0xaf, 0x3d, 0xd9, 0x4f, 0xe5, 0xe5, 0x3b, 0x8d, 0x45, 0x47, 0x35, 0x60, 0x79, 0x02, + 0x70, 0xd1, 0xfb, 0x90, 0x77, 0x46, 0x1d, 0xcd, 0x0f, 0xcf, 0xc4, 0x5b, 0x83, 0xcf, 0x7b, 0x46, + 0x9d, 0x81, 0xd9, 0x7d, 0x40, 0x4e, 0xfd, 0xc1, 0x38, 0xa3, 0xce, 0x03, 0x11, 0x45, 0xf1, 0x2b, + 0xa9, 0xf0, 0xaf, 0x1c, 0x43, 0xc1, 0xdf, 0x14, 0xe8, 0x87, 0xa0, 0x04, 0x58, 0x1e, 0x94, 0x24, + 0x63, 0x0f, 0x01, 0xe9, 0x7e, 0x6c, 0xc2, 0x48, 0xb4, 0x67, 0xf6, 0x2c, 0x62, 0x68, 0x63, 0x7e, + 0xcc, 0x7f, 0xad, 0x80, 0x97, 0xc5, 0x87, 0x5d, 0x9f, 0x1c, 0xab, 0xff, 0x4e, 0x42, 0xc1, 0x2f, + 0x3d, 0xa1, 0x77, 0x43, 0xfb, 0xae, 0x3c, 0xe3, 0xa2, 0xee, 0x2b, 0x8e, 0xcb, 0x47, 0xd1, 0xb1, + 0xa6, 0xce, 0x3f, 0xd6, 0xb8, 0x3a, 0xa0, 0x5f, 0x91, 0xcd, 0x9c, 0xbb, 0x22, 0xfb, 0x36, 0x20, + 0x6a, 0x53, 0x7d, 0xa0, 0x1d, 0xdb, 0xd4, 0xb4, 0x7a, 0x9a, 0x08, 0xb6, 0xe0, 0x02, 0x15, 0xfe, + 0xe5, 0x11, 0xff, 0x70, 0xc0, 0xe3, 0xfe, 0xcb, 0x24, 0x14, 0x02, 0x50, 0x3f, 0x6f, 0x35, 0xe8, + 0x22, 0xe4, 0x24, 0x6e, 0x89, 0x72, 0x90, 0xec, 0x05, 0x85, 0xc9, 0x4c, 0xa8, 0x30, 0x59, 0x83, + 0xc2, 0x90, 0x50, 0x9d, 0x9f, 0x6c, 0xe2, 0x8a, 0x12, 0xf4, 0x6f, 0xbe, 0x07, 0xc5, 0x50, 0x61, + 0x8e, 0x65, 0xde, 0x5e, 0xf3, 0xa3, 0x4a, 0xa2, 0x96, 0x7f, 0xf6, 0xe5, 0xd5, 0xf4, 0x1e, 0x79, + 0xca, 0xf6, 0x2c, 0x6e, 0x36, 0x5a, 0xcd, 0xc6, 0x83, 0x4a, 0xb2, 0x56, 0x7c, 0xf6, 0xe5, 0xd5, + 0x3c, 0x26, 0xbc, 0x48, 0x70, 0xb3, 0x05, 0x4b, 0xe1, 0x55, 0x89, 0x42, 0x1f, 0x82, 0xf2, 0xbd, + 0x87, 0x07, 0xbb, 0x3b, 0x8d, 0xed, 0x76, 0x53, 0x7b, 0xb4, 0xdf, 0x6e, 0x56, 0x92, 0xe8, 0x55, + 0xb8, 0xb0, 0xbb, 0xf3, 0xe3, 0x56, 0x5b, 0x6b, 0xec, 0xee, 0x34, 0xf7, 0xda, 0xda, 0x76, 0xbb, + 0xbd, 0xdd, 0x78, 0x50, 0x49, 0x6d, 0xfd, 0x5e, 0x81, 0xe5, 0xed, 0x7a, 0x63, 0x87, 0xc1, 0xb6, + 0xd9, 0xd5, 0xf9, 0xfd, 0xb1, 0x01, 0x19, 0x7e, 0x43, 0x3c, 0xf3, 0xd9, 0xae, 0x76, 0x76, 0xf9, + 0x08, 0xdd, 0x87, 0x2c, 0xbf, 0x3c, 0xa2, 0xb3, 0xdf, 0xf1, 0x6a, 0x73, 0xea, 0x49, 0x6c, 0x30, + 0x3c, 0x3d, 0xce, 0x7c, 0xd8, 0xab, 0x9d, 0x5d, 0x5e, 0x42, 0x18, 0x94, 0x31, 0xf9, 0x9c, 0xff, + 0xd0, 0x55, 0x5b, 0x00, 0x6c, 0xd0, 0x2e, 0xe4, 0xfd, 0xfb, 0xc2, 0xbc, 0xa7, 0xb7, 0xda, 0xdc, + 0xfa, 0x0f, 0x0b, 0x97, 0xb8, 0xd7, 0x9d, 0xfd, 0x8e, 0x58, 0x9b, 0x53, 0xcc, 0x42, 0x3b, 0x90, + 0x93, 0x84, 0x6a, 0xce, 0x73, 0x5a, 0x6d, 0x5e, 0x3d, 0x87, 0x05, 0x6d, 0x7c, 0x63, 0x9e, 0xff, + 0x3a, 0x5a, 0x5b, 0xa0, 0x4e, 0x87, 0x1e, 0x02, 0x84, 0x6e, 0x71, 0x0b, 0x3c, 0x7b, 0xd6, 0x16, + 0xa9, 0xbf, 0xa1, 0x7d, 0x28, 0x04, 0xa4, 0x7a, 0xee, 0x23, 0x64, 0x6d, 0x7e, 0x21, 0x0c, 0x3d, + 0x86, 0x52, 0x94, 0x4c, 0x2e, 0xf6, 0xb4, 0x58, 0x5b, 0xb0, 0xc2, 0xc5, 0xfc, 0x47, 0x99, 0xe5, + 0x62, 0x4f, 0x8d, 0xb5, 0x05, 0x0b, 0x5e, 0xe8, 0x33, 0x58, 0x99, 0x66, 0x7e, 0x8b, 0xbf, 0x3c, + 0xd6, 0xce, 0x51, 0x02, 0x43, 0x43, 0x40, 0x33, 0x18, 0xe3, 0x39, 0x1e, 0x22, 0x6b, 0xe7, 0xa9, + 0x88, 0xd5, 0x9b, 0x5f, 0xbd, 0x58, 0x4b, 0x7e, 0xfd, 0x62, 0x2d, 0xf9, 0x8f, 0x17, 0x6b, 0xc9, + 0xe7, 0x2f, 0xd7, 0x12, 0x5f, 0xbf, 0x5c, 0x4b, 0xfc, 0xf5, 0xe5, 0x5a, 0xe2, 0x67, 0x6f, 0xf5, + 0x4c, 0xda, 0x1f, 0x75, 0x36, 0xba, 0xf6, 0x70, 0x33, 0xfc, 0x0f, 0x87, 0x59, 0xff, 0xba, 0xe8, + 0xe4, 0xf8, 0xa1, 0x72, 0xeb, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x92, 0xa5, 0x39, 0xcc, 0x95, + 0x21, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -5229,6 +5239,13 @@ func (m *ResponseCheckTx) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.MempoolError) > 0 { + i -= len(m.MempoolError) + copy(dAtA[i:], m.MempoolError) + i = encodeVarintTypes(dAtA, i, uint64(len(m.MempoolError))) + i-- + dAtA[i] = 0x5a + } if m.Priority != 0 { i = encodeVarintTypes(dAtA, i, uint64(m.Priority)) i-- @@ -6842,6 +6859,10 @@ func (m *ResponseCheckTx) Size() (n int) { if m.Priority != 0 { n += 1 + sovTypes(uint64(m.Priority)) } + l = len(m.MempoolError) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } return n } @@ -11107,6 +11128,38 @@ func (m *ResponseCheckTx) Unmarshal(dAtA []byte) error { break } } + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MempoolError", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MempoolError = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/internal/mempool/v1/mempool.go b/internal/mempool/v1/mempool.go index 534b7ff55..aab3020ef 100644 --- a/internal/mempool/v1/mempool.go +++ b/internal/mempool/v1/mempool.go @@ -504,104 +504,110 @@ func (txmp *TxMempool) Update( // - An explicit lock is NOT required. func (txmp *TxMempool) initTxCallback(wtx *WrappedTx, res *abci.Response, txInfo mempool.TxInfo) { checkTxRes, ok := res.Value.(*abci.Response_CheckTx) - if ok { - var err error - if txmp.postCheck != nil { - err = txmp.postCheck(wtx.tx, checkTxRes.CheckTx) + if !ok { + return + } + + var err error + if txmp.postCheck != nil { + err = txmp.postCheck(wtx.tx, checkTxRes.CheckTx) + } + + if err != nil || checkTxRes.CheckTx.Code != abci.CodeTypeOK { + // ignore bad transactions + txmp.logger.Info( + "rejected bad transaction", + "priority", wtx.priority, + "tx", fmt.Sprintf("%X", wtx.tx.Hash()), + "peer_id", txInfo.SenderNodeID, + "code", checkTxRes.CheckTx.Code, + "post_check_err", err, + ) + + txmp.metrics.FailedTxs.Add(1) + + if !txmp.config.KeepInvalidTxsInCache { + txmp.cache.Remove(wtx.tx) } + if err != nil { + checkTxRes.CheckTx.MempoolError = err.Error() + } + return + } - if checkTxRes.CheckTx.Code == abci.CodeTypeOK && err == nil { - sender := checkTxRes.CheckTx.Sender - priority := checkTxRes.CheckTx.Priority + sender := checkTxRes.CheckTx.Sender + priority := checkTxRes.CheckTx.Priority - if len(sender) > 0 { - if wtx := txmp.txStore.GetTxBySender(sender); wtx != nil { - txmp.logger.Error( - "rejected incoming good transaction; tx already exists for sender", - "tx", fmt.Sprintf("%X", wtx.tx.Hash()), - "sender", sender, - ) - txmp.metrics.RejectedTxs.Add(1) - return - } - } - - if err := txmp.canAddTx(wtx); err != nil { - evictTxs := txmp.priorityIndex.GetEvictableTxs( - priority, - int64(wtx.Size()), - txmp.SizeBytes(), - txmp.config.MaxTxsBytes, - ) - if len(evictTxs) == 0 { - // No room for the new incoming transaction so we just remove it from - // the cache. - txmp.cache.Remove(wtx.tx) - txmp.logger.Error( - "rejected incoming good transaction; mempool full", - "tx", fmt.Sprintf("%X", wtx.tx.Hash()), - "err", err.Error(), - ) - txmp.metrics.RejectedTxs.Add(1) - return - } - - // evict an existing transaction(s) - // - // NOTE: - // - The transaction, toEvict, can be removed while a concurrent - // reCheckTx callback is being executed for the same transaction. - for _, toEvict := range evictTxs { - txmp.removeTx(toEvict, true) - txmp.logger.Debug( - "evicted existing good transaction; mempool full", - "old_tx", fmt.Sprintf("%X", toEvict.tx.Hash()), - "old_priority", toEvict.priority, - "new_tx", fmt.Sprintf("%X", wtx.tx.Hash()), - "new_priority", wtx.priority, - ) - txmp.metrics.EvictedTxs.Add(1) - } - } - - wtx.gasWanted = checkTxRes.CheckTx.GasWanted - wtx.priority = priority - wtx.sender = sender - wtx.peers = map[uint16]struct{}{ - txInfo.SenderID: {}, - } - - txmp.metrics.TxSizeBytes.Observe(float64(wtx.Size())) - txmp.metrics.Size.Set(float64(txmp.Size())) - - txmp.insertTx(wtx) - txmp.logger.Debug( - "inserted good transaction", - "priority", wtx.priority, + if len(sender) > 0 { + if wtx := txmp.txStore.GetTxBySender(sender); wtx != nil { + txmp.logger.Error( + "rejected incoming good transaction; tx already exists for sender", "tx", fmt.Sprintf("%X", wtx.tx.Hash()), - "height", txmp.height, - "num_txs", txmp.Size(), + "sender", sender, ) - txmp.notifyTxsAvailable() - - } else { - // ignore bad transactions - txmp.logger.Info( - "rejected bad transaction", - "priority", wtx.priority, - "tx", fmt.Sprintf("%X", wtx.tx.Hash()), - "peer_id", txInfo.SenderNodeID, - "code", checkTxRes.CheckTx.Code, - "post_check_err", err, - ) - - txmp.metrics.FailedTxs.Add(1) - - if !txmp.config.KeepInvalidTxsInCache { - txmp.cache.Remove(wtx.tx) - } + txmp.metrics.RejectedTxs.Add(1) + return } } + + if err := txmp.canAddTx(wtx); err != nil { + evictTxs := txmp.priorityIndex.GetEvictableTxs( + priority, + int64(wtx.Size()), + txmp.SizeBytes(), + txmp.config.MaxTxsBytes, + ) + if len(evictTxs) == 0 { + // No room for the new incoming transaction so we just remove it from + // the cache. + txmp.cache.Remove(wtx.tx) + txmp.logger.Error( + "rejected incoming good transaction; mempool full", + "tx", fmt.Sprintf("%X", wtx.tx.Hash()), + "err", err.Error(), + ) + txmp.metrics.RejectedTxs.Add(1) + return + } + + // evict an existing transaction(s) + // + // NOTE: + // - The transaction, toEvict, can be removed while a concurrent + // reCheckTx callback is being executed for the same transaction. + for _, toEvict := range evictTxs { + txmp.removeTx(toEvict, true) + txmp.logger.Debug( + "evicted existing good transaction; mempool full", + "old_tx", fmt.Sprintf("%X", toEvict.tx.Hash()), + "old_priority", toEvict.priority, + "new_tx", fmt.Sprintf("%X", wtx.tx.Hash()), + "new_priority", wtx.priority, + ) + txmp.metrics.EvictedTxs.Add(1) + } + } + + wtx.gasWanted = checkTxRes.CheckTx.GasWanted + wtx.priority = priority + wtx.sender = sender + wtx.peers = map[uint16]struct{}{ + txInfo.SenderID: {}, + } + + txmp.metrics.TxSizeBytes.Observe(float64(wtx.Size())) + txmp.metrics.Size.Set(float64(txmp.Size())) + + txmp.insertTx(wtx) + txmp.logger.Debug( + "inserted good transaction", + "priority", wtx.priority, + "tx", fmt.Sprintf("%X", wtx.tx.Hash()), + "height", txmp.height, + "num_txs", txmp.Size(), + ) + txmp.notifyTxsAvailable() + } // defaultTxCallback performs the default CheckTx application callback. This is diff --git a/internal/mempool/v1/mempool_test.go b/internal/mempool/v1/mempool_test.go index eb8385d7e..df26be726 100644 --- a/internal/mempool/v1/mempool_test.go +++ b/internal/mempool/v1/mempool_test.go @@ -3,11 +3,13 @@ package v1 import ( "bytes" "context" + "errors" "fmt" "math/rand" "os" "sort" "strconv" + "strings" "sync" "testing" "time" @@ -70,13 +72,13 @@ func (app *application) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { } } -func setup(t testing.TB, cacheSize int) *TxMempool { +func setup(t testing.TB, cacheSize int, options ...TxMempoolOption) *TxMempool { t.Helper() app := &application{kvstore.NewApplication()} cc := proxy.NewLocalClientCreator(app) - cfg := config.ResetTestRoot(t.Name()) + cfg := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|")) cfg.Mempool.CacheSize = cacheSize appConnMem, err := cc.NewABCIClient() @@ -88,7 +90,7 @@ func setup(t testing.TB, cacheSize int) *TxMempool { require.NoError(t, appConnMem.Stop()) }) - return NewTxMempool(log.TestingLogger().With("test", t.Name()), cfg.Mempool, appConnMem, 0) + return NewTxMempool(log.TestingLogger().With("test", t.Name()), cfg.Mempool, appConnMem, 0, options...) } func checkTxs(t *testing.T, txmp *TxMempool, numTxs int, peerID uint16) []testTx { @@ -320,11 +322,17 @@ func TestTxMempool_CheckTxExceedsMaxSize(t *testing.T) { txmp := setup(t, 0) rng := rand.New(rand.NewSource(time.Now().UnixNano())) - tx := make([]byte, txmp.config.MaxTxsBytes+1) + tx := make([]byte, txmp.config.MaxTxBytes+1) _, err := rng.Read(tx) require.NoError(t, err) require.Error(t, txmp.CheckTx(context.Background(), tx, nil, mempool.TxInfo{SenderID: 0})) + + tx = make([]byte, txmp.config.MaxTxBytes-1) + _, err = rng.Read(tx) + require.NoError(t, err) + + require.NoError(t, txmp.CheckTx(context.Background(), tx, nil, mempool.TxInfo{SenderID: 0})) } func TestTxMempool_CheckTxSamePeer(t *testing.T) { @@ -477,3 +485,43 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { require.GreaterOrEqual(t, txmp.Size(), 45) require.GreaterOrEqual(t, txmp.heightIndex.Size(), 45) } + +func TestTxMempool_CheckTxPostCheckError(t *testing.T) { + cases := []struct { + name string + err error + }{ + { + name: "error", + err: errors.New("test error"), + }, + { + name: "no error", + err: nil, + }, + } + for _, tc := range cases { + testCase := tc + t.Run(testCase.name, func(t *testing.T) { + postCheckFn := func(_ types.Tx, _ *abci.ResponseCheckTx) error { + return testCase.err + } + txmp := setup(t, 0, WithPostCheck(postCheckFn)) + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + tx := make([]byte, txmp.config.MaxTxBytes-1) + _, err := rng.Read(tx) + require.NoError(t, err) + + callback := func(res *abci.Response) { + checkTxRes, ok := res.Value.(*abci.Response_CheckTx) + require.True(t, ok) + expectedErrString := "" + if testCase.err != nil { + expectedErrString = testCase.err.Error() + } + require.Equal(t, expectedErrString, checkTxRes.CheckTx.MempoolError) + } + require.NoError(t, txmp.CheckTx(context.Background(), tx, callback, mempool.TxInfo{SenderID: 0})) + }) + } +} diff --git a/proto/tendermint/abci/types.proto b/proto/tendermint/abci/types.proto index bf088e8b1..f0d70548e 100644 --- a/proto/tendermint/abci/types.proto +++ b/proto/tendermint/abci/types.proto @@ -186,16 +186,17 @@ message ResponseBeginBlock { } message ResponseCheckTx { - uint32 code = 1; - bytes data = 2; - string log = 3; // nondeterministic - string info = 4; // nondeterministic - int64 gas_wanted = 5 [json_name = "gas_wanted"]; - int64 gas_used = 6 [json_name = "gas_used"]; - repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; - string codespace = 8; - string sender = 9; - int64 priority = 10; + uint32 code = 1; + bytes data = 2; + string log = 3; // nondeterministic + string info = 4; // nondeterministic + int64 gas_wanted = 5 [json_name = "gas_wanted"]; + int64 gas_used = 6 [json_name = "gas_used"]; + repeated Event events = 7 [(gogoproto.nullable) = false, (gogoproto.jsontag) = "events,omitempty"]; + string codespace = 8; + string sender = 9; + int64 priority = 10; + string mempool_error = 11; } message ResponseDeliverTx { diff --git a/proto/tendermint/p2p/types.pb.go b/proto/tendermint/p2p/types.pb.go index 4e63e7dd0..bffa6884f 100644 --- a/proto/tendermint/p2p/types.pb.go +++ b/proto/tendermint/p2p/types.pb.go @@ -5,15 +5,14 @@ package p2p import ( fmt "fmt" - io "io" - math "math" - math_bits "math/bits" - time "time" - _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" _ "github.com/gogo/protobuf/types" github_com_gogo_protobuf_types "github.com/gogo/protobuf/types" + io "io" + math "math" + math_bits "math/bits" + time "time" ) // Reference imports to suppress errors if they are not otherwise used. @@ -370,8 +369,8 @@ func (m *PeerAddressInfo) GetDialFailures() uint32 { func init() { proto.RegisterType((*ProtocolVersion)(nil), "tendermint.p2p.ProtocolVersion") - proto.RegisterType((*NodeInfo)(nil), "tendermint.types.NodeInfo") - proto.RegisterType((*NodeInfoOther)(nil), "tendermint.types.NodeInfoOther") + proto.RegisterType((*NodeInfo)(nil), "tendermint.p2p.NodeInfo") + proto.RegisterType((*NodeInfoOther)(nil), "tendermint.p2p.NodeInfoOther") proto.RegisterType((*PeerInfo)(nil), "tendermint.p2p.PeerInfo") proto.RegisterType((*PeerAddressInfo)(nil), "tendermint.p2p.PeerAddressInfo") } diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index 392453777..690b0a295 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -48,11 +48,12 @@ func (env *Environment) BroadcastTxSync(ctx *rpctypes.Context, tx types.Tx) (*ct r := res.GetCheckTx() return &ctypes.ResultBroadcastTx{ - Code: r.Code, - Data: r.Data, - Log: r.Log, - Codespace: r.Codespace, - Hash: tx.Hash(), + Code: r.Code, + Data: r.Data, + Log: r.Log, + Codespace: r.Codespace, + MempoolError: r.MempoolError, + Hash: tx.Hash(), }, nil } diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index e5ac55f04..a49e3c0d9 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -187,10 +187,11 @@ type ResultConsensusState struct { // CheckTx result type ResultBroadcastTx struct { - Code uint32 `json:"code"` - Data bytes.HexBytes `json:"data"` - Log string `json:"log"` - Codespace string `json:"codespace"` + Code uint32 `json:"code"` + Data bytes.HexBytes `json:"data"` + Log string `json:"log"` + Codespace string `json:"codespace"` + MempoolError string `json:"mempool_error"` Hash bytes.HexBytes `json:"hash"` } From 97a8f125e064fa328266dc60f3c22b8b4ea47fe2 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 22 Jul 2021 17:59:02 +0200 Subject: [PATCH 07/14] e2e: allow for both v0 and v1 mempool implementations (#6752) --- test/e2e/generator/generate.go | 2 ++ test/e2e/pkg/manifest.go | 3 +++ test/e2e/pkg/testnet.go | 7 +++++++ test/e2e/runner/setup.go | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/test/e2e/generator/generate.go b/test/e2e/generator/generate.go index 554384d97..1e134cf71 100644 --- a/test/e2e/generator/generate.go +++ b/test/e2e/generator/generate.go @@ -31,6 +31,7 @@ var ( nodePrivvalProtocols = uniformChoice{"file", "unix", "tcp", "grpc"} // FIXME: v2 disabled due to flake nodeFastSyncs = uniformChoice{"v0"} // "v2" + nodeMempools = uniformChoice{"v0", "v1"} nodeStateSyncs = uniformChoice{false, true} nodePersistIntervals = uniformChoice{0, 1, 5} nodeSnapshotIntervals = uniformChoice{0, 3} @@ -273,6 +274,7 @@ func generateNode( ABCIProtocol: nodeABCIProtocols.Choose(r).(string), PrivvalProtocol: nodePrivvalProtocols.Choose(r).(string), FastSync: nodeFastSyncs.Choose(r).(string), + Mempool: nodeMempools.Choose(r).(string), StateSync: nodeStateSyncs.Choose(r).(bool) && startAt > 0, PersistInterval: ptrUint64(uint64(nodePersistIntervals.Choose(r).(int))), SnapshotInterval: uint64(nodeSnapshotIntervals.Choose(r).(int)), diff --git a/test/e2e/pkg/manifest.go b/test/e2e/pkg/manifest.go index 5320c4637..89b38e02a 100644 --- a/test/e2e/pkg/manifest.go +++ b/test/e2e/pkg/manifest.go @@ -110,6 +110,9 @@ type ManifestNode struct { // Defaults to disabled. FastSync string `toml:"fast_sync"` + // Mempool specifies which version of mempool to use. Either "v0" or "v1" + Mempool string `toml:"mempool_version"` + // StateSync enables state sync. The runner automatically configures trusted // block hashes and RPC servers. At least one node in the network must have // SnapshotInterval set to non-zero, and the state syncing node must have diff --git a/test/e2e/pkg/testnet.go b/test/e2e/pkg/testnet.go index a5cec0147..d54df4406 100644 --- a/test/e2e/pkg/testnet.go +++ b/test/e2e/pkg/testnet.go @@ -80,6 +80,7 @@ type Node struct { ProxyPort uint32 StartAt int64 FastSync string + Mempool string StateSync bool Database string ABCIProtocol Protocol @@ -168,6 +169,7 @@ func LoadTestnet(file string) (*Testnet, error) { PrivvalProtocol: ProtocolFile, StartAt: nodeManifest.StartAt, FastSync: nodeManifest.FastSync, + Mempool: nodeManifest.Mempool, StateSync: nodeManifest.StateSync, PersistInterval: 1, SnapshotInterval: nodeManifest.SnapshotInterval, @@ -331,6 +333,11 @@ func (n Node) Validate(testnet Testnet) error { default: return fmt.Errorf("invalid fast sync setting %q", n.FastSync) } + switch n.Mempool { + case "", "v0", "v1": + default: + return fmt.Errorf("invalid mempool version %q", n.Mempool) + } switch n.QueueType { case "", "priority", "wdrr", "fifo": default: diff --git a/test/e2e/runner/setup.go b/test/e2e/runner/setup.go index c2b61617e..c80d05b22 100644 --- a/test/e2e/runner/setup.go +++ b/test/e2e/runner/setup.go @@ -292,6 +292,10 @@ func MakeConfig(node *e2e.Node) (*config.Config, error) { return nil, fmt.Errorf("unexpected mode %q", node.Mode) } + if node.Mempool != "" { + cfg.Mempool.Version = node.Mempool + } + if node.FastSync == "" { cfg.FastSyncMode = false } else { From 0e2752ae42208a8becdb95dba6ebcb105c4398f1 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Thu, 22 Jul 2021 18:12:34 +0200 Subject: [PATCH 08/14] light: improve error handling and allow providers to be added (#6733) --- internal/statesync/snapshots.go | 2 +- light/client.go | 141 ++++++++++++++++++++------------ light/client_test.go | 30 +++++++ light/detector.go | 12 ++- light/verifier.go | 17 ++-- light/verifier_test.go | 4 +- 6 files changed, 144 insertions(+), 62 deletions(-) diff --git a/internal/statesync/snapshots.go b/internal/statesync/snapshots.go index 11cd78394..9058304a9 100644 --- a/internal/statesync/snapshots.go +++ b/internal/statesync/snapshots.go @@ -85,7 +85,7 @@ func (p *snapshotPool) Add(peerID types.NodeID, snapshot *snapshot) (bool, error appHash, err := p.stateProvider.AppHash(ctx, snapshot.Height) if err != nil { - return false, err + return false, fmt.Errorf("failed to get app hash: %w", err) } snapshot.trustedAppHash = appHash key := snapshot.Key() diff --git a/light/client.go b/light/client.go index 122130224..52bbdf981 100644 --- a/light/client.go +++ b/light/client.go @@ -478,7 +478,7 @@ func (c *Client) VerifyHeader(ctx context.Context, newHeader *types.Header, now } if !bytes.Equal(l.Hash(), newHeader.Hash()) { - return fmt.Errorf("light block header %X does not match newHeader %X", l.Hash(), newHeader.Hash()) + return fmt.Errorf("header from primary %X does not match newHeader %X", l.Hash(), newHeader.Hash()) } return c.verifyLightBlock(ctx, l, now) @@ -586,7 +586,7 @@ func (c *Client) verifySequential( } // If some intermediate header is invalid, remove the primary and try again. - c.logger.Error("primary sent invalid header -> removing", "err", err, "primary", c.primary) + c.logger.Info("primary sent invalid header -> removing", "err", err, "primary", c.primary) replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) if removeErr != nil { @@ -630,6 +630,10 @@ func (c *Client) verifySequential( // requested from source is kept such that when a verification is made, and the // light client tries again to verify the new light block in the middle, the light // client does not need to ask for all the same light blocks again. +// +// If this function errors, it should always wrap it in a `ErrVerifcationFailed` +// struct so that the calling function can determine where it failed and handle +// it accordingly. func (c *Client) verifySkipping( ctx context.Context, source provider.Provider, @@ -690,20 +694,10 @@ func (c *Client) verifySkipping( // schedule what the next height we need to fetch is pivotHeight := c.schedule(verifiedBlock.Height, blockCache[depth].Height) interimBlock, providerErr := source.LightBlock(ctx, pivotHeight) - switch providerErr { - case nil: - blockCache = append(blockCache, interimBlock) - - // if the error is benign, the client does not need to replace the primary - case provider.ErrLightBlockNotFound, provider.ErrNoResponse, provider.ErrHeightTooHigh: - return nil, err - - // all other errors such as ErrBadLightBlock or ErrUnreliableProvider are seen as malevolent and the - // provider is removed - default: + if providerErr != nil { return nil, ErrVerificationFailed{From: verifiedBlock.Height, To: pivotHeight, Reason: providerErr} } - + blockCache = append(blockCache, interimBlock) } depth++ @@ -729,33 +723,8 @@ func (c *Client) verifySkippingAgainstPrimary( now time.Time) error { trace, err := c.verifySkipping(ctx, c.primary, trustedBlock, newLightBlock, now) - - switch errors.Unwrap(err).(type) { - case ErrInvalidHeader: - // If the target header is invalid, return immediately. - invalidHeaderHeight := err.(ErrVerificationFailed).To - if invalidHeaderHeight == newLightBlock.Height { - c.logger.Debug("target header is invalid", "err", err) - return err - } - - // If some intermediate header is invalid, remove the primary and try again. - c.logger.Error("primary sent invalid header -> replacing", "err", err, "primary", c.primary) - replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, true) - if removeErr != nil { - c.logger.Error("failed to replace primary. Returning original error", "err", removeErr) - return err - } - - if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) { - c.logger.Debug("replaced primary but new primary has a different block to the initial one. Returning original error") - return err - } - - // attempt to verify the header again - return c.verifySkippingAgainstPrimary(ctx, trustedBlock, replacementBlock, now) - case nil: - // Compare header with the witnesses to ensure it's not a fork. + if err == nil { + // Success! Now compare the header with the witnesses to ensure it's not a fork. // More witnesses we have, more chance to notice one. // // CORRECTNESS ASSUMPTION: there's at least 1 correct full node @@ -763,11 +732,62 @@ func (c *Client) verifySkippingAgainstPrimary( if cmpErr := c.detectDivergence(ctx, trace, now); cmpErr != nil { return cmpErr } - default: + } + + var e = &ErrVerificationFailed{} + // all errors from verify skipping should be `ErrVerificationFailed` + // if it's not we just return the error directly + if !errors.As(err, e) { return err } - return nil + replace := true + switch e.Reason.(type) { + // Verification returned an invalid header + case ErrInvalidHeader: + // If it was the target header, return immediately. + if e.To == newLightBlock.Height { + c.logger.Debug("target header is invalid", "err", err) + return err + } + + // If some intermediate header is invalid, remove the primary and try + // again. + + // An intermediate header expired. We can no longer validate it as there is + // no longer the ability to punish invalid blocks as evidence of misbehavior + case ErrOldHeaderExpired: + return err + + // This happens if there was a problem in finding the next block or a + // context was canceled. + default: + if errors.Is(e.Reason, context.Canceled) || errors.Is(e.Reason, context.DeadlineExceeded) { + return e.Reason + } + + if !c.providerShouldBeRemoved(e.Reason) { + replace = false + } + } + + // if we've reached here we're attempting to retry verification with a + // different provider + c.logger.Info("primary returned error", "err", e, "primary", c.primary, "replace", replace) + + replacementBlock, removeErr := c.findNewPrimary(ctx, newLightBlock.Height, replace) + if removeErr != nil { + c.logger.Error("failed to replace primary. Returning original error", "err", removeErr) + return e.Reason + } + + if !bytes.Equal(replacementBlock.Hash(), newLightBlock.Hash()) { + c.logger.Debug("replaced primary but new primary has a different block to the initial one. Returning original error") + return e.Reason + } + + // attempt to verify the header again from the trusted block + return c.verifySkippingAgainstPrimary(ctx, trustedBlock, replacementBlock, now) } // LastTrustedHeight returns a last trusted height. -1 and nil are returned if @@ -811,6 +831,15 @@ func (c *Client) Witnesses() []provider.Provider { return c.witnesses } +// AddProvider adds a providers to the light clients set +// +// NOTE: The light client does not check for uniqueness +func (c *Client) AddProvider(p provider.Provider) { + c.providerMutex.Lock() + defer c.providerMutex.Unlock() + c.witnesses = append(c.witnesses, p) +} + // Cleanup removes all the data (headers and validator sets) stored. Note: the // client must be stopped at this point. func (c *Client) Cleanup() error { @@ -865,7 +894,7 @@ func (c *Client) backwards( "newHash", interimHeader.Hash()) if err := VerifyBackwards(interimHeader, verifiedHeader); err != nil { // verification has failed - c.logger.Error("backwards verification failed, replacing primary...", "err", err, "primary", c.primary) + c.logger.Info("backwards verification failed, replacing primary...", "err", err, "primary", c.primary) // the client tries to see if it can get a witness to continue with the request newPrimarysBlock, replaceErr := c.findNewPrimary(ctx, newHeader.Height, true) @@ -915,14 +944,14 @@ func (c *Client) lightBlockFromPrimary(ctx context.Context, height int64) (*type case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: // we find a new witness to replace the primary - c.logger.Debug("error from light block request from primary, replacing...", + c.logger.Info("error from light block request from primary, replacing...", "error", err, "height", height, "primary", c.primary) return c.findNewPrimary(ctx, height, false) default: // The light client has most likely received either provider.ErrUnreliableProvider or provider.ErrBadLightBlock // These errors mean that the light client should drop the primary and try with another provider instead - c.logger.Error("error from light block request from primary, removing...", + c.logger.Info("error from light block request from primary, removing...", "error", err, "height", height, "primary", c.primary) return c.findNewPrimary(ctx, height, true) } @@ -1022,7 +1051,7 @@ func (c *Client) findNewPrimary(ctx context.Context, height int64, remove bool) // process benign errors by logging them only case provider.ErrNoResponse, provider.ErrLightBlockNotFound, provider.ErrHeightTooHigh: lastError = response.err - c.logger.Debug("error on light block request from witness", + c.logger.Info("error on light block request from witness", "error", response.err, "primary", c.witnesses[response.witnessIndex]) continue @@ -1066,13 +1095,13 @@ func (c *Client) compareFirstHeaderWithWitnesses(ctx context.Context, h *types.S case nil: continue case errConflictingHeaders: - c.logger.Error(fmt.Sprintf(`witness #%d has a different header. Please check primary is correct -and remove witness. Otherwise, use a different primary`, e.WitnessIndex), "witness", c.witnesses[e.WitnessIndex]) + c.logger.Error(`witness has a different header. Please check primary is +correct and remove witness. Otherwise, use a different primary`, + "Witness", c.witnesses[e.WitnessIndex], "ExpHeader", h.Hash(), "GotHeader", e.Block.Hash()) return err case errBadWitness: // If witness sent us an invalid header, then remove it - c.logger.Info("witness sent an invalid light block or didn't respond, removing...", - "witness", c.witnesses[e.WitnessIndex], + c.logger.Info("witness returned an error, removing...", "err", err) witnessesToRemove = append(witnessesToRemove, e.WitnessIndex) default: @@ -1082,7 +1111,7 @@ and remove witness. Otherwise, use a different primary`, e.WitnessIndex), "witne } // the witness either didn't respond or didn't have the block. We ignore it. - c.logger.Debug("unable to compare first header with witness", + c.logger.Debug("unable to compare first header with witness, ignoring", "err", err) } @@ -1091,3 +1120,11 @@ and remove witness. Otherwise, use a different primary`, e.WitnessIndex), "witne // remove all witnesses that misbehaved return c.removeWitnesses(witnessesToRemove) } + +// providerShouldBeRemoved analyzes the nature of the error and whether the provider +// should be removed from the light clients set +func (c *Client) providerShouldBeRemoved(err error) bool { + return errors.As(err, &provider.ErrUnreliableProvider{}) || + errors.As(err, &provider.ErrBadLightBlock{}) || + errors.Is(err, provider.ErrConnectionClosed) +} diff --git a/light/client_test.go b/light/client_test.go index b826eaead..67e0525b8 100644 --- a/light/client_test.go +++ b/light/client_test.go @@ -589,6 +589,36 @@ func TestClient_Concurrency(t *testing.T) { wg.Wait() } +func TestClient_AddProviders(t *testing.T) { + c, err := light.NewClient( + ctx, + chainID, + trustOptions, + fullNode, + []provider.Provider{fullNode}, + dbs.New(dbm.NewMemDB()), + light.Logger(log.TestingLogger()), + ) + require.NoError(t, err) + + closeCh := make(chan struct{}) + go func() { + // run verification concurrently to make sure it doesn't dead lock + _, err = c.VerifyLightBlockAtHeight(ctx, 2, bTime.Add(2*time.Hour)) + require.NoError(t, err) + close(closeCh) + }() + + // NOTE: the light client doesn't check uniqueness of providers + c.AddProvider(fullNode) + require.Len(t, c.Witnesses(), 2) + select { + case <-closeCh: + case <-time.After(5 * time.Second): + t.Fatal("concurent light block verification failed to finish in 5s") + } +} + func TestClientReplacesPrimaryWithWitnessIfPrimaryIsUnavailable(t *testing.T) { c, err := light.NewClient( ctx, diff --git a/light/detector.go b/light/detector.go index 4d7301cb0..32a0c3f1e 100644 --- a/light/detector.go +++ b/light/detector.go @@ -131,7 +131,11 @@ func (c *Client) compareNewHeaderWithWitness(ctx context.Context, errc chan erro var isTargetHeight bool isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) if err != nil { - errc <- err + if c.providerShouldBeRemoved(err) { + errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} + } else { + errc <- err + } return } @@ -155,7 +159,11 @@ func (c *Client) compareNewHeaderWithWitness(ctx context.Context, errc chan erro time.Sleep(2*c.maxClockDrift + c.maxBlockLag) isTargetHeight, lightBlock, err = c.getTargetBlockOrLatest(ctx, h.Height, witness) if err != nil { - errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} + if c.providerShouldBeRemoved(err) { + errc <- errBadWitness{Reason: err, WitnessIndex: witnessIndex} + } else { + errc <- err + } return } if isTargetHeight { diff --git a/light/verifier.go b/light/verifier.go index d9c8019eb..ee4bfb053 100644 --- a/light/verifier.go +++ b/light/verifier.go @@ -50,8 +50,9 @@ func VerifyNonAdjacent( return err } - if HeaderExpired(trustedHeader, trustingPeriod, now) { - return ErrOldHeaderExpired{trustedHeader.Time.Add(trustingPeriod), now} + // check if the untrusted header is within the trust period + if HeaderExpired(untrustedHeader, trustingPeriod, now) { + return ErrOldHeaderExpired{untrustedHeader.Time.Add(trustingPeriod), now} } if err := verifyNewHeaderAndVals( @@ -117,8 +118,9 @@ func VerifyAdjacent( return errors.New("headers must be adjacent in height") } - if HeaderExpired(trustedHeader, trustingPeriod, now) { - return ErrOldHeaderExpired{trustedHeader.Time.Add(trustingPeriod), now} + // check if the untrusted header is within the trust period + if HeaderExpired(untrustedHeader, trustingPeriod, now) { + return ErrOldHeaderExpired{untrustedHeader.Time.Add(trustingPeriod), now} } if err := verifyNewHeaderAndVals( @@ -192,7 +194,10 @@ func HeaderExpired(h *types.SignedHeader, trustingPeriod time.Duration, now time // of the trusted header // // For any of these cases ErrInvalidHeader is returned. -// NOTE: This does not check whether the trusted header has expired or not. +// NOTE: This does not check whether the trusted or untrusted header has expired +// or not. These checks are not necessary because the detector never runs during +// backwards verification and thus evidence that needs to be within a certain +// time bound is never sent. func VerifyBackwards(untrustedHeader, trustedHeader *types.Header) error { if err := untrustedHeader.ValidateBasic(); err != nil { return ErrInvalidHeader{err} @@ -220,6 +225,8 @@ func VerifyBackwards(untrustedHeader, trustedHeader *types.Header) error { return nil } +// NOTE: This function assumes that untrustedHeader is after trustedHeader. +// Do not use for backwards verification. func verifyNewHeaderAndVals( untrustedHeader *types.SignedHeader, untrustedVals *types.ValidatorSet, diff --git a/light/verifier_test.go b/light/verifier_test.go index 556beee75..0432c130d 100644 --- a/light/verifier_test.go +++ b/light/verifier_test.go @@ -64,7 +64,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) { keys.GenSignedHeader(chainID, nextHeight, bTime.Add(-1*time.Hour), nil, vals, vals, hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)), vals, - 3 * time.Hour, + 4 * time.Hour, bTime.Add(2 * time.Hour), nil, "to be after old header time", @@ -146,7 +146,7 @@ func TestVerifyAdjacentHeaders(t *testing.T) { hash("app_hash"), hash("cons_hash"), hash("results_hash"), 0, len(keys)), keys.ToValidators(10, 1), 1 * time.Hour, - bTime.Add(1 * time.Hour), + bTime.Add(2 * time.Hour), nil, "old header has expired", }, From a393cf8babfafbe598ad9cdd47d313fdfdd37be2 Mon Sep 17 00:00:00 2001 From: Aleksandr Bezobchuk Date: Fri, 23 Jul 2021 08:15:57 -0400 Subject: [PATCH 09/14] internal: update blockchain reactor godoc (#6749) --- internal/blockchain/doc.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/blockchain/doc.go b/internal/blockchain/doc.go index b5d9858d0..f1db243c6 100644 --- a/internal/blockchain/doc.go +++ b/internal/blockchain/doc.go @@ -1,17 +1,36 @@ /* -Package blockchain provides two implementations of the fast-sync protocol. +Package blockchain implements two versions of a reactor Service that are +responsible for block propagation and gossip between peers. This mechanism is +more formally known as fast-sync. -- v0 was the very first implementation. it's battle tested, but does not have a -lot of test coverage. -- v2 is the newest implementation, with a focus on testability and readability. +In order for a full node to successfully participate in consensus, it must have +the latest view of state. The fast-sync protocol is a mechanism in which peers +may exchange and gossip entire blocks with one another, in a request/response +type model, until they've successfully synced to the latest head block. Once +succussfully synced, the full node can switch to an active role in consensus and +will no longer fast-sync and thus no longer run the fast-sync process. -Check out ADR-40 for the formal model and requirements. +Note, the blockchain reactor Service gossips entire block and relevant data such +that each receiving peer may construct the entire view of the blockchain state. -# Termination criteria +There are two versions of the blockchain reactor Service, i.e. fast-sync: -1. the maximum peer height is reached -2. termination timeout is triggered, which is set if the peer set is empty or -there are no pending requests. +- v0: The initial implementation that is battle-tested, but whose test coverage + is lacking and is not formally verifiable. +- v2: The latest implementation that has much higher test coverage and is formally + verified. However, the current implementation of v2 is not as battle-tested and + is known to have various bugs that could make it unreliable in production + environments. +The v0 blockchain reactor Service has one p2p channel, BlockchainChannel. This +channel is responsible for handling messages that both request blocks and respond +to block requests from peers. For every block request from a peer, the reactor +will execute respondToPeer which will fetch the block from the node's state store +and respond to the peer. For every block response, the node will add the block +to its pool via AddBlock. + +Internally, v0 runs a poolRoutine that constantly checks for what blocks it needs +and requests them. The poolRoutine is also responsible for taking blocks from the +pool, saving and executing each block. */ package blockchain From c3ae6f5b58e07b29c62bfdc5715b6bf8ae5ee951 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:29:19 -0400 Subject: [PATCH 10/14] p2p: add coverage for mConnConnection.TrySendMessage (#6754) This change adds additional coverage to the `mConnConnection.TrySendMessage` code path. Adds test to ensure it returns `io.EOF` when closed. Addresses: #6570 --- internal/p2p/conn/connection.go | 1 - internal/p2p/transport_mconn_test.go | 48 +++++++++++++++++----------- internal/p2p/transport_test.go | 6 ++++ 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index 105288bdc..dc5bacc39 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -421,7 +421,6 @@ func (c *MConnection) CanSend(chID byte) bool { // sendRoutine polls for packets to send from channels. func (c *MConnection) sendRoutine() { defer c._recover() - protoWriter := protoio.NewDelimitedWriter(c.bufConnWriter) FOR_LOOP: diff --git a/internal/p2p/transport_mconn_test.go b/internal/p2p/transport_mconn_test.go index 541f4767e..06cd93c0a 100644 --- a/internal/p2p/transport_mconn_test.go +++ b/internal/p2p/transport_mconn_test.go @@ -151,9 +151,6 @@ func TestMConnTransport_Listen(t *testing.T) { []*p2p.ChannelDescriptor{{ID: byte(chID), Priority: 1}}, p2p.MConnTransportOptions{}, ) - t.Cleanup(func() { - _ = transport.Close() - }) // Transport should not listen on any endpoints yet. require.Empty(t, transport.Endpoints()) @@ -166,19 +163,6 @@ func TestMConnTransport_Listen(t *testing.T) { } require.NoError(t, err) - // Start a goroutine to just accept any connections. - go func() { - for { - conn, err := transport.Accept() - if err != nil { - return - } - defer func() { - _ = conn.Close() - }() - } - }() - // Check the endpoint. endpoints := transport.Endpoints() require.Len(t, endpoints, 1) @@ -195,14 +179,40 @@ func TestMConnTransport_Listen(t *testing.T) { require.NotZero(t, endpoint.Port) require.Empty(t, endpoint.Path) - // Dialing the endpoint should work. - conn, err := transport.Dial(ctx, endpoint) + dialedChan := make(chan struct{}) + + var peerConn p2p.Connection + go func() { + // Dialing the endpoint should work. + var err error + peerConn, err = transport.Dial(ctx, endpoint) + require.NoError(t, err) + close(dialedChan) + }() + + conn, err := transport.Accept() require.NoError(t, err) - require.NoError(t, conn.Close()) + _ = conn.Close() + <-dialedChan + + time.Sleep(time.Minute) + // closing the connection should not error + require.NoError(t, peerConn.Close()) + + // try to read from the connection should error + _, _, err = peerConn.ReceiveMessage() + require.Error(t, err) // Trying to listen again should error. err = transport.Listen(tc.endpoint) require.Error(t, err) + + // close the transport + _ = transport.Close() + + // Dialing the closed endpoint should error + _, err = transport.Dial(ctx, endpoint) + require.Error(t, err) }) } } diff --git a/internal/p2p/transport_test.go b/internal/p2p/transport_test.go index 2d9d9f9f3..1b8ab77f5 100644 --- a/internal/p2p/transport_test.go +++ b/internal/p2p/transport_test.go @@ -403,6 +403,9 @@ func TestConnection_SendReceive(t *testing.T) { _, _, err = ab.ReceiveMessage() require.Error(t, err) require.Equal(t, io.EOF, err) + _, err = ab.TrySendMessage(chID, []byte("closed try")) + require.Error(t, err) + require.Equal(t, io.EOF, err) _, err = ab.SendMessage(chID, []byte("closed")) require.Error(t, err) require.Equal(t, io.EOF, err) @@ -410,6 +413,9 @@ func TestConnection_SendReceive(t *testing.T) { _, _, err = ba.ReceiveMessage() require.Error(t, err) require.Equal(t, io.EOF, err) + _, err = ba.TrySendMessage(chID, []byte("closed try")) + require.Error(t, err) + require.Equal(t, io.EOF, err) _, err = ba.SendMessage(chID, []byte("closed")) require.Error(t, err) require.Equal(t, io.EOF, err) From a341a626e0e3431b657535ddc4f43456b25a040f Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 26 Jul 2021 15:09:07 +0200 Subject: [PATCH 11/14] p2p: avoid blocking on the dequeCh (#6765) --- internal/p2p/pqueue.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/p2p/pqueue.go b/internal/p2p/pqueue.go index 0e62c87d9..e4560c7bd 100644 --- a/internal/p2p/pqueue.go +++ b/internal/p2p/pqueue.go @@ -257,7 +257,11 @@ func (s *pqScheduler) process() { s.metrics.PeerSendBytesTotal.With( "chID", chIDStr, "peer_id", string(pqEnv.envelope.To)).Add(float64(pqEnv.size)) - s.dequeueCh <- pqEnv.envelope + select { + case s.dequeueCh <- pqEnv.envelope: + case <-s.closer.Done(): + return + } } case <-s.closer.Done(): From 91e277d7b7d087c78397353d4d29eec3c84af13f Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 26 Jul 2021 15:22:12 +0200 Subject: [PATCH 12/14] enable pex reactor depending on config param (#6762) --- node/node.go | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/node/node.go b/node/node.go index cb32c5ce6..994132b29 100644 --- a/node/node.go +++ b/node/node.go @@ -380,35 +380,37 @@ func makeNode(config *cfg.Config, pexCh := pex.ChannelDescriptor() transport.AddChannelDescriptors([]*p2p.ChannelDescriptor{&pexCh}) - if config.P2P.DisableLegacy { - addrBook = nil - pexReactorV2, err = createPEXReactorV2(config, logger, peerManager, router) - if err != nil { - return nil, err - } - } else { - // setup Transport and Switch - sw = createSwitch( - config, transport, p2pMetrics, mpReactorShim, bcReactorForSwitch, - stateSyncReactorShim, csReactorShim, evReactorShim, proxyApp, nodeInfo, nodeKey, p2pLogger, - ) + if config.P2P.PexReactor { + if config.P2P.DisableLegacy { + addrBook = nil + pexReactorV2, err = createPEXReactorV2(config, logger, peerManager, router) + if err != nil { + return nil, err + } + } else { + // setup Transport and Switch + sw = createSwitch( + config, transport, p2pMetrics, mpReactorShim, bcReactorForSwitch, + stateSyncReactorShim, csReactorShim, evReactorShim, proxyApp, nodeInfo, nodeKey, p2pLogger, + ) - err = sw.AddPersistentPeers(strings.SplitAndTrimEmpty(config.P2P.PersistentPeers, ",", " ")) - if err != nil { - return nil, fmt.Errorf("could not add peers from persistent-peers field: %w", err) - } + err = sw.AddPersistentPeers(strings.SplitAndTrimEmpty(config.P2P.PersistentPeers, ",", " ")) + if err != nil { + return nil, fmt.Errorf("could not add peers from persistent-peers field: %w", err) + } - err = sw.AddUnconditionalPeerIDs(strings.SplitAndTrimEmpty(config.P2P.UnconditionalPeerIDs, ",", " ")) - if err != nil { - return nil, fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err) - } + err = sw.AddUnconditionalPeerIDs(strings.SplitAndTrimEmpty(config.P2P.UnconditionalPeerIDs, ",", " ")) + if err != nil { + return nil, fmt.Errorf("could not add peer ids from unconditional_peer_ids field: %w", err) + } - addrBook, err = createAddrBookAndSetOnSwitch(config, sw, p2pLogger, nodeKey) - if err != nil { - return nil, fmt.Errorf("could not create addrbook: %w", err) - } + addrBook, err = createAddrBookAndSetOnSwitch(config, sw, p2pLogger, nodeKey) + if err != nil { + return nil, fmt.Errorf("could not create addrbook: %w", err) + } - pexReactor = createPEXReactorAndAddToSwitch(addrBook, config, sw, logger) + pexReactor = createPEXReactorAndAddToSwitch(addrBook, config, sw, logger) + } } if config.RPC.PprofListenAddress != "" { From 93f462ef869b1b3fc36bfa6af89cd073f3f66031 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 14:17:02 +0000 Subject: [PATCH 13/14] build(deps): Bump codecov/codecov-action from 2.0.1 to 2.0.2 (#6764) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 2.0.1 to 2.0.2. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v2.0.1...v2.0.2) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Marko --- .github/workflows/coverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ee6b0068f..7d7ab8113 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -121,7 +121,7 @@ jobs: - run: | cat ./*profile.out | grep -v "mode: atomic" >> coverage.txt if: env.GIT_DIFF - - uses: codecov/codecov-action@v2.0.1 + - uses: codecov/codecov-action@v2.0.2 with: file: ./coverage.txt if: env.GIT_DIFF From c5dc3b267f68e167c0e58bdcf50a29ff4df48349 Mon Sep 17 00:00:00 2001 From: William Banfield <4561443+williambanfield@users.noreply.github.com> Date: Mon, 26 Jul 2021 10:58:51 -0400 Subject: [PATCH 14/14] test/fuzz: add test to reproduce found fuzz errors (#6757) This change does two things: 1. It fixes the json fuzzer to account for receiving array results. Arrays are returned by the rpc server when the input data is an array. 2. Adds a `fuzz_test.go` file and corresponding `testdata` directory containing the failing test case. This seems like a reasonable way to add and track previous crash issues in our fuzz test cases. The upcoming stdlib go fuzz tool does effectively this automatically. --- test/fuzz/rpc/jsonrpc/server/fuzz_test.go | 33 +++++++++++++++++++ test/fuzz/rpc/jsonrpc/server/handler.go | 23 ++++++++++--- .../1184f5b8d4b6dd08709cf1513f26744167065e0d | 1 + ...d-fuzz_rpc_jsonrpc_server-4738572803506176 | 1 + 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 test/fuzz/rpc/jsonrpc/server/fuzz_test.go create mode 100644 test/fuzz/rpc/jsonrpc/server/testdata/1184f5b8d4b6dd08709cf1513f26744167065e0d create mode 100644 test/fuzz/rpc/jsonrpc/server/testdata/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176 diff --git a/test/fuzz/rpc/jsonrpc/server/fuzz_test.go b/test/fuzz/rpc/jsonrpc/server/fuzz_test.go new file mode 100644 index 000000000..005248375 --- /dev/null +++ b/test/fuzz/rpc/jsonrpc/server/fuzz_test.go @@ -0,0 +1,33 @@ +package server_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/test/fuzz/rpc/jsonrpc/server" +) + +const testdataDir = "testdata" + +func TestServerOnTestData(t *testing.T) { + entries, err := os.ReadDir(testdataDir) + require.NoError(t, err) + + for _, e := range entries { + entry := e + t.Run(entry.Name(), func(t *testing.T) { + defer func() { + r := recover() + require.Nilf(t, r, "testdata test panic") + }() + f, err := os.Open(filepath.Join(testdataDir, entry.Name())) + require.NoError(t, err) + input, err := ioutil.ReadAll(f) + require.NoError(t, err) + server.Fuzz(input) + }) + } +} diff --git a/test/fuzz/rpc/jsonrpc/server/handler.go b/test/fuzz/rpc/jsonrpc/server/handler.go index cab1b6ed0..c30a8890c 100644 --- a/test/fuzz/rpc/jsonrpc/server/handler.go +++ b/test/fuzz/rpc/jsonrpc/server/handler.go @@ -1,4 +1,4 @@ -package handler +package server import ( "bytes" @@ -39,11 +39,26 @@ func Fuzz(data []byte) int { if err := res.Body.Close(); err != nil { panic(err) } - if len(blob) > 0 { - recv := new(types.RPCResponse) - if err := json.Unmarshal(blob, recv); err != nil { + if len(blob) == 0 { + return 1 + } + + if inputJSONIsMultiElementSlice(data) { + recv := []types.RPCResponse{} + if err := json.Unmarshal(blob, &recv); err != nil { panic(err) } + return 1 + } + recv := &types.RPCResponse{} + if err := json.Unmarshal(blob, recv); err != nil { + panic(err) } return 1 } + +func inputJSONIsMultiElementSlice(input []byte) bool { + slice := []interface{}{} + err := json.Unmarshal(input, &slice) + return err == nil && len(slice) > 1 +} diff --git a/test/fuzz/rpc/jsonrpc/server/testdata/1184f5b8d4b6dd08709cf1513f26744167065e0d b/test/fuzz/rpc/jsonrpc/server/testdata/1184f5b8d4b6dd08709cf1513f26744167065e0d new file mode 100644 index 000000000..6e7ea636e --- /dev/null +++ b/test/fuzz/rpc/jsonrpc/server/testdata/1184f5b8d4b6dd08709cf1513f26744167065e0d @@ -0,0 +1 @@ +[0] \ No newline at end of file diff --git a/test/fuzz/rpc/jsonrpc/server/testdata/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176 b/test/fuzz/rpc/jsonrpc/server/testdata/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176 new file mode 100644 index 000000000..0f7836d2f --- /dev/null +++ b/test/fuzz/rpc/jsonrpc/server/testdata/clusterfuzz-testcase-minimized-fuzz_rpc_jsonrpc_server-4738572803506176 @@ -0,0 +1 @@ +[{"iD":7},{"iD":7}] \ No newline at end of file