From da3449599f1306e2d36a6d4f5e0c9eaad01e4e44 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Thu, 25 Nov 2021 07:45:38 -0800 Subject: [PATCH 01/21] Update example code in OpenAPI docs. (#7318) The event examples for the query filter language were not updated after the change of key and value types from []byte to string. Also, the attributes need to be a slice not a bare value. --- rpc/openapi/openapi.yaml | 42 ++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/rpc/openapi/openapi.yaml b/rpc/openapi/openapi.yaml index 83d85be8f..9e4f79dfa 100644 --- a/rpc/openapi/openapi.yaml +++ b/rpc/openapi/openapi.yaml @@ -296,28 +296,28 @@ paths: Events: []abci.Event{ { Type: "rewards.withdraw", - Attributes: abci.EventAttribute{ - {Key: []byte("address"), Value: []byte("AddrA"), Index: true}, - {Key: []byte("source"), Value: []byte("SrcX"), Index: true}, - {Key: []byte("amount"), Value: []byte("..."), Index: true}, - {Key: []byte("balance"), Value: []byte("..."), Index: true}, + Attributes: []abci.EventAttribute{ + {Key: "address", Value: "AddrA", Index: true}, + {Key: "source", Value: "SrcX", Index: true}, + {Key: "amount", Value: "...", Index: true}, + {Key: "balance", Value: "...", Index: true}, }, }, { Type: "rewards.withdraw", - Attributes: abci.EventAttribute{ - {Key: []byte("address"), Value: []byte("AddrB"), Index: true}, - {Key: []byte("source"), Value: []byte("SrcY"), Index: true}, - {Key: []byte("amount"), Value: []byte("..."), Index: true}, - {Key: []byte("balance"), Value: []byte("..."), Index: true}, + Attributes: []abci.EventAttribute{ + {Key: "address", Value: "AddrB", Index: true}, + {Key: "source", Value: "SrcY", Index: true}, + {Key: "amount", Value: "...", Index: true}, + {Key: "balance", Value: "...", Index: true}, }, }, { Type: "transfer", - Attributes: abci.EventAttribute{ - {Key: []byte("sender"), Value: []byte("AddrC"), Index: true}, - {Key: []byte("recipient"), Value: []byte("AddrD"), Index: true}, - {Key: []byte("amount"), Value: []byte("..."), Index: true}, + Attributes: []abci.EventAttribute{ + {Key: "sender", Value: "AddrC", Index: true}, + {Key: "recipient", Value: "AddrD", Index: true}, + {Key: "amount", Value: "...", Index: true}, }, }, }, @@ -1423,25 +1423,25 @@ components: example: "0" total_snapshots: type: string - example: "10" + example: "10" chunk_process_avg_time: type: string - example: "1000000000" + example: "1000000000" snapshot_height: type: string - example: "1262196" + example: "1262196" snapshot_chunks_count: type: string - example: "10" + example: "10" snapshot_chunks_total: type: string - example: "100" + example: "100" backfilled_blocks: type: string - example: "10" + example: "10" backfill_blocks_total: type: string - example: "100" + example: "100" ValidatorInfo: type: object properties: From a36dd49eae474e2b07b8c982e2a2ee28c152f126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20P=C4=99dziwiatr?= <84311757+ppedziwiatr@users.noreply.github.com> Date: Sat, 27 Nov 2021 18:16:15 +0100 Subject: [PATCH 02/21] docs: go tutorial fixed for 0.35.0 version (#7329) (#7330) --- docs/tutorials/go.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/go.md b/docs/tutorials/go.md index 9461f2cea..6614cffe5 100644 --- a/docs/tutorials/go.md +++ b/docs/tutorials/go.md @@ -367,7 +367,7 @@ func main() { flag.Parse() - logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + logger := log.MustNewDefaultLogger(log.LogFormatPlain, log.LogLevelInfo, false) server := abciserver.NewSocketServer(socketAddr, app) server.SetLogger(logger) @@ -438,7 +438,7 @@ This should create a `go.mod` file. The current tutorial only works with the master branch of Tendermint, so let's make sure we're using the latest version: ```sh -go get github.com/tendermint/tendermint@master +go get github.com/tendermint/tendermint@97a3e44e0724f2017079ce24d36433f03124c09e ``` This will populate the `go.mod` with a release number followed by a hash for Tendermint. From d1f07047ec8c765b44632cb6986a205bafc19cc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 13:04:54 +0000 Subject: [PATCH 03/21] build(deps): Bump actions/cache from 2.1.6 to 2.1.7 (#7334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 2.1.6 to 2.1.7.
Release notes

Sourced from actions/cache's releases.

v2.1.7

Support 10GB cache upload using the latest version 1.0.8 of @actions/cache

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=2.1.6&new-version=2.1.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 83f753a5e..df069382c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: - name: install run: make install install_abci if: "env.GIT_DIFF != ''" - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} @@ -36,7 +36,7 @@ jobs: ${{ runner.os }}-go- if: env.GIT_DIFF # Cache binaries for use by other jobs - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/bin key: ${{ runner.os }}-${{ github.sha }}-tm-binary @@ -57,14 +57,14 @@ jobs: **/**.go go.mod go.sum - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- if: env.GIT_DIFF - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/bin key: ${{ runner.os }}-${{ github.sha }}-tm-binary @@ -88,14 +88,14 @@ jobs: **/**.go go.mod go.sum - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/pkg/mod key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- if: env.GIT_DIFF - - uses: actions/cache@v2.1.6 + - uses: actions/cache@v2.1.7 with: path: ~/go/bin key: ${{ runner.os }}-${{ github.sha }}-tm-binary From 1dca1a8f97b112f765d7b7de471ec8e6686bf1c2 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 29 Nov 2021 13:08:48 -0800 Subject: [PATCH 04/21] Performance improvements for the event query API (#7319) Rework the implementation of event query parsing and execution to improve performance and reduce memory usage. Previous memory and CPU profiles of the pubsub service showed query processing as a significant hotspot. While we don't have evidence that this is visibly hurting users, fixing it is fairly easy and self-contained. Updates #6439. Typical benchmark results comparing the original implementation (PEG) with the reworked implementation (Custom): ``` TEST TIME/OP BYTES/OP ALLOCS/OP SPEEDUP MEM SAVING BenchmarkParsePEG-12 51716 ns 526832 27 BenchmarkParseCustom-12 2167 ns 4616 17 23.8x 99.1% BenchmarkMatchPEG-12 3086 ns 1097 22 BenchmarkMatchCustom-12 294.2 ns 64 3 10.5x 94.1% ``` Components: * Add a basic parsing benchmark. * Move the original query implementation to a subdirectory. * Add lexical scanner for Query expressions. * Add a parser for Query expressions. * Implement query compiler. * Add test cases based on OpenAPI examples. * Add MustCompile to replace the original MustParse, and update usage. --- .github/workflows/coverage.yml | 2 +- abci/client/mocks/client.go | 16 - internal/eventbus/event_bus_test.go | 12 +- internal/inspect/inspect_test.go | 2 +- internal/state/indexer/block/kv/kv.go | 22 +- internal/state/indexer/block/kv/kv_test.go | 18 +- internal/state/indexer/block/kv/util.go | 8 +- internal/state/indexer/query_range.go | 44 +- internal/state/indexer/sink/kv/kv_test.go | 26 +- internal/state/indexer/tx/kv/kv.go | 36 +- internal/state/indexer/tx/kv/kv_bench_test.go | 2 +- internal/state/indexer/tx/kv/kv_test.go | 10 +- libs/pubsub/example_test.go | 2 +- libs/pubsub/pubsub_test.go | 43 +- libs/pubsub/query/bench_test.go | 84 ++ libs/pubsub/query/{ => oldquery}/Makefile | 3 +- libs/pubsub/query/{ => oldquery}/empty.go | 0 .../pubsub/query/{ => oldquery}/empty_test.go | 2 +- .../query/{ => oldquery}/fuzz_test/main.go | 2 +- .../query/{ => oldquery}/parser_test.go | 2 +- libs/pubsub/query/oldquery/peg.go | 3 + libs/pubsub/query/oldquery/query.go | 527 ++++++++++++ libs/pubsub/query/{ => oldquery}/query.peg | 0 libs/pubsub/query/{ => oldquery}/query.peg.go | 0 libs/pubsub/query/oldquery/query_test.go | 205 +++++ libs/pubsub/query/peg.go | 3 - libs/pubsub/query/query.go | 760 +++++++----------- libs/pubsub/query/query_test.go | 457 ++++++----- libs/pubsub/query/syntax/doc.go | 34 + libs/pubsub/query/syntax/parser.go | 213 +++++ libs/pubsub/query/syntax/scanner.go | 312 +++++++ libs/pubsub/query/syntax/syntax_test.go | 190 +++++ types/events.go | 4 +- types/events_test.go | 6 +- 34 files changed, 2219 insertions(+), 831 deletions(-) create mode 100644 libs/pubsub/query/bench_test.go rename libs/pubsub/query/{ => oldquery}/Makefile (81%) rename libs/pubsub/query/{ => oldquery}/empty.go (100%) rename libs/pubsub/query/{ => oldquery}/empty_test.go (92%) rename libs/pubsub/query/{ => oldquery}/fuzz_test/main.go (85%) rename libs/pubsub/query/{ => oldquery}/parser_test.go (97%) create mode 100644 libs/pubsub/query/oldquery/peg.go create mode 100644 libs/pubsub/query/oldquery/query.go rename libs/pubsub/query/{ => oldquery}/query.peg (100%) rename libs/pubsub/query/{ => oldquery}/query.peg.go (100%) create mode 100644 libs/pubsub/query/oldquery/query_test.go delete mode 100644 libs/pubsub/query/peg.go create mode 100644 libs/pubsub/query/syntax/doc.go create mode 100644 libs/pubsub/query/syntax/parser.go create mode 100644 libs/pubsub/query/syntax/scanner.go create mode 100644 libs/pubsub/query/syntax/syntax_test.go diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1e2749b3c..491836106 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -88,7 +88,7 @@ jobs: go-version: "1.17" - name: test & coverage report creation run: | - cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 8m -race -coverprofile=${{ matrix.part }}profile.out + cat pkgs.txt.part.${{ matrix.part }} | xargs go test -mod=readonly -timeout 15m -race -coverprofile=${{ matrix.part }}profile.out if: env.GIT_DIFF - uses: actions/upload-artifact@v2 with: diff --git a/abci/client/mocks/client.go b/abci/client/mocks/client.go index f0d82a50e..e727fd544 100644 --- a/abci/client/mocks/client.go +++ b/abci/client/mocks/client.go @@ -680,22 +680,6 @@ func (_m *Client) QuerySync(_a0 context.Context, _a1 types.RequestQuery) (*types return r0, r1 } -// Quit provides a mock function with given fields: -func (_m *Client) Quit() <-chan struct{} { - ret := _m.Called() - - var r0 <-chan struct{} - if rf, ok := ret.Get(0).(func() <-chan struct{}); ok { - r0 = rf() - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(<-chan struct{}) - } - } - - return r0 -} - // SetResponseCallback provides a mock function with given fields: _a0 func (_m *Client) SetResponseCallback(_a0 abciclient.Callback) { _m.Called(_a0) diff --git a/internal/eventbus/event_bus_test.go b/internal/eventbus/event_bus_test.go index 3e9069718..c65c8cdd6 100644 --- a/internal/eventbus/event_bus_test.go +++ b/internal/eventbus/event_bus_test.go @@ -38,7 +38,7 @@ func TestEventBusPublishEventTx(t *testing.T) { query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X' AND testType.baz=1", tx.Hash()) txsSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: "test", - Query: tmquery.MustParse(query), + Query: tmquery.MustCompile(query), }) require.NoError(t, err) @@ -96,7 +96,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { query := "tm.event='NewBlock' AND testType.baz=1 AND testType.foz=2" blocksSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: "test", - Query: tmquery.MustParse(query), + Query: tmquery.MustCompile(query), }) require.NoError(t, err) @@ -205,7 +205,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { sub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: fmt.Sprintf("client-%d", i), - Query: tmquery.MustParse(tc.query), + Query: tmquery.MustCompile(tc.query), }) require.NoError(t, err) @@ -269,7 +269,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { query := "tm.event='NewBlockHeader' AND testType.baz=1 AND testType.foz=2" headersSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: "test", - Query: tmquery.MustParse(query), + Query: tmquery.MustCompile(query), }) require.NoError(t, err) @@ -312,7 +312,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { const query = `tm.event='NewEvidence'` evSub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: "test", - Query: tmquery.MustParse(query), + Query: tmquery.MustCompile(query), }) require.NoError(t, err) @@ -352,7 +352,7 @@ func TestEventBusPublish(t *testing.T) { sub, err := eventBus.SubscribeWithArgs(ctx, tmpubsub.SubscribeArgs{ ClientID: "test", - Query: tmquery.Empty{}, + Query: tmquery.All, Limit: numEventsExpected, }) require.NoError(t, err) diff --git a/internal/inspect/inspect_test.go b/internal/inspect/inspect_test.go index 15a555ab0..ff6ade0d0 100644 --- a/internal/inspect/inspect_test.go +++ b/internal/inspect/inspect_test.go @@ -115,7 +115,7 @@ func TestBlock(t *testing.T) { func TestTxSearch(t *testing.T) { testHash := []byte("test") testTx := []byte("tx") - testQuery := fmt.Sprintf("tx.hash='%s'", string(testHash)) + testQuery := fmt.Sprintf("tx.hash = '%s'", string(testHash)) testTxResult := &abcitypes.TxResult{ Height: 1, Index: 100, diff --git a/internal/state/indexer/block/kv/kv.go b/internal/state/indexer/block/kv/kv.go index d52f06c96..26fdcf1fc 100644 --- a/internal/state/indexer/block/kv/kv.go +++ b/internal/state/indexer/block/kv/kv.go @@ -14,6 +14,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -91,10 +92,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, default: } - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("failed to parse query conditions: %w", err) - } + conditions := q.Syntax() // If there is an exact height query, return the result immediately // (if it exists). @@ -158,7 +156,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, continue } - startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand)) + startKey, err := orderedcode.Append(nil, c.Tag, c.Arg.Value()) if err != nil { return nil, err } @@ -327,7 +325,7 @@ iter: // matched. func (idx *BlockerIndexer) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHeights map[string][]byte, firstRun bool, @@ -342,7 +340,7 @@ func (idx *BlockerIndexer) match( tmpHeights := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(idx.store, startKeyBz) if err != nil { return nil, fmt.Errorf("failed to create prefix iterator: %w", err) @@ -361,8 +359,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpExists: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TExists: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -389,8 +387,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpContains: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TContains: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -408,7 +406,7 @@ func (idx *BlockerIndexer) match( continue } - if strings.Contains(eventValue, c.Operand.(string)) { + if strings.Contains(eventValue, c.Arg.Value()) { tmpHeights[string(it.Value())] = it.Value() } diff --git a/internal/state/indexer/block/kv/kv_test.go b/internal/state/indexer/block/kv/kv_test.go index 024df332c..650723dbf 100644 --- a/internal/state/indexer/block/kv/kv_test.go +++ b/internal/state/indexer/block/kv/kv_test.go @@ -94,39 +94,39 @@ func TestBlockIndexer(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } diff --git a/internal/state/indexer/block/kv/util.go b/internal/state/indexer/block/kv/util.go index c0b88018e..fff88046c 100644 --- a/internal/state/indexer/block/kv/util.go +++ b/internal/state/indexer/block/kv/util.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/google/orderedcode" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -85,10 +85,10 @@ func parseValueFromEventKey(key []byte) (string, error) { return eventValue, nil } -func lookForHeight(conditions []query.Condition) (int64, bool) { +func lookForHeight(conditions []syntax.Condition) (int64, bool) { for _, c := range conditions { - if c.CompositeKey == types.BlockHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64), true + if c.Tag == types.BlockHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()), true } } diff --git a/internal/state/indexer/query_range.go b/internal/state/indexer/query_range.go index b4edf53c5..4c026955d 100644 --- a/internal/state/indexer/query_range.go +++ b/internal/state/indexer/query_range.go @@ -3,7 +3,7 @@ package indexer import ( "time" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) // QueryRanges defines a mapping between a composite event key and a QueryRange. @@ -77,32 +77,32 @@ func (qr QueryRange) UpperBoundValue() interface{} { // LookForRanges returns a mapping of QueryRanges and the matching indexes in // the provided query conditions. -func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes []int) { +func LookForRanges(conditions []syntax.Condition) (ranges QueryRanges, indexes []int) { ranges = make(QueryRanges) for i, c := range conditions { if IsRangeOperation(c.Op) { - r, ok := ranges[c.CompositeKey] + r, ok := ranges[c.Tag] if !ok { - r = QueryRange{Key: c.CompositeKey} + r = QueryRange{Key: c.Tag} } switch c.Op { - case query.OpGreater: - r.LowerBound = c.Operand + case syntax.TGt: + r.LowerBound = conditionArg(c) - case query.OpGreaterEqual: + case syntax.TGeq: r.IncludeLowerBound = true - r.LowerBound = c.Operand + r.LowerBound = conditionArg(c) - case query.OpLess: - r.UpperBound = c.Operand + case syntax.TLt: + r.UpperBound = conditionArg(c) - case query.OpLessEqual: + case syntax.TLeq: r.IncludeUpperBound = true - r.UpperBound = c.Operand + r.UpperBound = conditionArg(c) } - ranges[c.CompositeKey] = r + ranges[c.Tag] = r indexes = append(indexes, i) } } @@ -112,12 +112,26 @@ func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes [] // IsRangeOperation returns a boolean signifying if a query Operator is a range // operation or not. -func IsRangeOperation(op query.Operator) bool { +func IsRangeOperation(op syntax.Token) bool { switch op { - case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: + case syntax.TGt, syntax.TGeq, syntax.TLt, syntax.TLeq: return true default: return false } } + +func conditionArg(c syntax.Condition) interface{} { + if c.Arg == nil { + return nil + } + switch c.Arg.Type { + case syntax.TNumber: + return int64(c.Arg.Number()) + case syntax.TTime, syntax.TDate: + return c.Arg.Time() + default: + return c.Arg.Value() // string + } +} diff --git a/internal/state/indexer/sink/kv/kv_test.go b/internal/state/indexer/sink/kv/kv_test.go index 7d7552946..47b1f5364 100644 --- a/internal/state/indexer/sink/kv/kv_test.go +++ b/internal/state/indexer/sink/kv/kv_test.go @@ -111,39 +111,39 @@ func TestBlockFuncs(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } @@ -175,7 +175,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number = 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -249,7 +249,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.SearchTxEvents(ctx, query.MustParse(tc.q)) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -273,7 +273,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -330,7 +330,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.SearchTxEvents(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.SearchTxEvents(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/internal/state/indexer/tx/kv/kv.go b/internal/state/indexer/tx/kv/kv.go index f0550f8f3..4bcff958b 100644 --- a/internal/state/indexer/tx/kv/kv.go +++ b/internal/state/indexer/tx/kv/kv.go @@ -14,6 +14,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" indexer "github.com/tendermint/tendermint/internal/state/indexer" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -148,10 +149,7 @@ func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResul filteredHashes := make(map[string][]byte) // get a list of conditions (like "tx.height > 5") - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("error during parsing conditions from query: %w", err) - } + conditions := q.Syntax() // if there is a hash condition, return the result immediately hash, ok, err := lookForHash(conditions) @@ -238,10 +236,10 @@ hashes: return results, nil } -func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { +func lookForHash(conditions []syntax.Condition) (hash []byte, ok bool, err error) { for _, c := range conditions { - if c.CompositeKey == types.TxHashKey { - decoded, err := hex.DecodeString(c.Operand.(string)) + if c.Tag == types.TxHashKey { + decoded, err := hex.DecodeString(c.Arg.Value()) return decoded, true, err } } @@ -249,10 +247,10 @@ func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) } // lookForHeight returns a height if there is an "height=X" condition. -func lookForHeight(conditions []query.Condition) (height int64) { +func lookForHeight(conditions []syntax.Condition) (height int64) { for _, c := range conditions { - if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64) + if c.Tag == types.TxHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()) } } return 0 @@ -265,7 +263,7 @@ func lookForHeight(conditions []query.Condition) (height int64) { // NOTE: filteredHashes may be empty if no previous condition has matched. func (txi *TxIndex) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHashes map[string][]byte, firstRun bool, @@ -279,7 +277,7 @@ func (txi *TxIndex) match( tmpHashes := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(txi.store, startKeyBz) if err != nil { panic(err) @@ -301,10 +299,10 @@ func (txi *TxIndex) match( panic(err) } - case c.Op == query.OpExists: + case c.Op == syntax.TExists: // XXX: can't use startKeyBz here because c.Operand is nil // (e.g. "account.owner//" won't match w/ a single row) - it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.Tag)) if err != nil { panic(err) } @@ -325,11 +323,11 @@ func (txi *TxIndex) match( panic(err) } - case c.Op == query.OpContains: + case c.Op == syntax.TContains: // XXX: startKey does not apply here. // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" - it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, prefixFromCompositeKey(c.Tag)) if err != nil { panic(err) } @@ -341,7 +339,7 @@ func (txi *TxIndex) match( if err != nil { continue } - if strings.Contains(value, c.Operand.(string)) { + if strings.Contains(value, c.Arg.Value()) { tmpHashes[string(it.Value())] = it.Value() } @@ -577,8 +575,8 @@ func prefixFromCompositeKeyAndValue(compositeKey, value string) []byte { } // a small utility function for getting a keys prefix based on a condition and a height -func prefixForCondition(c query.Condition, height int64) []byte { - key := prefixFromCompositeKeyAndValue(c.CompositeKey, fmt.Sprintf("%v", c.Operand)) +func prefixForCondition(c syntax.Condition, height int64) []byte { + key := prefixFromCompositeKeyAndValue(c.Tag, c.Arg.Value()) if height > 0 { var err error key, err = orderedcode.Append(key, height) diff --git a/internal/state/indexer/tx/kv/kv_bench_test.go b/internal/state/indexer/tx/kv/kv_bench_test.go index e8504ebcc..b8f72b2fa 100644 --- a/internal/state/indexer/tx/kv/kv_bench_test.go +++ b/internal/state/indexer/tx/kv/kv_bench_test.go @@ -60,7 +60,7 @@ func BenchmarkTxSearch(b *testing.B) { } } - txQuery := query.MustParse("transfer.address = 'address_43' AND transfer.amount = 50") + txQuery := query.MustCompile(`transfer.address = 'address_43' AND transfer.amount = 50`) b.ResetTimer() diff --git a/internal/state/indexer/tx/kv/kv_test.go b/internal/state/indexer/tx/kv/kv_test.go index 985d58f42..e65f9ca2d 100644 --- a/internal/state/indexer/tx/kv/kv_test.go +++ b/internal/state/indexer/tx/kv/kv_test.go @@ -131,7 +131,7 @@ func TestTxSearch(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) @@ -157,7 +157,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.Search(ctx, query.MustParse("account.number = 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -230,7 +230,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -254,7 +254,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -311,7 +311,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index 4d317215f..0d608ceff 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -18,7 +18,7 @@ func TestExample(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: "example-client", - Query: query.MustParse("abci.account.name='John'"), + Query: query.MustCompile(`abci.account.name='John'`), })) events := []abci.Event{ diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index be7f3e6e0..926a263a4 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -27,7 +27,7 @@ func TestSubscribeWithArgs(t *testing.T) { t.Run("DefaultLimit", func(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.Empty{}, + Query: query.All, })) require.Equal(t, 1, s.NumClients()) @@ -39,7 +39,7 @@ func TestSubscribeWithArgs(t *testing.T) { t.Run("PositiveLimit", func(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID + "-2", - Query: query.Empty{}, + Query: query.All, Limit: 10, })) require.NoError(t, s.Publish(ctx, "Aggamon")) @@ -73,9 +73,9 @@ func TestObserverErrors(t *testing.T) { s := newTestServer(ctx, t) - require.Error(t, s.Observe(ctx, nil, query.Empty{})) + require.Error(t, s.Observe(ctx, nil, query.All)) require.NoError(t, s.Observe(ctx, func(pubsub.Message) error { return nil })) - require.Error(t, s.Observe(ctx, func(pubsub.Message) error { return nil }, query.Empty{})) + require.Error(t, s.Observe(ctx, func(pubsub.Message) error { return nil }, query.All)) } func TestPublishDoesNotBlock(t *testing.T) { @@ -86,7 +86,7 @@ func TestPublishDoesNotBlock(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.Empty{}, + Query: query.All, })) published := make(chan struct{}) go func() { @@ -119,7 +119,7 @@ func TestSubscribeErrors(t *testing.T) { t.Run("NegativeLimitErr", func(t *testing.T) { _, err := s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.Empty{}, + Query: query.All, Limit: -5, }) require.Error(t, err) @@ -134,7 +134,7 @@ func TestSlowSubscriber(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.Empty{}, + Query: query.All, })) require.NoError(t, s.Publish(ctx, "Fat Cobra")) @@ -155,7 +155,7 @@ func TestDifferentClients(t *testing.T) { sub1 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: "client-1", - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) events := []abci.Event{{ @@ -168,7 +168,7 @@ func TestDifferentClients(t *testing.T) { sub2 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: "client-2", - Query: query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), + Query: query.MustCompile(`tm.events.type='NewBlock' AND abci.account.name='Igor'`), })) events = []abci.Event{ @@ -188,7 +188,8 @@ func TestDifferentClients(t *testing.T) { sub3 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: "client-3", - Query: query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), + Query: query.MustCompile( + `tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10`), })) events = []abci.Event{{ @@ -218,7 +219,7 @@ func TestSubscribeDuplicateKeys(t *testing.T) { for i, tc := range testCases { id := fmt.Sprintf("client-%d", i) - q := query.MustParse(tc.query) + q := query.MustCompile(tc.query) t.Run(id, func(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: id, @@ -261,7 +262,7 @@ func TestClientSubscribesTwice(t *testing.T) { s := newTestServer(ctx, t) - q := query.MustParse("tm.events.type='NewBlock'") + q := query.MustCompile(`tm.events.type='NewBlock'`) events := []abci.Event{{ Type: "tm.events", Attributes: []abci.EventAttribute{{Key: "type", Value: "NewBlock"}}, @@ -298,13 +299,13 @@ func TestUnsubscribe(t *testing.T) { sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) // Removing the subscription we just made should succeed. require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) // Publishing should still work. @@ -322,15 +323,15 @@ func TestClientUnsubscribesTwice(t *testing.T) { newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) require.ErrorIs(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), }), pubsub.ErrSubscriptionNotFound) require.ErrorIs(t, s.UnsubscribeAll(ctx, clientID), pubsub.ErrSubscriptionNotFound) } @@ -343,13 +344,13 @@ func TestResubscribe(t *testing.T) { args := pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.Empty{}, + Query: query.All, } newTestSub(t).must(s.SubscribeWithArgs(ctx, args)) require.NoError(t, s.Unsubscribe(ctx, pubsub.UnsubscribeArgs{ Subscriber: clientID, - Query: query.Empty{}, + Query: query.All, })) sub := newTestSub(t).must(s.SubscribeWithArgs(ctx, args)) @@ -366,11 +367,11 @@ func TestUnsubscribeAll(t *testing.T) { sub1 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.MustParse("tm.events.type='NewBlock'"), + Query: query.MustCompile(`tm.events.type='NewBlock'`), })) sub2 := newTestSub(t).must(s.SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ ClientID: clientID, - Query: query.MustParse("tm.events.type='NewBlockHeader'"), + Query: query.MustCompile(`tm.events.type='NewBlockHeader'`), })) require.NoError(t, s.UnsubscribeAll(ctx, clientID)) diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go new file mode 100644 index 000000000..2e49b58ae --- /dev/null +++ b/libs/pubsub/query/bench_test.go @@ -0,0 +1,84 @@ +package query_test + +import ( + "testing" + + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query" + oldquery "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" +) + +const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` + +var testEvents = []types.Event{ + { + Type: "tm.events", + Attributes: []types.EventAttribute{{ + Key: "index", + Value: "25", + }, { + Key: "type", + Value: "NewBlock", + }}, + }, + { + Type: "abci.account", + Attributes: []types.EventAttribute{{ + Key: "name", + Value: "Anya", + }, { + Key: "name", + Value: "Igor", + }}, + }, +} + +func BenchmarkParsePEG(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := oldquery.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkParseCustom(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMatchPEG(b *testing.B) { + q, err := oldquery.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} + +func BenchmarkMatchCustom(b *testing.B) { + q, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} diff --git a/libs/pubsub/query/Makefile b/libs/pubsub/query/oldquery/Makefile similarity index 81% rename from libs/pubsub/query/Makefile rename to libs/pubsub/query/oldquery/Makefile index aef42b2df..8ec72981e 100644 --- a/libs/pubsub/query/Makefile +++ b/libs/pubsub/query/oldquery/Makefile @@ -1,6 +1,5 @@ gen_query_parser: - go get -u -v github.com/pointlander/peg - peg -inline -switch query.peg + go generate . fuzzy_test: go get -u -v github.com/dvyukov/go-fuzz/go-fuzz diff --git a/libs/pubsub/query/empty.go b/libs/pubsub/query/oldquery/empty.go similarity index 100% rename from libs/pubsub/query/empty.go rename to libs/pubsub/query/oldquery/empty.go diff --git a/libs/pubsub/query/empty_test.go b/libs/pubsub/query/oldquery/empty_test.go similarity index 92% rename from libs/pubsub/query/empty_test.go rename to libs/pubsub/query/oldquery/empty_test.go index 4bb3067d6..81f313d30 100644 --- a/libs/pubsub/query/empty_test.go +++ b/libs/pubsub/query/oldquery/empty_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) func TestEmptyQueryMatchesAnything(t *testing.T) { diff --git a/libs/pubsub/query/fuzz_test/main.go b/libs/pubsub/query/oldquery/fuzz_test/main.go similarity index 85% rename from libs/pubsub/query/fuzz_test/main.go rename to libs/pubsub/query/oldquery/fuzz_test/main.go index 7a46116b5..8bbcaa25f 100644 --- a/libs/pubsub/query/fuzz_test/main.go +++ b/libs/pubsub/query/oldquery/fuzz_test/main.go @@ -3,7 +3,7 @@ package fuzz_test import ( "fmt" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) func Fuzz(data []byte) int { diff --git a/libs/pubsub/query/parser_test.go b/libs/pubsub/query/oldquery/parser_test.go similarity index 97% rename from libs/pubsub/query/parser_test.go rename to libs/pubsub/query/oldquery/parser_test.go index a08a0d16d..661a80f93 100644 --- a/libs/pubsub/query/parser_test.go +++ b/libs/pubsub/query/oldquery/parser_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/tendermint/tendermint/libs/pubsub/query" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) // TODO: fuzzy testing? diff --git a/libs/pubsub/query/oldquery/peg.go b/libs/pubsub/query/oldquery/peg.go new file mode 100644 index 000000000..bf6789b58 --- /dev/null +++ b/libs/pubsub/query/oldquery/peg.go @@ -0,0 +1,3 @@ +package query + +//go:generate go run github.com/pointlander/peg@v1.0.0 -inline -switch query.peg diff --git a/libs/pubsub/query/oldquery/query.go b/libs/pubsub/query/oldquery/query.go new file mode 100644 index 000000000..7b1dfe0f9 --- /dev/null +++ b/libs/pubsub/query/oldquery/query.go @@ -0,0 +1,527 @@ +// Package query provides a parser for a custom query format: +// +// abci.invoice.number=22 AND abci.invoice.owner=Ivan +// +// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. +// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics +// +// It has a support for numbers (integer and floating point), dates and times. +package query + +import ( + "fmt" + "reflect" + "regexp" + "strconv" + "strings" + "time" + + "github.com/tendermint/tendermint/abci/types" +) + +var ( + numRegex = regexp.MustCompile(`([0-9\.]+)`) +) + +// Query holds the query string and the query parser. +type Query struct { + str string + parser *QueryParser +} + +// Condition represents a single condition within a query and consists of composite key +// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). +type Condition struct { + CompositeKey string + Op Operator + Operand interface{} +} + +// New parses the given string and returns a query or error if the string is +// invalid. +func New(s string) (*Query, error) { + p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} + p.Init() + if err := p.Parse(); err != nil { + return nil, err + } + return &Query{str: s, parser: p}, nil +} + +// MustParse turns the given string into a query or panics; for tests or others +// cases where you know the string is valid. +func MustParse(s string) *Query { + q, err := New(s) + if err != nil { + panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + } + return q +} + +// String returns the original string. +func (q *Query) String() string { + return q.str +} + +// Operator is an operator that defines some kind of relation between composite key and +// operand (equality, etc.). +type Operator uint8 + +const ( + // "<=" + OpLessEqual Operator = iota + // ">=" + OpGreaterEqual + // "<" + OpLess + // ">" + OpGreater + // "=" + OpEqual + // "CONTAINS"; used to check if a string contains a certain sub string. + OpContains + // "EXISTS"; used to check if a certain event attribute is present. + OpExists +) + +const ( + // DateLayout defines a layout for all dates (`DATE date`) + DateLayout = "2006-01-02" + // TimeLayout defines a layout for all times (`TIME time`) + TimeLayout = time.RFC3339 +) + +// Conditions returns a list of conditions. It returns an error if there is any +// error with the provided grammar in the Query. +func (q *Query) Conditions() ([]Condition, error) { + var ( + eventAttr string + op Operator + ) + + conditions := make([]Condition, 0) + buffer, begin, end := q.parser.Buffer, 0, 0 + + // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") + for token := range q.parser.Tokens() { + switch token.pegRule { + case rulePegText: + begin, end = int(token.begin), int(token.end) + + case ruletag: + eventAttr = buffer[begin:end] + + case rulele: + op = OpLessEqual + + case rulege: + op = OpGreaterEqual + + case rulel: + op = OpLess + + case ruleg: + op = OpGreater + + case ruleequal: + op = OpEqual + + case rulecontains: + op = OpContains + + case ruleexists: + op = OpExists + conditions = append(conditions, Condition{eventAttr, op, nil}) + + case rulevalue: + // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") + valueWithoutSingleQuotes := buffer[begin+1 : end-1] + conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) + + case rulenumber: + number := buffer[begin:end] + if strings.ContainsAny(number, ".") { // if it looks like a floating-point number + value, err := strconv.ParseFloat(number, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", + err, number, + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } else { + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", + err, number, + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } + + case ruletime: + value, err := time.Parse(TimeLayout, buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + + case ruledate: + value, err := time.Parse("2006-01-02", buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return nil, err + } + + conditions = append(conditions, Condition{eventAttr, op, value}) + } + } + + return conditions, nil +} + +// Matches returns true if the query matches against any event in the given set +// of events, false otherwise. For each event, a match exists if the query is +// matched against *any* value in a slice of values. An error is returned if +// any attempted event match returns an error. +// +// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. +// More examples could be found in parser_test.go and query_test.go. +func (q *Query) Matches(rawEvents []types.Event) (bool, error) { + if len(rawEvents) == 0 { + return false, nil + } + + events := flattenEvents(rawEvents) + + var ( + eventAttr string + op Operator + ) + + buffer, begin, end := q.parser.Buffer, 0, 0 + + // tokens must be in the following order: + + // tag ("tx.gas") -> operator ("=") -> operand ("7") + for token := range q.parser.Tokens() { + switch token.pegRule { + case rulePegText: + begin, end = int(token.begin), int(token.end) + + case ruletag: + eventAttr = buffer[begin:end] + + case rulele: + op = OpLessEqual + + case rulege: + op = OpGreaterEqual + + case rulel: + op = OpLess + + case ruleg: + op = OpGreater + + case ruleequal: + op = OpEqual + + case rulecontains: + op = OpContains + case ruleexists: + op = OpExists + if strings.Contains(eventAttr, ".") { + // Searching for a full "type.attribute" event. + _, ok := events[eventAttr] + if !ok { + return false, nil + } + } else { + foundEvent := false + + loop: + for compositeKey := range events { + if strings.Index(compositeKey, eventAttr) == 0 { + foundEvent = true + break loop + } + } + if !foundEvent { + return false, nil + } + } + + case rulevalue: + // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") + valueWithoutSingleQuotes := buffer[begin+1 : end-1] + + // see if the triplet (event attribute, operator, operand) matches any event + // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } + match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + + case rulenumber: + number := buffer[begin:end] + if strings.ContainsAny(number, ".") { // if it looks like a floating-point number + value, err := strconv.ParseFloat(number, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", + err, number, + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } else { + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", + err, number, + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } + + case ruletime: + value, err := time.Parse(TimeLayout, buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + + case ruledate: + value, err := time.Parse("2006-01-02", buffer[begin:end]) + if err != nil { + err = fmt.Errorf( + "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", + err, buffer[begin:end], + ) + return false, err + } + + match, err := match(eventAttr, op, reflect.ValueOf(value), events) + if err != nil { + return false, err + } + + if !match { + return false, nil + } + } + } + + return true, nil +} + +// match returns true if the given triplet (attribute, operator, operand) matches +// any value in an event for that attribute. If any match fails with an error, +// that error is returned. +// +// First, it looks up the key in the events and if it finds one, tries to compare +// all the values from it to the operand using the operator. +// +// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} +func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { + // look up the tag from the query in tags + values, ok := events[attr] + if !ok { + return false, nil + } + + for _, value := range values { + // return true if any value in the set of the event's values matches + match, err := matchValue(value, op, operand) + if err != nil { + return false, err + } + + if match { + return true, nil + } + } + + return false, nil +} + +// matchValue will attempt to match a string value against an operator an +// operand. A boolean is returned representing the match result. It will return +// an error if the value cannot be parsed and matched against the operand type. +func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { + switch operand.Kind() { + case reflect.Struct: // time + operandAsTime := operand.Interface().(time.Time) + + // try our best to convert value from events to time.Time + var ( + v time.Time + err error + ) + + if strings.ContainsAny(value, "T") { + v, err = time.Parse(TimeLayout, value) + } else { + v, err = time.Parse(DateLayout, value) + } + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) + } + + switch op { + case OpLessEqual: + return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil + case OpGreaterEqual: + return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil + case OpLess: + return v.Before(operandAsTime), nil + case OpGreater: + return v.After(operandAsTime), nil + case OpEqual: + return v.Equal(operandAsTime), nil + } + + case reflect.Float64: + var v float64 + + operandFloat64 := operand.Interface().(float64) + filteredValue := numRegex.FindString(value) + + // try our best to convert value from tags to float64 + v, err := strconv.ParseFloat(filteredValue, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) + } + + switch op { + case OpLessEqual: + return v <= operandFloat64, nil + case OpGreaterEqual: + return v >= operandFloat64, nil + case OpLess: + return v < operandFloat64, nil + case OpGreater: + return v > operandFloat64, nil + case OpEqual: + return v == operandFloat64, nil + } + + case reflect.Int64: + var v int64 + + operandInt := operand.Interface().(int64) + filteredValue := numRegex.FindString(value) + + // if value looks like float, we try to parse it as float + if strings.ContainsAny(filteredValue, ".") { + v1, err := strconv.ParseFloat(filteredValue, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) + } + + v = int64(v1) + } else { + var err error + // try our best to convert value from tags to int64 + v, err = strconv.ParseInt(filteredValue, 10, 64) + if err != nil { + return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) + } + } + + switch op { + case OpLessEqual: + return v <= operandInt, nil + case OpGreaterEqual: + return v >= operandInt, nil + case OpLess: + return v < operandInt, nil + case OpGreater: + return v > operandInt, nil + case OpEqual: + return v == operandInt, nil + } + + case reflect.String: + switch op { + case OpEqual: + return value == operand.String(), nil + case OpContains: + return strings.Contains(value, operand.String()), nil + } + + default: + return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) + } + + return false, nil +} + +func flattenEvents(events []types.Event) map[string][]string { + flattened := make(map[string][]string) + + for _, event := range events { + if len(event.Type) == 0 { + continue + } + + for _, attr := range event.Attributes { + if len(attr.Key) == 0 { + continue + } + + compositeEvent := fmt.Sprintf("%s.%s", event.Type, attr.Key) + flattened[compositeEvent] = append(flattened[compositeEvent], attr.Value) + } + } + + return flattened +} diff --git a/libs/pubsub/query/query.peg b/libs/pubsub/query/oldquery/query.peg similarity index 100% rename from libs/pubsub/query/query.peg rename to libs/pubsub/query/oldquery/query.peg diff --git a/libs/pubsub/query/query.peg.go b/libs/pubsub/query/oldquery/query.peg.go similarity index 100% rename from libs/pubsub/query/query.peg.go rename to libs/pubsub/query/oldquery/query.peg.go diff --git a/libs/pubsub/query/oldquery/query_test.go b/libs/pubsub/query/oldquery/query_test.go new file mode 100644 index 000000000..9db7651c7 --- /dev/null +++ b/libs/pubsub/query/oldquery/query_test.go @@ -0,0 +1,205 @@ +package query_test + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" +) + +func expandEvents(flattenedEvents map[string][]string) []abci.Event { + events := make([]abci.Event, len(flattenedEvents)) + + for composite, values := range flattenedEvents { + tokens := strings.Split(composite, ".") + + attrs := make([]abci.EventAttribute, len(values)) + for i, v := range values { + attrs[i] = abci.EventAttribute{ + Key: tokens[len(tokens)-1], + Value: v, + } + } + + events = append(events, abci.Event{ + Type: strings.Join(tokens[:len(tokens)-1], "."), + Attributes: attrs, + }) + } + + return events +} + +func TestMatches(t *testing.T) { + var ( + txDate = "2017-01-01" + txTime = "2018-05-03T14:45:00Z" + ) + + testCases := []struct { + s string + events map[string][]string + matches bool + }{ + {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, true}, + {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, true}, + {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, true}, + {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, + {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, + {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false}, + {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, true}, + {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, true}, + {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, true}, + {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, true}, + {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, true}, + { + "oranges.kg < 4 AND watermellons.kg > 10", + map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, + true, + }, + {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false}, + { + "tx.date > DATE 2017-01-01", + map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, + true, + }, + {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, true}, + {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false}, + { + "tx.time >= TIME 2013-05-03T14:45:00Z", + map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, + true, + }, + {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false}, + {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, true}, + {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false}, + {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, true}, + { + "abci.owner.name = 'Ivan'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + true, + }, + { + "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + true, + }, + { + "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", + map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, + false, + }, + { + "tm.events.type='NewBlock'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "tm.events.type='NewBlock' AND app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + true, + }, + { + "tm.events.type='NewHeader' AND app.name = 'fuzzed'", + map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, + false, + }, + {"slash EXISTS", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"sl EXISTS", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"slash EXISTS", + map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, + "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, + false, + }, + {"slash.reason EXISTS AND slash.power > 1000", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, + true, + }, + {"slash.reason EXISTS AND slash.power > 1000", + map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, + false, + }, + {"slash.reason EXISTS", + map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, + "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, + false, + }, + } + + for _, tc := range testCases { + q, err := query.New(tc.s) + require.Nil(t, err) + require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + + rawEvents := expandEvents(tc.events) + match, err := q.Matches(rawEvents) + require.Nil(t, err, "Query '%s' should not error on input %v", tc.s, tc.events) + require.Equal(t, tc.matches, match, "Query '%s' on input %v: got %v, want %v", + tc.s, tc.events, match, tc.matches) + } +} + +func TestMustParse(t *testing.T) { + require.Panics(t, func() { query.MustParse("=") }) + require.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +} + +func TestConditions(t *testing.T) { + txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") + require.NoError(t, err) + + testCases := []struct { + s string + conditions []query.Condition + }{ + { + s: "tm.events.type='NewBlock'", + conditions: []query.Condition{ + {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, + }, + }, + { + s: "tx.gas > 7 AND tx.gas < 9", + conditions: []query.Condition{ + {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, + {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, + }, + }, + { + s: "tx.time >= TIME 2013-05-03T14:45:00Z", + conditions: []query.Condition{ + {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, + }, + }, + { + s: "slashing EXISTS", + conditions: []query.Condition{ + {CompositeKey: "slashing", Op: query.OpExists}, + }, + }, + } + + for _, tc := range testCases { + q, err := query.New(tc.s) + require.Nil(t, err) + + c, err := q.Conditions() + require.NoError(t, err) + require.Equal(t, tc.conditions, c) + } +} diff --git a/libs/pubsub/query/peg.go b/libs/pubsub/query/peg.go deleted file mode 100644 index 816589f02..000000000 --- a/libs/pubsub/query/peg.go +++ /dev/null @@ -1,3 +0,0 @@ -package query - -//go:generate peg -inline -switch query.peg diff --git a/libs/pubsub/query/query.go b/libs/pubsub/query/query.go index 7b1dfe0f9..e874f037c 100644 --- a/libs/pubsub/query/query.go +++ b/libs/pubsub/query/query.go @@ -1,527 +1,327 @@ -// Package query provides a parser for a custom query format: +// Package query implements the custom query format used to filter event +// subscriptions in Tendermint. // -// abci.invoice.number=22 AND abci.invoice.owner=Ivan +// Query expressions describe properties of events and their attributes, using +// strings like: // -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics +// abci.invoice.number = 22 AND abci.invoice.owner = 'Ivan' +// +// Query expressions can handle attribute values encoding numbers, strings, +// dates, and timestamps. The complete query grammar is described in the +// query/syntax package. // -// It has a support for numbers (integer and floating point), dates and times. package query import ( "fmt" - "reflect" "regexp" "strconv" "strings" "time" "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) +// All is a query that matches all events. +var All *Query -// Query holds the query string and the query parser. +// A Query is the compiled form of a query. type Query struct { - str string - parser *QueryParser + ast syntax.Query + conds []condition } -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - p.Init() - if err := p.Parse(); err != nil { +// New parses and compiles the query expression into an executable query. +func New(query string) (*Query, error) { + ast, err := syntax.Parse(query) + if err != nil { return nil, err } - return &Query{str: s, parser: p}, nil + return Compile(ast) } -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) +// MustCompile compiles the query expression into an executable query. +// In case of error, MustCompile will panic. +// +// This is intended for use in program initialization; use query.New if you +// need to check errors. +func MustCompile(query string) *Query { + q, err := New(query) if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + panic(err) } return q } -// String returns the original string. +// Compile compiles the given query AST so it can be used to match events. +func Compile(ast syntax.Query) (*Query, error) { + conds := make([]condition, len(ast)) + for i, q := range ast { + cond, err := compileCondition(q) + if err != nil { + return nil, fmt.Errorf("compile %s: %w", q, err) + } + conds[i] = cond + } + return &Query{ast: ast, conds: conds}, nil +} + +// Matches satisfies part of the pubsub.Query interface. This implementation +// never reports an error. A nil *Query matches all events. +func (q *Query) Matches(events []types.Event) (bool, error) { + if q == nil { + return true, nil + } + return q.matchesEvents(events), nil +} + +// String matches part of the pubsub.Query interface. func (q *Query) String() string { - return q.str + if q == nil { + return "" + } + return q.ast.String() } -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 - -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) - -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } +// Syntax returns the syntax tree representation of q. +func (q *Query) Syntax() syntax.Query { + if q == nil { + return nil } - - return conditions, nil + return q.ast } -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. -func (q *Query) Matches(rawEvents []types.Event) (bool, error) { - if len(rawEvents) == 0 { - return false, nil - } - - events := flattenEvents(rawEvents) - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } +// matchesEvents reports whether all the conditions match the given events. +func (q *Query) matchesEvents(events []types.Event) bool { + for _, cond := range q.conds { + if !cond.matchesAny(events) { + return false } } - - return true, nil + return len(events) != 0 } -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil - } - - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil - } - } - - return false, nil +// A condition is a compiled match condition. A condition matches an event if +// the event has the designated type, contains an attribute with the given +// name, and the match function returns true for the attribute value. +type condition struct { + tag string // e.g., "tx.hash" + match func(s string) bool } -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) - - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) - - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) - } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) - } - - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil - } - - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil - } - - case reflect.Int64: - var v int64 - - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) - - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } - - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } - - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } - - default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) +// findAttr returns a slice of attribute values from event matching the +// condition tag, and reports whether the event type strictly equals the +// condition tag. +func (c condition) findAttr(event types.Event) ([]string, bool) { + if !strings.HasPrefix(c.tag, event.Type) { + return nil, false // type does not match tag + } else if len(c.tag) == len(event.Type) { + return nil, true // type == tag } - - return false, nil + var vals []string + for _, attr := range event.Attributes { + fullName := event.Type + "." + attr.Key + if fullName == c.tag { + vals = append(vals, attr.Value) + } + } + return vals, false } -func flattenEvents(events []types.Event) map[string][]string { - flattened := make(map[string][]string) - +// matchesAny reports whether c matches at least one of the given events. +func (c condition) matchesAny(events []types.Event) bool { for _, event := range events { - if len(event.Type) == 0 { - continue - } - - for _, attr := range event.Attributes { - if len(attr.Key) == 0 { - continue - } - - compositeEvent := fmt.Sprintf("%s.%s", event.Type, attr.Key) - flattened[compositeEvent] = append(flattened[compositeEvent], attr.Value) + if c.matchesEvent(event) { + return true } } - - return flattened + return false +} + +// matchesEvent reports whether c matches the given event. +func (c condition) matchesEvent(event types.Event) bool { + vs, tagEqualsType := c.findAttr(event) + if len(vs) == 0 { + // As a special case, a condition tag that exactly matches the event type + // is matched against an empty string. This allows existence checks to + // work for type-only queries. + if tagEqualsType { + return c.match("") + } + return false + } + + // At this point, we have candidate values. + for _, v := range vs { + if c.match(v) { + return true + } + } + return false +} + +func compileCondition(cond syntax.Condition) (condition, error) { + out := condition{tag: cond.Tag} + + // Handle existence checks separately to simplify the logic below for + // comparisons that take arguments. + if cond.Op == syntax.TExists { + out.match = func(string) bool { return true } + return out, nil + } + + // All the other operators require an argument. + if cond.Arg == nil { + return condition{}, fmt.Errorf("missing argument for %v", cond.Op) + } + + // Precompile the argument value matcher. + argType := cond.Arg.Type + var argValue interface{} + + switch argType { + case syntax.TString: + argValue = cond.Arg.Value() + case syntax.TNumber: + argValue = cond.Arg.Number() + case syntax.TTime, syntax.TDate: + argValue = cond.Arg.Time() + default: + return condition{}, fmt.Errorf("unknown argument type %v", argType) + } + + mcons := opTypeMap[cond.Op][argType] + if mcons == nil { + return condition{}, fmt.Errorf("invalid op/arg combination (%v, %v)", cond.Op, argType) + } + out.match = mcons(argValue) + return out, nil +} + +// TODO(creachadair): The existing implementation allows anything number shaped +// to be treated as a number. This preserves the parts of that behavior we had +// tests for, but we should probably get rid of that. +var extractNum = regexp.MustCompile(`^\d+(\.\d+)?`) + +func parseNumber(s string) (float64, error) { + return strconv.ParseFloat(extractNum.FindString(s), 64) +} + +// A map of operator ⇒ argtype ⇒ match-constructor. +// An entry does not exist if the combination is not valid. +// +// Disable the dupl lint for this map. The result isn't even correct. +//nolint:dupl +var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{ + syntax.TContains: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { + return strings.Contains(s, v.(string)) + } + }, + }, + syntax.TEq: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { return s == v.(string) } + }, + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w == v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + }, + syntax.TLt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w < v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + }, + syntax.TLeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w <= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + }, + syntax.TGt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w > v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + }, + syntax.TGeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w >= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + }, } diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index 87f61aafe..883e771c6 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -6,242 +6,271 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -func expandEvents(flattenedEvents map[string][]string) []abci.Event { - events := make([]abci.Event, len(flattenedEvents)) +var _ pubsub.Query = (*query.Query)(nil) - for composite, values := range flattenedEvents { - tokens := strings.Split(composite, ".") - - attrs := make([]abci.EventAttribute, len(values)) - for i, v := range values { - attrs[i] = abci.EventAttribute{ - Key: tokens[len(tokens)-1], - Value: v, - } - } - - events = append(events, abci.Event{ - Type: strings.Join(tokens[:len(tokens)-1], "."), - Attributes: attrs, - }) - } - - return events +// Example events from the OpenAPI documentation: +// https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml +// +// Redactions: +// +// - Add an explicit "tm" event for the built-in attributes. +// - Remove Index fields (not relevant to tests). +// - Add explicit balance values (to use in tests). +// +var apiEvents = []types.Event{ + { + Type: "tm", + Attributes: []types.EventAttribute{ + {Key: "event", Value: "Tx"}, + {Key: "hash", Value: "XYZ"}, + {Key: "height", Value: "5"}, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + {Key: "address", Value: "AddrA"}, + {Key: "source", Value: "SrcX"}, + {Key: "amount", Value: "100"}, + {Key: "balance", Value: "1500"}, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + {Key: "address", Value: "AddrB"}, + {Key: "source", Value: "SrcY"}, + {Key: "amount", Value: "45"}, + {Key: "balance", Value: "999"}, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + {Key: "sender", Value: "AddrC"}, + {Key: "recipient", Value: "AddrD"}, + {Key: "amount", Value: "160"}, + }, + }, } -func TestMatches(t *testing.T) { +func TestCompiledMatches(t *testing.T) { var ( txDate = "2017-01-01" txTime = "2018-05-03T14:45:00Z" ) + //nolint:lll testCases := []struct { - s string - events map[string][]string - err bool - matches bool - matchErr bool + s string + events []types.Event + matches bool }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, false, true, false}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false, false, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, false, true, false}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, false, true, false}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, false, true, false}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, false, true, false}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - false, - true, - false, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false, false, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - false, - true, - false, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, false, true, false}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false, false, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - false, - true, - false, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false, false, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, false, true, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false, false, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true, false}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - false, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - false, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - false, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`), + true}, + {`tx.gas > 7`, + newTestEvents(`tx|gas=8`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8stake`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8.045`), + true}, + {`transfer.amount > 7.043`, + newTestEvents(`transfer|amount=8.045stake`), + true}, + {`transfer.amount > 8.045`, + newTestEvents(`transfer|amount=8.045stake`), + false}, + {`tx.gas > 7 AND tx.gas < 9`, + newTestEvents(`tx|gas=8`), + true}, + {`body.weight >= 3.5`, + newTestEvents(`body|weight=3.5`), + true}, + {`account.balance < 1000.0`, + newTestEvents(`account|balance=900`), + true}, + {`apples.kg <= 4`, + newTestEvents(`apples|kg=4.0`), + true}, + {`body.weight >= 4.5`, + newTestEvents(`body|weight=4.5`), + true}, + {`oranges.kg < 4 AND watermellons.kg > 10`, + newTestEvents(`oranges|kg=3`, `watermellons|kg=12`), + true}, + {`peaches.kg < 4`, + newTestEvents(`peaches|kg=5`), + false}, + {`tx.date > DATE 2017-01-01`, + newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)), + true}, + {`tx.date = DATE 2017-01-01`, + newTestEvents(`tx|date=` + txDate), + true}, + {`tx.date = DATE 2018-01-01`, + newTestEvents(`tx|date=` + txDate), + false}, + {`tx.time >= TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)), + true}, + {`tx.time = TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + txTime), + false}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`), + false}, + {`abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + false}, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewBlock' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewHeader' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + false}, + {`slash EXISTS`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=500`), + false}, + {`slash.reason EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + + // Test cases based on the OpenAPI examples. + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`, + apiEvents, true}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrA'`, + apiEvents, false}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrC'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`, + apiEvents, false}, } - for _, tc := range testCases { - q, err := query.New(tc.s) - if !tc.err { - require.Nil(t, err) - } - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + // NOTE: The original implementation allowed arbitrary prefix matches on + // attribute tags, e.g., "sl" would match "slash". + // + // That is weird and probably wrong: "foo.ba" should not match "foo.bar", + // or there is no way to distinguish the case where there were two values + // for "foo.bar" or one value each for "foo.ba" and "foo.bar". + // + // Apart from a single test case, I could not find any attested usage of + // this implementation detail. It isn't documented in the OpenAPI docs and + // is not shown in any of the example inputs. + // + // On that basis, I removed that test case. This implementation still does + // correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z") + // since that was required by the original "flattened" event representation. - rawEvents := expandEvents(tc.events) + for i, tc := range testCases { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + c, err := query.New(tc.s) + if err != nil { + t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err) + } - if tc.matches { - match, err := q.Matches(rawEvents) - require.Nil(t, err, "Query '%s' should not error on match %v", tc.s, tc.events) - require.True(t, match, "Query '%s' should match %v", tc.s, tc.events) - } else { - match, err := q.Matches(rawEvents) - require.Equal(t, tc.matchErr, err != nil, "Unexpected error for query '%s' match %v", tc.s, tc.events) - require.False(t, match, "Query '%s' should not match %v", tc.s, tc.events) + got, err := c.Matches(tc.events) + if err != nil { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v", + tc.s, tc.events, err) + } + if got != tc.matches { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v", + tc.s, tc.events, got, tc.matches) + } + }) + } +} + +func TestAllMatchesAll(t *testing.T) { + events := newTestEvents( + ``, + `Asher|Roth=`, + `Route|66=`, + `Rilly|Blue=`, + ) + for i := 0; i < len(events); i++ { + match, err := query.All.Matches(events[:i]) + if err != nil { + t.Errorf("Matches failed: %v", err) + } else if !match { + t.Errorf("Did not match on %+v ", events[:i]) } } } -func TestMustParse(t *testing.T) { - require.Panics(t, func() { query.MustParse("=") }) - require.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +// newTestEvent constructs an Event message from a template string. +// The format is "type|attr1=val1|attr2=val2|...". +func newTestEvent(s string) types.Event { + var event types.Event + parts := strings.Split(s, "|") + event.Type = parts[0] + if len(parts) == 1 { + return event // type only, no attributes + } + for _, kv := range parts[1:] { + key, val := splitKV(kv) + event.Attributes = append(event.Attributes, types.EventAttribute{ + Key: key, + Value: val, + }) + } + return event } -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") - require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - require.Equal(t, tc.conditions, c) +// newTestEvents constructs a slice of Event messages by applying newTestEvent +// to each element of ss. +func newTestEvents(ss ...string) []types.Event { + events := make([]types.Event, len(ss)) + for i, s := range ss { + events[i] = newTestEvent(s) } + return events +} + +func splitKV(s string) (key, value string) { + kv := strings.SplitN(s, "=", 2) + return kv[0], kv[1] } diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go new file mode 100644 index 000000000..e7a9896c4 --- /dev/null +++ b/libs/pubsub/query/syntax/doc.go @@ -0,0 +1,34 @@ +// Package syntax defines a scanner and parser for the Tendermint event filter +// query language. A query selects events by their types and attribute values. +// +// Grammar +// +// The grammar of the query language is defined by the following EBNF: +// +// query = conditions EOF +// conditions = condition {"AND" condition} +// condition = tag comparison +// comparison = equal / order / contains / "EXISTS" +// equal = "=" (date / number / time / value) +// order = cmp (date / number / time) +// contains = "CONTAINS" value +// cmp = "<" / "<=" / ">" / ">=" +// +// The lexical terms are defined here using RE2 regular expression notation: +// +// // The name of an event attribute (type.value) +// tag = #'\w+(\.\w+)*' +// +// // A datestamp (YYYY-MM-DD) +// date = #'DATE \d{4}-\d{2}-\d{2}' +// +// // A number with optional fractional parts (0, 10, 3.25) +// number = #'\d+(\.\d+)?' +// +// // An RFC3339 timestamp (2021-11-23T22:04:19-09:00) +// time = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)' +// +// // A quoted literal string value ('a b c') +// value = #'\'[^\']*\'' +// +package syntax diff --git a/libs/pubsub/query/syntax/parser.go b/libs/pubsub/query/syntax/parser.go new file mode 100644 index 000000000..a100ec79c --- /dev/null +++ b/libs/pubsub/query/syntax/parser.go @@ -0,0 +1,213 @@ +package syntax + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" +) + +// Parse parses the specified query string. It is shorthand for constructing a +// parser for s and calling its Parse method. +func Parse(s string) (Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Query is the root of the parse tree for a query. A query is the conjunction +// of one or more conditions. +type Query []Condition + +func (q Query) String() string { + ss := make([]string, len(q)) + for i, cond := range q { + ss[i] = cond.String() + } + return strings.Join(ss, " AND ") +} + +// A Condition is a single conditional expression, consisting of a tag, a +// comparison operator, and an optional argument. The type of the argument +// depends on the operator. +type Condition struct { + Tag string + Op Token + Arg *Arg + + opText string +} + +func (c Condition) String() string { + s := c.Tag + " " + c.opText + if c.Arg != nil { + return s + " " + c.Arg.String() + } + return s +} + +// An Arg is the argument of a comparison operator. +type Arg struct { + Type Token + text string +} + +func (a *Arg) String() string { + if a == nil { + return "" + } + switch a.Type { + case TString: + return "'" + a.text + "'" + case TTime: + return "TIME " + a.text + case TDate: + return "DATE " + a.text + default: + return a.text + } +} + +// Number returns the value of the argument text as a number, or a NaN if the +// text does not encode a valid number value. +func (a *Arg) Number() float64 { + if a == nil { + return -1 + } + v, err := strconv.ParseFloat(a.text, 64) + if err == nil && v >= 0 { + return v + } + return math.NaN() +} + +// Time returns the value of the argument text as a time, or the zero value if +// the text does not encode a timestamp or datestamp. +func (a *Arg) Time() time.Time { + var ts time.Time + if a == nil { + return ts + } + var err error + switch a.Type { + case TDate: + ts, err = ParseDate(a.text) + case TTime: + ts, err = ParseTime(a.text) + } + if err == nil { + return ts + } + return time.Time{} +} + +// Value returns the value of the argument text as a string, or "". +func (a *Arg) Value() string { + if a == nil { + return "" + } + return a.text +} + +// Parser is a query expression parser. The grammar for query expressions is +// defined in the syntax package documentation. +type Parser struct { + scanner *Scanner +} + +// NewParser constructs a new parser that reads the input from r. +func NewParser(r io.Reader) *Parser { + return &Parser{scanner: NewScanner(r)} +} + +// Parse parses the complete input and returns the resulting query. +func (p *Parser) Parse() (Query, error) { + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds := []Condition{cond} + for p.scanner.Next() != io.EOF { + if tok := p.scanner.Token(); tok != TAnd { + return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd) + } + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds = append(conds, cond) + } + return conds, nil +} + +// parseCond parses a conditional expression: tag OP value. +func (p *Parser) parseCond() (Condition, error) { + var cond Condition + if err := p.require(TTag); err != nil { + return cond, err + } + cond.Tag = p.scanner.Text() + if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil { + return cond, err + } + cond.Op = p.scanner.Token() + cond.opText = p.scanner.Text() + + var err error + switch cond.Op { + case TLeq, TGeq, TLt, TGt: + err = p.require(TNumber, TTime, TDate) + case TEq: + err = p.require(TNumber, TTime, TDate, TString) + case TContains: + err = p.require(TString) + case TExists: + // no argument + return cond, nil + default: + return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op) + } + if err != nil { + return cond, err + } + cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()} + return cond, nil +} + +// require advances the scanner and requires that the resulting token is one of +// the specified token types. +func (p *Parser) require(tokens ...Token) error { + if err := p.scanner.Next(); err != nil { + return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err) + } + got := p.scanner.Token() + for _, tok := range tokens { + if tok == got { + return nil + } + } + return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens)) +} + +// tokLabel makes a human-readable summary string for the given token types. +func tokLabel(tokens []Token) string { + if len(tokens) == 1 { + return tokens[0].String() + } + last := len(tokens) - 1 + ss := make([]string, len(tokens)-1) + for i, tok := range tokens[:last] { + ss[i] = tok.String() + } + return strings.Join(ss, ", ") + " or " + tokens[last].String() +} + +// ParseDate parses s as a date string in the format used by DATE values. +func ParseDate(s string) (time.Time, error) { + return time.Parse("2006-01-02", s) +} + +// ParseTime parses s as a timestamp in the format used by TIME values. +func ParseTime(s string) (time.Time, error) { + return time.Parse(time.RFC3339, s) +} diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go new file mode 100644 index 000000000..332e3f7b1 --- /dev/null +++ b/libs/pubsub/query/syntax/scanner.go @@ -0,0 +1,312 @@ +package syntax + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "time" + "unicode" +) + +// Token is the type of a lexical token in the query grammar. +type Token byte + +const ( + TInvalid = iota // invalid or unknown token + TTag // field tag: x.y + TString // string value: 'foo bar' + TNumber // number: 0, 15.5, 100 + TTime // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z) + TDate // datestamp: DATE yyyy-mm-dd + TAnd // operator: AND + TContains // operator: CONTAINS + TExists // operator: EXISTS + TEq // operator: = + TLt // operator: < + TLeq // operator: <= + TGt // operator: > + TGeq // operator: >= + + // Do not reorder these values without updating the scanner code. +) + +var tString = [...]string{ + TInvalid: "invalid token", + TTag: "tag", + TString: "string", + TNumber: "number", + TTime: "timestamp", + TDate: "datestamp", + TAnd: "AND operator", + TContains: "CONTAINS operator", + TExists: "EXISTS operator", + TEq: "= operator", + TLt: "< operator", + TLeq: "<= operator", + TGt: "> operator", + TGeq: ">= operator", +} + +func (t Token) String() string { + v := int(t) + if v > len(tString) { + return "unknown token type" + } + return tString[v] +} + +const ( + // TimeFormat is the format string used for timestamp values. + TimeFormat = time.RFC3339 + + // DateFormat is the format string used for datestamp values. + DateFormat = "2006-01-02" +) + +// Scanner reads lexical tokens of the query language from an input stream. +// Each call to Next advances the scanner to the next token, or reports an +// error. +type Scanner struct { + r *bufio.Reader + buf bytes.Buffer + tok Token + err error + + pos, last, end int +} + +// NewScanner constructs a new scanner that reads from r. +func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} } + +// Next advances s to the next token in the input, or reports an error. At the +// end of input, Next returns io.EOF. +func (s *Scanner) Next() error { + s.buf.Reset() + s.pos = s.end + s.tok = TInvalid + s.err = nil + + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } + if unicode.IsSpace(ch) { + s.pos = s.end + continue // skip whitespace + } + if '0' <= ch && ch <= '9' { + return s.scanNumber(ch) + } else if isTagRune(ch) { + return s.scanTagLike(ch) + } + switch ch { + case '\'': + return s.scanString(ch) + case '<', '>', '=': + return s.scanCompare(ch) + default: + return s.invalid(ch) + } + } +} + +// Token returns the type of the current input token. +func (s *Scanner) Token() Token { return s.tok } + +// Text returns the text of the current input token. +func (s *Scanner) Text() string { return s.buf.String() } + +// Pos returns the start offset of the current token in the input. +func (s *Scanner) Pos() int { return s.pos } + +// Err returns the last error reported by Next, if any. +func (s *Scanner) Err() error { return s.err } + +// scanNumber scans for numbers with optional fractional parts. +// Examples: 0, 1, 3.14 +func (s *Scanner) scanNumber(first rune) error { + s.buf.WriteRune(first) + if err := s.scanWhile(isDigit); err != nil { + return err + } + + ch, err := s.rune() + if err != nil && err != io.EOF { + return err + } + if ch == '.' { + s.buf.WriteRune(ch) + if err := s.scanWhile(isDigit); err != nil { + return err + } + } else { + s.unrune() + } + s.tok = TNumber + return nil +} + +func (s *Scanner) scanString(first rune) error { + // discard opening quote + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } else if ch == first { + // discard closing quote + s.tok = TString + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) scanCompare(first rune) error { + s.buf.WriteRune(first) + switch first { + case '=': + s.tok = TEq + return nil + case '<': + s.tok = TLt + case '>': + s.tok = TGt + default: + return s.invalid(first) + } + + ch, err := s.rune() + if err == io.EOF { + return nil // the assigned token is correct + } else if err != nil { + return s.fail(err) + } + if ch == '=' { + s.buf.WriteRune(ch) + s.tok++ // depends on token order + return nil + } + s.unrune() + return nil +} + +func (s *Scanner) scanTagLike(first rune) error { + s.buf.WriteRune(first) + var hasSpace bool + for { + ch, err := s.rune() + if err == io.EOF { + break + } else if err != nil { + return s.fail(err) + } + if !isTagRune(ch) { + hasSpace = ch == ' ' // to check for TIME, DATE + break + } + s.buf.WriteRune(ch) + } + + text := s.buf.String() + switch text { + case "TIME": + if hasSpace { + return s.scanTimestamp() + } + s.tok = TTag + case "DATE": + if hasSpace { + return s.scanDatestamp() + } + s.tok = TTag + case "AND": + s.tok = TAnd + case "EXISTS": + s.tok = TExists + case "CONTAINS": + s.tok = TContains + default: + s.tok = TTag + } + s.unrune() + return nil +} + +func (s *Scanner) scanTimestamp() error { + s.buf.Reset() // discard "TIME" label + if err := s.scanWhile(isTimeRune); err != nil { + return err + } + if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid TIME value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year())) + } + s.tok = TTime + return nil +} + +func (s *Scanner) scanDatestamp() error { + s.buf.Reset() // discard "DATE" label + if err := s.scanWhile(isDateRune); err != nil { + return err + } + if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid DATE value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year())) + } + s.tok = TDate + return nil +} + +func (s *Scanner) scanWhile(ok func(rune) bool) error { + for { + ch, err := s.rune() + if err == io.EOF { + return nil + } else if err != nil { + return s.fail(err) + } else if !ok(ch) { + s.unrune() + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) rune() (rune, error) { + ch, nb, err := s.r.ReadRune() + s.last = nb + s.end += nb + return ch, err +} + +func (s *Scanner) unrune() { + _ = s.r.UnreadRune() + s.end -= s.last +} + +func (s *Scanner) fail(err error) error { + s.err = err + return err +} + +func (s *Scanner) invalid(ch rune) error { + return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end)) +} + +func isDigit(r rune) bool { return '0' <= r && r <= '9' } + +func isTagRune(r rune) bool { + return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +func isTimeRune(r rune) bool { + return strings.ContainsRune("-T:+Z", r) || isDigit(r) +} + +func isDateRune(r rune) bool { return isDigit(r) || r == '-' } diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go new file mode 100644 index 000000000..ac95fd8b1 --- /dev/null +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -0,0 +1,190 @@ +package syntax_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" +) + +func TestScanner(t *testing.T) { + tests := []struct { + input string + want []syntax.Token + }{ + // Empty inputs + {"", nil}, + {" ", nil}, + {"\t\n ", nil}, + + // Numbers + {`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + {`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + + // Tags + {`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + + // Strings (values) + {` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}}, + {` 'you are not your job' `, []syntax.Token{syntax.TString}}, + + // Comparison operators + {`< <= = > >=`, []syntax.Token{ + syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq, + }}, + + // Mixed values of various kinds. + {`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}}, + {`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}}, + {`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}}, + {`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}}, + + // Timestamp + {`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}}, + + // Datestamp + {`DATE 2021-11-23`, []syntax.Token{syntax.TDate}}, + } + + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + var got []syntax.Token + for s.Next() == nil { + got = append(got, s.Token()) + } + if err := s.Err(); err != io.EOF { + t.Errorf("Next: unexpected error: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Logf("Scanner input: %q", test.input) + t.Errorf("Wrong tokens:\ngot: %+v\nwant: %+v", got, test.want) + } + } +} + +func TestScannerErrors(t *testing.T) { + tests := []struct { + input string + }{ + {`'incomplete string`}, + {`-23`}, + {`&`}, + {`DATE xyz-pdq`}, + {`DATE xyzp-dq-zv`}, + {`DATE 0000-00-00`}, + {`DATE 0000-00-000`}, + {`DATE 2021-01-99`}, + {`TIME 2021-01-01T34:56:78Z`}, + {`TIME 2021-01-99T14:56:08Z`}, + {`TIME 2021-01-99T34:56:08`}, + {`TIME 2021-01-99T34:56:11+3`}, + } + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + if err := s.Next(); err == nil { + t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text()) + } + } +} + +// These parser tests were copied from the original implementation of the query +// parser, and are preserved here as a compatibility check. +func TestParseValid(t *testing.T) { + tests := []struct { + input string + valid bool + }{ + {"tm.events.type='NewBlock'", true}, + {"tm.events.type = 'NewBlock'", true}, + {"tm.events.name = ''", true}, + {"tm.events.type='TIME'", true}, + {"tm.events.type='DATE'", true}, + {"tm.events.type='='", true}, + {"tm.events.type='TIME", false}, + {"tm.events.type=TIME'", false}, + {"tm.events.type==", false}, + {"tm.events.type=NewBlock", false}, + {">==", false}, + {"tm.events.type 'NewBlock' =", false}, + {"tm.events.type>'NewBlock'", false}, + {"", false}, + {"=", false}, + {"='NewBlock'", false}, + {"tm.events.type=", false}, + + {"tm.events.typeNewBlock", false}, + {"tm.events.type'NewBlock'", false}, + {"'NewBlock'", false}, + {"NewBlock", false}, + {"", false}, + + {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, + {"tm.events.type='NewBlock' AND", false}, + {"tm.events.type='NewBlock' AN", false}, + {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, + {"AND tm.events.type='NewBlock' ", false}, + + {"abci.account.name CONTAINS 'Igor'", true}, + + {"tx.date > DATE 2013-05-03", true}, + {"tx.date < DATE 2013-05-03", true}, + {"tx.date <= DATE 2013-05-03", true}, + {"tx.date >= DATE 2013-05-03", true}, + {"tx.date >= DAT 2013-05-03", false}, + {"tx.date <= DATE2013-05-03", false}, + {"tx.date <= DATE -05-03", false}, + {"tx.date >= DATE 20130503", false}, + {"tx.date >= DATE 2013+01-03", false}, + // incorrect year, month, day + {"tx.date >= DATE 0013-01-03", false}, + {"tx.date >= DATE 2013-31-03", false}, + {"tx.date >= DATE 2013-01-83", false}, + + {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, + {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, + {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME2013-05-03T14:45:00Z", false}, + {"tx.date = IME 2013-05-03T14:45:00Z", false}, + {"tx.date = TIME 2013-05-:45:00Z", false}, + {"tx.date >= TIME 2013-05-03T14:45:00", false}, + {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, + {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, + + {"account.balance=100", true}, + {"account.balance >= 200", true}, + {"account.balance >= -300", false}, + {"account.balance >>= 400", false}, + {"account.balance=33.22.1", false}, + + {"slashing.amount EXISTS", true}, + {"slashing.amount EXISTS AND account.balance=100", true}, + {"account.balance=100 AND slashing.amount EXISTS", true}, + {"slashing EXISTS", true}, + + {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, + {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, + } + + for _, test := range tests { + q, err := syntax.Parse(test.input) + if test.valid != (err == nil) { + t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err) + } + + // For valid queries, check that the query round-trips. + if test.valid { + qstr := q.String() + r, err := syntax.Parse(qstr) + if err != nil { + t.Errorf("Reparse %#q failed: %v", qstr, err) + } + if rstr := r.String(); rstr != qstr { + t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr) + } + } + } +} diff --git a/types/events.go b/types/events.go index 46f150abd..7bb183fb7 100644 --- a/types/events.go +++ b/types/events.go @@ -232,11 +232,11 @@ var ( ) func EventQueryTxFor(tx Tx) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTxValue, TxHashKey, tx.Hash())) + return tmquery.MustCompile(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTxValue, TxHashKey, tx.Hash())) } func QueryForEvent(eventValue string) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s'", EventTypeKey, eventValue)) + return tmquery.MustCompile(fmt.Sprintf("%s='%s'", EventTypeKey, eventValue)) } // BlockEventPublisher publishes all block related events diff --git a/types/events_test.go b/types/events_test.go index dcd998ace..bd4bde264 100644 --- a/types/events_test.go +++ b/types/events_test.go @@ -10,18 +10,18 @@ import ( func TestQueryTxFor(t *testing.T) { tx := Tx("foo") assert.Equal(t, - fmt.Sprintf("tm.event='Tx' AND tx.hash='%X'", tx.Hash()), + fmt.Sprintf("tm.event = 'Tx' AND tx.hash = '%X'", tx.Hash()), EventQueryTxFor(tx).String(), ) } func TestQueryForEvent(t *testing.T) { assert.Equal(t, - "tm.event='NewBlock'", + "tm.event = 'NewBlock'", QueryForEvent(EventNewBlockValue).String(), ) assert.Equal(t, - "tm.event='NewEvidence'", + "tm.event = 'NewEvidence'", QueryForEvent(EventNewEvidenceValue).String(), ) } From abc697b46ca57687d7c0e80bf8eee338490fda42 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Mon, 29 Nov 2021 17:02:27 -0500 Subject: [PATCH 05/21] adr: lib2p implementation plan (#7282) --- docs/architecture/README.md | 5 +- .../adr-072-request-for-comments.md | 2 +- docs/architecture/adr-073-libp2p.md | 235 ++++++++++++++++++ 3 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 docs/architecture/adr-073-libp2p.md diff --git a/docs/architecture/README.md b/docs/architecture/README.md index f6c12996f..7256e5038 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -65,7 +65,9 @@ Note the context/background should be written in the present tense. - [ADR-059: Evidence-Composition-and-Lifecycle](./adr-059-evidence-composition-and-lifecycle.md) - [ADR-062: P2P-Architecture](./adr-062-p2p-architecture.md) - [ADR-063: Privval-gRPC](./adr-063-privval-grpc.md) -- [ADR-066-E2E-Testing](./adr-066-e2e-testing.md) +- [ADR-066: E2E-Testing](./adr-066-e2e-testing.md) +- [ADR-072: Restore Requests for Comments](./adr-072-request-for-comments.md) + ### Accepted - [ADR-006: Trust-Metric](./adr-006-trust-metric.md) @@ -99,4 +101,3 @@ Note the context/background should be written in the present tense. - [ADR-057: RPC](./adr-057-RPC.md) - [ADR-069: Node Initialization](./adr-069-flexible-node-initialization.md) - [ADR-071: Proposer-Based Timestamps](adr-071-proposer-based-timestamps.md) -- [ADR-072: Restore Requests for Comments](./adr-072-request-for-comments.md) diff --git a/docs/architecture/adr-072-request-for-comments.md b/docs/architecture/adr-072-request-for-comments.md index 7eb22ebc9..7b656d05e 100644 --- a/docs/architecture/adr-072-request-for-comments.md +++ b/docs/architecture/adr-072-request-for-comments.md @@ -6,7 +6,7 @@ ## Status -Proposed +Implemented ## Context diff --git a/docs/architecture/adr-073-libp2p.md b/docs/architecture/adr-073-libp2p.md new file mode 100644 index 000000000..a73443d67 --- /dev/null +++ b/docs/architecture/adr-073-libp2p.md @@ -0,0 +1,235 @@ +# ADR 073: Adopt LibP2P + +## Changelog + +- 2021-11-02: Initial Draft (@tychoish) + +## Status + +Proposed. + +## Context + + +As part of the 0.35 development cycle, the Tendermint team completed +the first phase of the work described in ADRs 61 and 62, which included a +large scale refactoring of the reactors and the p2p message +routing. This replaced the switch and many of the other legacy +components without breaking protocol or network-level +interoperability and left the legacy connection/socket handling code. + +Following the release, the team has reexamined the state of the code +and the design, as well as Tendermint's requirements. The notes +from that process are available in the [P2P Roadmap +RFC][rfc]. + +This ADR supersedes the decisions made in ADRs 60 and 61, but +builds on the completed portions of this work. Previously, the +boundaries of peer management, message handling, and the higher level +business logic (e.g., "the reactors") were intermingled, and core +elements of the p2p system were responsible for the orchestration of +higher-level business logic. Refactoring the legacy components +made it more obvious that this entanglement of responsibilities +had outsized influence on the entire implementation, making +it difficult to iterate within the current abstractions. +It would not be viable to maintain interoperability with legacy +systems while also achieving many of our broader objectives. + +LibP2P is a thoroughly-specified implementation of a peer-to-peer +networking stack, designed specifically for systems such as +ours. Adopting LibP2P as the basis of Tendermint will allow the +Tendermint team to focus more of their time on other differentiating +aspects of the system, and make it possible for the ecosystem as a +whole to take advantage of tooling and efforts of the LibP2P +platform. + +## Alternative Approaches + +As discussed in the [P2P Roadmap RFC][rfc], the primary alternative would be to +continue development of Tendermint's home-grown peer-to-peer +layer. While that would give the Tendermint team maximal control +over the peer system, the current design is unexceptional on its +own merits, and the prospective maintenance burden for this system +exceeds our tolerances for the medium term. + +Tendermint can and should differentiate itself not on the basis of +its networking implementation or peer management tools, but providing +a consistent operator experience, a battle-tested consensus algorithm, +and an ergonomic user experience. + +## Decision + +Tendermint will adopt libp2p during the 0.37 development cycle, +replacing the bespoke Tendermint P2P stack. This will remove the +`Endpoint`, `Transport`, `Connection`, and `PeerManager` abstractions +and leave the reactors, `p2p.Router` and `p2p.Channel` +abstractions. + +LibP2P may obviate the need for a dedicated peer exchange (PEX) +reactor, which would also in turn obviate the need for a dedicated +seed mode. If this is the case, then all of this functionality would +be removed. + +If it turns out (based on the advice of Protocol Labs) that it makes +sense to maintain separate pubsub or gossipsub topics +per-message-type, then the `Router` abstraction could also +be entirely subsumed. + +## Detailed Design + +### Implementation Changes + +The seams in the P2P implementation between the higher level +constructs (reactors), the routing layer (`Router`) and the lower +level connection and peer management code make this operation +relatively straightforward to implement. A key +goal in this design is to minimize the impact on the reactors +(potentially entirely,) and completely remove the lower level +components (e.g., `Transport`, `Connection` and `PeerManager`) using the +separation afforded by the `Router` layer. The current state of the +code makes these changes relatively surgical, and limited to a small +number of methods: + +- `p2p.Router.OpenChannel` will still return a `Channel` structure + which will continue to serve as a pipe between the reactors and the + `Router`. The implementation will no longer need the queue + implementation, and will instead start goroutines that + are responsible for routing the messages from the channel to libp2p + fundamentals, replacing the current `p2p.Router.routeChannel`. + +- The current `p2p.Router.dialPeers` and `p2p.Router.acceptPeers`, + are responsible for establishing outbound and inbound connections, + respectively. These methods will be removed, along with + `p2p.Router.openConnection`, and the libp2p connection manager will + be responsible for maintaining network connectivity. + +- The `p2p.Channel` interface will change to replace Go + channels with a more functional interface for sending messages. + New methods on this object will take contexts to support safe + cancellation, and return errors, and will block rather than + running asynchronously. The `Out` channel through which + reactors send messages to Peers, will be replaced by a `Send` + method, and the Error channel will be replaced by an `Error` + method. + +- Reactors will be passed an interface that will allow them to + access Peer information from libp2p. This will supplant the + `p2p.PeerUpdates` subscription. + +- Add some kind of heartbeat message at the application level + (e.g. with a reactor,) potentially connected to libp2p's DHT to be + used by reactors for service discovery, message targeting, or other + features. + +- Replace the existing/legacy handshake protocol with [Noise](http://www.noiseprotocol.org/noise.html). + +This project will initially use the TCP-based transport protocols within +libp2p. QUIC is also available as an option that we may implement later. +We will not support mixed networks in the initial release, but will +revisit that possibility later if there is a demonstrated need. + +### Upgrade and Compatibility + +Because the routers and all current P2P libraries are `internal` +packages and not part of the public API, the only changes to the public +API surface area of Tendermint will be different configuration +file options, replacing the current P2P options with options relevant +to libp2p. + +However, it will not be possible to run a network with both networking +stacks active at once, so the upgrade to the version of Tendermint +will need to be coordinated between all nodes of the network. This is +consistent with the expectations around upgrades for Tendermint moving +forward, and will help manage both the complexity of the project and +the implementation timeline. + +## Open Questions + +- What is the role of Protocol Labs in the implementation of libp2p in + tendermint, both during the initial implementation and on an ongoing + basis thereafter? + +- Should all P2P traffic for a given node be pushed to a single topic, + so that a topic maps to a specific ChainID, or should + each reactor (or type of message) have its own topic? How many + topics can a libp2p network support? Is there testing that validates + the capabilities? + +- Tendermint presently provides a very coarse QoS-like functionality + using priorities based on message-type. + This intuitively/theoretically ensures that evidence and consensus + messages don't get starved by blocksync/statesync messages. It's + unclear if we can or should attempt to replicate this with libp2p. + +- What kind of QoS functionality does libp2p provide and what kind of + metrics does libp2p provide about it's QoS functionality? + +- Is it possible to store additional (and potentially arbitrary) + information into the DHT as part of the heartbeats between nodes, + such as the latest height, and then access that in the + reactors. How frequently can the DHT be updated? + +- Does it make sense to have reactors continue to consume inbound + messages from a Channel (`In`) or is there another interface or + pattern that we should consider? + + - We should avoid exposing Go channels when possible, and likely + some kind of alternate iterator likely makes sense for processing + messages within the reactors. + +- What are the security and protocol implications of tracking + information from peer heartbeats and exposing that to reactors? + +- How much (or how little) configuration can Tendermint provide for + libp2p, particularly on the first release? + + - In general, we should not support byo-functionality for libp2p + components within Tendermint, and reduce the configuration surface + area, as much as possible. + +- What are the best ways to provide request/response semantics for + reactors on top of libp2p? Will it be possible to add + request/response semantics in a future release or is there + anticipatory work that needs to be done as part of the initial + release? + +## Consequences + +### Positive + +- Reduce the maintenance burden for the Tendermint Core team by + removing a large swath of legacy code that has proven to be + difficult to modify safely. + +- Remove the responsibility for maintaining and developing the entire + peer management system (p2p) and stack. + +- Provide users with a more stable peer and networking system, + Tendermint can improve operator experience and network stability. + +### Negative + +- By deferring to library implementations for peer management and + networking, Tendermint loses some flexibility for innovating at the + peer and networking level. However, Tendermint should be innovating + primarily at the consensus layer, and libp2p does not preclude + optimization or development in the peer layer. + +- Libp2p is a large dependency and Tendermint would become dependent + upon Protocol Labs' release cycle and prioritization for bug + fixes. If this proves onerous, it's possible to maintain a vendor + fork of relevant components as needed. + +### Neutral + +- N/A + +## References + +- [ADR 61: P2P Refactor Scope][adr61] +- [ADR 62: P2P Architecture][adr62] +- [P2P Roadmap RFC][rfc] + +[adr61]: ./adr-061-p2p-refactor-scope.md +[adr62]: ./adr-062-p2p-architecture.md +[rfc]: ../rfc/rfc-000-p2p.rst From 99c51b354cf591efc4d4c54e255207e85d9a9d27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 22:12:35 +0000 Subject: [PATCH 06/21] build(deps-dev): Bump watchpack from 2.2.0 to 2.3.0 in /docs (#7335) Bumps [watchpack](https://github.com/webpack/watchpack) from 2.2.0 to 2.3.0.
Release notes

Sourced from watchpack's releases.

v2.3.0

Features

  • allow to grab separate file and directory time info objects
  • allow functions passed to the ignored option

Bugfixes

  • ignore EACCESS errors during initial scan

Performance

  • improve performance of watcher update

Contributing

  • CI tests node.js 17 too
Commits
  • b903dd3 2.3.0
  • c82a595 Merge pull request #211 from webpack/bugfix/missing-info
  • 1e56026 fix missing time info in files
  • 89d5f48 Merge pull request #205 from markjm/markjm/split
  • f1f3586 Merge pull request #210 from webpack/ci/no-macos-polling
  • 8d14e94 Merge branch 'main' into markjm/split
  • e71b62b Merge pull request #197 from markjm/markjm/watch-change
  • a3b2b82 provide additional method instead of changing existing one
  • aab3697 disable testing polling for macos ci
  • 523793e Merge pull request #208 from webpack/perf/update-watchers
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=watchpack&package-manager=npm_and_yarn&previous-version=2.2.0&new-version=2.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--- docs/package-lock.json | 6 +++--- docs/package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/package-lock.json b/docs/package-lock.json index c6f21bd6b..204ffc9ba 100644 --- a/docs/package-lock.json +++ b/docs/package-lock.json @@ -10069,9 +10069,9 @@ } }, "watchpack": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.2.0.tgz", - "integrity": "sha512-up4YAn/XHgZHIxFBVCdlMiWDj6WaLKpwVeGQk2I5thdYxF/KmF0aaz6TfJZ/hfl1h/XlcDr7k1KH7ThDagpFaA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.3.0.tgz", + "integrity": "sha512-MnN0Q1OsvB/GGHETrFeZPQaOelWh/7O+EiFlj8sM9GPjtQkis7k01aAxrg/18kTfoIVcLL+haEVFlXDaSRwKRw==", "dev": true, "requires": { "glob-to-regexp": "^0.4.1", diff --git a/docs/package.json b/docs/package.json index 89798cf28..2ae446af9 100644 --- a/docs/package.json +++ b/docs/package.json @@ -7,7 +7,7 @@ "vuepress-theme-cosmos": "^1.0.182" }, "devDependencies": { - "watchpack": "^2.2.0" + "watchpack": "^2.3.0" }, "scripts": { "preserve": "./pre.sh", From 99ee730ee77037f002503e0e833760a21a4bf5e7 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Mon, 29 Nov 2021 15:00:01 -0800 Subject: [PATCH 07/21] Remove the PEG query implementation. (#7336) A follow-up to #7319. --- libs/pubsub/query/bench_test.go | 26 - libs/pubsub/query/oldquery/Makefile | 10 - libs/pubsub/query/oldquery/empty.go | 18 - libs/pubsub/query/oldquery/empty_test.go | 55 - libs/pubsub/query/oldquery/fuzz_test/main.go | 30 - libs/pubsub/query/oldquery/parser_test.go | 97 - libs/pubsub/query/oldquery/peg.go | 3 - libs/pubsub/query/oldquery/query.go | 527 ----- libs/pubsub/query/oldquery/query.peg | 35 - libs/pubsub/query/oldquery/query.peg.go | 1871 ------------------ libs/pubsub/query/oldquery/query_test.go | 205 -- 11 files changed, 2877 deletions(-) delete mode 100644 libs/pubsub/query/oldquery/Makefile delete mode 100644 libs/pubsub/query/oldquery/empty.go delete mode 100644 libs/pubsub/query/oldquery/empty_test.go delete mode 100644 libs/pubsub/query/oldquery/fuzz_test/main.go delete mode 100644 libs/pubsub/query/oldquery/parser_test.go delete mode 100644 libs/pubsub/query/oldquery/peg.go delete mode 100644 libs/pubsub/query/oldquery/query.go delete mode 100644 libs/pubsub/query/oldquery/query.peg delete mode 100644 libs/pubsub/query/oldquery/query.peg.go delete mode 100644 libs/pubsub/query/oldquery/query_test.go diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go index 2e49b58ae..894c16628 100644 --- a/libs/pubsub/query/bench_test.go +++ b/libs/pubsub/query/bench_test.go @@ -5,7 +5,6 @@ import ( "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/pubsub/query" - oldquery "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" ) const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` @@ -33,15 +32,6 @@ var testEvents = []types.Event{ }, } -func BenchmarkParsePEG(b *testing.B) { - for i := 0; i < b.N; i++ { - _, err := oldquery.New(testQuery) - if err != nil { - b.Fatal(err) - } - } -} - func BenchmarkParseCustom(b *testing.B) { for i := 0; i < b.N; i++ { _, err := query.New(testQuery) @@ -51,22 +41,6 @@ func BenchmarkParseCustom(b *testing.B) { } } -func BenchmarkMatchPEG(b *testing.B) { - q, err := oldquery.New(testQuery) - if err != nil { - b.Fatal(err) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - ok, err := q.Matches(testEvents) - if err != nil { - b.Fatal(err) - } else if !ok { - b.Error("no match") - } - } -} - func BenchmarkMatchCustom(b *testing.B) { q, err := query.New(testQuery) if err != nil { diff --git a/libs/pubsub/query/oldquery/Makefile b/libs/pubsub/query/oldquery/Makefile deleted file mode 100644 index 8ec72981e..000000000 --- a/libs/pubsub/query/oldquery/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -gen_query_parser: - go generate . - -fuzzy_test: - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test - go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output - -.PHONY: gen_query_parser fuzzy_test diff --git a/libs/pubsub/query/oldquery/empty.go b/libs/pubsub/query/oldquery/empty.go deleted file mode 100644 index dd6b3f3b2..000000000 --- a/libs/pubsub/query/oldquery/empty.go +++ /dev/null @@ -1,18 +0,0 @@ -package query - -import ( - "github.com/tendermint/tendermint/abci/types" -) - -// Empty query matches any set of events. -type Empty struct { -} - -// Matches always returns true. -func (Empty) Matches(events []types.Event) (bool, error) { - return true, nil -} - -func (Empty) String() string { - return "empty" -} diff --git a/libs/pubsub/query/oldquery/empty_test.go b/libs/pubsub/query/oldquery/empty_test.go deleted file mode 100644 index 81f313d30..000000000 --- a/libs/pubsub/query/oldquery/empty_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func TestEmptyQueryMatchesAnything(t *testing.T) { - q := query.Empty{} - - testCases := []struct { - events []abci.Event - }{ - { - []abci.Event{}, - }, - { - []abci.Event{ - { - Type: "Asher", - Attributes: []abci.EventAttribute{{Key: "Roth"}}, - }, - }, - }, - { - []abci.Event{ - { - Type: "Route", - Attributes: []abci.EventAttribute{{Key: "66"}}, - }, - }, - }, - { - []abci.Event{ - { - Type: "Route", - Attributes: []abci.EventAttribute{{Key: "66"}}, - }, - { - Type: "Billy", - Attributes: []abci.EventAttribute{{Key: "Blue"}}, - }, - }, - }, - } - - for _, tc := range testCases { - match, err := q.Matches(tc.events) - require.Nil(t, err) - require.True(t, match) - } -} diff --git a/libs/pubsub/query/oldquery/fuzz_test/main.go b/libs/pubsub/query/oldquery/fuzz_test/main.go deleted file mode 100644 index 8bbcaa25f..000000000 --- a/libs/pubsub/query/oldquery/fuzz_test/main.go +++ /dev/null @@ -1,30 +0,0 @@ -package fuzz_test - -import ( - "fmt" - - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func Fuzz(data []byte) int { - sdata := string(data) - q0, err := query.New(sdata) - if err != nil { - return 0 - } - - sdata1 := q0.String() - q1, err := query.New(sdata1) - if err != nil { - panic(err) - } - - sdata2 := q1.String() - if sdata1 != sdata2 { - fmt.Printf("q0: %q\n", sdata1) - fmt.Printf("q1: %q\n", sdata2) - panic("query changed") - } - - return 1 -} diff --git a/libs/pubsub/query/oldquery/parser_test.go b/libs/pubsub/query/oldquery/parser_test.go deleted file mode 100644 index 661a80f93..000000000 --- a/libs/pubsub/query/oldquery/parser_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -// TODO: fuzzy testing? -func TestParser(t *testing.T) { - cases := []struct { - query string - valid bool - }{ - {"tm.events.type='NewBlock'", true}, - {"tm.events.type = 'NewBlock'", true}, - {"tm.events.name = ''", true}, - {"tm.events.type='TIME'", true}, - {"tm.events.type='DATE'", true}, - {"tm.events.type='='", true}, - {"tm.events.type='TIME", false}, - {"tm.events.type=TIME'", false}, - {"tm.events.type==", false}, - {"tm.events.type=NewBlock", false}, - {">==", false}, - {"tm.events.type 'NewBlock' =", false}, - {"tm.events.type>'NewBlock'", false}, - {"", false}, - {"=", false}, - {"='NewBlock'", false}, - {"tm.events.type=", false}, - - {"tm.events.typeNewBlock", false}, - {"tm.events.type'NewBlock'", false}, - {"'NewBlock'", false}, - {"NewBlock", false}, - {"", false}, - - {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, - {"tm.events.type='NewBlock' AND", false}, - {"tm.events.type='NewBlock' AN", false}, - {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, - {"AND tm.events.type='NewBlock' ", false}, - - {"abci.account.name CONTAINS 'Igor'", true}, - - {"tx.date > DATE 2013-05-03", true}, - {"tx.date < DATE 2013-05-03", true}, - {"tx.date <= DATE 2013-05-03", true}, - {"tx.date >= DATE 2013-05-03", true}, - {"tx.date >= DAT 2013-05-03", false}, - {"tx.date <= DATE2013-05-03", false}, - {"tx.date <= DATE -05-03", false}, - {"tx.date >= DATE 20130503", false}, - {"tx.date >= DATE 2013+01-03", false}, - // incorrect year, month, day - {"tx.date >= DATE 0013-01-03", false}, - {"tx.date >= DATE 2013-31-03", false}, - {"tx.date >= DATE 2013-01-83", false}, - - {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, - {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, - {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME2013-05-03T14:45:00Z", false}, - {"tx.date = IME 2013-05-03T14:45:00Z", false}, - {"tx.date = TIME 2013-05-:45:00Z", false}, - {"tx.date >= TIME 2013-05-03T14:45:00", false}, - {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, - {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, - - {"account.balance=100", true}, - {"account.balance >= 200", true}, - {"account.balance >= -300", false}, - {"account.balance >>= 400", false}, - {"account.balance=33.22.1", false}, - - {"slashing.amount EXISTS", true}, - {"slashing.amount EXISTS AND account.balance=100", true}, - {"account.balance=100 AND slashing.amount EXISTS", true}, - {"slashing EXISTS", true}, - - {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, - {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, - } - - for _, c := range cases { - _, err := query.New(c.query) - if c.valid { - assert.NoErrorf(t, err, "Query was '%s'", c.query) - } else { - assert.Errorf(t, err, "Query was '%s'", c.query) - } - } -} diff --git a/libs/pubsub/query/oldquery/peg.go b/libs/pubsub/query/oldquery/peg.go deleted file mode 100644 index bf6789b58..000000000 --- a/libs/pubsub/query/oldquery/peg.go +++ /dev/null @@ -1,3 +0,0 @@ -package query - -//go:generate go run github.com/pointlander/peg@v1.0.0 -inline -switch query.peg diff --git a/libs/pubsub/query/oldquery/query.go b/libs/pubsub/query/oldquery/query.go deleted file mode 100644 index 7b1dfe0f9..000000000 --- a/libs/pubsub/query/oldquery/query.go +++ /dev/null @@ -1,527 +0,0 @@ -// Package query provides a parser for a custom query format: -// -// abci.invoice.number=22 AND abci.invoice.owner=Ivan -// -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics -// -// It has a support for numbers (integer and floating point), dates and times. -package query - -import ( - "fmt" - "reflect" - "regexp" - "strconv" - "strings" - "time" - - "github.com/tendermint/tendermint/abci/types" -) - -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) - -// Query holds the query string and the query parser. -type Query struct { - str string - parser *QueryParser -} - -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - p.Init() - if err := p.Parse(); err != nil { - return nil, err - } - return &Query{str: s, parser: p}, nil -} - -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) - if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) - } - return q -} - -// String returns the original string. -func (q *Query) String() string { - return q.str -} - -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 - -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) - -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } - } - - return conditions, nil -} - -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. -func (q *Query) Matches(rawEvents []types.Event) (bool, error) { - if len(rawEvents) == 0 { - return false, nil - } - - events := flattenEvents(rawEvents) - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - } - - return true, nil -} - -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil - } - - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil - } - } - - return false, nil -} - -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) - - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) - - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) - } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) - } - - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil - } - - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil - } - - case reflect.Int64: - var v int64 - - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) - - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } - - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } - - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } - - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } - - default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) - } - - return false, nil -} - -func flattenEvents(events []types.Event) map[string][]string { - flattened := make(map[string][]string) - - for _, event := range events { - if len(event.Type) == 0 { - continue - } - - for _, attr := range event.Attributes { - if len(attr.Key) == 0 { - continue - } - - compositeEvent := fmt.Sprintf("%s.%s", event.Type, attr.Key) - flattened[compositeEvent] = append(flattened[compositeEvent], attr.Value) - } - } - - return flattened -} diff --git a/libs/pubsub/query/oldquery/query.peg b/libs/pubsub/query/oldquery/query.peg deleted file mode 100644 index e2cfd0826..000000000 --- a/libs/pubsub/query/oldquery/query.peg +++ /dev/null @@ -1,35 +0,0 @@ -package query - -type QueryParser Peg { -} - -e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !. - -condition <- tag ' '* (le ' '* (number / time / date) - / ge ' '* (number / time / date) - / l ' '* (number / time / date) - / g ' '* (number / time / date) - / equal ' '* (number / time / date / value) - / contains ' '* value - / exists - ) - -tag <- < (![ \t\n\r\\()"'=><] .)+ > -value <- < '\'' (!["'] .)* '\''> -number <- < ('0' - / [1-9] digit* ('.' digit*)?) > -digit <- [0-9] -time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') > -date <- "DATE " < year '-' month '-' day > -year <- ('1' / '2') digit digit digit -month <- ('0' / '1') digit -day <- ('0' / '1' / '2' / '3') digit -and <- "AND" - -equal <- "=" -contains <- "CONTAINS" -exists <- "EXISTS" -le <- "<=" -ge <- ">=" -l <- "<" -g <- ">" diff --git a/libs/pubsub/query/oldquery/query.peg.go b/libs/pubsub/query/oldquery/query.peg.go deleted file mode 100644 index a8e14c869..000000000 --- a/libs/pubsub/query/oldquery/query.peg.go +++ /dev/null @@ -1,1871 +0,0 @@ -// nolint -package query - -import ( - "fmt" - "math" - "sort" - "strconv" -) - -const endSymbol rune = 1114112 - -/* The rule types inferred from the grammar are below. */ -type pegRule uint8 - -const ( - ruleUnknown pegRule = iota - rulee - rulecondition - ruletag - rulevalue - rulenumber - ruledigit - ruletime - ruledate - ruleyear - rulemonth - ruleday - ruleand - ruleequal - rulecontains - ruleexists - rulele - rulege - rulel - ruleg - rulePegText - - rulePre - ruleIn - ruleSuf -) - -var rul3s = [...]string{ - "Unknown", - "e", - "condition", - "tag", - "value", - "number", - "digit", - "time", - "date", - "year", - "month", - "day", - "and", - "equal", - "contains", - "exists", - "le", - "ge", - "l", - "g", - "PegText", - - "Pre_", - "_In_", - "_Suf", -} - -type node32 struct { - token32 - up, next *node32 -} - -func (node *node32) print(depth int, buffer string) { - for node != nil { - for c := 0; c < depth; c++ { - fmt.Printf(" ") - } - fmt.Printf("\x1B[34m%v\x1B[m %v\n", rul3s[node.pegRule], strconv.Quote(string(([]rune(buffer)[node.begin:node.end])))) - if node.up != nil { - node.up.print(depth+1, buffer) - } - node = node.next - } -} - -func (node *node32) Print(buffer string) { - node.print(0, buffer) -} - -type element struct { - node *node32 - down *element -} - -/* ${@} bit structure for abstract syntax tree */ -type token32 struct { - pegRule - begin, end, next uint32 -} - -func (t *token32) isZero() bool { - return t.pegRule == ruleUnknown && t.begin == 0 && t.end == 0 && t.next == 0 -} - -func (t *token32) isParentOf(u token32) bool { - return t.begin <= u.begin && t.end >= u.end && t.next > u.next -} - -func (t *token32) getToken32() token32 { - return token32{pegRule: t.pegRule, begin: uint32(t.begin), end: uint32(t.end), next: uint32(t.next)} -} - -func (t *token32) String() string { - return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v %v", rul3s[t.pegRule], t.begin, t.end, t.next) -} - -type tokens32 struct { - tree []token32 - ordered [][]token32 -} - -func (t *tokens32) trim(length int) { - t.tree = t.tree[0:length] -} - -func (t *tokens32) Print() { - for _, token := range t.tree { - fmt.Println(token.String()) - } -} - -func (t *tokens32) Order() [][]token32 { - if t.ordered != nil { - return t.ordered - } - - depths := make([]int32, 1, math.MaxInt16) - for i, token := range t.tree { - if token.pegRule == ruleUnknown { - t.tree = t.tree[:i] - break - } - depth := int(token.next) - if length := len(depths); depth >= length { - depths = depths[:depth+1] - } - depths[depth]++ - } - depths = append(depths, 0) - - ordered, pool := make([][]token32, len(depths)), make([]token32, len(t.tree)+len(depths)) - for i, depth := range depths { - depth++ - ordered[i], pool, depths[i] = pool[:depth], pool[depth:], 0 - } - - for i, token := range t.tree { - depth := token.next - token.next = uint32(i) - ordered[depth][depths[depth]] = token - depths[depth]++ - } - t.ordered = ordered - return ordered -} - -type state32 struct { - token32 - depths []int32 - leaf bool -} - -func (t *tokens32) AST() *node32 { - tokens := t.Tokens() - stack := &element{node: &node32{token32: <-tokens}} - for token := range tokens { - if token.begin == token.end { - continue - } - node := &node32{token32: token} - for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { - stack.node.next = node.up - node.up = stack.node - stack = stack.down - } - stack = &element{node: node, down: stack} - } - return stack.node -} - -func (t *tokens32) PreOrder() (<-chan state32, [][]token32) { - s, ordered := make(chan state32, 6), t.Order() - go func() { - var states [8]state32 - for i := range states { - states[i].depths = make([]int32, len(ordered)) - } - depths, state, depth := make([]int32, len(ordered)), 0, 1 - write := func(t token32, leaf bool) { - S := states[state] - state, S.pegRule, S.begin, S.end, S.next, S.leaf = (state+1)%8, t.pegRule, t.begin, t.end, uint32(depth), leaf - copy(S.depths, depths) - s <- S - } - - states[state].token32 = ordered[0][0] - depths[0]++ - state++ - a, b := ordered[depth-1][depths[depth-1]-1], ordered[depth][depths[depth]] - depthFirstSearch: - for { - for { - if i := depths[depth]; i > 0 { - if c, j := ordered[depth][i-1], depths[depth-1]; a.isParentOf(c) && - (j < 2 || !ordered[depth-1][j-2].isParentOf(c)) { - if c.end != b.begin { - write(token32{pegRule: ruleIn, begin: c.end, end: b.begin}, true) - } - break - } - } - - if a.begin < b.begin { - write(token32{pegRule: rulePre, begin: a.begin, end: b.begin}, true) - } - break - } - - next := depth + 1 - if c := ordered[next][depths[next]]; c.pegRule != ruleUnknown && b.isParentOf(c) { - write(b, false) - depths[depth]++ - depth, a, b = next, b, c - continue - } - - write(b, true) - depths[depth]++ - c, parent := ordered[depth][depths[depth]], true - for { - if c.pegRule != ruleUnknown && a.isParentOf(c) { - b = c - continue depthFirstSearch - } else if parent && b.end != a.end { - write(token32{pegRule: ruleSuf, begin: b.end, end: a.end}, true) - } - - depth-- - if depth > 0 { - a, b, c = ordered[depth-1][depths[depth-1]-1], a, ordered[depth][depths[depth]] - parent = a.isParentOf(b) - continue - } - - break depthFirstSearch - } - } - - close(s) - }() - return s, ordered -} - -func (t *tokens32) PrintSyntax() { - tokens, ordered := t.PreOrder() - max := -1 - for token := range tokens { - if !token.leaf { - fmt.Printf("%v", token.begin) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[36m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[36m%v\x1B[m\n", rul3s[token.pegRule]) - } else if token.begin == token.end { - fmt.Printf("%v", token.begin) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[31m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[31m%v\x1B[m\n", rul3s[token.pegRule]) - } else { - for c, end := token.begin, token.end; c < end; c++ { - if i := int(c); max+1 < i { - for j := max; j < i; j++ { - fmt.Printf("skip %v %v\n", j, token.String()) - } - max = i - } else if i := int(c); i <= max { - for j := i; j <= max; j++ { - fmt.Printf("dupe %v %v\n", j, token.String()) - } - } else { - max = int(c) - } - fmt.Printf("%v", c) - for i, leaf, depths := 0, int(token.next), token.depths; i < leaf; i++ { - fmt.Printf(" \x1B[34m%v\x1B[m", rul3s[ordered[i][depths[i]-1].pegRule]) - } - fmt.Printf(" \x1B[34m%v\x1B[m\n", rul3s[token.pegRule]) - } - fmt.Printf("\n") - } - } -} - -func (t *tokens32) PrintSyntaxTree(buffer string) { - tokens, _ := t.PreOrder() - for token := range tokens { - for c := 0; c < int(token.next); c++ { - fmt.Printf(" ") - } - fmt.Printf("\x1B[34m%v\x1B[m %v\n", rul3s[token.pegRule], strconv.Quote(string(([]rune(buffer)[token.begin:token.end])))) - } -} - -func (t *tokens32) Add(rule pegRule, begin, end, depth uint32, index int) { - t.tree[index] = token32{pegRule: rule, begin: uint32(begin), end: uint32(end), next: uint32(depth)} -} - -func (t *tokens32) Tokens() <-chan token32 { - s := make(chan token32, 16) - go func() { - for _, v := range t.tree { - s <- v.getToken32() - } - close(s) - }() - return s -} - -func (t *tokens32) Error() []token32 { - ordered := t.Order() - length := len(ordered) - tokens, length := make([]token32, length), length-1 - for i := range tokens { - o := ordered[length-i] - if len(o) > 1 { - tokens[i] = o[len(o)-2].getToken32() - } - } - return tokens -} - -func (t *tokens32) Expand(index int) { - tree := t.tree - if index >= len(tree) { - expanded := make([]token32, 2*len(tree)) - copy(expanded, tree) - t.tree = expanded - } -} - -type QueryParser struct { - Buffer string - buffer []rune - rules [21]func() bool - Parse func(rule ...int) error - Reset func() - Pretty bool - tokens32 -} - -type textPosition struct { - line, symbol int -} - -type textPositionMap map[int]textPosition - -func translatePositions(buffer []rune, positions []int) textPositionMap { - length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 - sort.Ints(positions) - -search: - for i, c := range buffer { - if c == '\n' { - line, symbol = line+1, 0 - } else { - symbol++ - } - if i == positions[j] { - translations[positions[j]] = textPosition{line, symbol} - for j++; j < length; j++ { - if i != positions[j] { - continue search - } - } - break search - } - } - - return translations -} - -type parseError struct { - p *QueryParser - max token32 -} - -func (e *parseError) Error() string { - tokens, error := []token32{e.max}, "\n" - positions, p := make([]int, 2*len(tokens)), 0 - for _, token := range tokens { - positions[p], p = int(token.begin), p+1 - positions[p], p = int(token.end), p+1 - } - translations := translatePositions(e.p.buffer, positions) - format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" - if e.p.Pretty { - format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" - } - for _, token := range tokens { - begin, end := int(token.begin), int(token.end) - error += fmt.Sprintf(format, - rul3s[token.pegRule], - translations[begin].line, translations[begin].symbol, - translations[end].line, translations[end].symbol, - strconv.Quote(string(e.p.buffer[begin:end]))) - } - - return error -} - -func (p *QueryParser) PrintSyntaxTree() { - p.tokens32.PrintSyntaxTree(p.Buffer) -} - -func (p *QueryParser) Highlighter() { - p.PrintSyntax() -} - -func (p *QueryParser) Init() { - p.buffer = []rune(p.Buffer) - if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { - p.buffer = append(p.buffer, endSymbol) - } - - tree := tokens32{tree: make([]token32, math.MaxInt16)} - var max token32 - position, depth, tokenIndex, buffer, _rules := uint32(0), uint32(0), 0, p.buffer, p.rules - - p.Parse = func(rule ...int) error { - r := 1 - if len(rule) > 0 { - r = rule[0] - } - matches := p.rules[r]() - p.tokens32 = tree - if matches { - p.trim(tokenIndex) - return nil - } - return &parseError{p, max} - } - - p.Reset = func() { - position, tokenIndex, depth = 0, 0, 0 - } - - add := func(rule pegRule, begin uint32) { - tree.Expand(tokenIndex) - tree.Add(rule, begin, position, depth, tokenIndex) - tokenIndex++ - if begin != position && position > max.end { - max = token32{rule, begin, position, depth} - } - } - - matchDot := func() bool { - if buffer[position] != endSymbol { - position++ - return true - } - return false - } - - /*matchChar := func(c byte) bool { - if buffer[position] == c { - position++ - return true - } - return false - }*/ - - /*matchRange := func(lower byte, upper byte) bool { - if c := buffer[position]; c >= lower && c <= upper { - position++ - return true - } - return false - }*/ - - _rules = [...]func() bool{ - nil, - /* 0 e <- <('"' condition (' '+ and ' '+ condition)* '"' !.)> */ - func() bool { - position0, tokenIndex0, depth0 := position, tokenIndex, depth - { - position1 := position - depth++ - if buffer[position] != rune('"') { - goto l0 - } - position++ - if !_rules[rulecondition]() { - goto l0 - } - l2: - { - position3, tokenIndex3, depth3 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l4: - { - position5, tokenIndex5, depth5 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l5 - } - position++ - goto l4 - l5: - position, tokenIndex, depth = position5, tokenIndex5, depth5 - } - { - position6 := position - depth++ - { - position7, tokenIndex7, depth7 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l8 - } - position++ - goto l7 - l8: - position, tokenIndex, depth = position7, tokenIndex7, depth7 - if buffer[position] != rune('A') { - goto l3 - } - position++ - } - l7: - { - position9, tokenIndex9, depth9 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l10 - } - position++ - goto l9 - l10: - position, tokenIndex, depth = position9, tokenIndex9, depth9 - if buffer[position] != rune('N') { - goto l3 - } - position++ - } - l9: - { - position11, tokenIndex11, depth11 := position, tokenIndex, depth - if buffer[position] != rune('d') { - goto l12 - } - position++ - goto l11 - l12: - position, tokenIndex, depth = position11, tokenIndex11, depth11 - if buffer[position] != rune('D') { - goto l3 - } - position++ - } - l11: - depth-- - add(ruleand, position6) - } - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l13: - { - position14, tokenIndex14, depth14 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l14 - } - position++ - goto l13 - l14: - position, tokenIndex, depth = position14, tokenIndex14, depth14 - } - if !_rules[rulecondition]() { - goto l3 - } - goto l2 - l3: - position, tokenIndex, depth = position3, tokenIndex3, depth3 - } - if buffer[position] != rune('"') { - goto l0 - } - position++ - { - position15, tokenIndex15, depth15 := position, tokenIndex, depth - if !matchDot() { - goto l15 - } - goto l0 - l15: - position, tokenIndex, depth = position15, tokenIndex15, depth15 - } - depth-- - add(rulee, position1) - } - return true - l0: - position, tokenIndex, depth = position0, tokenIndex0, depth0 - return false - }, - /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('E' | 'e') exists) | (&('=') (equal ' '* ((&('\'') value) | (&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ - func() bool { - position16, tokenIndex16, depth16 := position, tokenIndex, depth - { - position17 := position - depth++ - { - position18 := position - depth++ - { - position19 := position - depth++ - { - position22, tokenIndex22, depth22 := position, tokenIndex, depth - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l22 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l22 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l22 - } - position++ - break - case '\'': - if buffer[position] != rune('\'') { - goto l22 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l22 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l22 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l22 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l22 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l22 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l22 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l22 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l22 - } - position++ - break - } - } - - goto l16 - l22: - position, tokenIndex, depth = position22, tokenIndex22, depth22 - } - if !matchDot() { - goto l16 - } - l20: - { - position21, tokenIndex21, depth21 := position, tokenIndex, depth - { - position24, tokenIndex24, depth24 := position, tokenIndex, depth - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l24 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l24 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l24 - } - position++ - break - case '\'': - if buffer[position] != rune('\'') { - goto l24 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l24 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l24 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l24 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l24 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l24 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l24 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l24 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l24 - } - position++ - break - } - } - - goto l21 - l24: - position, tokenIndex, depth = position24, tokenIndex24, depth24 - } - if !matchDot() { - goto l21 - } - goto l20 - l21: - position, tokenIndex, depth = position21, tokenIndex21, depth21 - } - depth-- - add(rulePegText, position19) - } - depth-- - add(ruletag, position18) - } - l26: - { - position27, tokenIndex27, depth27 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l27 - } - position++ - goto l26 - l27: - position, tokenIndex, depth = position27, tokenIndex27, depth27 - } - { - position28, tokenIndex28, depth28 := position, tokenIndex, depth - { - position30 := position - depth++ - if buffer[position] != rune('<') { - goto l29 - } - position++ - if buffer[position] != rune('=') { - goto l29 - } - position++ - depth-- - add(rulele, position30) - } - l31: - { - position32, tokenIndex32, depth32 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l32 - } - position++ - goto l31 - l32: - position, tokenIndex, depth = position32, tokenIndex32, depth32 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l29 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l29 - } - break - default: - if !_rules[rulenumber]() { - goto l29 - } - break - } - } - - goto l28 - l29: - position, tokenIndex, depth = position28, tokenIndex28, depth28 - { - position35 := position - depth++ - if buffer[position] != rune('>') { - goto l34 - } - position++ - if buffer[position] != rune('=') { - goto l34 - } - position++ - depth-- - add(rulege, position35) - } - l36: - { - position37, tokenIndex37, depth37 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l37 - } - position++ - goto l36 - l37: - position, tokenIndex, depth = position37, tokenIndex37, depth37 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l34 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l34 - } - break - default: - if !_rules[rulenumber]() { - goto l34 - } - break - } - } - - goto l28 - l34: - position, tokenIndex, depth = position28, tokenIndex28, depth28 - { - switch buffer[position] { - case 'E', 'e': - { - position40 := position - depth++ - { - position41, tokenIndex41, depth41 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l42 - } - position++ - goto l41 - l42: - position, tokenIndex, depth = position41, tokenIndex41, depth41 - if buffer[position] != rune('E') { - goto l16 - } - position++ - } - l41: - { - position43, tokenIndex43, depth43 := position, tokenIndex, depth - if buffer[position] != rune('x') { - goto l44 - } - position++ - goto l43 - l44: - position, tokenIndex, depth = position43, tokenIndex43, depth43 - if buffer[position] != rune('X') { - goto l16 - } - position++ - } - l43: - { - position45, tokenIndex45, depth45 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l46 - } - position++ - goto l45 - l46: - position, tokenIndex, depth = position45, tokenIndex45, depth45 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l45: - { - position47, tokenIndex47, depth47 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l48 - } - position++ - goto l47 - l48: - position, tokenIndex, depth = position47, tokenIndex47, depth47 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l47: - { - position49, tokenIndex49, depth49 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l50 - } - position++ - goto l49 - l50: - position, tokenIndex, depth = position49, tokenIndex49, depth49 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l49: - { - position51, tokenIndex51, depth51 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l52 - } - position++ - goto l51 - l52: - position, tokenIndex, depth = position51, tokenIndex51, depth51 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l51: - depth-- - add(ruleexists, position40) - } - break - case '=': - { - position53 := position - depth++ - if buffer[position] != rune('=') { - goto l16 - } - position++ - depth-- - add(ruleequal, position53) - } - l54: - { - position55, tokenIndex55, depth55 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l55 - } - position++ - goto l54 - l55: - position, tokenIndex, depth = position55, tokenIndex55, depth55 - } - { - switch buffer[position] { - case '\'': - if !_rules[rulevalue]() { - goto l16 - } - break - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - case '>': - { - position57 := position - depth++ - if buffer[position] != rune('>') { - goto l16 - } - position++ - depth-- - add(ruleg, position57) - } - l58: - { - position59, tokenIndex59, depth59 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l59 - } - position++ - goto l58 - l59: - position, tokenIndex, depth = position59, tokenIndex59, depth59 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - case '<': - { - position61 := position - depth++ - if buffer[position] != rune('<') { - goto l16 - } - position++ - depth-- - add(rulel, position61) - } - l62: - { - position63, tokenIndex63, depth63 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l63 - } - position++ - goto l62 - l63: - position, tokenIndex, depth = position63, tokenIndex63, depth63 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - break - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - break - default: - if !_rules[rulenumber]() { - goto l16 - } - break - } - } - - break - default: - { - position65 := position - depth++ - { - position66, tokenIndex66, depth66 := position, tokenIndex, depth - if buffer[position] != rune('c') { - goto l67 - } - position++ - goto l66 - l67: - position, tokenIndex, depth = position66, tokenIndex66, depth66 - if buffer[position] != rune('C') { - goto l16 - } - position++ - } - l66: - { - position68, tokenIndex68, depth68 := position, tokenIndex, depth - if buffer[position] != rune('o') { - goto l69 - } - position++ - goto l68 - l69: - position, tokenIndex, depth = position68, tokenIndex68, depth68 - if buffer[position] != rune('O') { - goto l16 - } - position++ - } - l68: - { - position70, tokenIndex70, depth70 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l71 - } - position++ - goto l70 - l71: - position, tokenIndex, depth = position70, tokenIndex70, depth70 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l70: - { - position72, tokenIndex72, depth72 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l73 - } - position++ - goto l72 - l73: - position, tokenIndex, depth = position72, tokenIndex72, depth72 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l72: - { - position74, tokenIndex74, depth74 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l75 - } - position++ - goto l74 - l75: - position, tokenIndex, depth = position74, tokenIndex74, depth74 - if buffer[position] != rune('A') { - goto l16 - } - position++ - } - l74: - { - position76, tokenIndex76, depth76 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l77 - } - position++ - goto l76 - l77: - position, tokenIndex, depth = position76, tokenIndex76, depth76 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l76: - { - position78, tokenIndex78, depth78 := position, tokenIndex, depth - if buffer[position] != rune('n') { - goto l79 - } - position++ - goto l78 - l79: - position, tokenIndex, depth = position78, tokenIndex78, depth78 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l78: - { - position80, tokenIndex80, depth80 := position, tokenIndex, depth - if buffer[position] != rune('s') { - goto l81 - } - position++ - goto l80 - l81: - position, tokenIndex, depth = position80, tokenIndex80, depth80 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l80: - depth-- - add(rulecontains, position65) - } - l82: - { - position83, tokenIndex83, depth83 := position, tokenIndex, depth - if buffer[position] != rune(' ') { - goto l83 - } - position++ - goto l82 - l83: - position, tokenIndex, depth = position83, tokenIndex83, depth83 - } - if !_rules[rulevalue]() { - goto l16 - } - break - } - } - - } - l28: - depth-- - add(rulecondition, position17) - } - return true - l16: - position, tokenIndex, depth = position16, tokenIndex16, depth16 - return false - }, - /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('\'') '\'') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ - nil, - /* 3 value <- <<('\'' (!('"' / '\'') .)* '\'')>> */ - func() bool { - position85, tokenIndex85, depth85 := position, tokenIndex, depth - { - position86 := position - depth++ - { - position87 := position - depth++ - if buffer[position] != rune('\'') { - goto l85 - } - position++ - l88: - { - position89, tokenIndex89, depth89 := position, tokenIndex, depth - { - position90, tokenIndex90, depth90 := position, tokenIndex, depth - { - position91, tokenIndex91, depth91 := position, tokenIndex, depth - if buffer[position] != rune('"') { - goto l92 - } - position++ - goto l91 - l92: - position, tokenIndex, depth = position91, tokenIndex91, depth91 - if buffer[position] != rune('\'') { - goto l90 - } - position++ - } - l91: - goto l89 - l90: - position, tokenIndex, depth = position90, tokenIndex90, depth90 - } - if !matchDot() { - goto l89 - } - goto l88 - l89: - position, tokenIndex, depth = position89, tokenIndex89, depth89 - } - if buffer[position] != rune('\'') { - goto l85 - } - position++ - depth-- - add(rulePegText, position87) - } - depth-- - add(rulevalue, position86) - } - return true - l85: - position, tokenIndex, depth = position85, tokenIndex85, depth85 - return false - }, - /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ - func() bool { - position93, tokenIndex93, depth93 := position, tokenIndex, depth - { - position94 := position - depth++ - { - position95 := position - depth++ - { - position96, tokenIndex96, depth96 := position, tokenIndex, depth - if buffer[position] != rune('0') { - goto l97 - } - position++ - goto l96 - l97: - position, tokenIndex, depth = position96, tokenIndex96, depth96 - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l93 - } - position++ - l98: - { - position99, tokenIndex99, depth99 := position, tokenIndex, depth - if !_rules[ruledigit]() { - goto l99 - } - goto l98 - l99: - position, tokenIndex, depth = position99, tokenIndex99, depth99 - } - { - position100, tokenIndex100, depth100 := position, tokenIndex, depth - if buffer[position] != rune('.') { - goto l100 - } - position++ - l102: - { - position103, tokenIndex103, depth103 := position, tokenIndex, depth - if !_rules[ruledigit]() { - goto l103 - } - goto l102 - l103: - position, tokenIndex, depth = position103, tokenIndex103, depth103 - } - goto l101 - l100: - position, tokenIndex, depth = position100, tokenIndex100, depth100 - } - l101: - } - l96: - depth-- - add(rulePegText, position95) - } - depth-- - add(rulenumber, position94) - } - return true - l93: - position, tokenIndex, depth = position93, tokenIndex93, depth93 - return false - }, - /* 5 digit <- <[0-9]> */ - func() bool { - position104, tokenIndex104, depth104 := position, tokenIndex, depth - { - position105 := position - depth++ - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l104 - } - position++ - depth-- - add(ruledigit, position105) - } - return true - l104: - position, tokenIndex, depth = position104, tokenIndex104, depth104 - return false - }, - /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ - func() bool { - position106, tokenIndex106, depth106 := position, tokenIndex, depth - { - position107 := position - depth++ - { - position108, tokenIndex108, depth108 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l109 - } - position++ - goto l108 - l109: - position, tokenIndex, depth = position108, tokenIndex108, depth108 - if buffer[position] != rune('T') { - goto l106 - } - position++ - } - l108: - { - position110, tokenIndex110, depth110 := position, tokenIndex, depth - if buffer[position] != rune('i') { - goto l111 - } - position++ - goto l110 - l111: - position, tokenIndex, depth = position110, tokenIndex110, depth110 - if buffer[position] != rune('I') { - goto l106 - } - position++ - } - l110: - { - position112, tokenIndex112, depth112 := position, tokenIndex, depth - if buffer[position] != rune('m') { - goto l113 - } - position++ - goto l112 - l113: - position, tokenIndex, depth = position112, tokenIndex112, depth112 - if buffer[position] != rune('M') { - goto l106 - } - position++ - } - l112: - { - position114, tokenIndex114, depth114 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l115 - } - position++ - goto l114 - l115: - position, tokenIndex, depth = position114, tokenIndex114, depth114 - if buffer[position] != rune('E') { - goto l106 - } - position++ - } - l114: - if buffer[position] != rune(' ') { - goto l106 - } - position++ - { - position116 := position - depth++ - if !_rules[ruleyear]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[rulemonth]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[ruleday]() { - goto l106 - } - if buffer[position] != rune('T') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - { - position117, tokenIndex117, depth117 := position, tokenIndex, depth - { - position119, tokenIndex119, depth119 := position, tokenIndex, depth - if buffer[position] != rune('-') { - goto l120 - } - position++ - goto l119 - l120: - position, tokenIndex, depth = position119, tokenIndex119, depth119 - if buffer[position] != rune('+') { - goto l118 - } - position++ - } - l119: - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - if buffer[position] != rune(':') { - goto l118 - } - position++ - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - goto l117 - l118: - position, tokenIndex, depth = position117, tokenIndex117, depth117 - if buffer[position] != rune('Z') { - goto l106 - } - position++ - } - l117: - depth-- - add(rulePegText, position116) - } - depth-- - add(ruletime, position107) - } - return true - l106: - position, tokenIndex, depth = position106, tokenIndex106, depth106 - return false - }, - /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ - func() bool { - position121, tokenIndex121, depth121 := position, tokenIndex, depth - { - position122 := position - depth++ - { - position123, tokenIndex123, depth123 := position, tokenIndex, depth - if buffer[position] != rune('d') { - goto l124 - } - position++ - goto l123 - l124: - position, tokenIndex, depth = position123, tokenIndex123, depth123 - if buffer[position] != rune('D') { - goto l121 - } - position++ - } - l123: - { - position125, tokenIndex125, depth125 := position, tokenIndex, depth - if buffer[position] != rune('a') { - goto l126 - } - position++ - goto l125 - l126: - position, tokenIndex, depth = position125, tokenIndex125, depth125 - if buffer[position] != rune('A') { - goto l121 - } - position++ - } - l125: - { - position127, tokenIndex127, depth127 := position, tokenIndex, depth - if buffer[position] != rune('t') { - goto l128 - } - position++ - goto l127 - l128: - position, tokenIndex, depth = position127, tokenIndex127, depth127 - if buffer[position] != rune('T') { - goto l121 - } - position++ - } - l127: - { - position129, tokenIndex129, depth129 := position, tokenIndex, depth - if buffer[position] != rune('e') { - goto l130 - } - position++ - goto l129 - l130: - position, tokenIndex, depth = position129, tokenIndex129, depth129 - if buffer[position] != rune('E') { - goto l121 - } - position++ - } - l129: - if buffer[position] != rune(' ') { - goto l121 - } - position++ - { - position131 := position - depth++ - if !_rules[ruleyear]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[rulemonth]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[ruleday]() { - goto l121 - } - depth-- - add(rulePegText, position131) - } - depth-- - add(ruledate, position122) - } - return true - l121: - position, tokenIndex, depth = position121, tokenIndex121, depth121 - return false - }, - /* 8 year <- <(('1' / '2') digit digit digit)> */ - func() bool { - position132, tokenIndex132, depth132 := position, tokenIndex, depth - { - position133 := position - depth++ - { - position134, tokenIndex134, depth134 := position, tokenIndex, depth - if buffer[position] != rune('1') { - goto l135 - } - position++ - goto l134 - l135: - position, tokenIndex, depth = position134, tokenIndex134, depth134 - if buffer[position] != rune('2') { - goto l132 - } - position++ - } - l134: - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - depth-- - add(ruleyear, position133) - } - return true - l132: - position, tokenIndex, depth = position132, tokenIndex132, depth132 - return false - }, - /* 9 month <- <(('0' / '1') digit)> */ - func() bool { - position136, tokenIndex136, depth136 := position, tokenIndex, depth - { - position137 := position - depth++ - { - position138, tokenIndex138, depth138 := position, tokenIndex, depth - if buffer[position] != rune('0') { - goto l139 - } - position++ - goto l138 - l139: - position, tokenIndex, depth = position138, tokenIndex138, depth138 - if buffer[position] != rune('1') { - goto l136 - } - position++ - } - l138: - if !_rules[ruledigit]() { - goto l136 - } - depth-- - add(rulemonth, position137) - } - return true - l136: - position, tokenIndex, depth = position136, tokenIndex136, depth136 - return false - }, - /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ - func() bool { - position140, tokenIndex140, depth140 := position, tokenIndex, depth - { - position141 := position - depth++ - { - switch buffer[position] { - case '3': - if buffer[position] != rune('3') { - goto l140 - } - position++ - break - case '2': - if buffer[position] != rune('2') { - goto l140 - } - position++ - break - case '1': - if buffer[position] != rune('1') { - goto l140 - } - position++ - break - default: - if buffer[position] != rune('0') { - goto l140 - } - position++ - break - } - } - - if !_rules[ruledigit]() { - goto l140 - } - depth-- - add(ruleday, position141) - } - return true - l140: - position, tokenIndex, depth = position140, tokenIndex140, depth140 - return false - }, - /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ - nil, - /* 12 equal <- <'='> */ - nil, - /* 13 contains <- <(('c' / 'C') ('o' / 'O') ('n' / 'N') ('t' / 'T') ('a' / 'A') ('i' / 'I') ('n' / 'N') ('s' / 'S'))> */ - nil, - /* 14 exists <- <(('e' / 'E') ('x' / 'X') ('i' / 'I') ('s' / 'S') ('t' / 'T') ('s' / 'S'))> */ - nil, - /* 15 le <- <('<' '=')> */ - nil, - /* 16 ge <- <('>' '=')> */ - nil, - /* 17 l <- <'<'> */ - nil, - /* 18 g <- <'>'> */ - nil, - nil, - } - p.rules = _rules -} diff --git a/libs/pubsub/query/oldquery/query_test.go b/libs/pubsub/query/oldquery/query_test.go deleted file mode 100644 index 9db7651c7..000000000 --- a/libs/pubsub/query/oldquery/query_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package query_test - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - abci "github.com/tendermint/tendermint/abci/types" - query "github.com/tendermint/tendermint/libs/pubsub/query/oldquery" -) - -func expandEvents(flattenedEvents map[string][]string) []abci.Event { - events := make([]abci.Event, len(flattenedEvents)) - - for composite, values := range flattenedEvents { - tokens := strings.Split(composite, ".") - - attrs := make([]abci.EventAttribute, len(values)) - for i, v := range values { - attrs[i] = abci.EventAttribute{ - Key: tokens[len(tokens)-1], - Value: v, - } - } - - events = append(events, abci.Event{ - Type: strings.Join(tokens[:len(tokens)-1], "."), - Attributes: attrs, - }) - } - - return events -} - -func TestMatches(t *testing.T) { - var ( - txDate = "2017-01-01" - txTime = "2018-05-03T14:45:00Z" - ) - - testCases := []struct { - s string - events map[string][]string - matches bool - }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, true}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, true}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, true}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, true}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, true}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, true}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, true}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, true}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, true}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - true, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - true, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, true}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - true, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, true}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, true}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - true, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - true, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - true, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - true, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) - - rawEvents := expandEvents(tc.events) - match, err := q.Matches(rawEvents) - require.Nil(t, err, "Query '%s' should not error on input %v", tc.s, tc.events) - require.Equal(t, tc.matches, match, "Query '%s' on input %v: got %v, want %v", - tc.s, tc.events, match, tc.matches) - } -} - -func TestMustParse(t *testing.T) { - require.Panics(t, func() { query.MustParse("=") }) - require.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) -} - -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") - require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - require.Equal(t, tc.conditions, c) - } -} From c9f90953a2e9320bd23edda102e6756cbcc9b460 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 30 Nov 2021 06:08:21 -0800 Subject: [PATCH 08/21] Add pending change log entry for #7319. (#7339) --- CHANGELOG_PENDING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index cb1a79cac..84abf5b00 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -44,6 +44,8 @@ Special thanks to external contributors on this release: ### IMPROVEMENTS +- [pubsub] \#7319 Performance improvements for the event query API (@creachadair) + ### BUG FIXES - fix: assignment copies lock value in `BitArray.UnmarshalJSON()` (@lklimek) From 4af2dbd03bc6d8b5025ffaece95ec79c6b120054 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 09:24:11 -0500 Subject: [PATCH 09/21] eventbus: plumb contexts (#7337) * eventbus: plumb contexts * fix lint --- internal/blocksync/reactor.go | 2 +- internal/blocksync/reactor_test.go | 2 +- internal/consensus/byzantine_test.go | 2 +- internal/consensus/common_test.go | 12 +- internal/consensus/reactor.go | 4 +- internal/consensus/reactor_test.go | 2 +- internal/consensus/replay.go | 34 ++-- internal/consensus/replay_file.go | 14 +- internal/consensus/replay_test.go | 21 ++- internal/consensus/state.go | 176 ++++++++++-------- internal/consensus/state_test.go | 22 +-- internal/consensus/wal_generator.go | 2 +- internal/eventbus/event_bus.go | 87 ++++----- internal/eventbus/event_bus_test.go | 44 ++--- internal/state/execution.go | 21 ++- internal/state/execution_test.go | 10 +- internal/state/helpers_test.go | 20 +- .../state/indexer/indexer_service_test.go | 6 +- internal/state/validation_test.go | 4 +- node/node.go | 8 +- node/setup.go | 3 +- types/events.go | 13 +- 22 files changed, 276 insertions(+), 233 deletions(-) diff --git a/internal/blocksync/reactor.go b/internal/blocksync/reactor.go index a6845b719..ac5d45fb7 100644 --- a/internal/blocksync/reactor.go +++ b/internal/blocksync/reactor.go @@ -580,7 +580,7 @@ FOR_LOOP: // TODO: Same thing for app - but we would need a way to get the hash // without persisting the state. - state, err = r.blockExec.ApplyBlock(state, firstID, first) + state, err = r.blockExec.ApplyBlock(ctx, state, firstID, first) if err != nil { // TODO: This is bad, are we zombie? panic(fmt.Sprintf("failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) diff --git a/internal/blocksync/reactor_test.go b/internal/blocksync/reactor_test.go index 5792d9e78..c5f76066a 100644 --- a/internal/blocksync/reactor_test.go +++ b/internal/blocksync/reactor_test.go @@ -158,7 +158,7 @@ func (rts *reactorTestSuite) addNode( thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes) blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()} - state, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, err = blockExec.ApplyBlock(ctx, state, blockID, thisBlock) require.NoError(t, err) blockStore.SaveBlock(thisBlock, thisParts, lastCommit) diff --git a/internal/consensus/byzantine_test.go b/internal/consensus/byzantine_test.go index d9e5e46b8..4c9ccbb2e 100644 --- a/internal/consensus/byzantine_test.go +++ b/internal/consensus/byzantine_test.go @@ -90,7 +90,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { // Make State blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) - cs := NewState(logger, thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) + cs := NewState(ctx, logger, thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) // set private validator pv := privVals[i] cs.SetPrivValidator(pv) diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index bcbbe7c88..30729c038 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -224,7 +224,7 @@ func (vss ValidatorStubsByPower) Swap(i, j int) { // Functions for transitioning the consensus state func startTestRound(ctx context.Context, cs *State, height int64, round int32) { - cs.enterNewRound(height, round) + cs.enterNewRound(ctx, height, round) cs.startRoutines(ctx, 0) } @@ -467,7 +467,15 @@ func newStateWithConfigAndBlockStore( } blockExec := sm.NewBlockExecutor(stateStore, logger, proxyAppConnCon, mempool, evpool, blockStore) - cs := NewState(logger.With("module", "consensus"), thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) + cs := NewState(ctx, + logger.With("module", "consensus"), + thisConfig.Consensus, + state, + blockExec, + blockStore, + mempool, + evpool, + ) cs.SetPrivValidator(pv) eventBus := eventbus.NewDefault(logger.With("module", "events")) diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 03865d13d..7e46444b9 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -275,7 +275,7 @@ func (r *Reactor) SwitchToConsensus(ctx context.Context, state sm.State, skipWAL // NOTE: The line below causes broadcastNewRoundStepRoutine() to broadcast a // NewRoundStepMessage. - r.state.updateToState(state) + r.state.updateToState(ctx, state) r.mtx.Lock() r.waitSync = false @@ -299,7 +299,7 @@ conR: } d := types.EventDataBlockSyncStatus{Complete: true, Height: state.LastBlockHeight} - if err := r.eventBus.PublishEventBlockSyncStatus(d); err != nil { + if err := r.eventBus.PublishEventBlockSyncStatus(ctx, d); err != nil { r.Logger.Error("failed to emit the blocksync complete event", "err", err) } } diff --git a/internal/consensus/reactor_test.go b/internal/consensus/reactor_test.go index de6465f23..9e7c498cd 100644 --- a/internal/consensus/reactor_test.go +++ b/internal/consensus/reactor_test.go @@ -421,7 +421,7 @@ func TestReactorWithEvidence(t *testing.T) { evpool2 := sm.EmptyEvidencePool{} blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) - cs := NewState(logger.With("validator", i, "module", "consensus"), + cs := NewState(ctx, logger.With("validator", i, "module", "consensus"), thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2) cs.SetPrivValidator(pv) diff --git a/internal/consensus/replay.go b/internal/consensus/replay.go index f40389f2b..2408b03f1 100644 --- a/internal/consensus/replay.go +++ b/internal/consensus/replay.go @@ -38,7 +38,7 @@ var crc32c = crc32.MakeTable(crc32.Castagnoli) // Unmarshal and apply a single message to the consensus state as if it were // received in receiveRoutine. Lines that start with "#" are ignored. // NOTE: receiveRoutine should not be running. -func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub eventbus.Subscription) error { +func (cs *State) readReplayMessage(ctx context.Context, msg *TimedWALMessage, newStepSub eventbus.Subscription) error { // Skip meta messages which exist for demarcating boundaries. if _, ok := msg.Msg.(EndHeightMessage); ok { return nil @@ -81,10 +81,10 @@ func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub eventbus.Sub "blockID", v.BlockID, "peer", peerID) } - cs.handleMsg(m) + cs.handleMsg(ctx, m) case timeoutInfo: cs.Logger.Info("Replay: Timeout", "height", m.Height, "round", m.Round, "step", m.Step, "dur", m.Duration) - cs.handleTimeout(m, cs.RoundState) + cs.handleTimeout(ctx, m, cs.RoundState) default: return fmt.Errorf("replay: Unknown TimedWALMessage type: %v", reflect.TypeOf(msg.Msg)) } @@ -93,7 +93,7 @@ func (cs *State) readReplayMessage(msg *TimedWALMessage, newStepSub eventbus.Sub // Replay only those messages since the last block. `timeoutRoutine` should // run concurrently to read off tickChan. -func (cs *State) catchupReplay(csHeight int64) error { +func (cs *State) catchupReplay(ctx context.Context, csHeight int64) error { // Set replayMode to true so we don't log signing errors. cs.replayMode = true @@ -160,7 +160,7 @@ LOOP: // NOTE: since the priv key is set when the msgs are received // it will attempt to eg double sign but we can just ignore it // since the votes will be replayed and we'll get to the next step - if err := cs.readReplayMessage(msg, nil); err != nil { + if err := cs.readReplayMessage(ctx, msg, nil); err != nil { return err } } @@ -390,7 +390,7 @@ func (h *Handshaker) ReplayBlocks( // Either the app is asking for replay, or we're all synced up. if appBlockHeight < storeBlockHeight { // the app is behind, so replay blocks, but no need to go through WAL (state is already synced to store) - return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, false) + return h.replayBlocks(ctx, state, proxyApp, appBlockHeight, storeBlockHeight, false) } else if appBlockHeight == storeBlockHeight { // We're good! @@ -405,7 +405,7 @@ func (h *Handshaker) ReplayBlocks( case appBlockHeight < stateBlockHeight: // the app is further behind than it should be, so replay blocks // but leave the last block to go through the WAL - return h.replayBlocks(state, proxyApp, appBlockHeight, storeBlockHeight, true) + return h.replayBlocks(ctx, state, proxyApp, appBlockHeight, storeBlockHeight, true) case appBlockHeight == stateBlockHeight: // We haven't run Commit (both the state and app are one block behind), @@ -413,7 +413,7 @@ func (h *Handshaker) ReplayBlocks( // NOTE: We could instead use the cs.WAL on cs.Start, // but we'd have to allow the WAL to replay a block that wrote it's #ENDHEIGHT h.logger.Info("Replay last block using real app") - state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) + state, err = h.replayBlock(ctx, state, storeBlockHeight, proxyApp.Consensus()) return state.AppHash, err case appBlockHeight == storeBlockHeight: @@ -424,7 +424,7 @@ func (h *Handshaker) ReplayBlocks( } mockApp := newMockProxyApp(ctx, h.logger, appHash, abciResponses) h.logger.Info("Replay last block using mock app") - state, err = h.replayBlock(state, storeBlockHeight, mockApp) + state, err = h.replayBlock(ctx, state, storeBlockHeight, mockApp) return state.AppHash, err } @@ -435,6 +435,7 @@ func (h *Handshaker) ReplayBlocks( } func (h *Handshaker) replayBlocks( + ctx context.Context, state sm.State, proxyApp proxy.AppConns, appBlockHeight, @@ -474,13 +475,13 @@ func (h *Handshaker) replayBlocks( blockExec := sm.NewBlockExecutor( h.stateStore, h.logger, proxyApp.Consensus(), emptyMempool{}, sm.EmptyEvidencePool{}, h.store) blockExec.SetEventBus(h.eventBus) - appHash, err = sm.ExecCommitBlock( + appHash, err = sm.ExecCommitBlock(ctx, blockExec, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state) if err != nil { return nil, err } } else { - appHash, err = sm.ExecCommitBlock( + appHash, err = sm.ExecCommitBlock(ctx, nil, proxyApp.Consensus(), block, h.logger, h.stateStore, h.genDoc.InitialHeight, state) if err != nil { return nil, err @@ -492,7 +493,7 @@ func (h *Handshaker) replayBlocks( if mutateState { // sync the final block - state, err = h.replayBlock(state, storeBlockHeight, proxyApp.Consensus()) + state, err = h.replayBlock(ctx, state, storeBlockHeight, proxyApp.Consensus()) if err != nil { return nil, err } @@ -504,7 +505,12 @@ func (h *Handshaker) replayBlocks( } // ApplyBlock on the proxyApp with the last block. -func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.AppConnConsensus) (sm.State, error) { +func (h *Handshaker) replayBlock( + ctx context.Context, + state sm.State, + height int64, + proxyApp proxy.AppConnConsensus, +) (sm.State, error) { block := h.store.LoadBlock(height) meta := h.store.LoadBlockMeta(height) @@ -514,7 +520,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap blockExec.SetEventBus(h.eventBus) var err error - state, err = blockExec.ApplyBlock(state, meta.BlockID, block) + state, err = blockExec.ApplyBlock(ctx, state, meta.BlockID, block) if err != nil { return sm.State{}, err } diff --git a/internal/consensus/replay_file.go b/internal/consensus/replay_file.go index 1de0ffa0e..ef5e88730 100644 --- a/internal/consensus/replay_file.go +++ b/internal/consensus/replay_file.go @@ -104,7 +104,7 @@ func (cs *State) ReplayFile(ctx context.Context, file string, console bool) erro return err } - if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil { + if err := pb.cs.readReplayMessage(ctx, msg, newStepSub); err != nil { return err } @@ -141,13 +141,13 @@ func newPlayback(fileName string, fp *os.File, cs *State, genState sm.State) *pl } // go back count steps by resetting the state and running (pb.count - count) steps -func (pb *playback) replayReset(count int, newStepSub eventbus.Subscription) error { +func (pb *playback) replayReset(ctx context.Context, count int, newStepSub eventbus.Subscription) error { if err := pb.cs.Stop(); err != nil { return err } pb.cs.Wait() - newCS := NewState(pb.cs.Logger, pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec, + newCS := NewState(ctx, pb.cs.Logger, pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec, pb.cs.blockStore, pb.cs.txNotifier, pb.cs.evpool) newCS.SetEventBus(pb.cs.eventBus) newCS.startForReplay() @@ -173,7 +173,7 @@ func (pb *playback) replayReset(count int, newStepSub eventbus.Subscription) err } else if err != nil { return err } - if err := pb.cs.readReplayMessage(msg, newStepSub); err != nil { + if err := pb.cs.readReplayMessage(ctx, msg, newStepSub); err != nil { return err } pb.count++ @@ -254,7 +254,7 @@ func (pb *playback) replayConsoleLoop() (int, error) { }() if len(tokens) == 1 { - if err := pb.replayReset(1, newStepSub); err != nil { + if err := pb.replayReset(ctx, 1, newStepSub); err != nil { pb.cs.Logger.Error("Replay reset error", "err", err) } } else { @@ -263,7 +263,7 @@ func (pb *playback) replayConsoleLoop() (int, error) { fmt.Println("back takes an integer argument") } else if i > pb.count { fmt.Printf("argument to back must not be larger than the current count (%d)\n", pb.count) - } else if err := pb.replayReset(i, newStepSub); err != nil { + } else if err := pb.replayReset(ctx, i, newStepSub); err != nil { pb.cs.Logger.Error("Replay reset error", "err", err) } } @@ -359,7 +359,7 @@ func newConsensusStateForReplay( mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{} blockExec := sm.NewBlockExecutor(stateStore, logger, proxyApp.Consensus(), mempool, evpool, blockStore) - consensusState := NewState(logger, csConfig, state.Copy(), blockExec, + consensusState := NewState(ctx, logger, csConfig, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetEventBus(eventBus) diff --git a/internal/consensus/replay_test.go b/internal/consensus/replay_test.go index bf1927742..6c33268a7 100644 --- a/internal/consensus/replay_test.go +++ b/internal/consensus/replay_test.go @@ -855,18 +855,21 @@ func testHandshakeReplay( } } -func applyBlock(stateStore sm.Store, +func applyBlock( + ctx context.Context, + stateStore sm.Store, mempool mempool.Mempool, evpool sm.EvidencePool, st sm.State, blk *types.Block, proxyApp proxy.AppConns, - blockStore *mockBlockStore) sm.State { + blockStore *mockBlockStore, +) sm.State { testPartSize := types.BlockPartSizeBytes blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) blkID := types.BlockID{Hash: blk.Hash(), PartSetHeader: blk.MakePartSet(testPartSize).Header()} - newState, err := blockExec.ApplyBlock(st, blkID, blk) + newState, err := blockExec.ApplyBlock(ctx, st, blkID, blk) if err != nil { panic(err) } @@ -904,18 +907,18 @@ func buildAppStateFromChain( case 0: for i := 0; i < nBlocks; i++ { block := chain[i] - state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) + state = applyBlock(ctx, stateStore, mempool, evpool, state, block, proxyApp, blockStore) } case 1, 2, 3: for i := 0; i < nBlocks-1; i++ { block := chain[i] - state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) + state = applyBlock(ctx, stateStore, mempool, evpool, state, block, proxyApp, blockStore) } if mode == 2 || mode == 3 { // update the kvstore height and apphash // as if we ran commit but not - state = applyBlock(stateStore, mempool, evpool, state, chain[nBlocks-1], proxyApp, blockStore) + state = applyBlock(ctx, stateStore, mempool, evpool, state, chain[nBlocks-1], proxyApp, blockStore) } default: panic(fmt.Sprintf("unknown mode %v", mode)) @@ -961,19 +964,19 @@ func buildTMStateFromChain( case 0: // sync right up for _, block := range chain { - state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) + state = applyBlock(ctx, stateStore, mempool, evpool, state, block, proxyApp, blockStore) } case 1, 2, 3: // sync up to the penultimate as if we stored the block. // whether we commit or not depends on the appHash for _, block := range chain[:len(chain)-1] { - state = applyBlock(stateStore, mempool, evpool, state, block, proxyApp, blockStore) + state = applyBlock(ctx, stateStore, mempool, evpool, state, block, proxyApp, blockStore) } // apply the final block to a state copy so we can // get the right next appHash but keep the state back - applyBlock(stateStore, mempool, evpool, state, chain[len(chain)-1], proxyApp, blockStore) + applyBlock(ctx, stateStore, mempool, evpool, state, chain[len(chain)-1], proxyApp, blockStore) default: panic(fmt.Sprintf("unknown mode %v", mode)) } diff --git a/internal/consensus/state.go b/internal/consensus/state.go index 8cf1a7f9f..2e9f97503 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -153,6 +153,7 @@ type StateOption func(*State) // NewState returns a new State. func NewState( + ctx context.Context, logger log.Logger, cfg *config.ConsensusConfig, state sm.State, @@ -190,7 +191,7 @@ func NewState( cs.reconstructLastCommit(state) } - cs.updateToState(state) + cs.updateToState(ctx, state) // NOTE: we do not call scheduleRound0 yet, we do that upon Start() @@ -345,7 +346,7 @@ func (cs *State) OnStart(ctx context.Context) error { LOOP: for { - err := cs.catchupReplay(cs.Height) + err := cs.catchupReplay(ctx, cs.Height) switch { case err == nil: break LOOP @@ -409,7 +410,7 @@ func (cs *State) OnStart(ctx context.Context) error { } // now start the receiveRoutine - go cs.receiveRoutine(0) + go cs.receiveRoutine(ctx, 0) // schedule the first round! // use GetRoundState so we don't race the receiveRoutine for access @@ -427,7 +428,7 @@ func (cs *State) startRoutines(ctx context.Context, maxSteps int) { return } - go cs.receiveRoutine(maxSteps) + go cs.receiveRoutine(ctx, maxSteps) } // loadWalFile loads WAL data from file. It overwrites cs.wal. @@ -625,7 +626,7 @@ func (cs *State) reconstructLastCommit(state sm.State) { // Updates State and increments height to match that of state. // The round becomes 0 and cs.Step becomes cstypes.RoundStepNewHeight. -func (cs *State) updateToState(state sm.State) { +func (cs *State) updateToState(ctx context.Context, state sm.State) { if cs.CommitRound > -1 && 0 < cs.Height && cs.Height != state.LastBlockHeight { panic(fmt.Sprintf( "updateToState() expected state height of %v but found %v", @@ -660,7 +661,7 @@ func (cs *State) updateToState(state sm.State) { "new_height", state.LastBlockHeight+1, "old_height", cs.state.LastBlockHeight+1, ) - cs.newStep() + cs.newStep(ctx) return } } @@ -729,10 +730,10 @@ func (cs *State) updateToState(state sm.State) { cs.state = state // Finally, broadcast RoundState - cs.newStep() + cs.newStep(ctx) } -func (cs *State) newStep() { +func (cs *State) newStep(ctx context.Context) { rs := cs.RoundStateEvent() if err := cs.wal.Write(rs); err != nil { cs.Logger.Error("failed writing to WAL", "err", err) @@ -742,7 +743,7 @@ func (cs *State) newStep() { // newStep is called by updateToState in NewState before the eventBus is set! if cs.eventBus != nil { - if err := cs.eventBus.PublishEventNewRoundStep(rs); err != nil { + if err := cs.eventBus.PublishEventNewRoundStep(ctx, rs); err != nil { cs.Logger.Error("failed publishing new round step", "err", err) } @@ -758,7 +759,7 @@ func (cs *State) newStep() { // It keeps the RoundState and is the only thing that updates it. // Updates (state transitions) happen on timeouts, complete proposals, and 2/3 majorities. // State must be locked before any internal state is updated. -func (cs *State) receiveRoutine(maxSteps int) { +func (cs *State) receiveRoutine(ctx context.Context, maxSteps int) { onExit := func(cs *State) { // NOTE: the internalMsgQueue may have signed messages from our // priv_val that haven't hit the WAL, but its ok because @@ -804,7 +805,7 @@ func (cs *State) receiveRoutine(maxSteps int) { select { case <-cs.txNotifier.TxsAvailable(): - cs.handleTxsAvailable() + cs.handleTxsAvailable(ctx) case mi = <-cs.peerMsgQueue: if err := cs.wal.Write(mi); err != nil { @@ -813,7 +814,7 @@ func (cs *State) receiveRoutine(maxSteps int) { // handles proposals, block parts, votes // may generate internal events (votes, complete proposals, 2/3 majorities) - cs.handleMsg(mi) + cs.handleMsg(ctx, mi) case mi = <-cs.internalMsgQueue: err := cs.wal.WriteSync(mi) // NOTE: fsync @@ -833,7 +834,7 @@ func (cs *State) receiveRoutine(maxSteps int) { } // handles proposals, block parts, votes - cs.handleMsg(mi) + cs.handleMsg(ctx, mi) case ti := <-cs.timeoutTicker.Chan(): // tockChan: if err := cs.wal.Write(ti); err != nil { @@ -842,17 +843,18 @@ func (cs *State) receiveRoutine(maxSteps int) { // if the timeout is relevant to the rs // go to the next step - cs.handleTimeout(ti, rs) + cs.handleTimeout(ctx, ti, rs) case <-cs.Quit(): onExit(cs) return } + // TODO should we handle context cancels here? } } // state transitions on complete-proposal, 2/3-any, 2/3-one -func (cs *State) handleMsg(mi msgInfo) { +func (cs *State) handleMsg(ctx context.Context, mi msgInfo) { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -871,7 +873,7 @@ func (cs *State) handleMsg(mi msgInfo) { case *BlockPartMessage: // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit - added, err = cs.addProposalBlockPart(msg, peerID) + added, err = cs.addProposalBlockPart(ctx, msg, peerID) if added { cs.statsMsgQueue <- mi } @@ -889,7 +891,7 @@ func (cs *State) handleMsg(mi msgInfo) { case *VoteMessage: // attempt to add the vote and dupeout the validator if its a duplicate signature // if the vote gives us a 2/3-any or 2/3-one, we transition - added, err = cs.tryAddVote(msg.Vote, peerID) + added, err = cs.tryAddVote(ctx, msg.Vote, peerID) if added { cs.statsMsgQueue <- mi } @@ -926,7 +928,11 @@ func (cs *State) handleMsg(mi msgInfo) { } } -func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { +func (cs *State) handleTimeout( + ctx context.Context, + ti timeoutInfo, + rs cstypes.RoundState, +) { cs.Logger.Debug("received tock", "timeout", ti.Duration, "height", ti.Height, "round", ti.Round, "step", ti.Step) // timeouts must be for current height, round, step @@ -943,32 +949,32 @@ func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { case cstypes.RoundStepNewHeight: // NewRound event fired from enterNewRound. // XXX: should we fire timeout here (for timeout commit)? - cs.enterNewRound(ti.Height, 0) + cs.enterNewRound(ctx, ti.Height, 0) case cstypes.RoundStepNewRound: - cs.enterPropose(ti.Height, 0) + cs.enterPropose(ctx, ti.Height, 0) case cstypes.RoundStepPropose: - if err := cs.eventBus.PublishEventTimeoutPropose(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventTimeoutPropose(ctx, cs.RoundStateEvent()); err != nil { cs.Logger.Error("failed publishing timeout propose", "err", err) } - cs.enterPrevote(ti.Height, ti.Round) + cs.enterPrevote(ctx, ti.Height, ti.Round) case cstypes.RoundStepPrevoteWait: - if err := cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventTimeoutWait(ctx, cs.RoundStateEvent()); err != nil { cs.Logger.Error("failed publishing timeout wait", "err", err) } - cs.enterPrecommit(ti.Height, ti.Round) + cs.enterPrecommit(ctx, ti.Height, ti.Round) case cstypes.RoundStepPrecommitWait: - if err := cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventTimeoutWait(ctx, cs.RoundStateEvent()); err != nil { cs.Logger.Error("failed publishing timeout wait", "err", err) } - cs.enterPrecommit(ti.Height, ti.Round) - cs.enterNewRound(ti.Height, ti.Round+1) + cs.enterPrecommit(ctx, ti.Height, ti.Round) + cs.enterNewRound(ctx, ti.Height, ti.Round+1) default: panic(fmt.Sprintf("invalid timeout step: %v", ti.Step)) @@ -976,7 +982,7 @@ func (cs *State) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { } -func (cs *State) handleTxsAvailable() { +func (cs *State) handleTxsAvailable(ctx context.Context) { cs.mtx.Lock() defer cs.mtx.Unlock() @@ -997,7 +1003,7 @@ func (cs *State) handleTxsAvailable() { cs.scheduleTimeout(timeoutCommit, cs.Height, 0, cstypes.RoundStepNewRound) case cstypes.RoundStepNewRound: // after timeoutCommit - cs.enterPropose(cs.Height, 0) + cs.enterPropose(ctx, cs.Height, 0) } } @@ -1011,7 +1017,7 @@ func (cs *State) handleTxsAvailable() { // Enter: +2/3 precommits for nil at (height,round-1) // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round) // NOTE: cs.StartTime was already set for height. -func (cs *State) enterNewRound(height int64, round int32) { +func (cs *State) enterNewRound(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cs.Step != cstypes.RoundStepNewHeight) { @@ -1054,7 +1060,7 @@ func (cs *State) enterNewRound(height int64, round int32) { cs.Votes.SetRound(tmmath.SafeAddInt32(round, 1)) // also track next round (round+1) to allow round-skipping cs.TriggeredTimeoutPrecommit = false - if err := cs.eventBus.PublishEventNewRound(cs.NewRoundEvent()); err != nil { + if err := cs.eventBus.PublishEventNewRound(ctx, cs.NewRoundEvent()); err != nil { cs.Logger.Error("failed publishing new round", "err", err) } @@ -1070,7 +1076,7 @@ func (cs *State) enterNewRound(height int64, round int32) { cstypes.RoundStepNewRound) } } else { - cs.enterPropose(height, round) + cs.enterPropose(ctx, height, round) } } @@ -1093,7 +1099,7 @@ func (cs *State) needProofBlock(height int64) bool { // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): // after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool -func (cs *State) enterPropose(height int64, round int32) { +func (cs *State) enterPropose(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPropose <= cs.Step) { @@ -1109,13 +1115,13 @@ func (cs *State) enterPropose(height int64, round int32) { defer func() { // Done enterPropose: cs.updateRoundStep(round, cstypes.RoundStepPropose) - cs.newStep() + cs.newStep(ctx) // If we have the whole proposal + POL, then goto Prevote now. // else, we'll enterPrevote when the rest of the proposal is received (in AddProposalBlockPart), // or else after timeoutPropose if cs.isProposalComplete() { - cs.enterPrevote(height, cs.Round) + cs.enterPrevote(ctx, height, cs.Round) } }() @@ -1271,7 +1277,7 @@ func (cs *State) createProposalBlock() (block *types.Block, blockParts *types.Pa // Enter: proposal block and POL is ready. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid. // Otherwise vote nil. -func (cs *State) enterPrevote(height int64, round int32) { +func (cs *State) enterPrevote(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevote <= cs.Step) { @@ -1285,7 +1291,7 @@ func (cs *State) enterPrevote(height int64, round int32) { defer func() { // Done enterPrevote: cs.updateRoundStep(round, cstypes.RoundStepPrevote) - cs.newStep() + cs.newStep(ctx) }() logger.Debug("entering prevote step", "current", fmt.Sprintf("%v/%v/%v", cs.Height, cs.Round, cs.Step)) @@ -1331,7 +1337,7 @@ func (cs *State) defaultDoPrevote(height int64, round int32) { } // Enter: any +2/3 prevotes at next round. -func (cs *State) enterPrevoteWait(height int64, round int32) { +func (cs *State) enterPrevoteWait(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrevoteWait <= cs.Step) { @@ -1354,7 +1360,7 @@ func (cs *State) enterPrevoteWait(height int64, round int32) { defer func() { // Done enterPrevoteWait: cs.updateRoundStep(round, cstypes.RoundStepPrevoteWait) - cs.newStep() + cs.newStep(ctx) }() // Wait for some more prevotes; enterPrecommit @@ -1367,7 +1373,7 @@ func (cs *State) enterPrevoteWait(height int64, round int32) { // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round) // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil, // else, precommit nil otherwise. -func (cs *State) enterPrecommit(height int64, round int32) { +func (cs *State) enterPrecommit(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommit <= cs.Step) { @@ -1383,7 +1389,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { defer func() { // Done enterPrecommit: cs.updateRoundStep(round, cstypes.RoundStepPrecommit) - cs.newStep() + cs.newStep(ctx) }() // check for a polka @@ -1402,7 +1408,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { } // At this point +2/3 prevoted for a particular block or nil. - if err := cs.eventBus.PublishEventPolka(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventPolka(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing polka", "err", err) } @@ -1422,7 +1428,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { cs.LockedBlock = nil cs.LockedBlockParts = nil - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventUnlock(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing event unlock", "err", err) } } @@ -1438,7 +1444,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { logger.Debug("precommit step; +2/3 prevoted locked block; relocking") cs.LockedRound = round - if err := cs.eventBus.PublishEventRelock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventRelock(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing event relock", "err", err) } @@ -1459,7 +1465,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { cs.LockedBlock = cs.ProposalBlock cs.LockedBlockParts = cs.ProposalBlockParts - if err := cs.eventBus.PublishEventLock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventLock(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing event lock", "err", err) } @@ -1481,7 +1487,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) } - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventUnlock(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing event unlock", "err", err) } @@ -1489,7 +1495,7 @@ func (cs *State) enterPrecommit(height int64, round int32) { } // Enter: any +2/3 precommits for next round. -func (cs *State) enterPrecommitWait(height int64, round int32) { +func (cs *State) enterPrecommitWait(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) { @@ -1513,7 +1519,7 @@ func (cs *State) enterPrecommitWait(height int64, round int32) { defer func() { // Done enterPrecommitWait: cs.TriggeredTimeoutPrecommit = true - cs.newStep() + cs.newStep(ctx) }() // wait for some more precommits; enterNewRound @@ -1521,7 +1527,7 @@ func (cs *State) enterPrecommitWait(height int64, round int32) { } // Enter: +2/3 precommits for block -func (cs *State) enterCommit(height int64, commitRound int32) { +func (cs *State) enterCommit(ctx context.Context, height int64, commitRound int32) { logger := cs.Logger.With("height", height, "commit_round", commitRound) if cs.Height != height || cstypes.RoundStepCommit <= cs.Step { @@ -1540,10 +1546,10 @@ func (cs *State) enterCommit(height int64, commitRound int32) { cs.updateRoundStep(cs.Round, cstypes.RoundStepCommit) cs.CommitRound = commitRound cs.CommitTime = tmtime.Now() - cs.newStep() + cs.newStep(ctx) // Maybe finalize immediately. - cs.tryFinalizeCommit(height) + cs.tryFinalizeCommit(ctx, height) }() blockID, ok := cs.Votes.Precommits(commitRound).TwoThirdsMajority() @@ -1574,7 +1580,7 @@ func (cs *State) enterCommit(height int64, commitRound int32) { cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartSetHeader) - if err := cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventValidBlock(ctx, cs.RoundStateEvent()); err != nil { logger.Error("failed publishing valid block", "err", err) } @@ -1584,7 +1590,7 @@ func (cs *State) enterCommit(height int64, commitRound int32) { } // If we have the block AND +2/3 commits for it, finalize. -func (cs *State) tryFinalizeCommit(height int64) { +func (cs *State) tryFinalizeCommit(ctx context.Context, height int64) { logger := cs.Logger.With("height", height) if cs.Height != height { @@ -1608,11 +1614,11 @@ func (cs *State) tryFinalizeCommit(height int64) { return } - cs.finalizeCommit(height) + cs.finalizeCommit(ctx, height) } // Increment height and goto cstypes.RoundStepNewHeight -func (cs *State) finalizeCommit(height int64) { +func (cs *State) finalizeCommit(ctx context.Context, height int64) { logger := cs.Logger.With("height", height) if cs.Height != height || cs.Step != cstypes.RoundStepCommit { @@ -1692,7 +1698,7 @@ func (cs *State) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // NOTE The block.AppHash wont reflect these txs until the next block. - stateCopy, err := cs.blockExec.ApplyBlock( + stateCopy, err := cs.blockExec.ApplyBlock(ctx, stateCopy, types.BlockID{ Hash: block.Hash(), @@ -1711,7 +1717,7 @@ func (cs *State) finalizeCommit(height int64) { cs.RecordMetrics(height, block) // NewHeightStep! - cs.updateToState(stateCopy) + cs.updateToState(ctx, stateCopy) fail.Fail() // XXX @@ -1864,7 +1870,11 @@ func (cs *State) defaultSetProposal(proposal *types.Proposal) error { // NOTE: block is not necessarily valid. // Asynchronously triggers either enterPrevote (before we timeout of propose) or tryFinalizeCommit, // once we have the full block. -func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID types.NodeID) (added bool, err error) { +func (cs *State) addProposalBlockPart( + ctx context.Context, + msg *BlockPartMessage, + peerID types.NodeID, +) (added bool, err error) { height, round, part := msg.Height, msg.Round, msg.Part // Blocks might be reused, so round mismatch is OK @@ -1918,7 +1928,7 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID types.NodeID // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) - if err := cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent()); err != nil { + if err := cs.eventBus.PublishEventCompleteProposal(ctx, cs.CompleteProposalEvent()); err != nil { cs.Logger.Error("failed publishing event complete proposal", "err", err) } @@ -1946,13 +1956,13 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID types.NodeID if cs.Step <= cstypes.RoundStepPropose && cs.isProposalComplete() { // Move onto the next step - cs.enterPrevote(height, cs.Round) + cs.enterPrevote(ctx, height, cs.Round) if hasTwoThirds { // this is optimisation as this will be triggered when prevote is added - cs.enterPrecommit(height, cs.Round) + cs.enterPrecommit(ctx, height, cs.Round) } } else if cs.Step == cstypes.RoundStepCommit { // If we're waiting on the proposal block... - cs.tryFinalizeCommit(height) + cs.tryFinalizeCommit(ctx, height) } return added, nil @@ -1962,8 +1972,8 @@ func (cs *State) addProposalBlockPart(msg *BlockPartMessage, peerID types.NodeID } // Attempt to add the vote. if its a duplicate signature, dupeout the validator -func (cs *State) tryAddVote(vote *types.Vote, peerID types.NodeID) (bool, error) { - added, err := cs.addVote(vote, peerID) +func (cs *State) tryAddVote(ctx context.Context, vote *types.Vote, peerID types.NodeID) (bool, error) { + added, err := cs.addVote(ctx, vote, peerID) if err != nil { // If the vote height is off, we'll just ignore it, // But if it's a conflicting sig, add it to the cs.evpool. @@ -2010,7 +2020,11 @@ func (cs *State) tryAddVote(vote *types.Vote, peerID types.NodeID) (bool, error) return added, nil } -func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err error) { +func (cs *State) addVote( + ctx context.Context, + vote *types.Vote, + peerID types.NodeID, +) (added bool, err error) { cs.Logger.Debug( "adding vote", "vote_height", vote.Height, @@ -2034,7 +2048,7 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err } cs.Logger.Debug("added vote to last precommits", "last_commit", cs.LastCommit.StringShort()) - if err := cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}); err != nil { + if err := cs.eventBus.PublishEventVote(ctx, types.EventDataVote{Vote: vote}); err != nil { return added, err } @@ -2044,7 +2058,7 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err if cs.config.SkipTimeoutCommit && cs.LastCommit.HasAll() { // go straight to new round (skip timeout commit) // cs.scheduleTimeout(time.Duration(0), cs.Height, 0, cstypes.RoundStepNewHeight) - cs.enterNewRound(cs.Height, 0) + cs.enterNewRound(ctx, cs.Height, 0) } return @@ -2064,7 +2078,7 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err return } - if err := cs.eventBus.PublishEventVote(types.EventDataVote{Vote: vote}); err != nil { + if err := cs.eventBus.PublishEventVote(ctx, types.EventDataVote{Vote: vote}); err != nil { return added, err } cs.evsw.FireEvent(types.EventVoteValue, vote) @@ -2093,7 +2107,7 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err cs.LockedBlock = nil cs.LockedBlockParts = nil - if err := cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventUnlock(ctx, cs.RoundStateEvent()); err != nil { return added, err } } @@ -2122,7 +2136,7 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err } cs.evsw.FireEvent(types.EventValidBlockValue, &cs.RoundState) - if err := cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()); err != nil { + if err := cs.eventBus.PublishEventValidBlock(ctx, cs.RoundStateEvent()); err != nil { return added, err } } @@ -2132,20 +2146,20 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err switch { case cs.Round < vote.Round && prevotes.HasTwoThirdsAny(): // Round-skip if there is any 2/3+ of votes ahead of us - cs.enterNewRound(height, vote.Round) + cs.enterNewRound(ctx, height, vote.Round) case cs.Round == vote.Round && cstypes.RoundStepPrevote <= cs.Step: // current round blockID, ok := prevotes.TwoThirdsMajority() if ok && (cs.isProposalComplete() || len(blockID.Hash) == 0) { - cs.enterPrecommit(height, vote.Round) + cs.enterPrecommit(ctx, height, vote.Round) } else if prevotes.HasTwoThirdsAny() { - cs.enterPrevoteWait(height, vote.Round) + cs.enterPrevoteWait(ctx, height, vote.Round) } case cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round: // If the proposal is now complete, enter prevote of cs.Round. if cs.isProposalComplete() { - cs.enterPrevote(height, cs.Round) + cs.enterPrevote(ctx, height, cs.Round) } } @@ -2161,20 +2175,20 @@ func (cs *State) addVote(vote *types.Vote, peerID types.NodeID) (added bool, err blockID, ok := precommits.TwoThirdsMajority() if ok { // Executed as TwoThirdsMajority could be from a higher round - cs.enterNewRound(height, vote.Round) - cs.enterPrecommit(height, vote.Round) + cs.enterNewRound(ctx, height, vote.Round) + cs.enterPrecommit(ctx, height, vote.Round) if len(blockID.Hash) != 0 { - cs.enterCommit(height, vote.Round) + cs.enterCommit(ctx, height, vote.Round) if cs.config.SkipTimeoutCommit && precommits.HasAll() { - cs.enterNewRound(cs.Height, 0) + cs.enterNewRound(ctx, cs.Height, 0) } } else { - cs.enterPrecommitWait(height, vote.Round) + cs.enterPrecommitWait(ctx, height, vote.Round) } } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() { - cs.enterNewRound(height, vote.Round) - cs.enterPrecommitWait(height, vote.Round) + cs.enterNewRound(ctx, height, vote.Round) + cs.enterPrecommitWait(ctx, height, vote.Round) } default: diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index 9216577d5..5d09908aa 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -191,7 +191,7 @@ func TestStateEnterProposeYesPrivValidator(t *testing.T) { timeoutCh := subscribe(ctx, t, cs.eventBus, types.EventQueryTimeoutPropose) proposalCh := subscribe(ctx, t, cs.eventBus, types.EventQueryCompleteProposal) - cs.enterNewRound(height, round) + cs.enterNewRound(ctx, height, round) cs.startRoutines(ctx, 3) ensureNewProposal(proposalCh, height, round) @@ -399,7 +399,7 @@ func TestStateFullRoundNil(t *testing.T) { voteCh := subscribe(ctx, t, cs.eventBus, types.EventQueryVote) - cs.enterPrevote(height, round) + cs.enterPrevote(ctx, height, round) cs.startRoutines(ctx, 4) ensurePrevote(voteCh, height, round) // prevote @@ -479,7 +479,7 @@ func TestStateLockNoPOL(t *testing.T) { */ // start round and wait for prevote - cs1.enterNewRound(height, round) + cs1.enterNewRound(ctx, height, round) cs1.startRoutines(ctx, 0) ensureNewRound(newRoundCh, height, round) @@ -1986,26 +1986,26 @@ func TestStateOutputsBlockPartsStats(t *testing.T) { } cs.ProposalBlockParts = types.NewPartSetFromHeader(parts.Header()) - cs.handleMsg(msgInfo{msg, peerID}) + cs.handleMsg(ctx, msgInfo{msg, peerID}) statsMessage := <-cs.statsMsgQueue require.Equal(t, msg, statsMessage.Msg, "") require.Equal(t, peerID, statsMessage.PeerID, "") // sending the same part from different peer - cs.handleMsg(msgInfo{msg, "peer2"}) + cs.handleMsg(ctx, msgInfo{msg, "peer2"}) // sending the part with the same height, but different round msg.Round = 1 - cs.handleMsg(msgInfo{msg, peerID}) + cs.handleMsg(ctx, msgInfo{msg, peerID}) // sending the part from the smaller height msg.Height = 0 - cs.handleMsg(msgInfo{msg, peerID}) + cs.handleMsg(ctx, msgInfo{msg, peerID}) // sending the part from the bigger height msg.Height = 3 - cs.handleMsg(msgInfo{msg, peerID}) + cs.handleMsg(ctx, msgInfo{msg, peerID}) select { case <-cs.statsMsgQueue: @@ -2031,20 +2031,20 @@ func TestStateOutputVoteStats(t *testing.T) { vote := signVote(ctx, vss[1], config, tmproto.PrecommitType, randBytes, types.PartSetHeader{}) voteMessage := &VoteMessage{vote} - cs.handleMsg(msgInfo{voteMessage, peerID}) + cs.handleMsg(ctx, msgInfo{voteMessage, peerID}) statsMessage := <-cs.statsMsgQueue require.Equal(t, voteMessage, statsMessage.Msg, "") require.Equal(t, peerID, statsMessage.PeerID, "") // sending the same part from different peer - cs.handleMsg(msgInfo{&VoteMessage{vote}, "peer2"}) + cs.handleMsg(ctx, msgInfo{&VoteMessage{vote}, "peer2"}) // sending the vote for the bigger height incrementHeight(vss[1]) vote = signVote(ctx, vss[1], config, tmproto.PrecommitType, randBytes, types.PartSetHeader{}) - cs.handleMsg(msgInfo{&VoteMessage{vote}, peerID}) + cs.handleMsg(ctx, msgInfo{&VoteMessage{vote}, peerID}) select { case <-cs.statsMsgQueue: diff --git a/internal/consensus/wal_generator.go b/internal/consensus/wal_generator.go index 35a539d64..d1130defb 100644 --- a/internal/consensus/wal_generator.go +++ b/internal/consensus/wal_generator.go @@ -80,7 +80,7 @@ func WALGenerateNBlocks(ctx context.Context, t *testing.T, wr io.Writer, numBloc mempool := emptyMempool{} evpool := sm.EmptyEvidencePool{} blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) - consensusState := NewState(logger, cfg.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) + consensusState := NewState(ctx, logger, cfg.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetEventBus(eventBus) if privValidator != nil && privValidator != (*privval.FilePV)(nil) { consensusState.SetPrivValidator(privValidator) diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index 4b28f6fcd..61473c713 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -82,10 +82,7 @@ func (b *EventBus) Observe(ctx context.Context, observe func(tmpubsub.Message) e return b.pubsub.Observe(ctx, observe, queries...) } -func (b *EventBus) Publish(eventValue string, eventData types.TMEventData) error { - // no explicit deadline for publishing events - ctx := context.Background() - +func (b *EventBus) Publish(ctx context.Context, eventValue string, eventData types.TMEventData) error { tokens := strings.Split(types.EventTypeKey, ".") event := abci.Event{ Type: tokens[0], @@ -100,9 +97,7 @@ func (b *EventBus) Publish(eventValue string, eventData types.TMEventData) error return b.pubsub.PublishWithEvents(ctx, eventData, []abci.Event{event}) } -func (b *EventBus) PublishEventNewBlock(data types.EventDataNewBlock) error { - // no explicit deadline for publishing events - ctx := context.Background() +func (b *EventBus) PublishEventNewBlock(ctx context.Context, data types.EventDataNewBlock) error { events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) // add Tendermint-reserved new block event @@ -111,9 +106,9 @@ func (b *EventBus) PublishEventNewBlock(data types.EventDataNewBlock) error { return b.pubsub.PublishWithEvents(ctx, data, events) } -func (b *EventBus) PublishEventNewBlockHeader(data types.EventDataNewBlockHeader) error { +func (b *EventBus) PublishEventNewBlockHeader(ctx context.Context, data types.EventDataNewBlockHeader) error { // no explicit deadline for publishing events - ctx := context.Background() + events := append(data.ResultBeginBlock.Events, data.ResultEndBlock.Events...) // add Tendermint-reserved new block header event @@ -122,32 +117,30 @@ func (b *EventBus) PublishEventNewBlockHeader(data types.EventDataNewBlockHeader return b.pubsub.PublishWithEvents(ctx, data, events) } -func (b *EventBus) PublishEventNewEvidence(evidence types.EventDataNewEvidence) error { - return b.Publish(types.EventNewEvidenceValue, evidence) +func (b *EventBus) PublishEventNewEvidence(ctx context.Context, evidence types.EventDataNewEvidence) error { + return b.Publish(ctx, types.EventNewEvidenceValue, evidence) } -func (b *EventBus) PublishEventVote(data types.EventDataVote) error { - return b.Publish(types.EventVoteValue, data) +func (b *EventBus) PublishEventVote(ctx context.Context, data types.EventDataVote) error { + return b.Publish(ctx, types.EventVoteValue, data) } -func (b *EventBus) PublishEventValidBlock(data types.EventDataRoundState) error { - return b.Publish(types.EventValidBlockValue, data) +func (b *EventBus) PublishEventValidBlock(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventValidBlockValue, data) } -func (b *EventBus) PublishEventBlockSyncStatus(data types.EventDataBlockSyncStatus) error { - return b.Publish(types.EventBlockSyncStatusValue, data) +func (b *EventBus) PublishEventBlockSyncStatus(ctx context.Context, data types.EventDataBlockSyncStatus) error { + return b.Publish(ctx, types.EventBlockSyncStatusValue, data) } -func (b *EventBus) PublishEventStateSyncStatus(data types.EventDataStateSyncStatus) error { - return b.Publish(types.EventStateSyncStatusValue, data) +func (b *EventBus) PublishEventStateSyncStatus(ctx context.Context, data types.EventDataStateSyncStatus) error { + return b.Publish(ctx, types.EventStateSyncStatusValue, data) } // PublishEventTx publishes tx event with events from Result. Note it will add // predefined keys (EventTypeKey, TxHashKey). Existing events with the same keys // will be overwritten. -func (b *EventBus) PublishEventTx(data types.EventDataTx) error { - // no explicit deadline for publishing events - ctx := context.Background() +func (b *EventBus) PublishEventTx(ctx context.Context, data types.EventDataTx) error { events := data.Result.Events // add Tendermint-reserved events @@ -178,44 +171,44 @@ func (b *EventBus) PublishEventTx(data types.EventDataTx) error { return b.pubsub.PublishWithEvents(ctx, data, events) } -func (b *EventBus) PublishEventNewRoundStep(data types.EventDataRoundState) error { - return b.Publish(types.EventNewRoundStepValue, data) +func (b *EventBus) PublishEventNewRoundStep(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventNewRoundStepValue, data) } -func (b *EventBus) PublishEventTimeoutPropose(data types.EventDataRoundState) error { - return b.Publish(types.EventTimeoutProposeValue, data) +func (b *EventBus) PublishEventTimeoutPropose(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventTimeoutProposeValue, data) } -func (b *EventBus) PublishEventTimeoutWait(data types.EventDataRoundState) error { - return b.Publish(types.EventTimeoutWaitValue, data) +func (b *EventBus) PublishEventTimeoutWait(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventTimeoutWaitValue, data) } -func (b *EventBus) PublishEventNewRound(data types.EventDataNewRound) error { - return b.Publish(types.EventNewRoundValue, data) +func (b *EventBus) PublishEventNewRound(ctx context.Context, data types.EventDataNewRound) error { + return b.Publish(ctx, types.EventNewRoundValue, data) } -func (b *EventBus) PublishEventCompleteProposal(data types.EventDataCompleteProposal) error { - return b.Publish(types.EventCompleteProposalValue, data) +func (b *EventBus) PublishEventCompleteProposal(ctx context.Context, data types.EventDataCompleteProposal) error { + return b.Publish(ctx, types.EventCompleteProposalValue, data) } -func (b *EventBus) PublishEventPolka(data types.EventDataRoundState) error { - return b.Publish(types.EventPolkaValue, data) +func (b *EventBus) PublishEventPolka(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventPolkaValue, data) } -func (b *EventBus) PublishEventUnlock(data types.EventDataRoundState) error { - return b.Publish(types.EventUnlockValue, data) +func (b *EventBus) PublishEventUnlock(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventUnlockValue, data) } -func (b *EventBus) PublishEventRelock(data types.EventDataRoundState) error { - return b.Publish(types.EventRelockValue, data) +func (b *EventBus) PublishEventRelock(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventRelockValue, data) } -func (b *EventBus) PublishEventLock(data types.EventDataRoundState) error { - return b.Publish(types.EventLockValue, data) +func (b *EventBus) PublishEventLock(ctx context.Context, data types.EventDataRoundState) error { + return b.Publish(ctx, types.EventLockValue, data) } -func (b *EventBus) PublishEventValidatorSetUpdates(data types.EventDataValidatorSetUpdates) error { - return b.Publish(types.EventValidatorSetUpdatesValue, data) +func (b *EventBus) PublishEventValidatorSetUpdates(ctx context.Context, data types.EventDataValidatorSetUpdates) error { + return b.Publish(ctx, types.EventValidatorSetUpdatesValue, data) } //----------------------------------------------------------------------------- @@ -223,22 +216,22 @@ func (b *EventBus) PublishEventValidatorSetUpdates(data types.EventDataValidator // NopEventBus implements a types.BlockEventPublisher that discards all events. type NopEventBus struct{} -func (NopEventBus) PublishEventNewBlock(types.EventDataNewBlock) error { +func (NopEventBus) PublishEventNewBlock(context.Context, types.EventDataNewBlock) error { return nil } -func (NopEventBus) PublishEventNewBlockHeader(types.EventDataNewBlockHeader) error { +func (NopEventBus) PublishEventNewBlockHeader(context.Context, types.EventDataNewBlockHeader) error { return nil } -func (NopEventBus) PublishEventNewEvidence(types.EventDataNewEvidence) error { +func (NopEventBus) PublishEventNewEvidence(context.Context, types.EventDataNewEvidence) error { return nil } -func (NopEventBus) PublishEventTx(types.EventDataTx) error { +func (NopEventBus) PublishEventTx(context.Context, types.EventDataTx) error { return nil } -func (NopEventBus) PublishEventValidatorSetUpdates(types.EventDataValidatorSetUpdates) error { +func (NopEventBus) PublishEventValidatorSetUpdates(context.Context, types.EventDataValidatorSetUpdates) error { return nil } diff --git a/internal/eventbus/event_bus_test.go b/internal/eventbus/event_bus_test.go index c65c8cdd6..06f2bfa64 100644 --- a/internal/eventbus/event_bus_test.go +++ b/internal/eventbus/event_bus_test.go @@ -55,7 +55,7 @@ func TestEventBusPublishEventTx(t *testing.T) { assert.Equal(t, result, edt.Result) }() - err = eventBus.PublishEventTx(types.EventDataTx{ + err = eventBus.PublishEventTx(ctx, types.EventDataTx{ TxResult: abci.TxResult{ Height: 1, Index: 0, @@ -113,7 +113,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { assert.Equal(t, resultEndBlock, edt.ResultEndBlock) }() - err = eventBus.PublishEventNewBlock(types.EventDataNewBlock{ + err = eventBus.PublishEventNewBlock(ctx, types.EventDataNewBlock{ Block: block, BlockID: blockID, ResultBeginBlock: resultBeginBlock, @@ -225,7 +225,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } }() - assert.NoError(t, eventBus.PublishEventTx(types.EventDataTx{ + assert.NoError(t, eventBus.PublishEventTx(ctx, types.EventDataTx{ TxResult: abci.TxResult{ Height: 1, Index: 0, @@ -285,7 +285,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { assert.Equal(t, resultEndBlock, edt.ResultEndBlock) }() - err = eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + err = eventBus.PublishEventNewBlockHeader(ctx, types.EventDataNewBlockHeader{ Header: block.Header, ResultBeginBlock: resultBeginBlock, ResultEndBlock: resultEndBlock, @@ -327,7 +327,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { assert.Equal(t, int64(4), edt.Height) }() - err = eventBus.PublishEventNewEvidence(types.EventDataNewEvidence{ + err = eventBus.PublishEventNewEvidence(ctx, types.EventDataNewEvidence{ Evidence: ev, Height: 4, }) @@ -371,23 +371,23 @@ func TestEventBusPublish(t *testing.T) { } }() - require.NoError(t, eventBus.Publish(types.EventNewBlockHeaderValue, + require.NoError(t, eventBus.Publish(ctx, types.EventNewBlockHeaderValue, types.EventDataNewBlockHeader{})) - require.NoError(t, eventBus.PublishEventNewBlock(types.EventDataNewBlock{})) - require.NoError(t, eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{})) - require.NoError(t, eventBus.PublishEventVote(types.EventDataVote{})) - require.NoError(t, eventBus.PublishEventNewRoundStep(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventTimeoutPropose(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventTimeoutWait(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventNewRound(types.EventDataNewRound{})) - require.NoError(t, eventBus.PublishEventCompleteProposal(types.EventDataCompleteProposal{})) - require.NoError(t, eventBus.PublishEventPolka(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventUnlock(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventRelock(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventLock(types.EventDataRoundState{})) - require.NoError(t, eventBus.PublishEventValidatorSetUpdates(types.EventDataValidatorSetUpdates{})) - require.NoError(t, eventBus.PublishEventBlockSyncStatus(types.EventDataBlockSyncStatus{})) - require.NoError(t, eventBus.PublishEventStateSyncStatus(types.EventDataStateSyncStatus{})) + require.NoError(t, eventBus.PublishEventNewBlock(ctx, types.EventDataNewBlock{})) + require.NoError(t, eventBus.PublishEventNewBlockHeader(ctx, types.EventDataNewBlockHeader{})) + require.NoError(t, eventBus.PublishEventVote(ctx, types.EventDataVote{})) + require.NoError(t, eventBus.PublishEventNewRoundStep(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventTimeoutPropose(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventTimeoutWait(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventNewRound(ctx, types.EventDataNewRound{})) + require.NoError(t, eventBus.PublishEventCompleteProposal(ctx, types.EventDataCompleteProposal{})) + require.NoError(t, eventBus.PublishEventPolka(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventUnlock(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventRelock(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventLock(ctx, types.EventDataRoundState{})) + require.NoError(t, eventBus.PublishEventValidatorSetUpdates(ctx, types.EventDataValidatorSetUpdates{})) + require.NoError(t, eventBus.PublishEventBlockSyncStatus(ctx, types.EventDataBlockSyncStatus{})) + require.NoError(t, eventBus.PublishEventStateSyncStatus(ctx, types.EventDataStateSyncStatus{})) require.GreaterOrEqual(t, <-count, numEventsExpected) } @@ -473,7 +473,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes eventValue = randEventValue() } - err := eventBus.Publish(eventValue, types.EventDataString("Gamora")) + err := eventBus.Publish(ctx, eventValue, types.EventDataString("Gamora")) if err != nil { b.Error(err) } diff --git a/internal/state/execution.go b/internal/state/execution.go index e4a1ba6c3..85e96b017 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -150,7 +150,10 @@ func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) e // from outside this package to process and commit an entire block. // It takes a blockID to avoid recomputing the parts hash. func (blockExec *BlockExecutor) ApplyBlock( - state State, blockID types.BlockID, block *types.Block, + ctx context.Context, + state State, + blockID types.BlockID, + block *types.Block, ) (State, error) { // validate the block if we haven't already @@ -232,7 +235,7 @@ func (blockExec *BlockExecutor) ApplyBlock( // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay - fireEvents(blockExec.logger, blockExec.eventBus, block, blockID, abciResponses, validatorUpdates) + fireEvents(ctx, blockExec.logger, blockExec.eventBus, block, blockID, abciResponses, validatorUpdates) return state, nil } @@ -508,6 +511,7 @@ func updateState( // Fire TxEvent for every tx. // NOTE: if Tendermint crashes before commit, some or all of these events may be published again. func fireEvents( + ctx context.Context, logger log.Logger, eventBus types.BlockEventPublisher, block *types.Block, @@ -515,7 +519,7 @@ func fireEvents( abciResponses *tmstate.ABCIResponses, validatorUpdates []*types.Validator, ) { - if err := eventBus.PublishEventNewBlock(types.EventDataNewBlock{ + if err := eventBus.PublishEventNewBlock(ctx, types.EventDataNewBlock{ Block: block, BlockID: blockID, ResultBeginBlock: *abciResponses.BeginBlock, @@ -524,7 +528,7 @@ func fireEvents( logger.Error("failed publishing new block", "err", err) } - if err := eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + if err := eventBus.PublishEventNewBlockHeader(ctx, types.EventDataNewBlockHeader{ Header: block.Header, NumTxs: int64(len(block.Txs)), ResultBeginBlock: *abciResponses.BeginBlock, @@ -535,7 +539,7 @@ func fireEvents( if len(block.Evidence.Evidence) != 0 { for _, ev := range block.Evidence.Evidence { - if err := eventBus.PublishEventNewEvidence(types.EventDataNewEvidence{ + if err := eventBus.PublishEventNewEvidence(ctx, types.EventDataNewEvidence{ Evidence: ev, Height: block.Height, }); err != nil { @@ -545,7 +549,7 @@ func fireEvents( } for i, tx := range block.Data.Txs { - if err := eventBus.PublishEventTx(types.EventDataTx{TxResult: abci.TxResult{ + if err := eventBus.PublishEventTx(ctx, types.EventDataTx{TxResult: abci.TxResult{ Height: block.Height, Index: uint32(i), Tx: tx, @@ -556,7 +560,7 @@ func fireEvents( } if len(validatorUpdates) > 0 { - if err := eventBus.PublishEventValidatorSetUpdates( + if err := eventBus.PublishEventValidatorSetUpdates(ctx, types.EventDataValidatorSetUpdates{ValidatorUpdates: validatorUpdates}); err != nil { logger.Error("failed publishing event", "err", err) } @@ -569,6 +573,7 @@ func fireEvents( // ExecCommitBlock executes and commits a block on the proxyApp without validating or mutating the state. // It returns the application root hash (result of abci.Commit). func ExecCommitBlock( + ctx context.Context, be *BlockExecutor, appConnConsensus proxy.AppConnConsensus, block *types.Block, @@ -598,7 +603,7 @@ func ExecCommitBlock( } blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(types.BlockPartSizeBytes).Header()} - fireEvents(be.logger, be.eventBus, block, blockID, abciResponses, validatorUpdates) + fireEvents(ctx, be.logger, be.eventBus, block, blockID, abciResponses, validatorUpdates) } // Commit block, get hash back diff --git a/internal/state/execution_test.go b/internal/state/execution_test.go index 5da5adbb5..4f68bd016 100644 --- a/internal/state/execution_test.go +++ b/internal/state/execution_test.go @@ -56,7 +56,7 @@ func TestApplyBlock(t *testing.T) { block := sf.MakeBlock(state, 1, new(types.Commit)) blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(testPartSize).Header()} - state, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(ctx, state, blockID, block) require.Nil(t, err) // TODO check state and mempool @@ -111,7 +111,7 @@ func TestBeginBlockValidators(t *testing.T) { // block for height 2 block := sf.MakeBlock(state, 2, lastCommit) - _, err = sm.ExecCommitBlock(nil, proxyApp.Consensus(), block, log.TestingLogger(), stateStore, 1, state) + _, err = sm.ExecCommitBlock(ctx, nil, proxyApp.Consensus(), block, log.TestingLogger(), stateStore, 1, state) require.Nil(t, err, tc.desc) // -> app receives a list of validators with a bool indicating if they signed @@ -219,7 +219,7 @@ func TestBeginBlockByzantineValidators(t *testing.T) { block.Header.EvidenceHash = block.Evidence.Hash() blockID = types.BlockID{Hash: block.Hash(), PartSetHeader: block.MakePartSet(testPartSize).Header()} - _, err = blockExec.ApplyBlock(state, blockID, block) + _, err = blockExec.ApplyBlock(ctx, state, blockID, block) require.Nil(t, err) // TODO check state and mempool @@ -404,7 +404,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { {PubKey: pk, Power: 10}, } - state, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(ctx, state, blockID, block) require.Nil(t, err) // test new validator was added to NextValidators if assert.Equal(t, state.Validators.Size()+1, state.NextValidators.Size()) { @@ -462,7 +462,7 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { {PubKey: vp, Power: 0}, } - assert.NotPanics(t, func() { state, err = blockExec.ApplyBlock(state, blockID, block) }) + assert.NotPanics(t, func() { state, err = blockExec.ApplyBlock(ctx, state, blockID, block) }) assert.NotNil(t, err) assert.NotEmpty(t, state.NextValidators.Validators) } diff --git a/internal/state/helpers_test.go b/internal/state/helpers_test.go index 821c0757e..20d931f62 100644 --- a/internal/state/helpers_test.go +++ b/internal/state/helpers_test.go @@ -2,6 +2,7 @@ package state_test import ( "bytes" + "context" "fmt" "time" @@ -36,15 +37,17 @@ func newTestApp() proxy.AppConns { } func makeAndCommitGoodBlock( + ctx context.Context, state sm.State, height int64, lastCommit *types.Commit, proposerAddr []byte, blockExec *sm.BlockExecutor, privVals map[string]types.PrivValidator, - evidence []types.Evidence) (sm.State, types.BlockID, *types.Commit, error) { + evidence []types.Evidence, +) (sm.State, types.BlockID, *types.Commit, error) { // A good block passes - state, blockID, err := makeAndApplyGoodBlock(state, height, lastCommit, proposerAddr, blockExec, evidence) + state, blockID, err := makeAndApplyGoodBlock(ctx, state, height, lastCommit, proposerAddr, blockExec, evidence) if err != nil { return state, types.BlockID{}, nil, err } @@ -57,15 +60,22 @@ func makeAndCommitGoodBlock( return state, blockID, commit, nil } -func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commit, proposerAddr []byte, - blockExec *sm.BlockExecutor, evidence []types.Evidence) (sm.State, types.BlockID, error) { +func makeAndApplyGoodBlock( + ctx context.Context, + state sm.State, + height int64, + lastCommit *types.Commit, + proposerAddr []byte, + blockExec *sm.BlockExecutor, + evidence []types.Evidence, +) (sm.State, types.BlockID, error) { block, _ := state.MakeBlock(height, factory.MakeTenTxs(height), lastCommit, evidence, proposerAddr) if err := blockExec.ValidateBlock(state, block); err != nil { return state, types.BlockID{}, err } blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: types.PartSetHeader{Total: 3, Hash: tmrand.Bytes(32)}} - state, err := blockExec.ApplyBlock(state, blockID, block) + state, err := blockExec.ApplyBlock(ctx, state, blockID, block) if err != nil { return state, types.BlockID{}, err } diff --git a/internal/state/indexer/indexer_service_test.go b/internal/state/indexer/indexer_service_test.go index 879cf8820..d24744aa9 100644 --- a/internal/state/indexer/indexer_service_test.go +++ b/internal/state/indexer/indexer_service_test.go @@ -75,7 +75,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { t.Cleanup(service.Wait) // publish block with txs - err = eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + err = eventBus.PublishEventNewBlockHeader(ctx, types.EventDataNewBlockHeader{ Header: types.Header{Height: 1}, NumTxs: int64(2), }) @@ -86,7 +86,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { Tx: types.Tx("foo"), Result: abci.ResponseDeliverTx{Code: 0}, } - err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult1}) + err = eventBus.PublishEventTx(ctx, types.EventDataTx{TxResult: *txResult1}) require.NoError(t, err) txResult2 := &abci.TxResult{ Height: 1, @@ -94,7 +94,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { Tx: types.Tx("bar"), Result: abci.ResponseDeliverTx{Code: 0}, } - err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult2}) + err = eventBus.PublishEventTx(ctx, types.EventDataTx{TxResult: *txResult2}) require.NoError(t, err) time.Sleep(100 * time.Millisecond) diff --git a/internal/state/validation_test.go b/internal/state/validation_test.go index 65c0648d4..19b830709 100644 --- a/internal/state/validation_test.go +++ b/internal/state/validation_test.go @@ -103,7 +103,7 @@ func TestValidateBlockHeader(t *testing.T) { A good block passes */ var err error - state, _, lastCommit, err = makeAndCommitGoodBlock( + state, _, lastCommit, err = makeAndCommitGoodBlock(ctx, state, height, lastCommit, state.Validators.GetProposer().Address, blockExec, privVals, nil) require.NoError(t, err, "height %d", height) } @@ -186,6 +186,7 @@ func TestValidateBlockCommit(t *testing.T) { var err error var blockID types.BlockID state, blockID, lastCommit, err = makeAndCommitGoodBlock( + ctx, state, height, lastCommit, @@ -310,6 +311,7 @@ func TestValidateBlockEvidence(t *testing.T) { var err error state, _, lastCommit, err = makeAndCommitGoodBlock( + ctx, state, height, lastCommit, diff --git a/node/node.go b/node/node.go index 7d3b56b47..86ae173e5 100644 --- a/node/node.go +++ b/node/node.go @@ -313,7 +313,7 @@ func makeNode( sm.BlockExecutorWithMetrics(nodeMetrics.state), ) - csReactor, csState, err := createConsensusReactor( + csReactor, csState, err := createConsensusReactor(ctx, cfg, state, blockExec, blockStore, mp, evPool, privValidator, nodeMetrics.consensus, stateSync || blockSync, eventBus, peerManager, router, logger, @@ -599,7 +599,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { // At the beginning of the statesync start, we use the initialHeight as the event height // because of the statesync doesn't have the concreate state height before fetched the snapshot. d := types.EventDataStateSyncStatus{Complete: false, Height: state.InitialHeight} - if err := n.eventBus.PublishEventStateSyncStatus(d); err != nil { + if err := n.eventBus.PublishEventStateSyncStatus(ctx, d); err != nil { n.eventBus.Logger.Error("failed to emit the statesync start event", "err", err) } @@ -619,7 +619,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { n.consensusReactor.SetStateSyncingMetrics(0) - if err := n.eventBus.PublishEventStateSyncStatus( + if err := n.eventBus.PublishEventStateSyncStatus(ctx, types.EventDataStateSyncStatus{ Complete: true, Height: state.LastBlockHeight, @@ -638,7 +638,7 @@ func (n *nodeImpl) OnStart(ctx context.Context) error { return } - if err := n.eventBus.PublishEventBlockSyncStatus( + if err := n.eventBus.PublishEventBlockSyncStatus(ctx, types.EventDataBlockSyncStatus{ Complete: false, Height: state.LastBlockHeight, diff --git a/node/setup.go b/node/setup.go index 6ca991484..55ca592e0 100644 --- a/node/setup.go +++ b/node/setup.go @@ -302,6 +302,7 @@ func createBlockchainReactor( } func createConsensusReactor( + ctx context.Context, cfg *config.Config, state sm.State, blockExec *sm.BlockExecutor, @@ -318,7 +319,7 @@ func createConsensusReactor( ) (*consensus.Reactor, *consensus.State, error) { logger = logger.With("module", "consensus") - consensusState := consensus.NewState( + consensusState := consensus.NewState(ctx, logger, cfg.Consensus, state.Copy(), diff --git a/types/events.go b/types/events.go index 7bb183fb7..86935ba25 100644 --- a/types/events.go +++ b/types/events.go @@ -1,6 +1,7 @@ package types import ( + "context" "fmt" "strings" @@ -241,13 +242,13 @@ func QueryForEvent(eventValue string) tmpubsub.Query { // BlockEventPublisher publishes all block related events type BlockEventPublisher interface { - PublishEventNewBlock(block EventDataNewBlock) error - PublishEventNewBlockHeader(header EventDataNewBlockHeader) error - PublishEventNewEvidence(evidence EventDataNewEvidence) error - PublishEventTx(EventDataTx) error - PublishEventValidatorSetUpdates(EventDataValidatorSetUpdates) error + PublishEventNewBlock(ctx context.Context, block EventDataNewBlock) error + PublishEventNewBlockHeader(ctx context.Context, header EventDataNewBlockHeader) error + PublishEventNewEvidence(ctx context.Context, evidence EventDataNewEvidence) error + PublishEventTx(context.Context, EventDataTx) error + PublishEventValidatorSetUpdates(context.Context, EventDataValidatorSetUpdates) error } type TxEventPublisher interface { - PublishEventTx(EventDataTx) error + PublishEventTx(context.Context, EventDataTx) error } From 4734f7d80686e2c74d49d54745b52da4143830f1 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 30 Nov 2021 07:30:35 -0800 Subject: [PATCH 10/21] pubsub: Make the queue unwritable after shutdown. (#7316) Prior to this change, shutting down the pubsub server could cause any laggard publishers to race with the shutdown plumbing. Fix that race condition, and plumb in the service context to the runner so that it will respect the external signal directly. Remove now-redundant local shutdown plumbing. --- libs/pubsub/pubsub.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/libs/pubsub/pubsub.go b/libs/pubsub/pubsub.go index 930dd47bc..3ac2c1940 100644 --- a/libs/pubsub/pubsub.go +++ b/libs/pubsub/pubsub.go @@ -106,7 +106,6 @@ type Server struct { queue chan item done <-chan struct{} // closed when server should exit - stop func() // signal the server to exit pubs sync.RWMutex // excl: shutdown; shared: active publisher exited chan struct{} // server exited @@ -333,15 +332,15 @@ func (s *Server) PublishWithEvents(ctx context.Context, msg interface{}, events return s.publish(ctx, msg, events) } -// OnStop implements Service.OnStop by shutting down the server. -func (s *Server) OnStop() { s.stop() } +// OnStop implements part of the Service interface. It is a no-op. +func (s *Server) OnStop() {} // Wait implements Service.Wait by blocking until the server has exited, then // yielding to the base service wait. func (s *Server) Wait() { <-s.exited; s.BaseService.Wait() } // OnStart implements Service.OnStart by starting the server. -func (s *Server) OnStart(ctx context.Context) error { s.run(); return nil } +func (s *Server) OnStart(ctx context.Context) error { s.run(ctx); return nil } // OnReset implements Service.OnReset. It has no effect for this service. func (s *Server) OnReset() error { return nil } @@ -363,11 +362,10 @@ func (s *Server) publish(ctx context.Context, data interface{}, events []types.E } } -func (s *Server) run() { +func (s *Server) run(ctx context.Context) { // The server runs until ctx is canceled. - ctx, cancel := context.WithCancel(context.Background()) s.done = ctx.Done() - s.stop = cancel + queue := s.queue // Shutdown monitor: When the context ends, wait for any active publish // calls to exit, then close the queue to signal the sender to exit. @@ -376,6 +374,7 @@ func (s *Server) run() { s.pubs.Lock() defer s.pubs.Unlock() close(s.queue) + s.queue = nil }() s.exited = make(chan struct{}) @@ -383,7 +382,7 @@ func (s *Server) run() { defer close(s.exited) // Sender: Service the queue and forward messages to subscribers. - for it := range s.queue { + for it := range queue { if err := s.send(it.Data, it.Events); err != nil { s.Logger.Error("Error sending event", "err", err) } From 502f92bb97c9dd4475f732f28ae9c9afe9145d48 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 11:55:31 -0500 Subject: [PATCH 11/21] lint: remove lll check (#7346) --- .golangci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index e0f3fe163..7330f25ee 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,7 +24,7 @@ linters: - govet - ineffassign # - interfacer - - lll + # - lll # - maligned - misspell - nakedret @@ -46,9 +46,6 @@ issues: - path: _test\.go linters: - gosec - - linters: - - lll - source: "https://" max-same-issues: 50 linters-settings: From cf5c7be4d805c1b78f1723f48cf998b70098252e Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 12:19:47 -0500 Subject: [PATCH 12/21] e2e: control access to state in Info calls (#7345) This is probably belt and suspenders, but might help the apphash clarity. --- test/e2e/app/app.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index 395c87024..570df6676 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -97,6 +97,9 @@ func NewApplication(cfg *Config) (*Application, error) { // Info implements ABCI. func (app *Application) Info(req abci.RequestInfo) abci.ResponseInfo { + app.state.RLock() + defer app.state.RUnlock() + return abci.ResponseInfo{ Version: version.ABCIVersion, AppVersion: 1, From d5c18b68c806acc08af628e822a7e910ca703045 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 12:34:16 -0500 Subject: [PATCH 13/21] e2e: more clear height test (#7347) This is a more clear version of the stable parts of the apphash tests. --- test/e2e/tests/app_test.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/e2e/tests/app_test.go b/test/e2e/tests/app_test.go index ab6f9739e..50989bebe 100644 --- a/test/e2e/tests/app_test.go +++ b/test/e2e/tests/app_test.go @@ -46,6 +46,7 @@ func TestApp_Hash(t *testing.T) { status, err := client.Status(ctx) require.NoError(t, err) + require.NotZero(t, status.SyncInfo.LatestBlockHeight) block, err := client.Block(ctx, &info.Response.LastBlockHeight) require.NoError(t, err) @@ -60,6 +61,29 @@ func TestApp_Hash(t *testing.T) { }) } +// Tests that the app and blockstore have and report the same height. +func TestApp_Height(t *testing.T) { + testNode(t, func(t *testing.T, node e2e.Node) { + client, err := node.Client() + require.NoError(t, err) + info, err := client.ABCIInfo(ctx) + require.NoError(t, err) + require.NotZero(t, info.Response.LastBlockHeight) + + status, err := client.Status(ctx) + require.NoError(t, err) + require.NotZero(t, status.SyncInfo.LatestBlockHeight) + + block, err := client.Block(ctx, &info.Response.LastBlockHeight) + require.NoError(t, err) + + require.Equal(t, info.Response.LastBlockHeight, block.Block.Height) + + require.True(t, status.SyncInfo.LatestBlockHeight >= info.Response.LastBlockHeight, + "status out of sync with application") + }) +} + // Tests that we can set a value and retrieve it. func TestApp_Tx(t *testing.T) { type broadcastFunc func(context.Context, types.Tx) error From 070445bc1088a8cf08bace8080a4549e93dff5a3 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 14:22:28 -0500 Subject: [PATCH 14/21] pex: improve goroutine lifecycle (#7343) I saw a race detected in a test here that I think would be better handled by just wiring up these threads. --- internal/p2p/pex/reactor.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/p2p/pex/reactor.go b/internal/p2p/pex/reactor.go index f6fcad5e1..53cd079bc 100644 --- a/internal/p2p/pex/reactor.go +++ b/internal/p2p/pex/reactor.go @@ -141,8 +141,8 @@ func NewReactor( // messages on that p2p channel accordingly. The caller must be sure to execute // OnStop to ensure the outbound p2p Channels are closed. func (r *Reactor) OnStart(ctx context.Context) error { - go r.processPexCh() - go r.processPeerUpdates() + go r.processPexCh(ctx) + go r.processPeerUpdates(ctx) return nil } @@ -162,17 +162,22 @@ func (r *Reactor) OnStop() { // processPexCh implements a blocking event loop where we listen for p2p // Envelope messages from the pexCh. -func (r *Reactor) processPexCh() { +func (r *Reactor) processPexCh(ctx context.Context) { defer r.pexCh.Close() - + timer := time.NewTimer(0) + defer timer.Stop() for { + timer.Reset(time.Until(r.nextRequestTime)) + select { + case <-ctx.Done(): + return case <-r.closeCh: r.Logger.Debug("stopped listening on PEX channel; closing...") return // outbound requests for new peers - case <-r.waitUntilNextRequest(): + case <-timer.C: r.sendRequestForPeers() // inbound requests for new peers or responses to requests sent by this @@ -192,11 +197,13 @@ func (r *Reactor) processPexCh() { // processPeerUpdates initiates a blocking process where we listen for and handle // PeerUpdate messages. When the reactor is stopped, we will catch the signal and // close the p2p PeerUpdatesCh gracefully. -func (r *Reactor) processPeerUpdates() { +func (r *Reactor) processPeerUpdates(ctx context.Context) { defer r.peerUpdates.Close() for { select { + case <-ctx.Done(): + return case peerUpdate := <-r.peerUpdates.Updates(): r.processPeerUpdate(peerUpdate) @@ -317,10 +324,6 @@ func (r *Reactor) processPeerUpdate(peerUpdate p2p.PeerUpdate) { } } -func (r *Reactor) waitUntilNextRequest() <-chan time.Time { - return time.After(time.Until(r.nextRequestTime)) -} - // sendRequestForPeers pops the first peerID off the list and sends the // peer a request for more peer addresses. The function then moves the // peer into the requestsSent bucket and calculates when the next request From c4033f95c168094c6af0667465f58a2f5d367407 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 14:35:21 -0500 Subject: [PATCH 15/21] e2e: stabilize validator update form (#7340) This might be a source of non-determinism in the e2e test. --- test/e2e/app/app.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/app/app.go b/test/e2e/app/app.go index 570df6676..353bb6b75 100644 --- a/test/e2e/app/app.go +++ b/test/e2e/app/app.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "path/filepath" + "sort" "strconv" "github.com/tendermint/tendermint/abci/example/code" @@ -282,6 +283,14 @@ func (app *Application) validatorUpdates(height uint64) (abci.ValidatorUpdates, } valUpdates = append(valUpdates, abci.UpdateValidator(keyBytes, int64(power), app.cfg.KeyType)) } + + // the validator updates could be returned in arbitrary order, + // and that seems potentially bad. This orders the validator + // set. + sort.Slice(valUpdates, func(i, j int) bool { + return valUpdates[i].PubKey.Compare(valUpdates[j].PubKey) < 0 + }) + return valUpdates, nil } From 24dcba923046bbc7fb558bee5920fcb02f3c457b Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 15:05:55 -0500 Subject: [PATCH 16/21] e2e: clarify apphash reporting (#7348) --- test/e2e/tests/app_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/tests/app_test.go b/test/e2e/tests/app_test.go index 50989bebe..91f3cd2a5 100644 --- a/test/e2e/tests/app_test.go +++ b/test/e2e/tests/app_test.go @@ -52,7 +52,9 @@ func TestApp_Hash(t *testing.T) { require.NoError(t, err) if info.Response.LastBlockHeight == block.Block.Height { - require.EqualValues(t, info.Response.LastBlockAppHash, block.Block.AppHash.Bytes(), + require.Equal(t, + fmt.Sprintf("%x", info.Response.LastBlockAppHash), + fmt.Sprintf("%x", block.Block.AppHash.Bytes()), "app hash does not match last block's app hash") } From babd3acb7018e12c2e10a3b69eb6ac712ce1bf28 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 15:21:48 -0500 Subject: [PATCH 17/21] e2e: generate keys for more stable load (#7344) --- test/e2e/runner/load.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/runner/load.go b/test/e2e/runner/load.go index f31b436dd..4d1f2dea1 100644 --- a/test/e2e/runner/load.go +++ b/test/e2e/runner/load.go @@ -97,9 +97,9 @@ func loadGenerate(ctx context.Context, r *rand.Rand, chTx chan<- types.Tx, txSiz case <-timer.C: } - // We keep generating the same 100 keys over and over, with different values. - // This gives a reasonable load without putting too much data in the app. - id := rand.Int63() % 100 // nolint: gosec + // Constrain the key space to avoid using too much + // space, while reduce the size of the data in the app. + id := r.Int63n(100) bz := make([]byte, txSize) _, err := r.Read(bz) From ab1788b922e9909a0fbc4a7680c7e73b8c6ed5b0 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 30 Nov 2021 12:57:44 -0800 Subject: [PATCH 18/21] Fix incorrect tests using the PSQL sink. (#7349) Some of our tests were creating a psql event sink and expecting it to report (or not report) certain kinds of errors. These tests were ill-founded in a couple of ways: 1. Tests that required the Postgres driver were not loading it. This led to spurious successes on tests that wanted "some error" from the sink constructor, but didn't exercise the right path. 2. Tests that wanted a Postgres sink to succeed without a database. These tests "passed" because they weren't actually establishing a connection to the database, but if they had would have failed for the lack of one. To fix this: - Load the postgres driver in tests that need it. - Verify connectivity before reporting successful creation of a PSQL event sink. - Remove tests that wanted a psql sink without a database, since that case is already tested elsewhere. --- cmd/tendermint/commands/reindex_event_test.go | 2 + internal/state/indexer/sink/psql/psql.go | 2 + node/node_test.go | 41 ++----------------- 3 files changed, 8 insertions(+), 37 deletions(-) diff --git a/cmd/tendermint/commands/reindex_event_test.go b/cmd/tendermint/commands/reindex_event_test.go index 452a6b2a8..c76ce6d9a 100644 --- a/cmd/tendermint/commands/reindex_event_test.go +++ b/cmd/tendermint/commands/reindex_event_test.go @@ -15,6 +15,8 @@ import ( "github.com/tendermint/tendermint/internal/state/mocks" prototmstate "github.com/tendermint/tendermint/proto/tendermint/state" "github.com/tendermint/tendermint/types" + + _ "github.com/lib/pq" // for the psql sink ) const ( diff --git a/internal/state/indexer/sink/psql/psql.go b/internal/state/indexer/sink/psql/psql.go index 4db6f4435..18e95b97d 100644 --- a/internal/state/indexer/sink/psql/psql.go +++ b/internal/state/indexer/sink/psql/psql.go @@ -39,6 +39,8 @@ func NewEventSink(connStr, chainID string) (*EventSink, error) { db, err := sql.Open(driverName, connStr) if err != nil { return nil, err + } else if err := db.Ping(); err != nil { + return nil, err } return &EventSink{ diff --git a/node/node_test.go b/node/node_test.go index 407bf93ea..3e4e83492 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -636,50 +636,17 @@ func TestNodeSetEventSink(t *testing.T) { assert.Contains(t, err.Error(), "the psql connection settings cannot be empty") t.Cleanup(cleanup(ns)) - var psqlConn = "test" - - cfg.TxIndex.Indexer = []string{"psql"} - cfg.TxIndex.PsqlConn = psqlConn - eventSinks = setupTest(t, cfg) - - assert.Equal(t, 1, len(eventSinks)) - assert.Equal(t, indexer.PSQL, eventSinks[0].Type()) - - cfg.TxIndex.Indexer = []string{"psql", "kv"} - cfg.TxIndex.PsqlConn = psqlConn - eventSinks = setupTest(t, cfg) - - assert.Equal(t, 2, len(eventSinks)) - // we use map to filter the duplicated sinks, so it's not guarantee the order when append sinks. - if eventSinks[0].Type() == indexer.KV { - assert.Equal(t, indexer.PSQL, eventSinks[1].Type()) - } else { - assert.Equal(t, indexer.PSQL, eventSinks[0].Type()) - assert.Equal(t, indexer.KV, eventSinks[1].Type()) - } - - cfg.TxIndex.Indexer = []string{"kv", "psql"} - cfg.TxIndex.PsqlConn = psqlConn - eventSinks = setupTest(t, cfg) - - assert.Equal(t, 2, len(eventSinks)) - if eventSinks[0].Type() == indexer.KV { - assert.Equal(t, indexer.PSQL, eventSinks[1].Type()) - } else { - assert.Equal(t, indexer.PSQL, eventSinks[0].Type()) - assert.Equal(t, indexer.KV, eventSinks[1].Type()) - } + // N.B. We can't create a PSQL event sink without starting a postgres + // instance for it to talk to. The indexer service tests exercise that case. var e = errors.New("found duplicated sinks, please check the tx-index section in the config.toml") - cfg.TxIndex.Indexer = []string{"psql", "kv", "Kv"} - cfg.TxIndex.PsqlConn = psqlConn + cfg.TxIndex.Indexer = []string{"null", "kv", "Kv"} ns, err = newDefaultNode(ctx, cfg, logger) require.Error(t, err) assert.Contains(t, err.Error(), e.Error()) t.Cleanup(cleanup(ns)) - cfg.TxIndex.Indexer = []string{"Psql", "kV", "kv", "pSql"} - cfg.TxIndex.PsqlConn = psqlConn + cfg.TxIndex.Indexer = []string{"Null", "kV", "kv", "nUlL"} ns, err = newDefaultNode(ctx, cfg, logger) require.Error(t, err) assert.Contains(t, err.Error(), e.Error()) From 76dea94a015d02e507538e8314b3b983d6adf3b0 Mon Sep 17 00:00:00 2001 From: "M. J. Fromberger" Date: Tue, 30 Nov 2021 13:32:21 -0800 Subject: [PATCH 19/21] Remove now-unused nolint:lll directives. (#7356) --- config/config_test.go | 1 - internal/consensus/msgs_test.go | 1 - internal/evidence/reactor_test.go | 1 - internal/inspect/inspect.go | 2 -- internal/inspect/rpc/rpc.go | 2 -- internal/p2p/conn/connection_test.go | 1 - internal/p2p/metrics.go | 2 -- internal/rpc/core/mempool.go | 2 +- internal/state/indexer/sink/sink.go | 2 -- libs/pubsub/query/query_test.go | 1 - light/proxy/routes.go | 2 +- light/rpc/client.go | 4 ++-- privval/msgs_test.go | 1 - proto/tendermint/blocksync/message_test.go | 1 - rpc/client/interface.go | 2 +- rpc/client/local/local.go | 4 ++-- rpc/client/mock/client.go | 4 ++-- types/proposal_test.go | 2 +- types/vote_test.go | 4 ++-- 19 files changed, 12 insertions(+), 27 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 304eeb0ce..b3dabfb08 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -105,7 +105,6 @@ func TestStateSyncConfigValidateBasic(t *testing.T) { } func TestConsensusConfig_ValidateBasic(t *testing.T) { - // nolint: lll testcases := map[string]struct { modify func(*ConsensusConfig) expectErr bool diff --git a/internal/consensus/msgs_test.go b/internal/consensus/msgs_test.go index c22ebf5c0..914d7dc4e 100644 --- a/internal/consensus/msgs_test.go +++ b/internal/consensus/msgs_test.go @@ -313,7 +313,6 @@ func TestWALMsgProto(t *testing.T) { } } -// nolint:lll //ignore line length for tests func TestConsMsgsVectors(t *testing.T) { date := time.Date(2018, 8, 30, 12, 0, 0, 0, time.UTC) psh := types.PartSetHeader{ diff --git a/internal/evidence/reactor_test.go b/internal/evidence/reactor_test.go index 764450cd6..7808e53fb 100644 --- a/internal/evidence/reactor_test.go +++ b/internal/evidence/reactor_test.go @@ -509,7 +509,6 @@ func TestReactorBroadcastEvidence_FullyConnected(t *testing.T) { } } -// nolint:lll func TestEvidenceListSerialization(t *testing.T) { exampleVote := func(msgType byte) *types.Vote { var stamp, err = time.Parse(types.TimeFormat, "2017-12-25T03:00:01.234Z") diff --git a/internal/inspect/inspect.go b/internal/inspect/inspect.go index 3bc744e4c..eb75d94a3 100644 --- a/internal/inspect/inspect.go +++ b/internal/inspect/inspect.go @@ -41,8 +41,6 @@ type Inspector struct { // The Inspector type does not modify the state or block stores. // The sinks are used to enable block and transaction querying via the RPC server. // The caller is responsible for starting and stopping the Inspector service. -/// -//nolint:lll func New(cfg *config.RPCConfig, bs state.BlockStore, ss state.Store, es []indexer.EventSink, logger log.Logger) *Inspector { eb := eventbus.NewDefault(logger.With("module", "events")) diff --git a/internal/inspect/rpc/rpc.go b/internal/inspect/rpc/rpc.go index 276bfe082..be83b1b1e 100644 --- a/internal/inspect/rpc/rpc.go +++ b/internal/inspect/rpc/rpc.go @@ -31,8 +31,6 @@ type eventBusUnsubscriber interface { } // Routes returns the set of routes used by the Inspector server. -// -//nolint: lll func Routes(cfg config.RPCConfig, s state.Store, bs state.BlockStore, es []indexer.EventSink, logger log.Logger) core.RoutesMap { env := &core.Environment{ Config: cfg, diff --git a/internal/p2p/conn/connection_test.go b/internal/p2p/conn/connection_test.go index dc198d8bd..f8b34bad6 100644 --- a/internal/p2p/conn/connection_test.go +++ b/internal/p2p/conn/connection_test.go @@ -555,7 +555,6 @@ func TestMConnectionTrySend(t *testing.T) { assert.Equal(t, "TrySend", <-resultCh) } -// nolint:lll //ignore line length for tests func TestConnVectors(t *testing.T) { testCases := []struct { diff --git a/internal/p2p/metrics.go b/internal/p2p/metrics.go index 3677180de..2780d221e 100644 --- a/internal/p2p/metrics.go +++ b/internal/p2p/metrics.go @@ -63,8 +63,6 @@ type Metrics struct { // PrometheusMetrics returns Metrics build using Prometheus client library. // Optionally, labels can be provided along with their values ("foo", // "fooValue"). -// -// nolint: lll func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { labels := []string{} for i := 0; i < len(labelsAndValues); i += 2 { diff --git a/internal/rpc/core/mempool.go b/internal/rpc/core/mempool.go index 5b12a6c2b..87136a532 100644 --- a/internal/rpc/core/mempool.go +++ b/internal/rpc/core/mempool.go @@ -59,7 +59,7 @@ func (env *Environment) BroadcastTxSync(ctx *rpctypes.Context, tx types.Tx) (*co // BroadcastTxCommit returns with the responses from CheckTx and DeliverTx. // More: https://docs.tendermint.com/master/rpc/#/Tx/broadcast_tx_commit -func (env *Environment) BroadcastTxCommit(ctx *rpctypes.Context, tx types.Tx) (*coretypes.ResultBroadcastTxCommit, error) { //nolint:lll +func (env *Environment) BroadcastTxCommit(ctx *rpctypes.Context, tx types.Tx) (*coretypes.ResultBroadcastTxCommit, error) { resCh := make(chan *abci.Response, 1) err := env.Mempool.CheckTx( ctx.Context(), diff --git a/internal/state/indexer/sink/sink.go b/internal/state/indexer/sink/sink.go index b4c41ec31..cae861416 100644 --- a/internal/state/indexer/sink/sink.go +++ b/internal/state/indexer/sink/sink.go @@ -13,8 +13,6 @@ import ( // EventSinksFromConfig constructs a slice of indexer.EventSink using the provided // configuration. -// -//nolint:lll func EventSinksFromConfig(cfg *config.Config, dbProvider config.DBProvider, chainID string) ([]indexer.EventSink, error) { if len(cfg.TxIndex.Indexer) == 0 { return []indexer.EventSink{null.NewEventSink()}, nil diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index 883e771c6..b0d1fb7fe 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -66,7 +66,6 @@ func TestCompiledMatches(t *testing.T) { txTime = "2018-05-03T14:45:00Z" ) - //nolint:lll testCases := []struct { s string events []types.Event diff --git a/light/proxy/routes.go b/light/proxy/routes.go index 436ae1b76..659cb5051 100644 --- a/light/proxy/routes.go +++ b/light/proxy/routes.go @@ -77,7 +77,7 @@ func makeNetInfoFunc(c *lrpc.Client) rpcNetInfoFunc { } } -type rpcBlockchainInfoFunc func(ctx *rpctypes.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) //nolint:lll +type rpcBlockchainInfoFunc func(ctx *rpctypes.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) func makeBlockchainInfoFunc(c *lrpc.Client) rpcBlockchainInfoFunc { return func(ctx *rpctypes.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { diff --git a/light/rpc/client.go b/light/rpc/client.go index 6143338f4..51676dd3e 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -122,7 +122,7 @@ func (c *Client) ABCIInfo(ctx context.Context) (*coretypes.ResultABCIInfo, error } // ABCIQuery requests proof by default. -func (c *Client) ABCIQuery(ctx context.Context, path string, data tmbytes.HexBytes) (*coretypes.ResultABCIQuery, error) { //nolint:lll +func (c *Client) ABCIQuery(ctx context.Context, path string, data tmbytes.HexBytes) (*coretypes.ResultABCIQuery, error) { return c.ABCIQueryWithOptions(ctx, path, data, rpcclient.DefaultABCIQueryOptions) } @@ -263,7 +263,7 @@ func (c *Client) Health(ctx context.Context) (*coretypes.ResultHealth, error) { // BlockchainInfo calls rpcclient#BlockchainInfo and then verifies every header // returned. -func (c *Client) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { //nolint:lll +func (c *Client) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { res, err := c.next.BlockchainInfo(ctx, minHeight, maxHeight) if err != nil { return nil, err diff --git a/privval/msgs_test.go b/privval/msgs_test.go index 7ac9f2c5d..43752749b 100644 --- a/privval/msgs_test.go +++ b/privval/msgs_test.go @@ -57,7 +57,6 @@ func exampleProposal() *types.Proposal { } } -// nolint:lll // ignore line length for tests func TestPrivvalVectors(t *testing.T) { pk := ed25519.GenPrivKeyFromSecret([]byte("it's a secret")).PubKey() ppk, err := encoding.PubKeyToProto(pk) diff --git a/proto/tendermint/blocksync/message_test.go b/proto/tendermint/blocksync/message_test.go index f81de149f..3406c6dff 100644 --- a/proto/tendermint/blocksync/message_test.go +++ b/proto/tendermint/blocksync/message_test.go @@ -84,7 +84,6 @@ func TestStatusResponse_Validate(t *testing.T) { } } -// nolint:lll func TestBlockchainMessageVectors(t *testing.T) { block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil) block.Version.Block = 11 // overwrite updated protocol version diff --git a/rpc/client/interface.go b/rpc/client/interface.go index 8d160b799..0fbc55717 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -134,7 +134,7 @@ type EventsClient interface { // // ctx cannot be used to unsubscribe. To unsubscribe, use either Unsubscribe // or UnsubscribeAll. - Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) //nolint:lll + Subscribe(ctx context.Context, subscriber, query string, outCapacity ...int) (out <-chan coretypes.ResultEvent, err error) // Unsubscribe unsubscribes given subscriber from query. Unsubscribe(ctx context.Context, subscriber, query string) error // UnsubscribeAll unsubscribes given subscriber from all the queries. diff --git a/rpc/client/local/local.go b/rpc/client/local/local.go index cb5a0a5ed..10b2b4be7 100644 --- a/rpc/client/local/local.go +++ b/rpc/client/local/local.go @@ -141,7 +141,7 @@ func (c *Local) Health(ctx context.Context) (*coretypes.ResultHealth, error) { return c.env.Health(c.ctx) } -func (c *Local) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { //nolint:lll +func (c *Local) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { return c.env.BlockchainInfo(c.ctx, minHeight, maxHeight) } @@ -169,7 +169,7 @@ func (c *Local) Commit(ctx context.Context, height *int64) (*coretypes.ResultCom return c.env.Commit(c.ctx, height) } -func (c *Local) Validators(ctx context.Context, height *int64, page, perPage *int) (*coretypes.ResultValidators, error) { //nolint:lll +func (c *Local) Validators(ctx context.Context, height *int64, page, perPage *int) (*coretypes.ResultValidators, error) { return c.env.Validators(c.ctx, height, page, perPage) } diff --git a/rpc/client/mock/client.go b/rpc/client/mock/client.go index a1a42e28d..b57e661cb 100644 --- a/rpc/client/mock/client.go +++ b/rpc/client/mock/client.go @@ -131,7 +131,7 @@ func (c Client) Health(ctx context.Context) (*coretypes.ResultHealth, error) { return c.env.Health(&rpctypes.Context{}) } -func (c Client) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { //nolint:lll +func (c Client) BlockchainInfo(ctx context.Context, minHeight, maxHeight int64) (*coretypes.ResultBlockchainInfo, error) { return c.env.BlockchainInfo(&rpctypes.Context{}, minHeight, maxHeight) } @@ -151,7 +151,7 @@ func (c Client) Commit(ctx context.Context, height *int64) (*coretypes.ResultCom return c.env.Commit(&rpctypes.Context{}, height) } -func (c Client) Validators(ctx context.Context, height *int64, page, perPage *int) (*coretypes.ResultValidators, error) { //nolint:lll +func (c Client) Validators(ctx context.Context, height *int64, page, perPage *int) (*coretypes.ResultValidators, error) { return c.env.Validators(&rpctypes.Context{}, height, page, perPage) } diff --git a/types/proposal_test.go b/types/proposal_test.go index 47f5a8890..e692a65a9 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -49,7 +49,7 @@ func TestProposalSignable(t *testing.T) { func TestProposalString(t *testing.T) { str := testProposal.String() - expected := `Proposal{12345/23456 (2D2D4A756E655F31355F323032305F616D696E6F5F7761735F72656D6F766564:111:2D2D4A756E65, -1) 000000000000 @ 2018-02-11T07:09:22.765Z}` //nolint:lll // ignore line length for tests + expected := `Proposal{12345/23456 (2D2D4A756E655F31355F323032305F616D696E6F5F7761735F72656D6F766564:111:2D2D4A756E65, -1) 000000000000 @ 2018-02-11T07:09:22.765Z}` if str != expected { t.Errorf("got unexpected string for Proposal. Expected:\n%v\nGot:\n%v", expected, str) } diff --git a/types/vote_test.go b/types/vote_test.go index bfd1c4164..3ffb60324 100644 --- a/types/vote_test.go +++ b/types/vote_test.go @@ -220,13 +220,13 @@ func TestVoteVerify(t *testing.T) { func TestVoteString(t *testing.T) { str := examplePrecommit().String() - expected := `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PRECOMMIT(Precommit) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` //nolint:lll //ignore line length for tests + expected := `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PRECOMMIT(Precommit) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` if str != expected { t.Errorf("got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str) } str2 := examplePrevote().String() - expected = `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PREVOTE(Prevote) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` //nolint:lll //ignore line length for tests + expected = `Vote{56789:6AF1F4111082 12345/02/SIGNED_MSG_TYPE_PREVOTE(Prevote) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` if str2 != expected { t.Errorf("got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str2) } From 3749c378475c177ea30e6adab48de7f161f4408d Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Tue, 30 Nov 2021 17:55:08 -0500 Subject: [PATCH 20/21] p2p: remove unused trust package (#7359) --- internal/p2p/trust/config.go | 55 ---- internal/p2p/trust/metric.go | 413 ------------------------------ internal/p2p/trust/metric_test.go | 128 --------- internal/p2p/trust/store.go | 222 ---------------- internal/p2p/trust/store_test.go | 176 ------------- internal/p2p/trust/ticker.go | 62 ----- 6 files changed, 1056 deletions(-) delete mode 100644 internal/p2p/trust/config.go delete mode 100644 internal/p2p/trust/metric.go delete mode 100644 internal/p2p/trust/metric_test.go delete mode 100644 internal/p2p/trust/store.go delete mode 100644 internal/p2p/trust/store_test.go delete mode 100644 internal/p2p/trust/ticker.go diff --git a/internal/p2p/trust/config.go b/internal/p2p/trust/config.go deleted file mode 100644 index 0f990a991..000000000 --- a/internal/p2p/trust/config.go +++ /dev/null @@ -1,55 +0,0 @@ -package trust - -import "time" - -// MetricConfig - Configures the weight functions and time intervals for the metric -type MetricConfig struct { - // Determines the percentage given to current behavior - ProportionalWeight float64 - - // Determines the percentage given to prior behavior - IntegralWeight float64 - - // The window of time that the trust metric will track events across. - // This can be set to cover many days without issue - TrackingWindow time.Duration - - // Each interval should be short for adapability. - // Less than 30 seconds is too sensitive, - // and greater than 5 minutes will make the metric numb - IntervalLength time.Duration -} - -// DefaultConfig returns a config with values that have been tested and produce desirable results -func DefaultConfig() MetricConfig { - return MetricConfig{ - ProportionalWeight: 0.4, - IntegralWeight: 0.6, - TrackingWindow: (time.Minute * 60 * 24) * 14, // 14 days. - IntervalLength: 1 * time.Minute, - } -} - -// Ensures that all configuration elements have valid values -func customConfig(tmc MetricConfig) MetricConfig { - config := DefaultConfig() - - // Check the config for set values, and setup appropriately - if tmc.ProportionalWeight > 0 { - config.ProportionalWeight = tmc.ProportionalWeight - } - - if tmc.IntegralWeight > 0 { - config.IntegralWeight = tmc.IntegralWeight - } - - if tmc.IntervalLength > time.Duration(0) { - config.IntervalLength = tmc.IntervalLength - } - - if tmc.TrackingWindow > time.Duration(0) && - tmc.TrackingWindow >= config.IntervalLength { - config.TrackingWindow = tmc.TrackingWindow - } - return config -} diff --git a/internal/p2p/trust/metric.go b/internal/p2p/trust/metric.go deleted file mode 100644 index 8ad814f10..000000000 --- a/internal/p2p/trust/metric.go +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright 2017 Tendermint. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package trust - -import ( - "context" - "math" - "time" - - tmsync "github.com/tendermint/tendermint/internal/libs/sync" - "github.com/tendermint/tendermint/libs/service" -) - -//--------------------------------------------------------------------------------------- - -const ( - // The weight applied to the derivative when current behavior is >= previous behavior - defaultDerivativeGamma1 = 0 - - // The weight applied to the derivative when current behavior is less than previous behavior - defaultDerivativeGamma2 = 1.0 - - // The weight applied to history data values when calculating the history value - defaultHistoryDataWeight = 0.8 -) - -// MetricHistoryJSON - history data necessary to save the trust metric -type MetricHistoryJSON struct { - NumIntervals int `json:"intervals"` - History []float64 `json:"history"` -} - -// Metric - keeps track of peer reliability -// See tendermint/docs/architecture/adr-006-trust-metric.md for details -type Metric struct { - service.BaseService - - // Mutex that protects the metric from concurrent access - mtx tmsync.Mutex - - // Determines the percentage given to current behavior - proportionalWeight float64 - - // Determines the percentage given to prior behavior - integralWeight float64 - - // Count of how many time intervals this metric has been tracking - numIntervals int - - // Size of the time interval window for this trust metric - maxIntervals int - - // The time duration for a single time interval - intervalLen time.Duration - - // Stores the trust history data for this metric - history []float64 - - // Weights applied to the history data when calculating the history value - historyWeights []float64 - - // The sum of the history weights used when calculating the history value - historyWeightSum float64 - - // The current number of history data elements - historySize int - - // The maximum number of history data elements - historyMaxSize int - - // The calculated history value for the current time interval - historyValue float64 - - // The number of recorded good and bad events for the current time interval - bad, good float64 - - // While true, history data is not modified - paused bool - - // Used during testing in order to control the passing of time intervals - testTicker MetricTicker -} - -// NewMetric returns a trust metric with the default configuration. -// Use Start to begin tracking the quality of peer behavior over time -func NewMetric() *Metric { - return NewMetricWithConfig(DefaultConfig()) -} - -// NewMetricWithConfig returns a trust metric with a custom configuration. -// Use Start to begin tracking the quality of peer behavior over time -func NewMetricWithConfig(tmc MetricConfig) *Metric { - tm := new(Metric) - config := customConfig(tmc) - - // Setup using the configuration values - tm.proportionalWeight = config.ProportionalWeight - tm.integralWeight = config.IntegralWeight - tm.intervalLen = config.IntervalLength - // The maximum number of time intervals is the tracking window / interval length - tm.maxIntervals = int(config.TrackingWindow / tm.intervalLen) - // The history size will be determined by the maximum number of time intervals - tm.historyMaxSize = intervalToHistoryOffset(tm.maxIntervals) + 1 - // This metric has a perfect history so far - tm.historyValue = 1.0 - - tm.BaseService = *service.NewBaseService(nil, "Metric", tm) - return tm -} - -// OnStart implements Service -func (tm *Metric) OnStart(ctx context.Context) error { - if err := tm.BaseService.OnStart(ctx); err != nil { - return err - } - go tm.processRequests() - return nil -} - -// OnStop implements Service -// Nothing to do since the goroutine shuts down by itself via BaseService.Quit() -func (tm *Metric) OnStop() {} - -// Returns a snapshot of the trust metric history data -func (tm *Metric) HistoryJSON() MetricHistoryJSON { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - return MetricHistoryJSON{ - NumIntervals: tm.numIntervals, - History: tm.history, - } -} - -// Instantiates a trust metric by loading the history data for a single peer. -// This is called only once and only right after creation, which is why the -// lock is not held while accessing the trust metric struct members -func (tm *Metric) Init(hist MetricHistoryJSON) { - // Restore the number of time intervals we have previously tracked - if hist.NumIntervals > tm.maxIntervals { - hist.NumIntervals = tm.maxIntervals - } - tm.numIntervals = hist.NumIntervals - // Restore the history and its current size - if len(hist.History) > tm.historyMaxSize { - // Keep the history no larger than historyMaxSize - last := len(hist.History) - tm.historyMaxSize - hist.History = hist.History[last:] - } - tm.history = hist.History - tm.historySize = len(tm.history) - // Create the history weight values and weight sum - for i := 1; i <= tm.numIntervals; i++ { - x := math.Pow(defaultHistoryDataWeight, float64(i)) // Optimistic weight - tm.historyWeights = append(tm.historyWeights, x) - } - - for _, v := range tm.historyWeights { - tm.historyWeightSum += v - } - // Calculate the history value based on the loaded history data - tm.historyValue = tm.calcHistoryValue() -} - -// Pause tells the metric to pause recording data over time intervals. -// All method calls that indicate events will unpause the metric -func (tm *Metric) Pause() { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - // Pause the metric for now - tm.paused = true -} - -// BadEvents indicates that an undesirable event(s) took place -func (tm *Metric) BadEvents(num int) { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - tm.unpause() - tm.bad += float64(num) -} - -// GoodEvents indicates that a desirable event(s) took place -func (tm *Metric) GoodEvents(num int) { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - tm.unpause() - tm.good += float64(num) -} - -// TrustValue gets the dependable trust value; always between 0 and 1 -func (tm *Metric) TrustValue() float64 { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - return tm.calcTrustValue() -} - -// TrustScore gets a score based on the trust value always between 0 and 100 -func (tm *Metric) TrustScore() int { - score := tm.TrustValue() * 100 - - return int(math.Floor(score)) -} - -// NextTimeInterval saves current time interval data and prepares for the following interval -func (tm *Metric) NextTimeInterval() { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - if tm.paused { - // Do not prepare for the next time interval while paused - return - } - - // Add the current trust value to the history data - newHist := tm.calcTrustValue() - tm.history = append(tm.history, newHist) - - // Update history and interval counters - if tm.historySize < tm.historyMaxSize { - tm.historySize++ - } else { - // Keep the history no larger than historyMaxSize - last := len(tm.history) - tm.historyMaxSize - tm.history = tm.history[last:] - } - - if tm.numIntervals < tm.maxIntervals { - tm.numIntervals++ - // Add the optimistic weight for the new time interval - wk := math.Pow(defaultHistoryDataWeight, float64(tm.numIntervals)) - tm.historyWeights = append(tm.historyWeights, wk) - tm.historyWeightSum += wk - } - - // Update the history data using Faded Memories - tm.updateFadedMemory() - // Calculate the history value for the upcoming time interval - tm.historyValue = tm.calcHistoryValue() - tm.good = 0 - tm.bad = 0 -} - -// SetTicker allows a TestTicker to be provided that will manually control -// the passing of time from the perspective of the Metric. -// The ticker must be set before Start is called on the metric -func (tm *Metric) SetTicker(ticker MetricTicker) { - tm.mtx.Lock() - defer tm.mtx.Unlock() - - tm.testTicker = ticker -} - -// Copy returns a new trust metric with members containing the same values -func (tm *Metric) Copy() *Metric { - if tm == nil { - return nil - } - - tm.mtx.Lock() - defer tm.mtx.Unlock() - - return &Metric{ - proportionalWeight: tm.proportionalWeight, - integralWeight: tm.integralWeight, - numIntervals: tm.numIntervals, - maxIntervals: tm.maxIntervals, - intervalLen: tm.intervalLen, - history: tm.history, - historyWeights: tm.historyWeights, - historyWeightSum: tm.historyWeightSum, - historySize: tm.historySize, - historyMaxSize: tm.historyMaxSize, - historyValue: tm.historyValue, - good: tm.good, - bad: tm.bad, - paused: tm.paused, - } - -} - -/* Private methods */ - -// This method is for a goroutine that handles all requests on the metric -func (tm *Metric) processRequests() { - t := tm.testTicker - if t == nil { - // No test ticker was provided, so we create a normal ticker - t = NewTicker(tm.intervalLen) - } - defer t.Stop() - // Obtain the raw channel - tick := t.GetChannel() -loop: - for { - select { - case <-tick: - tm.NextTimeInterval() - case <-tm.Quit(): - // Stop all further tracking for this metric - break loop - } - } -} - -// Wakes the trust metric up if it is currently paused -// This method needs to be called with the mutex locked -func (tm *Metric) unpause() { - // Check if this is the first experience with - // what we are tracking since being paused - if tm.paused { - tm.good = 0 - tm.bad = 0 - // New events cause us to unpause the metric - tm.paused = false - } -} - -// Calculates the trust value for the request processing -func (tm *Metric) calcTrustValue() float64 { - weightedP := tm.proportionalWeight * tm.proportionalValue() - weightedI := tm.integralWeight * tm.historyValue - weightedD := tm.weightedDerivative() - - tv := weightedP + weightedI + weightedD - // Do not return a negative value. - if tv < 0 { - tv = 0 - } - return tv -} - -// Calculates the current score for good/bad experiences -func (tm *Metric) proportionalValue() float64 { - value := 1.0 - - total := tm.good + tm.bad - if total > 0 { - value = tm.good / total - } - return value -} - -// Strengthens the derivative component when the change is negative -func (tm *Metric) weightedDerivative() float64 { - var weight float64 = defaultDerivativeGamma1 - - d := tm.derivativeValue() - if d < 0 { - weight = defaultDerivativeGamma2 - } - return weight * d -} - -// Calculates the derivative component -func (tm *Metric) derivativeValue() float64 { - return tm.proportionalValue() - tm.historyValue -} - -// Calculates the integral (history) component of the trust value -func (tm *Metric) calcHistoryValue() float64 { - var hv float64 - - for i := 0; i < tm.numIntervals; i++ { - hv += tm.fadedMemoryValue(i) * tm.historyWeights[i] - } - - return hv / tm.historyWeightSum -} - -// Retrieves the actual history data value that represents the requested time interval -func (tm *Metric) fadedMemoryValue(interval int) float64 { - first := tm.historySize - 1 - - if interval == 0 { - // Base case - return tm.history[first] - } - - offset := intervalToHistoryOffset(interval) - return tm.history[first-offset] -} - -// Performs the update for our Faded Memories process, which allows the -// trust metric tracking window to be large while maintaining a small -// number of history data values -func (tm *Metric) updateFadedMemory() { - if tm.historySize < 2 { - return - } - - end := tm.historySize - 1 - // Keep the most recent history element - for count := 1; count < tm.historySize; count++ { - i := end - count - // The older the data is, the more we spread it out - x := math.Pow(2, float64(count)) - // Two history data values are merged into a single value - tm.history[i] = ((tm.history[i] * (x - 1)) + tm.history[i+1]) / x - } -} - -// Map the interval value down to an offset from the beginning of history -func intervalToHistoryOffset(interval int) int { - // The system maintains 2^m interval values in the form of m history - // data values. Therefore, we access the ith interval by obtaining - // the history data index = the floor of log2(i) - return int(math.Floor(math.Log2(float64(interval)))) -} diff --git a/internal/p2p/trust/metric_test.go b/internal/p2p/trust/metric_test.go deleted file mode 100644 index b7d19da23..000000000 --- a/internal/p2p/trust/metric_test.go +++ /dev/null @@ -1,128 +0,0 @@ -package trust - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestTrustMetricScores(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tm := NewMetric() - err := tm.Start(ctx) - require.NoError(t, err) - - // Perfect score - tm.GoodEvents(1) - score := tm.TrustScore() - assert.Equal(t, 100, score) - - // Less than perfect score - tm.BadEvents(10) - score = tm.TrustScore() - assert.NotEqual(t, 100, score) - err = tm.Stop() - require.NoError(t, err) -} - -func TestTrustMetricConfig(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // 7 days - window := time.Minute * 60 * 24 * 7 - config := MetricConfig{ - TrackingWindow: window, - IntervalLength: 2 * time.Minute, - } - - tm := NewMetricWithConfig(config) - err := tm.Start(ctx) - require.NoError(t, err) - - // The max time intervals should be the TrackingWindow / IntervalLen - assert.Equal(t, int(config.TrackingWindow/config.IntervalLength), tm.maxIntervals) - - dc := DefaultConfig() - // These weights should still be the default values - assert.Equal(t, dc.ProportionalWeight, tm.proportionalWeight) - assert.Equal(t, dc.IntegralWeight, tm.integralWeight) - err = tm.Stop() - require.NoError(t, err) - tm.Wait() - - config.ProportionalWeight = 0.3 - config.IntegralWeight = 0.7 - tm = NewMetricWithConfig(config) - err = tm.Start(ctx) - require.NoError(t, err) - - // These weights should be equal to our custom values - assert.Equal(t, config.ProportionalWeight, tm.proportionalWeight) - assert.Equal(t, config.IntegralWeight, tm.integralWeight) - err = tm.Stop() - require.NoError(t, err) - tm.Wait() -} - -func TestTrustMetricCopyNilPointer(t *testing.T) { - var tm *Metric - - ctm := tm.Copy() - - assert.Nil(t, ctm) -} - -// XXX: This test fails non-deterministically -//nolint:unused,deadcode -func _TestTrustMetricStopPause(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // The TestTicker will provide manual control over - // the passing of time within the metric - tt := NewTestTicker() - tm := NewMetric() - tm.SetTicker(tt) - err := tm.Start(ctx) - require.NoError(t, err) - // Allow some time intervals to pass and pause - tt.NextTick() - tt.NextTick() - tm.Pause() - - // could be 1 or 2 because Pause and NextTick race - first := tm.Copy().numIntervals - - // Allow more time to pass and check the intervals are unchanged - tt.NextTick() - tt.NextTick() - assert.Equal(t, first, tm.Copy().numIntervals) - - // Get the trust metric activated again - tm.GoodEvents(5) - // Allow some time intervals to pass and stop - tt.NextTick() - tt.NextTick() - err = tm.Stop() - require.NoError(t, err) - tm.Wait() - - second := tm.Copy().numIntervals - // Allow more intervals to pass while the metric is stopped - // and check that the number of intervals match - tm.NextTimeInterval() - tm.NextTimeInterval() - // XXX: fails non-deterministically: - // expected 5, got 6 - assert.Equal(t, second+2, tm.Copy().numIntervals) - - if first > second { - t.Fatalf("numIntervals should always increase or stay the same over time") - } -} diff --git a/internal/p2p/trust/store.go b/internal/p2p/trust/store.go deleted file mode 100644 index a01cbab2e..000000000 --- a/internal/p2p/trust/store.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2017 Tendermint. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package trust - -import ( - "context" - "encoding/json" - "fmt" - "time" - - dbm "github.com/tendermint/tm-db" - - tmsync "github.com/tendermint/tendermint/internal/libs/sync" - "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/libs/service" -) - -const defaultStorePeriodicSaveInterval = 1 * time.Minute - -var trustMetricKey = []byte("trustMetricStore") - -// MetricStore - Manages all trust metrics for peers -type MetricStore struct { - service.BaseService - - // Maps a Peer.Key to that peer's TrustMetric - peerMetrics map[string]*Metric - - // Mutex that protects the map and history data file - mtx tmsync.Mutex - - // The db where peer trust metric history data will be stored - db dbm.DB - - // This configuration will be used when creating new TrustMetrics - config MetricConfig -} - -// NewTrustMetricStore returns a store that saves data to the DB -// and uses the config when creating new trust metrics. -// Use Start to to initialize the trust metric store -func NewTrustMetricStore(db dbm.DB, tmc MetricConfig, logger log.Logger) *MetricStore { - tms := &MetricStore{ - peerMetrics: make(map[string]*Metric), - db: db, - config: tmc, - } - - tms.BaseService = *service.NewBaseService(logger, "MetricStore", tms) - return tms -} - -// OnStart implements Service -func (tms *MetricStore) OnStart(ctx context.Context) error { - if err := tms.BaseService.OnStart(ctx); err != nil { - return err - } - - tms.mtx.Lock() - defer tms.mtx.Unlock() - - tms.loadFromDB(ctx) - go tms.saveRoutine() - return nil -} - -// OnStop implements Service -func (tms *MetricStore) OnStop() { - tms.BaseService.OnStop() - - tms.mtx.Lock() - defer tms.mtx.Unlock() - - // Stop all trust metric go-routines - for _, tm := range tms.peerMetrics { - if err := tm.Stop(); err != nil { - tms.Logger.Error("unable to stop metric store", "error", err) - } - } - - // Make the final trust history data save - tms.saveToDB() -} - -// Size returns the number of entries in the trust metric store -func (tms *MetricStore) Size() int { - tms.mtx.Lock() - defer tms.mtx.Unlock() - - return tms.size() -} - -// AddPeerTrustMetric takes an existing trust metric and associates it with a peer key. -// The caller is expected to call Start on the TrustMetric being added -func (tms *MetricStore) AddPeerTrustMetric(key string, tm *Metric) { - tms.mtx.Lock() - defer tms.mtx.Unlock() - - if key == "" || tm == nil { - return - } - tms.peerMetrics[key] = tm -} - -// GetPeerTrustMetric returns a trust metric by peer key -func (tms *MetricStore) GetPeerTrustMetric(ctx context.Context, key string) *Metric { - tms.mtx.Lock() - defer tms.mtx.Unlock() - - tm, ok := tms.peerMetrics[key] - if !ok { - // If the metric is not available, we will create it - tm = NewMetricWithConfig(tms.config) - if err := tm.Start(ctx); err != nil { - tms.Logger.Error("unable to start metric store", "error", err) - } - // The metric needs to be in the map - tms.peerMetrics[key] = tm - } - return tm -} - -// PeerDisconnected pauses the trust metric associated with the peer identified by the key -func (tms *MetricStore) PeerDisconnected(key string) { - tms.mtx.Lock() - defer tms.mtx.Unlock() - - // If the Peer that disconnected has a metric, pause it - if tm, ok := tms.peerMetrics[key]; ok { - tm.Pause() - } -} - -// Saves the history data for all peers to the store DB. -// This public method acquires the trust metric store lock -func (tms *MetricStore) SaveToDB() { - tms.mtx.Lock() - defer tms.mtx.Unlock() - - tms.saveToDB() -} - -/* Private methods */ - -// size returns the number of entries in the store without acquiring the mutex -func (tms *MetricStore) size() int { - return len(tms.peerMetrics) -} - -/* Loading & Saving */ -/* Both loadFromDB and savetoDB assume the mutex has been acquired */ - -// Loads the history data for all peers from the store DB -// cmn.Panics if file is corrupt -func (tms *MetricStore) loadFromDB(ctx context.Context) bool { - // Obtain the history data we have so far - bytes, err := tms.db.Get(trustMetricKey) - if err != nil { - panic(err) - } - if bytes == nil { - return false - } - - peers := make(map[string]MetricHistoryJSON) - err = json.Unmarshal(bytes, &peers) - if err != nil { - panic(fmt.Sprintf("Could not unmarshal Trust Metric Store DB data: %v", err)) - } - - // If history data exists in the file, - // load it into trust metric - for key, p := range peers { - tm := NewMetricWithConfig(tms.config) - - if err := tm.Start(ctx); err != nil { - tms.Logger.Error("unable to start metric", "error", err) - } - tm.Init(p) - // Load the peer trust metric into the store - tms.peerMetrics[key] = tm - } - return true -} - -// Saves the history data for all peers to the store DB -func (tms *MetricStore) saveToDB() { - tms.Logger.Debug("Saving TrustHistory to DB", "size", tms.size()) - - peers := make(map[string]MetricHistoryJSON) - - for key, tm := range tms.peerMetrics { - // Add an entry for the peer identified by key - peers[key] = tm.HistoryJSON() - } - - // Write all the data back to the DB - bytes, err := json.Marshal(peers) - if err != nil { - tms.Logger.Error("Failed to encode the TrustHistory", "err", err) - return - } - if err := tms.db.SetSync(trustMetricKey, bytes); err != nil { - tms.Logger.Error("failed to flush data to disk", "error", err) - } -} - -// Periodically saves the trust history data to the DB -func (tms *MetricStore) saveRoutine() { - t := time.NewTicker(defaultStorePeriodicSaveInterval) - defer t.Stop() -loop: - for { - select { - case <-t.C: - tms.SaveToDB() - case <-tms.Quit(): - break loop - } - } -} diff --git a/internal/p2p/trust/store_test.go b/internal/p2p/trust/store_test.go deleted file mode 100644 index a6178459f..000000000 --- a/internal/p2p/trust/store_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright 2017 Tendermint. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package trust - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - dbm "github.com/tendermint/tm-db" - - "github.com/tendermint/tendermint/libs/log" -) - -func TestTrustMetricStoreSaveLoad(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - dir := t.TempDir() - logger := log.TestingLogger() - - historyDB, err := dbm.NewDB("trusthistory", "goleveldb", dir) - require.NoError(t, err) - - // 0 peers saved - store := NewTrustMetricStore(historyDB, DefaultConfig(), logger) - store.saveToDB() - // Load the data from the file - store = NewTrustMetricStore(historyDB, DefaultConfig(), logger) - err = store.Start(ctx) - require.NoError(t, err) - // Make sure we still have 0 entries - assert.Zero(t, store.Size()) - - // 100 TestTickers - var tt []*TestTicker - for i := 0; i < 100; i++ { - // The TestTicker will provide manual control over - // the passing of time within the metric - tt = append(tt, NewTestTicker()) - } - // 100 peers - for i := 0; i < 100; i++ { - key := fmt.Sprintf("peer_%d", i) - tm := NewMetric() - - tm.SetTicker(tt[i]) - err = tm.Start(ctx) - require.NoError(t, err) - store.AddPeerTrustMetric(key, tm) - - tm.BadEvents(10) - tm.GoodEvents(1) - } - // Check that we have 100 entries and save - assert.Equal(t, 100, store.Size()) - // Give the 100 metrics time to process the history data - for i := 0; i < 100; i++ { - tt[i].NextTick() - tt[i].NextTick() - } - // Stop all the trust metrics and save - err = store.Stop() - require.NoError(t, err) - - // Load the data from the DB - store = NewTrustMetricStore(historyDB, DefaultConfig(), logger) - - err = store.Start(ctx) - require.NoError(t, err) - - // Check that we still have 100 peers with imperfect trust values - assert.Equal(t, 100, store.Size()) - for _, tm := range store.peerMetrics { - assert.NotEqual(t, 1.0, tm.TrustValue()) - } - - err = store.Stop() - require.NoError(t, err) -} - -func TestTrustMetricStoreConfig(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) - - config := MetricConfig{ - ProportionalWeight: 0.5, - IntegralWeight: 0.5, - } - - logger := log.TestingLogger() - // Create a store with custom config - store := NewTrustMetricStore(historyDB, config, logger) - - err = store.Start(ctx) - require.NoError(t, err) - - // Have the store make us a metric with the config - tm := store.GetPeerTrustMetric(ctx, "TestKey") - - // Check that the options made it to the metric - assert.Equal(t, 0.5, tm.proportionalWeight) - assert.Equal(t, 0.5, tm.integralWeight) - err = store.Stop() - require.NoError(t, err) -} - -func TestTrustMetricStoreLookup(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) - - store := NewTrustMetricStore(historyDB, DefaultConfig(), log.TestingLogger()) - - err = store.Start(ctx) - require.NoError(t, err) - - // Create 100 peers in the trust metric store - for i := 0; i < 100; i++ { - key := fmt.Sprintf("peer_%d", i) - store.GetPeerTrustMetric(ctx, key) - - // Check that the trust metric was successfully entered - ktm := store.peerMetrics[key] - assert.NotNil(t, ktm, "Expected to find TrustMetric %s but wasn't there.", key) - } - - err = store.Stop() - require.NoError(t, err) -} - -func TestTrustMetricStorePeerScore(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - historyDB, err := dbm.NewDB("", "memdb", "") - require.NoError(t, err) - - store := NewTrustMetricStore(historyDB, DefaultConfig(), log.TestingLogger()) - - err = store.Start(ctx) - require.NoError(t, err) - - key := "TestKey" - tm := store.GetPeerTrustMetric(ctx, key) - - // This peer is innocent so far - first := tm.TrustScore() - assert.Equal(t, 100, first) - - // Add some undesirable events and disconnect - tm.BadEvents(1) - first = tm.TrustScore() - assert.NotEqual(t, 100, first) - tm.BadEvents(10) - second := tm.TrustScore() - - if second > first { - t.Errorf("a greater number of bad events should lower the trust score") - } - store.PeerDisconnected(key) - - // We will remember our experiences with this peer - tm = store.GetPeerTrustMetric(ctx, key) - assert.NotEqual(t, 100, tm.TrustScore()) - err = store.Stop() - require.NoError(t, err) -} diff --git a/internal/p2p/trust/ticker.go b/internal/p2p/trust/ticker.go deleted file mode 100644 index 3f0f30919..000000000 --- a/internal/p2p/trust/ticker.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2017 Tendermint. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package trust - -import ( - "time" -) - -// MetricTicker provides a single ticker interface for the trust metric -type MetricTicker interface { - // GetChannel returns the receive only channel that fires at each time interval - GetChannel() <-chan time.Time - - // Stop will halt further activity on the ticker channel - Stop() -} - -// The ticker used during testing that provides manual control over time intervals -type TestTicker struct { - C chan time.Time - stopped bool -} - -// NewTestTicker returns our ticker used within test routines -func NewTestTicker() *TestTicker { - c := make(chan time.Time) - return &TestTicker{ - C: c, - } -} - -func (t *TestTicker) GetChannel() <-chan time.Time { - return t.C -} - -func (t *TestTicker) Stop() { - t.stopped = true -} - -// NextInterval manually sends Time on the ticker channel -func (t *TestTicker) NextTick() { - if t.stopped { - return - } - t.C <- time.Now() -} - -// Ticker is just a wrap around time.Ticker that allows it -// to meet the requirements of our interface -type Ticker struct { - *time.Ticker -} - -// NewTicker returns a normal time.Ticker wrapped to meet our interface -func NewTicker(d time.Duration) *Ticker { - return &Ticker{time.NewTicker(d)} -} - -func (t *Ticker) GetChannel() <-chan time.Time { - return t.C -} From a823d167bc615b3a482420efde98c239d14985d1 Mon Sep 17 00:00:00 2001 From: Sam Kleinman Date: Wed, 1 Dec 2021 09:28:06 -0500 Subject: [PATCH 21/21] service: cleanup base implementation and some caller implementations (#7301) --- abci/client/local_client.go | 5 +- abci/client/socket_client.go | 2 - internal/blocksync/pool.go | 37 +++++++++++---- internal/blocksync/reactor.go | 10 ++-- internal/consensus/common_test.go | 2 + internal/consensus/reactor.go | 6 ++- internal/consensus/replay.go | 2 +- internal/consensus/state.go | 41 +++++++++++------ internal/consensus/ticker.go | 14 +++--- internal/consensus/wal.go | 13 ++++-- internal/eventbus/event_bus.go | 9 +--- internal/eventbus/event_bus_test.go | 6 +-- internal/libs/autofile/cmd/logjack.go | 6 +-- internal/libs/autofile/group.go | 9 ++-- internal/mempool/mempool_test.go | 24 +++++----- internal/mempool/reactor_test.go | 10 ++-- internal/p2p/conn/connection.go | 8 ++-- internal/p2p/conn/connection_test.go | 58 ++++++++++++++---------- libs/service/service.go | 43 +++++++++--------- libs/service/service_test.go | 5 +- light/rpc/client.go | 17 +++++-- node/node_test.go | 45 +++++++----------- privval/signer_client_test.go | 6 +-- privval/signer_dialer_endpoint.go | 4 ++ privval/signer_listener_endpoint.go | 14 +++--- privval/signer_listener_endpoint_test.go | 6 +-- privval/signer_server.go | 2 - rpc/client/local/local.go | 2 +- 28 files changed, 222 insertions(+), 184 deletions(-) diff --git a/abci/client/local_client.go b/abci/client/local_client.go index 701108a3c..f534a1716 100644 --- a/abci/client/local_client.go +++ b/abci/client/local_client.go @@ -38,10 +38,13 @@ func NewLocalClient(mtx *tmsync.Mutex, app types.Application) Client { return cli } +func (*localClient) OnStart(context.Context) error { return nil } +func (*localClient) OnStop() {} + func (app *localClient) SetResponseCallback(cb Callback) { app.mtx.Lock() + defer app.mtx.Unlock() app.Callback = cb - app.mtx.Unlock() } // TODO: change types.Application to include Error()? diff --git a/abci/client/socket_client.go b/abci/client/socket_client.go index 8dfee0c8d..9173475c6 100644 --- a/abci/client/socket_client.go +++ b/abci/client/socket_client.go @@ -126,8 +126,6 @@ func (cli *socketClient) sendRequestsRoutine(ctx context.Context, conn io.Writer select { case <-ctx.Done(): return - case <-cli.Quit(): - return case reqres := <-cli.reqQueue: if ctx.Err() != nil { return diff --git a/internal/blocksync/pool.go b/internal/blocksync/pool.go index 6f06c9883..a83a119a6 100644 --- a/internal/blocksync/pool.go +++ b/internal/blocksync/pool.go @@ -84,6 +84,7 @@ type BlockPool struct { requestsCh chan<- BlockRequest errorsCh chan<- peerError + exitedCh chan struct{} startHeight int64 lastHundredBlockTimeStamp time.Time @@ -102,11 +103,11 @@ func NewBlockPool( bp := &BlockPool{ peers: make(map[types.NodeID]*bpPeer), - requesters: make(map[int64]*bpRequester), - height: start, - startHeight: start, - numPending: 0, - + requesters: make(map[int64]*bpRequester), + height: start, + startHeight: start, + numPending: 0, + exitedCh: make(chan struct{}), requestsCh: requestsCh, errorsCh: errorsCh, lastSyncRate: 0, @@ -121,9 +122,17 @@ func (pool *BlockPool) OnStart(ctx context.Context) error { pool.lastAdvance = time.Now() pool.lastHundredBlockTimeStamp = pool.lastAdvance go pool.makeRequestersRoutine(ctx) + + go func() { + defer close(pool.exitedCh) + pool.Wait() + }() + return nil } +func (*BlockPool) OnStop() {} + // spawns requesters as needed func (pool *BlockPool) makeRequestersRoutine(ctx context.Context) { for { @@ -572,10 +581,12 @@ func newBPRequester(pool *BlockPool, height int64) *bpRequester { } func (bpr *bpRequester) OnStart(ctx context.Context) error { - go bpr.requestRoutine() + go bpr.requestRoutine(ctx) return nil } +func (*bpRequester) OnStop() {} + // Returns true if the peer matches and block doesn't already exist. func (bpr *bpRequester) setBlock(block *types.Block, peerID types.NodeID) bool { bpr.mtx.Lock() @@ -630,7 +641,13 @@ func (bpr *bpRequester) redo(peerID types.NodeID) { // Responsible for making more requests as necessary // Returns only when a block is found (e.g. AddBlock() is called) -func (bpr *bpRequester) requestRoutine() { +func (bpr *bpRequester) requestRoutine(ctx context.Context) { + bprPoolDone := make(chan struct{}) + go func() { + defer close(bprPoolDone) + bpr.pool.Wait() + }() + OUTER_LOOP: for { // Pick a peer to send request to. @@ -656,13 +673,13 @@ OUTER_LOOP: WAIT_LOOP: for { select { - case <-bpr.pool.Quit(): + case <-ctx.Done(): + return + case <-bpr.pool.exitedCh: if err := bpr.Stop(); err != nil { bpr.Logger.Error("Error stopped requester", "err", err) } return - case <-bpr.Quit(): - return case peerID := <-bpr.redoCh: if peerID == bpr.peerID { bpr.reset() diff --git a/internal/blocksync/reactor.go b/internal/blocksync/reactor.go index ac5d45fb7..5fe8b2123 100644 --- a/internal/blocksync/reactor.go +++ b/internal/blocksync/reactor.go @@ -158,7 +158,7 @@ func (r *Reactor) OnStart(ctx context.Context) error { return err } r.poolWG.Add(1) - go r.requestRoutine() + go r.requestRoutine(ctx) r.poolWG.Add(1) go r.poolRoutine(false) @@ -375,7 +375,7 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { r.syncStartTime = time.Now() r.poolWG.Add(1) - go r.requestRoutine() + go r.requestRoutine(ctx) r.poolWG.Add(1) go r.poolRoutine(true) @@ -383,7 +383,7 @@ func (r *Reactor) SwitchToBlockSync(ctx context.Context, state sm.State) error { return nil } -func (r *Reactor) requestRoutine() { +func (r *Reactor) requestRoutine(ctx context.Context) { statusUpdateTicker := time.NewTicker(statusUpdateIntervalSeconds * time.Second) defer statusUpdateTicker.Stop() @@ -394,7 +394,7 @@ func (r *Reactor) requestRoutine() { case <-r.closeCh: return - case <-r.pool.Quit(): + case <-ctx.Done(): return case request := <-r.requestsCh: @@ -607,7 +607,7 @@ FOR_LOOP: case <-r.closeCh: break FOR_LOOP - case <-r.pool.Quit(): + case <-r.pool.exitedCh: break FOR_LOOP } } diff --git a/internal/consensus/common_test.go b/internal/consensus/common_test.go index 30729c038..27f9628d1 100644 --- a/internal/consensus/common_test.go +++ b/internal/consensus/common_test.go @@ -915,6 +915,8 @@ func (m *mockTicker) Stop() error { return nil } +func (m *mockTicker) IsRunning() bool { return false } + func (m *mockTicker) ScheduleTimeout(ti timeoutInfo) { m.mtx.Lock() defer m.mtx.Unlock() diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 7e46444b9..4e2b1ad3a 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "errors" "fmt" "runtime/debug" "time" @@ -205,11 +206,12 @@ func (r *Reactor) OnStart(ctx context.Context) error { // blocking until they all exit, as well as unsubscribing from events and stopping // state. func (r *Reactor) OnStop() { - r.unsubscribeFromBroadcastEvents() if err := r.state.Stop(); err != nil { - r.Logger.Error("failed to stop consensus state", "err", err) + if !errors.Is(err, service.ErrAlreadyStopped) { + r.Logger.Error("failed to stop consensus state", "err", err) + } } if !r.WaitSync() { diff --git a/internal/consensus/replay.go b/internal/consensus/replay.go index 2408b03f1..b797eabae 100644 --- a/internal/consensus/replay.go +++ b/internal/consensus/replay.go @@ -50,7 +50,7 @@ func (cs *State) readReplayMessage(ctx context.Context, msg *TimedWALMessage, ne cs.Logger.Info("Replay: New Step", "height", m.Height, "round", m.Round, "step", m.Step) // these are playback checks if newStepSub != nil { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) defer cancel() stepMsg, err := newStepSub.Next(ctx) if errors.Is(err, context.DeadlineExceeded) { diff --git a/internal/consensus/state.go b/internal/consensus/state.go index 2e9f97503..f71d08649 100644 --- a/internal/consensus/state.go +++ b/internal/consensus/state.go @@ -363,6 +363,7 @@ func (cs *State) OnStart(ctx context.Context) error { // 1) prep work if err := cs.wal.Stop(); err != nil { + return err } @@ -421,6 +422,8 @@ func (cs *State) OnStart(ctx context.Context) error { // timeoutRoutine: receive requests for timeouts on tickChan and fire timeouts on tockChan // receiveRoutine: serializes processing of proposoals, block parts, votes; coordinates state transitions +// +// this is only used in tests. func (cs *State) startRoutines(ctx context.Context, maxSteps int) { err := cs.timeoutTicker.Start(ctx) if err != nil { @@ -445,7 +448,6 @@ func (cs *State) loadWalFile(ctx context.Context) error { // OnStop implements service.Service. func (cs *State) OnStop() { - // If the node is committing a new block, wait until it is finished! if cs.GetRoundState().Step == cstypes.RoundStepCommit { select { @@ -457,15 +459,19 @@ func (cs *State) OnStop() { close(cs.onStopCh) - if err := cs.evsw.Stop(); err != nil { - if !errors.Is(err, service.ErrAlreadyStopped) { - cs.Logger.Error("failed trying to stop eventSwitch", "error", err) + if cs.evsw.IsRunning() { + if err := cs.evsw.Stop(); err != nil { + if !errors.Is(err, service.ErrAlreadyStopped) { + cs.Logger.Error("failed trying to stop eventSwitch", "error", err) + } } } - if err := cs.timeoutTicker.Stop(); err != nil { - if !errors.Is(err, service.ErrAlreadyStopped) { - cs.Logger.Error("failed trying to stop timeoutTicket", "error", err) + if cs.timeoutTicker.IsRunning() { + if err := cs.timeoutTicker.Stop(); err != nil { + if !errors.Is(err, service.ErrAlreadyStopped) { + cs.Logger.Error("failed trying to stop timeoutTicket", "error", err) + } } } // WAL is stopped in receiveRoutine. @@ -845,9 +851,10 @@ func (cs *State) receiveRoutine(ctx context.Context, maxSteps int) { // go to the next step cs.handleTimeout(ctx, ti, rs) - case <-cs.Quit(): + case <-ctx.Done(): onExit(cs) return + } // TODO should we handle context cancels here? } @@ -875,7 +882,11 @@ func (cs *State) handleMsg(ctx context.Context, mi msgInfo) { // if the proposal is complete, we'll enterPrevote or tryFinalizeCommit added, err = cs.addProposalBlockPart(ctx, msg, peerID) if added { - cs.statsMsgQueue <- mi + select { + case cs.statsMsgQueue <- mi: + case <-ctx.Done(): + return + } } if err != nil && msg.Round != cs.Round { @@ -893,7 +904,11 @@ func (cs *State) handleMsg(ctx context.Context, mi msgInfo) { // if the vote gives us a 2/3-any or 2/3-one, we transition added, err = cs.tryAddVote(ctx, msg.Vote, peerID) if added { - cs.statsMsgQueue <- mi + select { + case cs.statsMsgQueue <- mi: + case <-ctx.Done(): + return + } } // if err == ErrAddingVote { @@ -1012,7 +1027,7 @@ func (cs *State) handleTxsAvailable(ctx context.Context) { // Used internally by handleTimeout and handleMsg to make state transitions // Enter: `timeoutNewHeight` by startTime (commitTime+timeoutCommit), -// or, if SkipTimeoutCommit==true, after receiving all precommits from (height,round-1) +// or, if SkipTimeoutCommit==true, after receiving all precommits from (height,round-1) // Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1) // Enter: +2/3 precommits for nil at (height,round-1) // Enter: +2/3 prevotes any or +2/3 precommits for block or any from (height, round) @@ -1097,7 +1112,7 @@ func (cs *State) needProofBlock(height int64) bool { // Enter (CreateEmptyBlocks): from enterNewRound(height,round) // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): -// after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval +// after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool func (cs *State) enterPropose(ctx context.Context, height int64, round int32) { logger := cs.Logger.With("height", height, "round", round) @@ -2011,7 +2026,7 @@ func (cs *State) tryAddVote(ctx context.Context, vote *types.Vote, peerID types. // 1) bad peer OR // 2) not a bad peer? this can also err sometimes with "Unexpected step" OR // 3) tmkms use with multiple validators connecting to a single tmkms instance - // (https://github.com/tendermint/tendermint/issues/3839). + // (https://github.com/tendermint/tendermint/issues/3839). cs.Logger.Info("failed attempting to add vote", "err", err) return added, ErrAddingVote } diff --git a/internal/consensus/ticker.go b/internal/consensus/ticker.go index e8583932d..6e323b2d0 100644 --- a/internal/consensus/ticker.go +++ b/internal/consensus/ticker.go @@ -18,6 +18,7 @@ var ( type TimeoutTicker interface { Start(context.Context) error Stop() error + IsRunning() bool Chan() <-chan timeoutInfo // on which to receive a timeout ScheduleTimeout(ti timeoutInfo) // reset the timer } @@ -48,17 +49,14 @@ func NewTimeoutTicker(logger log.Logger) TimeoutTicker { } // OnStart implements service.Service. It starts the timeout routine. -func (t *timeoutTicker) OnStart(gctx context.Context) error { - go t.timeoutRoutine() +func (t *timeoutTicker) OnStart(ctx context.Context) error { + go t.timeoutRoutine(ctx) return nil } // OnStop implements service.Service. It stops the timeout routine. -func (t *timeoutTicker) OnStop() { - t.BaseService.OnStop() - t.stopTimer() -} +func (t *timeoutTicker) OnStop() { t.stopTimer() } // Chan returns a channel on which timeouts are sent. func (t *timeoutTicker) Chan() <-chan timeoutInfo { @@ -89,7 +87,7 @@ func (t *timeoutTicker) stopTimer() { // send on tickChan to start a new timer. // timers are interupted and replaced by new ticks from later steps // timeouts of 0 on the tickChan will be immediately relayed to the tockChan -func (t *timeoutTicker) timeoutRoutine() { +func (t *timeoutTicker) timeoutRoutine(ctx context.Context) { t.Logger.Debug("Starting timeout routine") var ti timeoutInfo for { @@ -125,7 +123,7 @@ func (t *timeoutTicker) timeoutRoutine() { // We can eliminate it by merging the timeoutRoutine into receiveRoutine // and managing the timeouts ourselves with a millisecond ticker go func(toi timeoutInfo) { t.tockChan <- toi }(ti) - case <-t.Quit(): + case <-ctx.Done(): return } } diff --git a/internal/consensus/wal.go b/internal/consensus/wal.go index 13f29a202..e89cf0992 100644 --- a/internal/consensus/wal.go +++ b/internal/consensus/wal.go @@ -131,18 +131,18 @@ func (wal *BaseWAL) OnStart(ctx context.Context) error { return err } wal.flushTicker = time.NewTicker(wal.flushInterval) - go wal.processFlushTicks() + go wal.processFlushTicks(ctx) return nil } -func (wal *BaseWAL) processFlushTicks() { +func (wal *BaseWAL) processFlushTicks(ctx context.Context) { for { select { case <-wal.flushTicker.C: if err := wal.FlushAndSync(); err != nil { wal.Logger.Error("Periodic WAL flush failed", "err", err) } - case <-wal.Quit(): + case <-ctx.Done(): return } } @@ -175,7 +175,12 @@ func (wal *BaseWAL) OnStop() { // Wait for the underlying autofile group to finish shutting down // so it's safe to cleanup files. func (wal *BaseWAL) Wait() { - wal.group.Wait() + if wal.IsRunning() { + wal.BaseService.Wait() + } + if wal.group.IsRunning() { + wal.group.Wait() + } } // Write is called in newStep and for each receive on the diff --git a/internal/eventbus/event_bus.go b/internal/eventbus/event_bus.go index 61473c713..0eb44c37b 100644 --- a/internal/eventbus/event_bus.go +++ b/internal/eventbus/event_bus.go @@ -2,7 +2,6 @@ package eventbus import ( "context" - "errors" "fmt" "strings" @@ -43,13 +42,7 @@ func (b *EventBus) OnStart(ctx context.Context) error { return b.pubsub.Start(ctx) } -func (b *EventBus) OnStop() { - if err := b.pubsub.Stop(); err != nil { - if !errors.Is(err, service.ErrAlreadyStopped) { - b.pubsub.Logger.Error("error trying to stop eventBus", "error", err) - } - } -} +func (b *EventBus) OnStop() {} func (b *EventBus) NumClients() int { return b.pubsub.NumClients() diff --git a/internal/eventbus/event_bus_test.go b/internal/eventbus/event_bus_test.go index 06f2bfa64..72b1094fb 100644 --- a/internal/eventbus/event_bus_test.go +++ b/internal/eventbus/event_bus_test.go @@ -436,11 +436,7 @@ func benchmarkEventBus(numClients int, randQueries bool, randEvents bool, b *tes if err != nil { b.Error(err) } - b.Cleanup(func() { - if err := eventBus.Stop(); err != nil { - b.Error(err) - } - }) + b.Cleanup(eventBus.Wait) q := types.EventQueryNewBlock diff --git a/internal/libs/autofile/cmd/logjack.go b/internal/libs/autofile/cmd/logjack.go index 0f412a366..72d816fcf 100644 --- a/internal/libs/autofile/cmd/logjack.go +++ b/internal/libs/autofile/cmd/logjack.go @@ -63,14 +63,10 @@ func main() { for { n, err := os.Stdin.Read(buf) if err != nil { - if err := group.Stop(); err != nil { - fmt.Fprintf(os.Stderr, "logjack stopped with error %v\n", headPath) - os.Exit(1) - } if err == io.EOF { os.Exit(0) } else { - fmt.Println("logjack errored") + fmt.Println("logjack errored:", err.Error()) os.Exit(1) } } diff --git a/internal/libs/autofile/group.go b/internal/libs/autofile/group.go index 0e208d8e9..969d101c3 100644 --- a/internal/libs/autofile/group.go +++ b/internal/libs/autofile/group.go @@ -138,7 +138,7 @@ func GroupTotalSizeLimit(limit int64) func(*Group) { // and group limits. func (g *Group) OnStart(ctx context.Context) error { g.ticker = time.NewTicker(g.groupCheckDuration) - go g.processTicks() + go g.processTicks(ctx) return nil } @@ -237,15 +237,16 @@ func (g *Group) FlushAndSync() error { return err } -func (g *Group) processTicks() { +func (g *Group) processTicks(ctx context.Context) { defer close(g.doneProcessTicks) + for { select { + case <-ctx.Done(): + return case <-g.ticker.C: g.checkHeadSizeLimit() g.checkTotalSizeLimit() - case <-g.Quit(): - return } } } diff --git a/internal/mempool/mempool_test.go b/internal/mempool/mempool_test.go index f06ee18d9..1613dce98 100644 --- a/internal/mempool/mempool_test.go +++ b/internal/mempool/mempool_test.go @@ -98,7 +98,9 @@ func setup(ctx context.Context, t testing.TB, cacheSize int, options ...TxMempoo return NewTxMempool(logger.With("test", t.Name()), cfg.Mempool, appConnMem, 0, options...) } -func checkTxs(t *testing.T, txmp *TxMempool, numTxs int, peerID uint16) []testTx { +func checkTxs(ctx context.Context, t *testing.T, txmp *TxMempool, numTxs int, peerID uint16) []testTx { + t.Helper() + txs := make([]testTx, numTxs) txInfo := TxInfo{SenderID: peerID} @@ -115,7 +117,7 @@ func checkTxs(t *testing.T, txmp *TxMempool, numTxs int, peerID uint16) []testTx tx: []byte(fmt.Sprintf("sender-%d-%d=%X=%d", i, peerID, prefix, priority)), priority: priority, } - require.NoError(t, txmp.CheckTx(context.Background(), txs[i].tx, nil, txInfo)) + require.NoError(t, txmp.CheckTx(ctx, txs[i].tx, nil, txInfo)) } return txs @@ -161,7 +163,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { // Execute CheckTx for some transactions and ensure TxsAvailable only fires // once. - txs := checkTxs(t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100, 0) ensureTxFire() ensureNoTxFire() @@ -184,7 +186,7 @@ func TestTxMempool_TxsAvailable(t *testing.T) { // Execute CheckTx for more transactions and ensure we do not fire another // event as we're still on the same height (1). - _ = checkTxs(t, txmp, 100, 0) + _ = checkTxs(ctx, t, txmp, 100, 0) ensureNoTxFire() } @@ -193,7 +195,7 @@ func TestTxMempool_Size(t *testing.T) { defer cancel() txmp := setup(ctx, t, 0) - txs := checkTxs(t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100, 0) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -220,7 +222,7 @@ func TestTxMempool_Flush(t *testing.T) { defer cancel() txmp := setup(ctx, t, 0) - txs := checkTxs(t, txmp, 100, 0) + txs := checkTxs(ctx, t, txmp, 100, 0) require.Equal(t, len(txs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -248,7 +250,7 @@ func TestTxMempool_ReapMaxBytesMaxGas(t *testing.T) { defer cancel() txmp := setup(ctx, t, 0) - tTxs := checkTxs(t, txmp, 100, 0) // all txs request 1 gas unit + tTxs := checkTxs(ctx, t, txmp, 100, 0) // all txs request 1 gas unit require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -301,7 +303,7 @@ func TestTxMempool_ReapMaxTxs(t *testing.T) { defer cancel() txmp := setup(ctx, t, 0) - tTxs := checkTxs(t, txmp, 100, 0) + tTxs := checkTxs(ctx, t, txmp, 100, 0) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, int64(5690), txmp.SizeBytes()) @@ -424,7 +426,7 @@ func TestTxMempool_ConcurrentTxs(t *testing.T) { wg.Add(1) go func() { for i := 0; i < 20; i++ { - _ = checkTxs(t, txmp, 100, 0) + _ = checkTxs(ctx, t, txmp, 100, 0) dur := rng.Intn(1000-500) + 500 time.Sleep(time.Duration(dur) * time.Millisecond) } @@ -486,7 +488,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { txmp.height = 100 txmp.config.TTLNumBlocks = 10 - tTxs := checkTxs(t, txmp, 100, 0) + tTxs := checkTxs(ctx, t, txmp, 100, 0) require.Equal(t, len(tTxs), txmp.Size()) require.Equal(t, 100, txmp.heightIndex.Size()) @@ -505,7 +507,7 @@ func TestTxMempool_ExpiredTxs_NumBlocks(t *testing.T) { require.Equal(t, 95, txmp.heightIndex.Size()) // check more txs at height 101 - _ = checkTxs(t, txmp, 50, 1) + _ = checkTxs(ctx, t, txmp, 50, 1) require.Equal(t, 145, txmp.Size()) require.Equal(t, 145, txmp.heightIndex.Size()) diff --git a/internal/mempool/reactor_test.go b/internal/mempool/reactor_test.go index 4456424b5..62cacfd10 100644 --- a/internal/mempool/reactor_test.go +++ b/internal/mempool/reactor_test.go @@ -203,7 +203,7 @@ func TestReactorBroadcastTxs(t *testing.T) { primary := rts.nodes[0] secondaries := rts.nodes[1:] - txs := checkTxs(t, rts.reactors[primary].mempool, numTxs, UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, UnknownPeerID) // run the router rts.start(t) @@ -238,7 +238,7 @@ func TestReactorConcurrency(t *testing.T) { // 1. submit a bunch of txs // 2. update the whole mempool - txs := checkTxs(t, rts.reactors[primary].mempool, numTxs, UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, numTxs, UnknownPeerID) go func() { defer wg.Done() @@ -257,7 +257,7 @@ func TestReactorConcurrency(t *testing.T) { // 1. submit a bunch of txs // 2. update none - _ = checkTxs(t, rts.reactors[secondary].mempool, numTxs, UnknownPeerID) + _ = checkTxs(ctx, t, rts.reactors[secondary].mempool, numTxs, UnknownPeerID) go func() { defer wg.Done() @@ -290,7 +290,7 @@ func TestReactorNoBroadcastToSender(t *testing.T) { secondary := rts.nodes[1] peerID := uint16(1) - _ = checkTxs(t, rts.mempools[primary], numTxs, peerID) + _ = checkTxs(ctx, t, rts.mempools[primary], numTxs, peerID) rts.start(t) @@ -430,7 +430,7 @@ func TestBroadcastTxForPeerStopsWhenPeerStops(t *testing.T) { } time.Sleep(500 * time.Millisecond) - txs := checkTxs(t, rts.reactors[primary].mempool, 4, UnknownPeerID) + txs := checkTxs(ctx, t, rts.reactors[primary].mempool, 4, UnknownPeerID) require.Equal(t, 4, len(txs)) require.Equal(t, 4, rts.mempools[primary].Size()) require.Equal(t, 0, rts.mempools[secondary].Size()) diff --git a/internal/p2p/conn/connection.go b/internal/p2p/conn/connection.go index 9fb330286..94f248a8c 100644 --- a/internal/p2p/conn/connection.go +++ b/internal/p2p/conn/connection.go @@ -101,6 +101,8 @@ type MConnection struct { // are safe to call concurrently. stopMtx tmsync.Mutex + cancel context.CancelFunc + flushTimer *timer.ThrottleTimer // flush writes as necessary but throttled. pingTimer *time.Ticker // send pings periodically @@ -187,6 +189,7 @@ func NewMConnectionWithConfig( onError: onError, config: config, created: time.Now(), + cancel: func() {}, } mconn.BaseService = *service.NewBaseService(logger, "MConnection", mconn) @@ -211,9 +214,6 @@ func NewMConnectionWithConfig( // OnStart implements BaseService func (c *MConnection) OnStart(ctx context.Context) error { - if err := c.BaseService.OnStart(ctx); err != nil { - return err - } c.flushTimer = timer.NewThrottleTimer("flush", c.config.FlushThrottle) c.pingTimer = time.NewTicker(c.config.PingInterval) c.pongTimeoutCh = make(chan bool, 1) @@ -247,7 +247,6 @@ func (c *MConnection) stopServices() (alreadyStopped bool) { default: } - c.BaseService.OnStop() c.flushTimer.Stop() c.pingTimer.Stop() c.chStatsTimer.Stop() @@ -296,6 +295,7 @@ func (c *MConnection) stopForError(r interface{}) { if err := c.Stop(); err != nil { c.Logger.Error("Error stopping connection", "err", err) } + if atomic.CompareAndSwapUint32(&c.errored, 0, 1) { if c.onError != nil { c.onError(r) diff --git a/internal/p2p/conn/connection_test.go b/internal/p2p/conn/connection_test.go index f8b34bad6..f1b2ae24c 100644 --- a/internal/p2p/conn/connection_test.go +++ b/internal/p2p/conn/connection_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/hex" "net" + "sync" "testing" "time" @@ -14,6 +15,7 @@ import ( "github.com/tendermint/tendermint/internal/libs/protoio" "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/service" tmp2p "github.com/tendermint/tendermint/proto/tendermint/p2p" "github.com/tendermint/tendermint/proto/tendermint/types" ) @@ -54,7 +56,7 @@ func TestMConnectionSendFlushStop(t *testing.T) { clientConn := createTestMConnection(log.TestingLogger(), client) err := clientConn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, clientConn)) + t.Cleanup(waitAll(clientConn)) msg := []byte("abc") assert.True(t, clientConn.Send(0x01, msg)) @@ -91,7 +93,7 @@ func TestMConnectionSend(t *testing.T) { mconn := createTestMConnection(log.TestingLogger(), client) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) msg := []byte("Ant-Man") assert.True(t, mconn.Send(0x01, msg)) @@ -132,12 +134,12 @@ func TestMConnectionReceive(t *testing.T) { mconn1 := createMConnectionWithCallbacks(logger, client, onReceive, onError) err := mconn1.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn1)) + t.Cleanup(waitAll(mconn1)) mconn2 := createTestMConnection(logger, server) err = mconn2.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn2)) + t.Cleanup(waitAll(mconn2)) msg := []byte("Cyclops") assert.True(t, mconn2.Send(0x01, msg)) @@ -171,7 +173,7 @@ func TestMConnectionPongTimeoutResultsInError(t *testing.T) { mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) serverGotPing := make(chan struct{}) go func() { @@ -212,7 +214,7 @@ func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) { mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) // sending 3 pongs in a row (abuse) protoWriter := protoio.NewDelimitedWriter(server) @@ -269,7 +271,7 @@ func TestMConnectionMultiplePings(t *testing.T) { mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) // sending 3 pings in a row (abuse) // see https://github.com/tendermint/tendermint/issues/1190 @@ -320,7 +322,7 @@ func TestMConnectionPingPongs(t *testing.T) { mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) serverGotPing := make(chan struct{}) go func() { @@ -380,7 +382,7 @@ func TestMConnectionStopsAndReturnsError(t *testing.T) { mconn := createMConnectionWithCallbacks(log.TestingLogger(), client, onReceive, onError) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) if err := client.Close(); err != nil { t.Error(err) @@ -454,7 +456,7 @@ func TestMConnectionReadErrorBadEncoding(t *testing.T) { _, err := client.Write([]byte{1, 2, 3, 4, 5}) require.NoError(t, err) assert.True(t, expectSend(chOnErr), "badly encoded msgPacket") - t.Cleanup(stopAll(t, mconnClient, mconnServer)) + t.Cleanup(waitAll(mconnClient, mconnServer)) } func TestMConnectionReadErrorUnknownChannel(t *testing.T) { @@ -473,7 +475,7 @@ func TestMConnectionReadErrorUnknownChannel(t *testing.T) { // should cause an error assert.True(t, mconnClient.Send(0x02, msg)) assert.True(t, expectSend(chOnErr), "unknown channel") - t.Cleanup(stopAll(t, mconnClient, mconnServer)) + t.Cleanup(waitAll(mconnClient, mconnServer)) } func TestMConnectionReadErrorLongMessage(t *testing.T) { @@ -484,7 +486,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { defer cancel() mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr) - t.Cleanup(stopAll(t, mconnClient, mconnServer)) + t.Cleanup(waitAll(mconnClient, mconnServer)) mconnServer.onReceive = func(chID ChannelID, msgBytes []byte) { chOnRcv <- struct{}{} @@ -522,7 +524,7 @@ func TestMConnectionReadErrorUnknownMsgType(t *testing.T) { chOnErr := make(chan struct{}) mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr) - t.Cleanup(stopAll(t, mconnClient, mconnServer)) + t.Cleanup(waitAll(mconnClient, mconnServer)) // send msg with unknown msg type _, err := protoio.NewDelimitedWriter(mconnClient.conn).WriteMsg(&types.Header{ChainID: "x"}) @@ -539,7 +541,7 @@ func TestMConnectionTrySend(t *testing.T) { mconn := createTestMConnection(log.TestingLogger(), client) err := mconn.Start(ctx) require.Nil(t, err) - t.Cleanup(stopAll(t, mconn)) + t.Cleanup(waitAll(mconn)) msg := []byte("Semicolon-Woman") resultCh := make(chan string, 2) @@ -586,7 +588,7 @@ func TestMConnectionChannelOverflow(t *testing.T) { defer cancel() mconnClient, mconnServer := newClientAndServerConnsForReadErrors(ctx, t, chOnErr) - t.Cleanup(stopAll(t, mconnClient, mconnServer)) + t.Cleanup(waitAll(mconnClient, mconnServer)) mconnServer.onReceive = func(chID ChannelID, msgBytes []byte) { chOnRcv <- struct{}{} @@ -611,16 +613,26 @@ func TestMConnectionChannelOverflow(t *testing.T) { } -type stopper interface { - Stop() error -} - -func stopAll(t *testing.T, stoppers ...stopper) func() { +func waitAll(waiters ...service.Service) func() { return func() { - for _, s := range stoppers { - if err := s.Stop(); err != nil { - t.Log(err) + switch len(waiters) { + case 0: + return + case 1: + waiters[0].Wait() + return + default: + wg := &sync.WaitGroup{} + + for _, w := range waiters { + wg.Add(1) + go func(s service.Service) { + defer wg.Done() + s.Wait() + }(w) } + + wg.Wait() } } } diff --git a/libs/service/service.go b/libs/service/service.go index f2b440e94..ad88ccf16 100644 --- a/libs/service/service.go +++ b/libs/service/service.go @@ -136,17 +136,31 @@ func (bs *BaseService) Start(ctx context.Context) error { } go func(ctx context.Context) { - <-ctx.Done() - if err := bs.Stop(); err != nil { - bs.Logger.Error("stopped service", - "err", err.Error(), + select { + case <-bs.quit: + // someone else explicitly called stop + // and then we shouldn't. + return + case <-ctx.Done(): + // if nothing is running, no need to + // shut down again. + if !bs.impl.IsRunning() { + return + } + + // the context was cancel and we + // should stop. + if err := bs.Stop(); err != nil { + bs.Logger.Error("stopped service", + "err", err.Error(), + "service", bs.name, + "impl", bs.impl.String()) + } + + bs.Logger.Info("stopped service", "service", bs.name, "impl", bs.impl.String()) } - - bs.Logger.Info("stopped service", - "service", bs.name, - "impl", bs.impl.String()) }(ctx) return nil @@ -156,11 +170,6 @@ func (bs *BaseService) Start(ctx context.Context) error { return ErrAlreadyStarted } -// OnStart implements Service by doing nothing. -// NOTE: Do not put anything in here, -// that way users don't need to call BaseService.OnStart() -func (bs *BaseService) OnStart(ctx context.Context) error { return nil } - // Stop implements Service by calling OnStop (if defined) and closing quit // channel. An error will be returned if the service is already stopped. func (bs *BaseService) Stop() error { @@ -182,11 +191,6 @@ func (bs *BaseService) Stop() error { return ErrAlreadyStopped } -// OnStop implements Service by doing nothing. -// NOTE: Do not put anything in here, -// that way users don't need to call BaseService.OnStop() -func (bs *BaseService) OnStop() {} - // IsRunning implements Service by returning true or false depending on the // service's state. func (bs *BaseService) IsRunning() bool { @@ -198,6 +202,3 @@ func (bs *BaseService) Wait() { <-bs.quit } // String implements Service by returning a string representation of the service. func (bs *BaseService) String() string { return bs.name } - -// Quit Implements Service by returning a quit channel. -func (bs *BaseService) Quit() <-chan struct{} { return bs.quit } diff --git a/libs/service/service_test.go b/libs/service/service_test.go index dc5d0ccb1..9630d358b 100644 --- a/libs/service/service_test.go +++ b/libs/service/service_test.go @@ -12,7 +12,8 @@ type testService struct { BaseService } -func (testService) OnReset() error { +func (testService) OnStop() {} +func (testService) OnStart(context.Context) error { return nil } @@ -31,7 +32,7 @@ func TestBaseServiceWait(t *testing.T) { waitFinished <- struct{}{} }() - go ts.Stop() //nolint:errcheck // ignore for tests + go cancel() select { case <-waitFinished: diff --git a/light/rpc/client.go b/light/rpc/client.go index 51676dd3e..a5317ca0b 100644 --- a/light/rpc/client.go +++ b/light/rpc/client.go @@ -47,6 +47,8 @@ type Client struct { // proof runtime used to verify values returned by ABCIQuery prt *merkle.ProofRuntime keyPathFn KeyPathFunc + + quitCh chan struct{} } var _ rpcclient.Client = (*Client)(nil) @@ -87,9 +89,10 @@ func DefaultMerkleKeyPathFn() KeyPathFunc { // NewClient returns a new client. func NewClient(next rpcclient.Client, lc LightClient, opts ...Option) *Client { c := &Client{ - next: next, - lc: lc, - prt: merkle.DefaultProofRuntime(), + next: next, + lc: lc, + prt: merkle.DefaultProofRuntime(), + quitCh: make(chan struct{}), } c.BaseService = *service.NewBaseService(nil, "Client", c) for _, o := range opts { @@ -102,6 +105,12 @@ func (c *Client) OnStart(ctx context.Context) error { if !c.next.IsRunning() { return c.next.Start(ctx) } + + go func() { + defer close(c.quitCh) + c.Wait() + }() + return nil } @@ -586,7 +595,7 @@ func (c *Client) SubscribeWS(ctx *rpctypes.Context, query string) (*coretypes.Re rpctypes.JSONRPCStringID(fmt.Sprintf("%v#event", ctx.JSONReq.ID)), resultEvent, )) - case <-c.Quit(): + case <-c.quitCh: return } } diff --git a/node/node_test.go b/node/node_test.go index 3e4e83492..16f8c44aa 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -48,45 +48,34 @@ func TestNodeStartStop(t *testing.T) { // create & start node ns, err := newDefaultNode(ctx, cfg, log.TestingLogger()) require.NoError(t, err) - require.NoError(t, ns.Start(ctx)) - - t.Cleanup(func() { - if ns.IsRunning() { - bcancel() - ns.Wait() - } - }) n, ok := ns.(*nodeImpl) require.True(t, ok) + t.Cleanup(func() { + if n.IsRunning() { + bcancel() + n.Wait() + } + }) + require.NoError(t, n.Start(ctx)) // wait for the node to produce a block - blocksSub, err := n.EventBus().SubscribeWithArgs(ctx, pubsub.SubscribeArgs{ + tctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + blocksSub, err := n.EventBus().SubscribeWithArgs(tctx, pubsub.SubscribeArgs{ ClientID: "node_test", Query: types.EventQueryNewBlock, }) require.NoError(t, err) - tctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if _, err := blocksSub.Next(tctx); err != nil { - t.Fatalf("Waiting for event: %v", err) - } + _, err = blocksSub.Next(tctx) + require.NoError(t, err, "waiting for event") - // stop the node - go func() { - bcancel() - n.Wait() - }() + cancel() // stop the subscription context + bcancel() // stop the base context + n.Wait() - select { - case <-n.Quit(): - return - case <-time.After(10 * time.Second): - if n.IsRunning() { - t.Fatal("timed out waiting for shutdown") - } - - } + require.False(t, n.IsRunning(), "node must shut down") } func getTestNode(ctx context.Context, t *testing.T, conf *config.Config, logger log.Logger) *nodeImpl { diff --git a/privval/signer_client_test.go b/privval/signer_client_test.go index f9272b004..7ff353dcd 100644 --- a/privval/signer_client_test.go +++ b/privval/signer_client_test.go @@ -60,10 +60,10 @@ func getSignerTestCases(ctx context.Context, t *testing.T) []signerTestCase { } func TestSignerClose(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + bctx, bcancel := context.WithCancel(context.Background()) + defer bcancel() - for _, tc := range getSignerTestCases(ctx, t) { + for _, tc := range getSignerTestCases(bctx, t) { t.Run(tc.name, func(t *testing.T) { defer tc.closer() diff --git a/privval/signer_dialer_endpoint.go b/privval/signer_dialer_endpoint.go index 93d26b043..cc605617f 100644 --- a/privval/signer_dialer_endpoint.go +++ b/privval/signer_dialer_endpoint.go @@ -1,6 +1,7 @@ package privval import ( + "context" "time" "github.com/tendermint/tendermint/libs/log" @@ -69,6 +70,9 @@ func NewSignerDialerEndpoint( return sd } +func (sd *SignerDialerEndpoint) OnStart(context.Context) error { return nil } +func (sd *SignerDialerEndpoint) OnStop() {} + func (sd *SignerDialerEndpoint) ensureConnection() error { if sd.IsConnected() { return nil diff --git a/privval/signer_listener_endpoint.go b/privval/signer_listener_endpoint.go index e2287c630..a825f635d 100644 --- a/privval/signer_listener_endpoint.go +++ b/privval/signer_listener_endpoint.go @@ -72,8 +72,8 @@ func (sl *SignerListenerEndpoint) OnStart(ctx context.Context) error { sl.pingInterval = time.Duration(sl.signerEndpoint.timeoutReadWrite.Milliseconds()*2/3) * time.Millisecond sl.pingTimer = time.NewTicker(sl.pingInterval) - go sl.serviceLoop() - go sl.pingLoop() + go sl.serviceLoop(ctx) + go sl.pingLoop(ctx) sl.connectRequestCh <- struct{}{} @@ -173,7 +173,7 @@ func (sl *SignerListenerEndpoint) triggerReconnect() { sl.triggerConnect() } -func (sl *SignerListenerEndpoint) serviceLoop() { +func (sl *SignerListenerEndpoint) serviceLoop(ctx context.Context) { for { select { case <-sl.connectRequestCh: @@ -185,7 +185,7 @@ func (sl *SignerListenerEndpoint) serviceLoop() { // We have a good connection, wait for someone that needs one otherwise cancellation select { case sl.connectionAvailableCh <- conn: - case <-sl.Quit(): + case <-ctx.Done(): return } } @@ -195,13 +195,13 @@ func (sl *SignerListenerEndpoint) serviceLoop() { default: } } - case <-sl.Quit(): + case <-ctx.Done(): return } } } -func (sl *SignerListenerEndpoint) pingLoop() { +func (sl *SignerListenerEndpoint) pingLoop(ctx context.Context) { for { select { case <-sl.pingTimer.C: @@ -212,7 +212,7 @@ func (sl *SignerListenerEndpoint) pingLoop() { sl.triggerReconnect() } } - case <-sl.Quit(): + case <-ctx.Done(): return } } diff --git a/privval/signer_listener_endpoint_test.go b/privval/signer_listener_endpoint_test.go index b92e0abe5..969590bc8 100644 --- a/privval/signer_listener_endpoint_test.go +++ b/privval/signer_listener_endpoint_test.go @@ -77,11 +77,7 @@ func TestSignerRemoteRetryTCPOnly(t *testing.T) { err = signerServer.Start(ctx) require.NoError(t, err) - t.Cleanup(func() { - if err := signerServer.Stop(); err != nil { - t.Error(err) - } - }) + t.Cleanup(signerServer.Wait) select { case attempts := <-attemptCh: diff --git a/privval/signer_server.go b/privval/signer_server.go index e31d3bdb4..dd924f0eb 100644 --- a/privval/signer_server.go +++ b/privval/signer_server.go @@ -94,8 +94,6 @@ func (ss *SignerServer) servicePendingRequest() { func (ss *SignerServer) serviceLoop(ctx context.Context) { for { select { - case <-ss.Quit(): - return case <-ctx.Done(): return default: diff --git a/rpc/client/local/local.go b/rpc/client/local/local.go index 10b2b4be7..9f5ba0072 100644 --- a/rpc/client/local/local.go +++ b/rpc/client/local/local.go @@ -220,7 +220,7 @@ func (c *Local) Subscribe( } ctx, cancel := context.WithCancel(ctx) - go func() { <-c.Quit(); cancel() }() + go func() { c.Wait(); cancel() }() subArgs := pubsub.SubscribeArgs{ ClientID: subscriber,