diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 15edf23fa..2473c5ded 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -55,7 +55,9 @@ updates: schedule: interval: weekly target-branch: "v0.37.x" - open-pull-requests-limit: 10 + # Only allow automated security-related dependency updates until we cut the + # final v0.37.0 release. + open-pull-requests-limit: 0 labels: - T:dependencies - S:automerge diff --git a/.github/workflows/e2e-nightly-34x.yml b/.github/workflows/e2e-nightly-34x.yml index 124a401f3..a01e769d2 100644 --- a/.github/workflows/e2e-nightly-34x.yml +++ b/.github/workflows/e2e-nightly-34x.yml @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -77,28 +77,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } diff --git a/.github/workflows/e2e-nightly-37x.yml b/.github/workflows/e2e-nightly-37x.yml index d11eea67d..c3f6b16aa 100644 --- a/.github/workflows/e2e-nightly-37x.yml +++ b/.github/workflows/e2e-nightly-37x.yml @@ -57,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -77,28 +77,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ needs.e2e-nightly-test.outputs.git-branch }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } diff --git a/.github/workflows/e2e-nightly-main.yml b/.github/workflows/e2e-nightly-main.yml index cda20df13..2bb00dc47 100644 --- a/.github/workflows/e2e-nightly-main.yml +++ b/.github/workflows/e2e-nightly-main.yml @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK @@ -66,28 +66,3 @@ jobs: } ] } - - e2e-nightly-success: # may turn this off once they seem to pass consistently - needs: e2e-nightly-test - if: ${{ success() }} - runs-on: ubuntu-latest - steps: - - name: Notify Slack on success - uses: slackapi/slack-github-action@v1.21.0 - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK - BRANCH: ${{ github.ref_name }} - with: - payload: | - { - "blocks": [ - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": ":white_check_mark: Nightly E2E tests for `${{ env.BRANCH }}` passed." - } - } - ] - } diff --git a/.github/workflows/fuzz-nightly.yml b/.github/workflows/fuzz-nightly.yml index 1673e99db..b7ac5168c 100644 --- a/.github/workflows/fuzz-nightly.yml +++ b/.github/workflows/fuzz-nightly.yml @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Notify Slack on failure - uses: slackapi/slack-github-action@v1.21.0 + uses: slackapi/slack-github-action@v1.22.0 env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/.github/workflows/gosec.yml b/.github/workflows/gosec.yml new file mode 100644 index 000000000..016234b60 --- /dev/null +++ b/.github/workflows/gosec.yml @@ -0,0 +1,41 @@ +name: Run Gosec +on: + pull_request: + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + push: + branches: + - main + - 'feature/*' + - 'v0.37.x' + - 'v0.34.x' + paths: + - '**/*.go' + - 'go.mod' + - 'go.sum' + +jobs: + Gosec: + permissions: + security-events: write + + runs-on: ubuntu-latest + env: + GO111MODULE: on + steps: + - name: Checkout Source + uses: actions/checkout@v3 + + - name: Run Gosec Security Scanner + uses: cosmos/gosec@master + with: + # Let the report trigger a failure with the Github Security scanner features. + args: "-no-fail -fmt sarif -out results.sarif ./..." + + - name: Upload SARIF file + uses: github/codeql-action/upload-sarif@v2 + with: + # Path to SARIF file relative to the root of the repository + sarif_file: results.sarif diff --git a/.github/workflows/janitor.yml b/.github/workflows/janitor.yml index ceb21941d..28ae05b51 100644 --- a/.github/workflows/janitor.yml +++ b/.github/workflows/janitor.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 3 steps: - - uses: styfle/cancel-workflow-action@0.10.0 + - uses: styfle/cancel-workflow-action@0.10.1 with: workflow_id: 1041851,1401230,2837803 access_token: ${{ github.token }} diff --git a/.github/workflows/proto-lint.yml b/.github/workflows/proto-lint.yml index 1ef085606..474a71ff1 100644 --- a/.github/workflows/proto-lint.yml +++ b/.github/workflows/proto-lint.yml @@ -15,7 +15,7 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v3 - - uses: bufbuild/buf-setup-action@v1.7.0 + - uses: bufbuild/buf-setup-action@v1.8.0 - uses: bufbuild/buf-lint-action@v1 with: input: 'proto' diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4089abfbc..51229cfb7 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v5 + - uses: actions/stale@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: "This pull request has been automatically marked as stale because it has not had diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index 2a593c458..ea745d4e1 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,5 +1,37 @@ # Unreleased Changes +## v0.38.0 + +### BREAKING CHANGES + +- CLI/RPC/Config + +- Apps + +- P2P Protocol + +- Go API + +- Blockchain Protocol + +- Data Storage + - [state] \#6541 Move pruneBlocks from consensus/state to state/execution. (@JayT106) + +- Tooling + - [tools/tm-signer-harness] \#6498 Set OS home dir to instead of the hardcoded PATH. (@JayT106) + +### FEATURES + +### IMPROVEMENTS + +- [pubsub] \#7319 Performance improvements for the event query API (@creachadair) +- [p2p/pex] \#6509 Improve addrBook.hash performance (@cuonglm) +- [crypto/merkle] \#6443 & \#6513 Improve HashAlternatives performance (@cuonglm, @marbar3778) + +### BUG FIXES + +- [docker] \#9462 ensure Docker image uses consistent version of Go + ## v0.37.0 Special thanks to external contributors on this release: @@ -33,6 +65,7 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi - Go API - [all] \#9144 Change spelling from British English to American (@cmwaters) - Rename "Subscription.Cancelled()" to "Subscription.Canceled()" in libs/pubsub + - [crypto/sr25519] \#6526 Do not re-execute the Ed25519-style key derivation step when doing signing and verification. The derivation is now done once and only once. This breaks `sr25519.GenPrivKeyFromSecret` output compatibility. (@Yawning) - Blockchain Protocol @@ -43,11 +76,24 @@ Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermi ### IMPROVEMENTS - [crypto] \#9250 Update to use btcec v2 and the latest btcutil. (@wcsiu) +- [cli] \#9171 add `--hard` flag to rollback command (and a boolean to the `RollbackState` method). This will rollback + state and remove the last block. This command can be triggered multiple times. The application must also rollback + state to the same height. (@tsutsu, @cmwaters) - [proto] \#9356 Migrate from `gogo/protobuf` to `cosmos/gogoproto` (@julienrbrt) - [rpc] \#9276 Added `header` and `header_by_hash` queries to the RPC client (@samricotta) - [abci] \#5706 Added `AbciVersion` to `RequestInfo` allowing applications to check ABCI version when connecting to Tendermint. (@marbar3778) +- [node] \#6059 Validate and complete genesis doc before saving to state store (@silasdavis) + +- [crypto/ed25519] \#5632 Adopt zip215 `ed25519` verification. (@marbar3778) +- [crypto/ed25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `ed25519` signing and verification. (@Yawning) +- [crypto/sr25519] \#6526 Use [curve25519-voi](https://github.com/oasisprotocol/curve25519-voi) for `sr25519` signing and verification. (@Yawning) +- [crypto] \#6120 Implement batch verification interface for ed25519 and sr25519. (@marbar3778 & @Yawning) +- [types] \#6120 use batch verification for verifying commits signatures. (@marbar3778 & @cmwaters & @Yawning) + - If the key type supports the batch verification API it will try to batch verify. If the verification fails we will single verify each signature. +- [state] \#9505 Added logic so when pruning, the evidence period is taken into consideration and only deletes unecessary data (@samricotta) ### BUG FIXES - [consensus] \#9229 fix round number of `enterPropose` when handling `RoundStepNewRound` timeout. (@fatcat22) - [docker] \#9073 enable cross platform build using docker buildx +- [blocksync] \#9518 handle the case when the sending queue is full: retry block request after a timeout \ No newline at end of file diff --git a/DOCKER/Dockerfile b/DOCKER/Dockerfile index 77d2ad991..df4e3f49a 100644 --- a/DOCKER/Dockerfile +++ b/DOCKER/Dockerfile @@ -1,5 +1,9 @@ +# Use a build arg to ensure that both stages use the same, +# hopefully current, go version. +ARG GOLANG_BASE_IMAGE=golang:1.18-alpine + # stage 1 Generate Tendermint Binary -FROM --platform=$BUILDPLATFORM golang:1.18-alpine as builder +FROM --platform=$BUILDPLATFORM $GOLANG_BASE_IMAGE as builder RUN apk update && \ apk upgrade && \ apk --no-cache add make @@ -8,7 +12,7 @@ WORKDIR /tendermint RUN TARGETPLATFORM=$TARGETPLATFORM make build-linux # stage 2 -FROM golang:1.15-alpine +FROM $GOLANG_BASE_IMAGE LABEL maintainer="hello@tendermint.com" # Tendermint will be looking for the genesis file in /tendermint/config/genesis.json diff --git a/Makefile b/Makefile index 11dfffb51..7a4ef6f9e 100644 --- a/Makefile +++ b/Makefile @@ -4,14 +4,8 @@ OUTPUT?=$(BUILDDIR)/tendermint BUILD_TAGS?=tendermint -# If building a release, please checkout the version tag to get the correct version setting -ifneq ($(shell git symbolic-ref -q --short HEAD),) -VERSION := unreleased-$(shell git symbolic-ref -q --short HEAD)-$(shell git rev-parse HEAD) -else -VERSION := $(shell git describe) -endif - -LD_FLAGS = -X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION) +COMMIT_HASH := $(shell git rev-parse --short HEAD) +LD_FLAGS = -X github.com/tendermint/tendermint/version.TMGitCommitHash=$(COMMIT_HASH) BUILD_FLAGS = -mod=readonly -ldflags "$(LD_FLAGS)" HTTPS_GIT := https://github.com/tendermint/tendermint.git CGO_ENABLED ?= 0 @@ -406,4 +400,4 @@ $(BUILDDIR)/packages.txt:$(GO_TEST_FILES) $(BUILDDIR) split-test-packages:$(BUILDDIR)/packages.txt split -d -n l/$(NUM_SPLIT) $< $<. test-group-%:split-test-packages - cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=5m -race -coverprofile=$(BUILDDIR)/$*.profile.out + cat $(BUILDDIR)/packages.txt.$* | xargs go test -mod=readonly -timeout=15m -race -coverprofile=$(BUILDDIR)/$*.profile.out diff --git a/README.md b/README.md index 6c255cc7b..732e53971 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,7 @@ for-profit entity that also maintains [tendermint.com](https://tendermint.com). [bft]: https://en.wikipedia.org/wiki/Byzantine_fault_tolerance [smr]: https://en.wikipedia.org/wiki/State_machine_replication [Blockchain]: https://en.wikipedia.org/wiki/Blockchain -[version-badge]: https://img.shields.io/github/tag/tendermint/tendermint.svg +[version-badge]: https://img.shields.io/github/v/release/tendermint/tendermint.svg [version-url]: https://github.com/tendermint/tendermint/releases/latest [api-badge]: https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667 [api-url]: https://pkg.go.dev/github.com/tendermint/tendermint diff --git a/RELEASES.md b/RELEASES.md index d17ac467c..3f8a9c721 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -93,22 +93,14 @@ the 0.38.x line. After doing these steps, go back to `main` and do the following: -1. Tag `main` as the dev branch for the _next_ minor version release and push - it up to GitHub. - For example: - ```sh - git tag -a v0.39.0-dev -m "Development base for Tendermint v0.39." - git push origin v0.39.0-dev - ``` - -2. Create a new workflow to run e2e nightlies for the new backport branch. (See +1. Create a new workflow to run e2e nightlies for the new backport branch. (See [e2e-nightly-main.yml][e2e] for an example.) -3. Add a new section to the Mergify config (`.github/mergify.yml`) to enable the +2. Add a new section to the Mergify config (`.github/mergify.yml`) to enable the backport bot to work on this branch, and add a corresponding `S:backport-to-v0.38.x` [label](https://github.com/tendermint/tendermint/labels) so the bot can be triggered. -4. Add a new section to the Dependabot config (`.github/dependabot.yml`) to +3. Add a new section to the Dependabot config (`.github/dependabot.yml`) to enable automatic update of Go dependencies on this branch. Copy and edit one of the existing branch configurations to set the correct `target-branch`. diff --git a/abci/version/version.go b/abci/version/version.go index f4dc4d235..2314c2852 100644 --- a/abci/version/version.go +++ b/abci/version/version.go @@ -6,4 +6,4 @@ import ( // TODO: eliminate this after some version refactor -const Version = version.ABCIVersion +const Version = version.ABCISemVer diff --git a/blocksync/msgs_test.go b/blocksync/msgs_test.go index 46983a2a1..d9d1d1066 100644 --- a/blocksync/msgs_test.go +++ b/blocksync/msgs_test.go @@ -80,7 +80,7 @@ func TestBcStatusResponseMessageValidateBasic(t *testing.T) { } //nolint:lll // ignore line length in tests -func TestBlockchainMessageVectors(t *testing.T) { +func TestBlocksyncMessageVectors(t *testing.T) { block := types.MakeBlock(int64(3), []types.Tx{types.Tx("Hello World")}, nil, nil) block.Version.Block = 11 // overwrite updated protocol version diff --git a/blocksync/pool.go b/blocksync/pool.go index 1a89cbe7d..57ba94ce8 100644 --- a/blocksync/pool.go +++ b/blocksync/pool.go @@ -32,6 +32,7 @@ const ( maxTotalRequesters = 600 maxPendingRequests = maxTotalRequesters maxPendingRequestsPerPeer = 20 + requestRetrySeconds = 30 // Minimum recv rate to ensure we're receiving blocks from a peer fast // enough. If a peer is not sending us data at at least that rate, we @@ -602,7 +603,7 @@ OUTER_LOOP: } peer = bpr.pool.pickIncrAvailablePeer(bpr.height) if peer == nil { - // log.Info("No peers available", "height", height) + bpr.Logger.Debug("No peers currently available; will retry shortly", "height", bpr.height) time.Sleep(requestIntervalMS * time.Millisecond) continue PICK_PEER_LOOP } @@ -612,6 +613,7 @@ OUTER_LOOP: bpr.peerID = peer.id bpr.mtx.Unlock() + to := time.NewTimer(requestRetrySeconds * time.Second) // Send request and wait. bpr.pool.sendRequest(bpr.height, peer.id) WAIT_LOOP: @@ -624,6 +626,11 @@ OUTER_LOOP: return case <-bpr.Quit(): return + case <-to.C: + bpr.Logger.Debug("Retrying block request after timeout", "height", bpr.height, "peer", bpr.peerID) + // Simulate a redo + bpr.reset() + continue OUTER_LOOP case peerID := <-bpr.redoCh: if peerID == bpr.peerID { bpr.reset() diff --git a/blocksync/reactor.go b/blocksync/reactor.go index 51e17630e..09dd2ef90 100644 --- a/blocksync/reactor.go +++ b/blocksync/reactor.go @@ -30,7 +30,7 @@ const ( ) type consensusReactor interface { - // for when we switch from blockchain reactor and block sync to + // for when we switch from blocksync reactor and block sync to // the consensus machine SwitchToConsensus(state sm.State, skipWAL bool) } @@ -406,7 +406,7 @@ FOR_LOOP: // TODO: same thing for app - but we would need a way to // get the hash without persisting the state - state, _, err = bcR.blockExec.ApplyBlock(state, firstID, first) + state, err = bcR.blockExec.ApplyBlock(state, firstID, first) if err != nil { // TODO This is bad, are we zombie? panic(fmt.Sprintf("Failed to process committed block (%d:%X): %v", first.Height, first.Hash(), err)) diff --git a/blocksync/reactor_test.go b/blocksync/reactor_test.go index 45d570182..ae04f478f 100644 --- a/blocksync/reactor_test.go +++ b/blocksync/reactor_test.go @@ -15,6 +15,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" mpmocks "github.com/tendermint/tendermint/mempool/mocks" "github.com/tendermint/tendermint/p2p" @@ -42,7 +43,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G return &types.GenesisDoc{ GenesisTime: tmtime.Now(), - ChainID: config.ChainID(), + ChainID: test.DefaultTestChainID, Validators: validators, }, privValidators } @@ -103,7 +104,7 @@ func newReactor( DiscardFinalizeBlockResponses: false, }) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, sm.EmptyEvidencePool{}) + mp, sm.EmptyEvidencePool{}, blockStore) if err = stateStore.Save(state); err != nil { panic(err) } @@ -136,7 +137,7 @@ func newReactor( require.NoError(t, err) blockID := types.BlockID{Hash: thisBlock.Hash(), PartSetHeader: thisParts.Header()} - state, _, err = blockExec.ApplyBlock(state, blockID, thisBlock) + state, err = blockExec.ApplyBlock(state, blockID, thisBlock) if err != nil { panic(fmt.Errorf("error apply block: %w", err)) } @@ -145,13 +146,13 @@ func newReactor( } bcReactor := NewReactor(state.Copy(), blockExec, blockStore, fastSync) - bcReactor.SetLogger(logger.With("module", "blockchain")) + bcReactor.SetLogger(logger.With("module", "blocksync")) return ReactorPair{bcReactor, proxyApp} } func TestNoBlockResponse(t *testing.T) { - config = cfg.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blocksync_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) @@ -163,7 +164,7 @@ func TestNoBlockResponse(t *testing.T) { reactorPairs[1] = newReactor(t, log.TestingLogger(), genDoc, privVals, 0) p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[i].reactor) return s }, p2p.Connect2Switches) @@ -213,7 +214,7 @@ func TestNoBlockResponse(t *testing.T) { // Alternatively we could actually dial a TCP conn but // that seems extreme. func TestBadBlockStopsPeer(t *testing.T) { - config = cfg.ResetTestRoot("blockchain_reactor_test") + config = test.ResetTestRoot("blocksync_reactor_test") defer os.RemoveAll(config.RootDir) genDoc, privVals := randGenesisDoc(1, false, 30) @@ -238,7 +239,7 @@ func TestBadBlockStopsPeer(t *testing.T) { reactorPairs[3] = newReactor(t, log.TestingLogger(), genDoc, privVals, 0) switches := p2p.MakeConnectedSwitches(config.P2P, 4, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[i].reactor) return s }, p2p.Connect2Switches) @@ -277,7 +278,7 @@ func TestBadBlockStopsPeer(t *testing.T) { reactorPairs = append(reactorPairs, lastReactorPair) switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch { - s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].reactor) + s.AddReactor("BLOCKSYNC", reactorPairs[len(reactorPairs)-1].reactor) return s }, p2p.Connect2Switches)...) diff --git a/cmd/tendermint/commands/reindex_event.go b/cmd/tendermint/commands/reindex_event.go index c7c664eb7..65070ef60 100644 --- a/cmd/tendermint/commands/reindex_event.go +++ b/cmd/tendermint/commands/reindex_event.go @@ -57,12 +57,18 @@ want to use this command. return } + state, err := ss.Load() + if err != nil { + fmt.Println(reindexFailed, err) + return + } + if err := checkValidHeight(bs); err != nil { fmt.Println(reindexFailed, err) return } - bi, ti, err := loadEventSinks(config) + bi, ti, err := loadEventSinks(config, state.ChainID) if err != nil { fmt.Println(reindexFailed, err) return @@ -94,7 +100,7 @@ func init() { ReIndexEventCmd.Flags().Int64Var(&endHeight, "end-height", 0, "the block height would like to finish for re-index") } -func loadEventSinks(cfg *tmcfg.Config) (indexer.BlockIndexer, txindex.TxIndexer, error) { +func loadEventSinks(cfg *tmcfg.Config, chainID string) (indexer.BlockIndexer, txindex.TxIndexer, error) { switch strings.ToLower(cfg.TxIndex.Indexer) { case "null": return nil, nil, errors.New("found null event sink, please check the tx-index section in the config.toml") @@ -103,7 +109,7 @@ func loadEventSinks(cfg *tmcfg.Config) (indexer.BlockIndexer, txindex.TxIndexer, if conn == "" { return nil, nil, errors.New("the psql connection settings cannot be empty") } - es, err := psql.NewEventSink(conn, cfg.ChainID()) + es, err := psql.NewEventSink(conn, chainID) if err != nil { return nil, nil, err } diff --git a/cmd/tendermint/commands/reindex_event_test.go b/cmd/tendermint/commands/reindex_event_test.go index 455a77855..0cd833db2 100644 --- a/cmd/tendermint/commands/reindex_event_test.go +++ b/cmd/tendermint/commands/reindex_event_test.go @@ -13,6 +13,7 @@ import ( abcitypes "github.com/tendermint/tendermint/abci/types" tmcfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" blockmocks "github.com/tendermint/tendermint/state/indexer/mocks" "github.com/tendermint/tendermint/state/mocks" txmocks "github.com/tendermint/tendermint/state/txindex/mocks" @@ -97,7 +98,7 @@ func TestLoadEventSink(t *testing.T) { cfg := tmcfg.TestConfig() cfg.TxIndex.Indexer = tc.sinks cfg.TxIndex.PsqlConn = tc.connURL - _, _, err := loadEventSinks(cfg) + _, _, err := loadEventSinks(cfg, test.DefaultTestChainID) if tc.loadErr { require.Error(t, err, idx) } else { diff --git a/cmd/tendermint/commands/rollback.go b/cmd/tendermint/commands/rollback.go index 07bdcb688..7683759ff 100644 --- a/cmd/tendermint/commands/rollback.go +++ b/cmd/tendermint/commands/rollback.go @@ -14,6 +14,12 @@ import ( "github.com/tendermint/tendermint/store" ) +var removeBlock bool = false + +func init() { + RollbackStateCmd.Flags().BoolVar(&removeBlock, "hard", false, "remove last block as well as state") +} + var RollbackStateCmd = &cobra.Command{ Use: "rollback", Short: "rollback tendermint state by one height", @@ -21,17 +27,23 @@ var RollbackStateCmd = &cobra.Command{ A state rollback is performed to recover from an incorrect application state transition, when Tendermint has persisted an incorrect app hash and is thus unable to make progress. Rollback overwrites a state at height n with the state at height n - 1. -The application should also roll back to height n - 1. No blocks are removed, so upon -restarting Tendermint the transactions in block n will be re-executed against the -application. +The application should also roll back to height n - 1. If the --hard flag is not used, +no blocks will be removed so upon restarting Tendermint the transactions in block n will be +re-executed against the application. Using --hard will also remove block n. This can +be done multiple times. `, RunE: func(cmd *cobra.Command, args []string) error { - height, hash, err := RollbackState(config) + height, hash, err := RollbackState(config, removeBlock) if err != nil { return fmt.Errorf("failed to rollback state: %w", err) } - fmt.Printf("Rolled back state to height %d and hash %v", height, hash) + if removeBlock { + fmt.Printf("Rolled back both state and block to height %d and hash %X\n", height, hash) + } else { + fmt.Printf("Rolled back state to height %d and hash %X\n", height, hash) + } + return nil }, } @@ -39,7 +51,7 @@ application. // RollbackState takes the state at the current height n and overwrites it with the state // at height n - 1. Note state here refers to tendermint state not application state. // Returns the latest state height and app hash alongside an error if there was one. -func RollbackState(config *cfg.Config) (int64, []byte, error) { +func RollbackState(config *cfg.Config, removeBlock bool) (int64, []byte, error) { // use the parsed config to load the block and state store blockStore, stateStore, err := loadStateAndBlockStore(config) if err != nil { @@ -51,7 +63,7 @@ func RollbackState(config *cfg.Config) (int64, []byte, error) { }() // rollback the last state - return state.Rollback(blockStore, stateStore) + return state.Rollback(blockStore, stateStore, removeBlock) } func loadStateAndBlockStore(config *cfg.Config) (*store.BlockStore, state.Store, error) { diff --git a/cmd/tendermint/commands/version.go b/cmd/tendermint/commands/version.go index d33a7c3a3..16fb878c9 100644 --- a/cmd/tendermint/commands/version.go +++ b/cmd/tendermint/commands/version.go @@ -14,6 +14,11 @@ var VersionCmd = &cobra.Command{ Use: "version", Short: "Show version info", Run: func(cmd *cobra.Command, args []string) { + tmVersion := version.TMCoreSemVer + if version.TMGitCommitHash != "" { + tmVersion += "+" + version.TMGitCommitHash + } + if verbose { values, _ := json.MarshalIndent(struct { Tendermint string `json:"tendermint"` @@ -21,14 +26,14 @@ var VersionCmd = &cobra.Command{ BlockProtocol uint64 `json:"block_protocol"` P2PProtocol uint64 `json:"p2p_protocol"` }{ - Tendermint: version.TMCoreSemVer, - ABCI: version.ABCIVersion, + Tendermint: tmVersion, + ABCI: version.ABCISemVer, BlockProtocol: version.BlockProtocol, P2PProtocol: version.P2PProtocol, }, "", " ") fmt.Println(string(values)) } else { - fmt.Println(version.TMCoreSemVer) + fmt.Println(tmVersion) } }, } diff --git a/config/config.go b/config/config.go index 6e164de3f..d76212f6c 100644 --- a/config/config.go +++ b/config/config.go @@ -7,7 +7,10 @@ import ( "net/http" "os" "path/filepath" + "regexp" "time" + + "github.com/tendermint/tendermint/version" ) const ( @@ -28,6 +31,19 @@ const ( // Default is v0. MempoolV0 = "v0" MempoolV1 = "v1" + + DefaultTendermintDir = ".tendermint" + DefaultConfigDir = "config" + DefaultDataDir = "data" + + DefaultConfigFileName = "config.toml" + DefaultGenesisJSONName = "genesis.json" + + DefaultPrivValKeyName = "priv_validator_key.json" + DefaultPrivValStateName = "priv_validator_state.json" + + DefaultNodeKeyName = "node_key.json" + DefaultAddrBookName = "addrbook.json" ) // NOTE: Most of the structs & relevant comments + the @@ -37,29 +53,19 @@ const ( // config/toml.go // NOTE: libs/cli must know to look in the config dir! var ( - DefaultTendermintDir = ".tendermint" - defaultConfigDir = "config" - defaultDataDir = "data" + defaultConfigFilePath = filepath.Join(DefaultConfigDir, DefaultConfigFileName) + defaultGenesisJSONPath = filepath.Join(DefaultConfigDir, DefaultGenesisJSONName) + defaultPrivValKeyPath = filepath.Join(DefaultConfigDir, DefaultPrivValKeyName) + defaultPrivValStatePath = filepath.Join(DefaultDataDir, DefaultPrivValStateName) - defaultConfigFileName = "config.toml" - defaultGenesisJSONName = "genesis.json" - - defaultPrivValKeyName = "priv_validator_key.json" - defaultPrivValStateName = "priv_validator_state.json" - - defaultNodeKeyName = "node_key.json" - defaultAddrBookName = "addrbook.json" - - defaultConfigFilePath = filepath.Join(defaultConfigDir, defaultConfigFileName) - defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName) - defaultPrivValKeyPath = filepath.Join(defaultConfigDir, defaultPrivValKeyName) - defaultPrivValStatePath = filepath.Join(defaultDataDir, defaultPrivValStateName) - - defaultNodeKeyPath = filepath.Join(defaultConfigDir, defaultNodeKeyName) - defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName) + defaultNodeKeyPath = filepath.Join(DefaultConfigDir, DefaultNodeKeyName) + defaultAddrBookPath = filepath.Join(DefaultConfigDir, DefaultAddrBookName) minSubscriptionBufferSize = 100 defaultSubscriptionBufferSize = 200 + + // taken from https://semver.org/ + semverRegexp = regexp.MustCompile(`^(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) ) // Config defines the top level configuration for a Tendermint node @@ -68,18 +74,15 @@ type Config struct { BaseConfig `mapstructure:",squash"` // Options for services - RPC *RPCConfig `mapstructure:"rpc"` - P2P *P2PConfig `mapstructure:"p2p"` - Mempool *MempoolConfig `mapstructure:"mempool"` - StateSync *StateSyncConfig `mapstructure:"statesync"` - BlockSync *BlockSyncConfig `mapstructure:"blocksync"` - //TODO(williambanfield): remove this field once v0.37 is released. - // https://github.com/tendermint/tendermint/issues/9279 - DeprecatedFastSyncConfig map[interface{}]interface{} `mapstructure:"fastsync"` - Consensus *ConsensusConfig `mapstructure:"consensus"` - Storage *StorageConfig `mapstructure:"storage"` - TxIndex *TxIndexConfig `mapstructure:"tx_index"` - Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` + RPC *RPCConfig `mapstructure:"rpc"` + P2P *P2PConfig `mapstructure:"p2p"` + Mempool *MempoolConfig `mapstructure:"mempool"` + StateSync *StateSyncConfig `mapstructure:"statesync"` + BlockSync *BlockSyncConfig `mapstructure:"blocksync"` + Consensus *ConsensusConfig `mapstructure:"consensus"` + Storage *StorageConfig `mapstructure:"storage"` + TxIndex *TxIndexConfig `mapstructure:"tx_index"` + Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` } // DefaultConfig returns a default configuration for a Tendermint node @@ -154,14 +157,9 @@ func (cfg *Config) ValidateBasic() error { return nil } +// CheckDeprecated returns any deprecation warnings. These are printed to the operator on startup func (cfg *Config) CheckDeprecated() []string { var warnings []string - if cfg.DeprecatedFastSyncConfig != nil { - warnings = append(warnings, "[fastsync] table detected. This section has been renamed to [blocksync]. The values in this deprecated section will be disregarded.") - } - if cfg.BaseConfig.DeprecatedFastSyncMode != nil { - warnings = append(warnings, "fast_sync key detected. This key has been renamed to block_sync. The value of this deprecated key will be disregarded.") - } return warnings } @@ -170,8 +168,10 @@ func (cfg *Config) CheckDeprecated() []string { // BaseConfig defines the base configuration for a Tendermint node type BaseConfig struct { //nolint: maligned - // chainID is unexposed and immutable but here for convenience - chainID string + + // The version of the Tendermint binary that created + // or last modified the config file + Version string `mapstructure:"version"` // The root directory for all data. // This should be set in viper so it can unmarshal into this struct @@ -189,10 +189,6 @@ type BaseConfig struct { //nolint: maligned // and verifying their commits BlockSyncMode bool `mapstructure:"block_sync"` - //TODO(williambanfield): remove this field once v0.37 is released. - // https://github.com/tendermint/tendermint/issues/9279 - DeprecatedFastSyncMode interface{} `mapstructure:"fast_sync"` - // Database backend: goleveldb | cleveldb | boltdb | rocksdb // * goleveldb (github.com/syndtr/goleveldb - most popular implementation) // - pure go @@ -250,6 +246,7 @@ type BaseConfig struct { //nolint: maligned // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ + Version: version.TMCoreSemVer, Genesis: defaultGenesisJSONPath, PrivValidatorKey: defaultPrivValKeyPath, PrivValidatorState: defaultPrivValStatePath, @@ -262,24 +259,19 @@ func DefaultBaseConfig() BaseConfig { BlockSyncMode: true, FilterPeers: false, DBBackend: "goleveldb", - DBPath: "data", + DBPath: DefaultDataDir, } } // TestBaseConfig returns a base configuration for testing a Tendermint node func TestBaseConfig() BaseConfig { cfg := DefaultBaseConfig() - cfg.chainID = "tendermint_test" cfg.ProxyApp = "kvstore" cfg.BlockSyncMode = false cfg.DBBackend = "memdb" return cfg } -func (cfg BaseConfig) ChainID() string { - return cfg.chainID -} - // GenesisFile returns the full path to the genesis.json file func (cfg BaseConfig) GenesisFile() string { return rootify(cfg.Genesis, cfg.RootDir) @@ -308,6 +300,12 @@ func (cfg BaseConfig) DBDir() string { // ValidateBasic performs basic validation (checking param bounds, etc.) and // returns an error if any check fails. func (cfg BaseConfig) ValidateBasic() error { + // version on old config files aren't set so we can't expect it + // always to exist + if cfg.Version != "" && !semverRegexp.MatchString(cfg.Version) { + return fmt.Errorf("invalid version string: %s", cfg.Version) + } + switch cfg.LogFormat { case LogFormatPlain, LogFormatJSON: default: @@ -513,7 +511,7 @@ func (cfg RPCConfig) KeyFile() string { if filepath.IsAbs(path) { return path } - return rootify(filepath.Join(defaultConfigDir, path), cfg.RootDir) + return rootify(filepath.Join(DefaultConfigDir, path), cfg.RootDir) } func (cfg RPCConfig) CertFile() string { @@ -521,7 +519,7 @@ func (cfg RPCConfig) CertFile() string { if filepath.IsAbs(path) { return path } - return rootify(filepath.Join(defaultConfigDir, path), cfg.RootDir) + return rootify(filepath.Join(DefaultConfigDir, path), cfg.RootDir) } func (cfg RPCConfig) IsTLSEnabled() bool { @@ -970,7 +968,7 @@ type ConsensusConfig struct { // DefaultConsensusConfig returns a default configuration for the consensus service func DefaultConsensusConfig() *ConsensusConfig { return &ConsensusConfig{ - WalPath: filepath.Join(defaultDataDir, "cs.wal", "wal"), + WalPath: filepath.Join(DefaultDataDir, "cs.wal", "wal"), TimeoutPropose: 3000 * time.Millisecond, TimeoutProposeDelta: 500 * time.Millisecond, TimeoutPrevote: 1000 * time.Millisecond, diff --git a/config/config_test.go b/config/config_test.go index 86b32c768..cd241e5aa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,4 +1,4 @@ -package config +package config_test import ( "reflect" @@ -7,13 +7,15 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/config" ) func TestDefaultConfig(t *testing.T) { assert := assert.New(t) // set up some defaults - cfg := DefaultConfig() + cfg := config.DefaultConfig() assert.NotNil(cfg.P2P) assert.NotNil(cfg.Mempool) assert.NotNil(cfg.Consensus) @@ -31,7 +33,7 @@ func TestDefaultConfig(t *testing.T) { } func TestConfigValidateBasic(t *testing.T) { - cfg := DefaultConfig() + cfg := config.DefaultConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with timeout_propose @@ -41,7 +43,7 @@ func TestConfigValidateBasic(t *testing.T) { func TestTLSConfiguration(t *testing.T) { assert := assert.New(t) - cfg := DefaultConfig() + cfg := config.DefaultConfig() cfg.SetRoot("/home/user") cfg.RPC.TLSCertFile = "file.crt" @@ -56,7 +58,7 @@ func TestTLSConfiguration(t *testing.T) { } func TestBaseConfigValidateBasic(t *testing.T) { - cfg := TestBaseConfig() + cfg := config.TestBaseConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with log format @@ -65,7 +67,7 @@ func TestBaseConfigValidateBasic(t *testing.T) { } func TestRPCConfigValidateBasic(t *testing.T) { - cfg := TestRPCConfig() + cfg := config.TestRPCConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -86,7 +88,7 @@ func TestRPCConfigValidateBasic(t *testing.T) { } func TestP2PConfigValidateBasic(t *testing.T) { - cfg := TestP2PConfig() + cfg := config.TestP2PConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -106,7 +108,7 @@ func TestP2PConfigValidateBasic(t *testing.T) { } func TestMempoolConfigValidateBasic(t *testing.T) { - cfg := TestMempoolConfig() + cfg := config.TestMempoolConfig() assert.NoError(t, cfg.ValidateBasic()) fieldsToTest := []string{ @@ -124,12 +126,12 @@ func TestMempoolConfigValidateBasic(t *testing.T) { } func TestStateSyncConfigValidateBasic(t *testing.T) { - cfg := TestStateSyncConfig() + cfg := config.TestStateSyncConfig() require.NoError(t, cfg.ValidateBasic()) } func TestBlockSyncConfigValidateBasic(t *testing.T) { - cfg := TestBlockSyncConfig() + cfg := config.TestBlockSyncConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with version @@ -143,33 +145,33 @@ func TestBlockSyncConfigValidateBasic(t *testing.T) { func TestConsensusConfig_ValidateBasic(t *testing.T) { //nolint: lll testcases := map[string]struct { - modify func(*ConsensusConfig) + modify func(*config.ConsensusConfig) expectErr bool }{ - "TimeoutPropose": {func(c *ConsensusConfig) { c.TimeoutPropose = time.Second }, false}, - "TimeoutPropose negative": {func(c *ConsensusConfig) { c.TimeoutPropose = -1 }, true}, - "TimeoutProposeDelta": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = time.Second }, false}, - "TimeoutProposeDelta negative": {func(c *ConsensusConfig) { c.TimeoutProposeDelta = -1 }, true}, - "TimeoutPrevote": {func(c *ConsensusConfig) { c.TimeoutPrevote = time.Second }, false}, - "TimeoutPrevote negative": {func(c *ConsensusConfig) { c.TimeoutPrevote = -1 }, true}, - "TimeoutPrevoteDelta": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = time.Second }, false}, - "TimeoutPrevoteDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrevoteDelta = -1 }, true}, - "TimeoutPrecommit": {func(c *ConsensusConfig) { c.TimeoutPrecommit = time.Second }, false}, - "TimeoutPrecommit negative": {func(c *ConsensusConfig) { c.TimeoutPrecommit = -1 }, true}, - "TimeoutPrecommitDelta": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = time.Second }, false}, - "TimeoutPrecommitDelta negative": {func(c *ConsensusConfig) { c.TimeoutPrecommitDelta = -1 }, true}, - "TimeoutCommit": {func(c *ConsensusConfig) { c.TimeoutCommit = time.Second }, false}, - "TimeoutCommit negative": {func(c *ConsensusConfig) { c.TimeoutCommit = -1 }, true}, - "PeerGossipSleepDuration": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = time.Second }, false}, - "PeerGossipSleepDuration negative": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, - "PeerQueryMaj23SleepDuration": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, - "PeerQueryMaj23SleepDuration negative": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, - "DoubleSignCheckHeight negative": {func(c *ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, + "TimeoutPropose": {func(c *config.ConsensusConfig) { c.TimeoutPropose = time.Second }, false}, + "TimeoutPropose negative": {func(c *config.ConsensusConfig) { c.TimeoutPropose = -1 }, true}, + "TimeoutProposeDelta": {func(c *config.ConsensusConfig) { c.TimeoutProposeDelta = time.Second }, false}, + "TimeoutProposeDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutProposeDelta = -1 }, true}, + "TimeoutPrevote": {func(c *config.ConsensusConfig) { c.TimeoutPrevote = time.Second }, false}, + "TimeoutPrevote negative": {func(c *config.ConsensusConfig) { c.TimeoutPrevote = -1 }, true}, + "TimeoutPrevoteDelta": {func(c *config.ConsensusConfig) { c.TimeoutPrevoteDelta = time.Second }, false}, + "TimeoutPrevoteDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutPrevoteDelta = -1 }, true}, + "TimeoutPrecommit": {func(c *config.ConsensusConfig) { c.TimeoutPrecommit = time.Second }, false}, + "TimeoutPrecommit negative": {func(c *config.ConsensusConfig) { c.TimeoutPrecommit = -1 }, true}, + "TimeoutPrecommitDelta": {func(c *config.ConsensusConfig) { c.TimeoutPrecommitDelta = time.Second }, false}, + "TimeoutPrecommitDelta negative": {func(c *config.ConsensusConfig) { c.TimeoutPrecommitDelta = -1 }, true}, + "TimeoutCommit": {func(c *config.ConsensusConfig) { c.TimeoutCommit = time.Second }, false}, + "TimeoutCommit negative": {func(c *config.ConsensusConfig) { c.TimeoutCommit = -1 }, true}, + "PeerGossipSleepDuration": {func(c *config.ConsensusConfig) { c.PeerGossipSleepDuration = time.Second }, false}, + "PeerGossipSleepDuration negative": {func(c *config.ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, + "PeerQueryMaj23SleepDuration": {func(c *config.ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, + "PeerQueryMaj23SleepDuration negative": {func(c *config.ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, + "DoubleSignCheckHeight negative": {func(c *config.ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, } for desc, tc := range testcases { tc := tc // appease linter t.Run(desc, func(t *testing.T) { - cfg := DefaultConsensusConfig() + cfg := config.DefaultConsensusConfig() tc.modify(cfg) err := cfg.ValidateBasic() @@ -183,7 +185,7 @@ func TestConsensusConfig_ValidateBasic(t *testing.T) { } func TestInstrumentationConfigValidateBasic(t *testing.T) { - cfg := TestInstrumentationConfig() + cfg := config.TestInstrumentationConfig() assert.NoError(t, cfg.ValidateBasic()) // tamper with maximum open connections diff --git a/config/toml.go b/config/toml.go index 53f01e8ce..c2de55642 100644 --- a/config/toml.go +++ b/config/toml.go @@ -2,8 +2,6 @@ package config import ( "bytes" - "fmt" - "os" "path/filepath" "strings" "text/template" @@ -34,10 +32,10 @@ func EnsureRoot(rootDir string) { if err := tmos.EnsureDir(rootDir, DefaultDirPerm); err != nil { panic(err.Error()) } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil { + if err := tmos.EnsureDir(filepath.Join(rootDir, DefaultConfigDir), DefaultDirPerm); err != nil { panic(err.Error()) } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil { + if err := tmos.EnsureDir(filepath.Join(rootDir, DefaultDataDir), DefaultDirPerm); err != nil { panic(err.Error()) } @@ -76,6 +74,10 @@ const defaultConfigTemplate = `# This is a TOML config file. # "$HOME/.tendermint" by default, but could be changed via $TMHOME env variable # or --home cmd flag. +# The version of the Tendermint binary that created or +# last modified the config file. Do not modify this. +version = "{{ .BaseConfig.Version }}" + ####################################################################### ### Main Base Config Options ### ####################################################################### @@ -485,6 +487,7 @@ peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" ####################################################### ### Storage Configuration Options ### ####################################################### +[storage] # Set to true to discard ABCI responses from the state store, which can save a # considerable amount of disk space. Set to false to ensure ABCI responses are @@ -536,101 +539,3 @@ max_open_connections = {{ .Instrumentation.MaxOpenConnections }} # Instrumentation namespace namespace = "{{ .Instrumentation.Namespace }}" ` - -/****** these are for test settings ***********/ - -func ResetTestRoot(testName string) *Config { - return ResetTestRootWithChainID(testName, "") -} - -func ResetTestRootWithChainID(testName string, chainID string) *Config { - // create a unique, concurrency-safe test directory under os.TempDir() - rootDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s_", chainID, testName)) - if err != nil { - panic(err) - } - // ensure config and data subdirs are created - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultConfigDir), DefaultDirPerm); err != nil { - panic(err) - } - if err := tmos.EnsureDir(filepath.Join(rootDir, defaultDataDir), DefaultDirPerm); err != nil { - panic(err) - } - - baseConfig := DefaultBaseConfig() - configFilePath := filepath.Join(rootDir, defaultConfigFilePath) - genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) - privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey) - privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState) - - // Write default config file if missing. - if !tmos.FileExists(configFilePath) { - writeDefaultConfigFile(configFilePath) - } - if !tmos.FileExists(genesisFilePath) { - if chainID == "" { - chainID = "tendermint_test" - } - testGenesis := fmt.Sprintf(testGenesisFmt, chainID) - tmos.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) - } - // we always overwrite the priv val - tmos.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) - tmos.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) - - config := TestConfig().SetRoot(rootDir) - return config -} - -var testGenesisFmt = `{ - "genesis_time": "2018-10-10T08:20:13.695936996Z", - "chain_id": "%s", - "initial_height": "1", - "consensus_params": { - "block": { - "max_bytes": "22020096", - "max_gas": "-1", - "time_iota_ms": "10" - }, - "evidence": { - "max_age_num_blocks": "100000", - "max_age_duration": "172800000000000", - "max_bytes": "1048576" - }, - "validator": { - "pub_key_types": [ - "ed25519" - ] - }, - "version": {} - }, - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" - }, - "power": "10", - "name": "" - } - ], - "app_hash": "" -}` - -var testPrivValidatorKey = `{ - "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" - }, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" - } -}` - -var testPrivValidatorState = `{ - "height": "0", - "round": 0, - "step": 0 -}` diff --git a/config/toml_test.go b/config/toml_test.go index d78f7eb9d..12b7f9c49 100644 --- a/config/toml_test.go +++ b/config/toml_test.go @@ -1,4 +1,4 @@ -package config +package config_test import ( "os" @@ -7,13 +7,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" ) func ensureFiles(t *testing.T, rootDir string, files ...string) { for _, f := range files { - p := rootify(rootDir, f) + p := filepath.Join(rootDir, f) _, err := os.Stat(p) - assert.Nil(t, err, p) + assert.NoError(t, err, p) } } @@ -26,10 +29,10 @@ func TestEnsureRoot(t *testing.T) { defer os.RemoveAll(tmpDir) // create root dir - EnsureRoot(tmpDir) + config.EnsureRoot(tmpDir) // make sure config is set properly - data, err := os.ReadFile(filepath.Join(tmpDir, defaultConfigFilePath)) + data, err := os.ReadFile(filepath.Join(tmpDir, config.DefaultConfigDir, config.DefaultConfigFileName)) require.Nil(err) assertValidConfig(t, string(data)) @@ -40,22 +43,20 @@ func TestEnsureRoot(t *testing.T) { func TestEnsureTestRoot(t *testing.T) { require := require.New(t) - testName := "ensureTestRoot" - // create root dir - cfg := ResetTestRoot(testName) + cfg := test.ResetTestRoot("ensureTestRoot") defer os.RemoveAll(cfg.RootDir) rootDir := cfg.RootDir // make sure config is set properly - data, err := os.ReadFile(filepath.Join(rootDir, defaultConfigFilePath)) + data, err := os.ReadFile(filepath.Join(rootDir, config.DefaultConfigDir, config.DefaultConfigFileName)) require.Nil(err) assertValidConfig(t, string(data)) // TODO: make sure the cfg returned and testconfig are the same! - baseConfig := DefaultBaseConfig() - ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState) + baseConfig := config.DefaultBaseConfig() + ensureFiles(t, rootDir, config.DefaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState) } func assertValidConfig(t *testing.T, configFile string) { diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index f134af5ac..84a2afad1 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -102,7 +102,7 @@ func TestByzantinePrevoteEquivocation(t *testing.T) { evpool.SetLogger(logger.With("module", "evidence")) // Make State - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) cs.SetLogger(cs.Logger) // set private validator diff --git a/consensus/common_test.go b/consensus/common_test.go index c9a685b67..e95df8e8c 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -24,6 +24,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" cstypes "github.com/tendermint/tendermint/consensus/types" + "github.com/tendermint/tendermint/internal/test" tmbytes "github.com/tendermint/tendermint/libs/bytes" "github.com/tendermint/tendermint/libs/log" tmos "github.com/tendermint/tendermint/libs/os" @@ -64,7 +65,7 @@ func ensureDir(dir string, mode os.FileMode) { } func ResetConfig(name string) *cfg.Config { - return cfg.ResetTestRoot(name) + return test.ResetTestRoot(name) } //------------------------------------------------------------------------------- @@ -109,7 +110,7 @@ func (vs *validatorStub) signVote( BlockID: types.BlockID{Hash: hash, PartSetHeader: header}, } v := vote.ToProto() - if err := vs.PrivValidator.SignVote(config.ChainID(), v); err != nil { + if err := vs.PrivValidator.SignVote(test.DefaultTestChainID, v); err != nil { return nil, fmt.Errorf("sign vote failed: %w", err) } @@ -370,7 +371,7 @@ func subscribeToVoter(cs *State, addr []byte) <-chan tmpubsub.Message { // consensus states func newState(state sm.State, pv types.PrivValidator, app abci.Application) *State { - config := cfg.ResetTestRoot("consensus_state_test") + config := test.ResetTestRoot("consensus_state_test") return newStateWithConfig(config, state, pv, app) } @@ -440,7 +441,7 @@ func newStateWithConfigAndBlockStore( panic(err) } - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool) cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) @@ -874,7 +875,7 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G return &types.GenesisDoc{ GenesisTime: tmtime.Now(), InitialHeight: 1, - ChainID: config.ChainID(), + ChainID: test.DefaultTestChainID, Validators: validators, }, privValidators } diff --git a/consensus/metrics.gen.go b/consensus/metrics.gen.go index bb9b068dd..6f1699cdd 100644 --- a/consensus/metrics.gen.go +++ b/consensus/metrics.gen.go @@ -179,13 +179,13 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Subsystem: MetricsSubsystem, Name: "round_voting_power_percent", Help: "RoundVotingPowerPercent is the percentage of the total voting power received with a round. The value begins at 0 for each round and approaches 1.0 as additional voting power is observed. The metric is labeled by vote type.", - }, labels).With(labelsAndValues...), + }, append(labels, "vote_type")).With(labelsAndValues...), LateVotes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "late_votes", Help: "LateVotes stores the number of votes that were received by this node that correspond to earlier heights and rounds than this node is currently in.", - }, labels).With(labelsAndValues...), + }, append(labels, "vote_type")).With(labelsAndValues...), } } diff --git a/consensus/metrics.go b/consensus/metrics.go index a2ee039d1..e6a8f284a 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -108,12 +108,12 @@ type Metrics struct { // RoundVotingPowerPercent is the percentage of the total voting power received // with a round. The value begins at 0 for each round and approaches 1.0 as // additional voting power is observed. The metric is labeled by vote type. - RoundVotingPowerPercent metrics.Gauge + RoundVotingPowerPercent metrics.Gauge `metrics_labels:"vote_type"` // LateVotes stores the number of votes that were received by this node that // correspond to earlier heights and rounds than this node is currently // in. - LateVotes metrics.Counter + LateVotes metrics.Counter `metrics_labels:"vote_type"` } // RecordConsMetrics uses for recording the block related metrics during fast-sync. diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index a329b304f..0e79e37aa 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -190,7 +190,7 @@ func TestReactorWithEvidence(t *testing.T) { // mock the evidence pool // everyone includes evidence of another double signing vIdx := (i + 1) % nValidators - ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], config.ChainID()) + ev, err := types.NewMockDuplicateVoteEvidenceWithValidator(1, defaultTestTime, privVals[vIdx], genDoc.ChainID) require.NoError(t, err) evpool := &statemocks.EvidencePool{} evpool.On("CheckEvidence", mock.AnythingOfType("types.EvidenceList")).Return(nil) @@ -201,7 +201,7 @@ func TestReactorWithEvidence(t *testing.T) { evpool2 := sm.EmptyEvidencePool{} // Make State - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyAppConnCon, mempool, evpool, blockStore) cs := NewState(thisConfig.Consensus, state, blockExec, blockStore, mempool, evpool2) cs.SetLogger(log.TestingLogger().With("module", "consensus")) cs.SetPrivValidator(pv) diff --git a/consensus/replay.go b/consensus/replay.go index 3dd198cee..450c1101d 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -497,11 +497,11 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap // Use stubs for both mempool and evidence pool since no transactions nor // evidence are needed here - block already exists. - blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}) + blockExec := sm.NewBlockExecutor(h.stateStore, h.logger, proxyApp, emptyMempool{}, sm.EmptyEvidencePool{}, h.store) blockExec.SetEventBus(h.eventBus) var err error - state, _, err = blockExec.ApplyBlock(state, meta.BlockID, block) + state, err = blockExec.ApplyBlock(state, meta.BlockID, block) if err != nil { return sm.State{}, err } diff --git a/consensus/replay_file.go b/consensus/replay_file.go index e5bc19514..905f363a0 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -330,7 +330,7 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo } mempool, evpool := emptyMempool{}, sm.EmptyEvidencePool{} - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) consensusState := NewState(csConfig, state.Copy(), blockExec, blockStore, mempool, evpool) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 83769ba9d..3ebff40fe 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -402,7 +402,7 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin state := genesisState.Copy() // run the chain through state.ApplyBlock to build up the tendermint state - state = buildTMStateFromChain(t, config, stateStore, mempool, evpool, state, chain, nBlocks, mode) + state = buildTMStateFromChain(t, config, stateStore, mempool, evpool, state, chain, nBlocks, mode, store) latestAppHash := state.AppHash // make a new client creator @@ -423,13 +423,13 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin }) err := stateStore.Save(genesisState) require.NoError(t, err) - buildAppStateFromChain(t, proxyApp, stateStore, mempool, evpool, genesisState, chain, nBlocks, mode) + buildAppStateFromChain(t, proxyApp, stateStore, mempool, evpool, genesisState, chain, nBlocks, mode, store) } // Prune block store if requested expectError := false if mode == 3 { - pruned, err := store.PruneBlocks(2) + pruned, _, err := store.PruneBlocks(2, state) require.NoError(t, err) require.EqualValues(t, 1, pruned) expectError = int64(nBlocks) < 2 @@ -483,20 +483,20 @@ func testHandshakeReplay(t *testing.T, config *cfg.Config, nBlocks int, mode uin } } -func applyBlock(t *testing.T, stateStore sm.Store, mempool mempool.Mempool, evpool sm.EvidencePool, st sm.State, blk *types.Block, proxyApp proxy.AppConns) sm.State { +func applyBlock(t *testing.T, stateStore sm.Store, mempool mempool.Mempool, evpool sm.EvidencePool, st sm.State, blk *types.Block, proxyApp proxy.AppConns, bs sm.BlockStore) sm.State { testPartSize := types.BlockPartSizeBytes - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, bs) bps, err := blk.MakePartSet(testPartSize) require.NoError(t, err) blkID := types.BlockID{Hash: blk.Hash(), PartSetHeader: bps.Header()} - newState, _, err := blockExec.ApplyBlock(st, blkID, blk) + newState, err := blockExec.ApplyBlock(st, blkID, blk) require.NoError(t, err) return newState } func buildAppStateFromChain(t *testing.T, proxyApp proxy.AppConns, stateStore sm.Store, mempool mempool.Mempool, evpool sm.EvidencePool, - state sm.State, chain []*types.Block, nBlocks int, mode uint) { + state sm.State, chain []*types.Block, nBlocks int, mode uint, bs sm.BlockStore) { // start a new app without handshake, play nBlocks blocks if err := proxyApp.Start(); err != nil { panic(err) @@ -517,18 +517,18 @@ func buildAppStateFromChain(t *testing.T, proxyApp proxy.AppConns, stateStore sm case 0: for i := 0; i < nBlocks; i++ { block := chain[i] - state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp) + state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp, bs) } case 1, 2, 3: for i := 0; i < nBlocks-1; i++ { block := chain[i] - state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp) + state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp, bs) } if mode == 2 || mode == 3 { // update the kvstore height and apphash // as if we ran commit but not - state = applyBlock(t, stateStore, mempool, evpool, state, chain[nBlocks-1], proxyApp) + state = applyBlock(t, stateStore, mempool, evpool, state, chain[nBlocks-1], proxyApp, bs) } default: panic(fmt.Sprintf("unknown mode %v", mode)) @@ -545,7 +545,8 @@ func buildTMStateFromChain( state sm.State, chain []*types.Block, nBlocks int, - mode uint) sm.State { + mode uint, + bs sm.BlockStore) sm.State { // run the whole chain against this client to build up the tendermint state clientCreator := proxy.NewLocalClientCreator( kvstore.NewPersistentApplication( @@ -570,19 +571,19 @@ func buildTMStateFromChain( case 0: // sync right up for _, block := range chain { - state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp) + state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp, bs) } case 1, 2, 3: // sync up to the penultimate as if we stored the block. // whether we commit or not depends on the appHash for _, block := range chain[:len(chain)-1] { - state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp) + state = applyBlock(t, stateStore, mempool, evpool, state, block, proxyApp, bs) } // apply the final block to a state copy so we can // get the right next appHash but keep the state back - applyBlock(t, stateStore, mempool, evpool, state, chain[len(chain)-1], proxyApp) + applyBlock(t, stateStore, mempool, evpool, state, chain[len(chain)-1], proxyApp, bs) default: panic(fmt.Sprintf("unknown mode %v", mode)) } @@ -880,7 +881,8 @@ func (bs *mockBlockStore) LoadSeenCommit(height int64) *types.Commit { return bs.commits[height-1] } -func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { +func (bs *mockBlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) { + evidencePoint := height pruned := uint64(0) for i := int64(0); i < height-1; i++ { bs.chain[i] = nil @@ -888,9 +890,11 @@ func (bs *mockBlockStore) PruneBlocks(height int64) (uint64, error) { pruned++ } bs.base = height - return pruned, nil + return pruned, evidencePoint, nil } +func (bs *mockBlockStore) DeleteLatestBlock() error { return nil } + //--------------------------------------- // Test handshake/init chain diff --git a/consensus/state.go b/consensus/state.go index e2e21a3d7..b1b64d7ef 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -1694,12 +1694,7 @@ func (cs *State) finalizeCommit(height int64) { // Execute and commit the block, update and save the state, and update the mempool. // NOTE The block.AppHash wont reflect these txs until the next block. - var ( - err error - retainHeight int64 - ) - - stateCopy, retainHeight, err = cs.blockExec.ApplyBlock( + stateCopy, err := cs.blockExec.ApplyBlock( stateCopy, types.BlockID{ Hash: block.Hash(), @@ -1714,16 +1709,6 @@ func (cs *State) finalizeCommit(height int64) { fail.Fail() // XXX - // Prune old heights, if requested by ABCI app. - if retainHeight > 0 { - pruned, err := cs.pruneBlocks(retainHeight) - if err != nil { - logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) - } else { - logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight) - } - } - // must be called before we update state cs.recordMetrics(height, block) @@ -1747,22 +1732,6 @@ func (cs *State) finalizeCommit(height int64) { // * cs.StartTime is set to when we will start round0. } -func (cs *State) pruneBlocks(retainHeight int64) (uint64, error) { - base := cs.blockStore.Base() - if retainHeight <= base { - return 0, nil - } - pruned, err := cs.blockStore.PruneBlocks(retainHeight) - if err != nil { - return 0, fmt.Errorf("failed to prune block store: %w", err) - } - err = cs.blockExec.Store().PruneStates(base, retainHeight) - if err != nil { - return 0, fmt.Errorf("failed to prune state database: %w", err) - } - return pruned, nil -} - func (cs *State) recordMetrics(height int64, block *types.Block) { cs.metrics.Validators.Set(float64(cs.Validators.Size())) cs.metrics.ValidatorsPower.Set(float64(cs.Validators.TotalVotingPower())) diff --git a/consensus/state_test.go b/consensus/state_test.go index 17f8e8364..211349598 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -214,7 +214,7 @@ func TestStateBadProposal(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(vs2.Height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(config.ChainID(), p); err != nil { + if err := vs2.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -276,7 +276,7 @@ func TestStateOversizedBlock(t *testing.T) { blockID := types.BlockID{Hash: propBlock.Hash(), PartSetHeader: propBlockParts.Header()} proposal := types.NewProposal(height, round, -1, blockID) p := proposal.ToProto() - if err := vs2.SignProposal(config.ChainID(), p); err != nil { + if err := vs2.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal("failed to sign bad proposal", err) } proposal.Signature = p.Signature @@ -1132,7 +1132,7 @@ func TestStateLockPOLSafety2(t *testing.T) { // in round 2 we see the polkad block from round 0 newProp := types.NewProposal(height, round, 0, propBlockID0) p := newProp.ToProto() - if err := vs3.SignProposal(config.ChainID(), p); err != nil { + if err := vs3.SignProposal(cs1.state.ChainID, p); err != nil { t.Fatal(err) } diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 68c4d98c0..ccbdbed9d 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -7,6 +7,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/types" @@ -16,7 +17,7 @@ import ( var config *cfg.Config // NOTE: must be reset for each _test.go file func TestMain(m *testing.M) { - config = cfg.ResetTestRoot("consensus_height_vote_set_test") + config = test.ResetTestRoot("consensus_height_vote_set_test") code := m.Run() os.RemoveAll(config.RootDir) os.Exit(code) @@ -25,7 +26,7 @@ func TestMain(m *testing.M) { func TestPeerCatchupRounds(t *testing.T) { valSet, privVals := types.RandValidatorSet(10, 1) - hvs := NewHeightVoteSet(config.ChainID(), 1, valSet) + hvs := NewHeightVoteSet(test.DefaultTestChainID, 1, valSet) vote999_0 := makeVoteHR(t, 1, 0, 999, privVals) added, err := hvs.AddVote(vote999_0, "peer1") @@ -73,10 +74,9 @@ func makeVoteHR(t *testing.T, height int64, valIndex, round int32, privVals []ty Type: tmproto.PrecommitType, BlockID: types.BlockID{Hash: randBytes, PartSetHeader: types.PartSetHeader{}}, } - chainID := config.ChainID() v := vote.ToProto() - err = privVal.SignVote(chainID, v) + err = privVal.SignVote(test.DefaultTestChainID, v) if err != nil { panic(fmt.Sprintf("Error signing vote: %v", err)) } diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 6fa479f49..a84ddc0b6 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/privval" @@ -84,7 +85,7 @@ func WALGenerateNBlocks(t *testing.T, wr io.Writer, numBlocks int) (err error) { }) mempool := emptyMempool{} evpool := sm.EmptyEvidencePool{} - blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool, blockStore) consensusState := NewState(config.Consensus, state.Copy(), blockExec, blockStore, mempool, evpool) consensusState.SetLogger(logger) consensusState.SetEventBus(eventBus) @@ -149,7 +150,7 @@ func makeAddrs() (string, string, string) { // getConfig returns a config for test cases func getConfig(t *testing.T) *cfg.Config { - c := cfg.ResetTestRoot(t.Name()) + c := test.ResetTestRoot(t.Name()) // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() diff --git a/crypto/batch/batch.go b/crypto/batch/batch.go new file mode 100644 index 000000000..459431e0a --- /dev/null +++ b/crypto/batch/batch.go @@ -0,0 +1,32 @@ +package batch + +import ( + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/sr25519" +) + +// CreateBatchVerifier checks if a key type implements the batch verifier interface. +// Currently only ed25519 & sr25519 supports batch verification. +func CreateBatchVerifier(pk crypto.PubKey) (crypto.BatchVerifier, bool) { + switch pk.Type() { + case ed25519.KeyType: + return ed25519.NewBatchVerifier(), true + case sr25519.KeyType: + return sr25519.NewBatchVerifier(), true + } + + // case where the key does not support batch verification + return nil, false +} + +// SupportsBatchVerifier checks if a key type implements the batch verifier +// interface. +func SupportsBatchVerifier(pk crypto.PubKey) bool { + switch pk.Type() { + case ed25519.KeyType, sr25519.KeyType: + return true + } + + return false +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 9a341f9ac..8d44b82f5 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -40,3 +40,15 @@ type Symmetric interface { Encrypt(plaintext []byte, secret []byte) (ciphertext []byte) Decrypt(ciphertext []byte, secret []byte) (plaintext []byte, err error) } + +// If a new key type implements batch verification, +// the key type must be registered in github.com/tendermint/tendermint/crypto/batch +type BatchVerifier interface { + // Add appends an entry into the BatchVerifier. + Add(key PubKey, message, signature []byte) error + // Verify verifies all the entries in the BatchVerifier, and returns + // if every signature in the batch is valid, and a vector of bools + // indicating the verification status of each signature (in the order + // that signatures were added to the batch). + Verify() (bool, []bool) +} diff --git a/crypto/ed25519/bench_test.go b/crypto/ed25519/bench_test.go index 47897cde6..49fcd1504 100644 --- a/crypto/ed25519/bench_test.go +++ b/crypto/ed25519/bench_test.go @@ -1,9 +1,12 @@ package ed25519 import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/internal/benchmarking" ) @@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) { priv := GenPrivKey() benchmarking.BenchmarkVerification(b, priv) } + +func BenchmarkVerifyBatch(b *testing.B) { + msg := []byte("BatchVerifyTest") + + for _, sigsCount := range []int{1, 8, 64, 1024} { + sigsCount := sigsCount + b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) { + // Pre-generate all of the keys, and signatures, but do not + // benchmark key-generation and signing. + pubs := make([]crypto.PubKey, 0, sigsCount) + sigs := make([][]byte, 0, sigsCount) + for i := 0; i < sigsCount; i++ { + priv := GenPrivKey() + sig, _ := priv.Sign(msg) + pubs = append(pubs, priv.PubKey().(PubKey)) + sigs = append(sigs, sig) + } + b.ResetTimer() + + b.ReportAllocs() + // NOTE: dividing by n so that metrics are per-signature + for i := 0; i < b.N/sigsCount; i++ { + // The benchmark could just benchmark the Verify() + // routine, but there is non-trivial overhead associated + // with BatchVerifier.Add(), which should be included + // in the benchmark. + v := NewBatchVerifier() + for i := 0; i < sigsCount; i++ { + err := v.Add(pubs[i], msg, sigs[i]) + require.NoError(b, err) + } + + if ok, _ := v.Verify(); !ok { + b.Fatal("signature set failed batch verification") + } + } + }) + } +} diff --git a/crypto/ed25519/ed25519.go b/crypto/ed25519/ed25519.go index 36095eece..1447ab273 100644 --- a/crypto/ed25519/ed25519.go +++ b/crypto/ed25519/ed25519.go @@ -3,10 +3,12 @@ package ed25519 import ( "bytes" "crypto/subtle" + "errors" "fmt" "io" - "golang.org/x/crypto/ed25519" + "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" + "github.com/oasisprotocol/curve25519-voi/primitives/ed25519/extra/cache" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" @@ -15,7 +17,19 @@ import ( //------------------------------------- -var _ crypto.PrivKey = PrivKey{} +var ( + _ crypto.PrivKey = PrivKey{} + _ crypto.BatchVerifier = &BatchVerifier{} + + // curve25519-voi's Ed25519 implementation supports configurable + // verification behavior, and tendermint uses the ZIP-215 verification + // semantics. + verifyOptions = &ed25519.Options{ + Verify: ed25519.VerifyOptionsZIP_215, + } + + cachingVerifier = cache.NewVerifier(cache.NewLRUCache(cacheSize)) +) const ( PrivKeyName = "tendermint/PrivKeyEd25519" @@ -32,6 +46,14 @@ const ( SeedSize = 32 KeyType = "ed25519" + + // cacheSize is the number of public keys that will be cached in + // an expanded format for repeated signature verification. + // + // TODO/perf: Either this should exclude single verification, or be + // tuned to `> validatorSize + maxTxnsPerBlock` to avoid cache + // thrashing. + cacheSize = 4096 ) func init() { @@ -105,14 +127,12 @@ func GenPrivKey() PrivKey { // genPrivKey generates a new ed25519 private key using the provided reader. func genPrivKey(rand io.Reader) PrivKey { - seed := make([]byte, SeedSize) - - _, err := io.ReadFull(rand, seed) + _, priv, err := ed25519.GenerateKey(rand) if err != nil { panic(err) } - return PrivKey(ed25519.NewKeyFromSeed(seed)) + return PrivKey(priv) } // GenPrivKeyFromSecret hashes the secret with SHA2, and uses @@ -129,7 +149,7 @@ func GenPrivKeyFromSecret(secret []byte) PrivKey { var _ crypto.PubKey = PubKey{} -// PubKeyEd25519 implements crypto.PubKey for the Ed25519 signature scheme. +// PubKey implements crypto.PubKey for the Ed25519 signature scheme. type PubKey []byte // Address is the SHA256-20 of the raw pubkey bytes. @@ -151,7 +171,7 @@ func (pubKey PubKey) VerifySignature(msg []byte, sig []byte) bool { return false } - return ed25519.Verify(ed25519.PublicKey(pubKey), msg, sig) + return cachingVerifier.VerifyWithOptions(ed25519.PublicKey(pubKey), msg, sig, verifyOptions) } func (pubKey PubKey) String() string { @@ -169,3 +189,40 @@ func (pubKey PubKey) Equals(other crypto.PubKey) bool { return false } + +//------------------------------------- + +// BatchVerifier implements batch verification for ed25519. +type BatchVerifier struct { + *ed25519.BatchVerifier +} + +func NewBatchVerifier() crypto.BatchVerifier { + return &BatchVerifier{ed25519.NewBatchVerifier()} +} + +func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error { + pkEd, ok := key.(PubKey) + if !ok { + return fmt.Errorf("pubkey is not Ed25519") + } + + pkBytes := pkEd.Bytes() + + if l := len(pkBytes); l != PubKeySize { + return fmt.Errorf("pubkey size is incorrect; expected: %d, got %d", PubKeySize, l) + } + + // check that the signature is the correct length + if len(signature) != SignatureSize { + return errors.New("invalid signature") + } + + cachingVerifier.AddWithOptions(b.BatchVerifier, ed25519.PublicKey(pkBytes), msg, signature, verifyOptions) + + return nil +} + +func (b *BatchVerifier) Verify() (bool, []bool) { + return b.BatchVerifier.Verify(crypto.CReader()) +} diff --git a/crypto/ed25519/ed25519_test.go b/crypto/ed25519/ed25519_test.go index 8c48847c0..3d329ea24 100644 --- a/crypto/ed25519/ed25519_test.go +++ b/crypto/ed25519/ed25519_test.go @@ -11,7 +11,6 @@ import ( ) func TestSignAndValidateEd25519(t *testing.T) { - privKey := ed25519.GenPrivKey() pubKey := privKey.PubKey() @@ -28,3 +27,28 @@ func TestSignAndValidateEd25519(t *testing.T) { assert.False(t, pubKey.VerifySignature(msg, sig)) } + +func TestBatchSafe(t *testing.T) { + v := ed25519.NewBatchVerifier() + + for i := 0; i <= 38; i++ { + priv := ed25519.GenPrivKey() + pub := priv.PubKey() + + var msg []byte + if i%2 == 0 { + msg = []byte("easter") + } else { + msg = []byte("egg") + } + + sig, err := priv.Sign(msg) + require.NoError(t, err) + + err = v.Add(pub, msg, sig) + require.NoError(t, err) + } + + ok, _ := v.Verify() + require.True(t, ok) +} diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go index d45130fe5..9c6df1786 100644 --- a/crypto/merkle/hash.go +++ b/crypto/merkle/hash.go @@ -1,6 +1,8 @@ package merkle import ( + "hash" + "github.com/tendermint/tendermint/crypto/tmhash" ) @@ -20,7 +22,27 @@ func leafHash(leaf []byte) []byte { return tmhash.Sum(append(leafPrefix, leaf...)) } +// returns tmhash(0x00 || leaf) +func leafHashOpt(s hash.Hash, leaf []byte) []byte { + s.Reset() + s.Write(leafPrefix) + s.Write(leaf) + return s.Sum(nil) +} + // returns tmhash(0x01 || left || right) func innerHash(left []byte, right []byte) []byte { - return tmhash.Sum(append(innerPrefix, append(left, right...)...)) + data := make([]byte, len(innerPrefix)+len(left)+len(right)) + n := copy(data, innerPrefix) + n += copy(data[n:], left) + copy(data[n:], right) + return tmhash.Sum(data) +} + +func innerHashOpt(s hash.Hash, left []byte, right []byte) []byte { + s.Reset() + s.Write(innerPrefix) + s.Write(left) + s.Write(right) + return s.Sum(nil) } diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index ab43f30e7..2994e8048 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -50,13 +50,13 @@ func ProofsFromByteSlices(items [][]byte) (rootHash []byte, proofs []*Proof) { // Verify that the Proof proves the root hash. // Check sp.Index/sp.Total manually if needed func (sp *Proof) Verify(rootHash []byte, leaf []byte) error { - leafHash := leafHash(leaf) if sp.Total < 0 { return errors.New("proof total must be positive") } if sp.Index < 0 { return errors.New("proof index cannot be negative") } + leafHash := leafHash(leaf) if !bytes.Equal(sp.LeafHash, leafHash) { return fmt.Errorf("invalid leaf hash: wanted %X got %X", leafHash, sp.LeafHash) } diff --git a/crypto/merkle/proof_key_path_test.go b/crypto/merkle/proof_key_path_test.go index 22e3e21ca..0cc947643 100644 --- a/crypto/merkle/proof_key_path_test.go +++ b/crypto/merkle/proof_key_path_test.go @@ -35,6 +35,7 @@ func TestKeyPath(t *testing.T) { res, err := KeyPathToKeys(path.String()) require.Nil(t, err) + require.Equal(t, len(keys), len(res)) for i, key := range keys { require.Equal(t, key, res[i]) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go index 22ab900f0..f0d2f8689 100644 --- a/crypto/merkle/proof_test.go +++ b/crypto/merkle/proof_test.go @@ -171,12 +171,12 @@ func TestProofValidateBasic(t *testing.T) { } } func TestVoteProtobuf(t *testing.T) { - _, proofs := ProofsFromByteSlices([][]byte{ []byte("apple"), []byte("watermelon"), []byte("kiwi"), }) + testCases := []struct { testName string v1 *Proof diff --git a/crypto/merkle/tree.go b/crypto/merkle/tree.go index 089c2f82e..896b67c59 100644 --- a/crypto/merkle/tree.go +++ b/crypto/merkle/tree.go @@ -1,22 +1,28 @@ package merkle import ( + "crypto/sha256" + "hash" "math/bits" ) // HashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. It follows RFC-6962. func HashFromByteSlices(items [][]byte) []byte { + return hashFromByteSlices(sha256.New(), items) +} + +func hashFromByteSlices(sha hash.Hash, items [][]byte) []byte { switch len(items) { case 0: return emptyHash() case 1: - return leafHash(items[0]) + return leafHashOpt(sha, items[0]) default: k := getSplitPoint(int64(len(items))) - left := HashFromByteSlices(items[:k]) - right := HashFromByteSlices(items[k:]) - return innerHash(left, right) + left := hashFromByteSlices(sha, items[:k]) + right := hashFromByteSlices(sha, items[k:]) + return innerHashOpt(sha, left, right) } } @@ -61,7 +67,7 @@ func HashFromByteSlices(items [][]byte) []byte { // implementation for so little benefit. func HashFromByteSlicesIterative(input [][]byte) []byte { items := make([][]byte, len(input)) - + sha := sha256.New() for i, leaf := range input { items[i] = leafHash(leaf) } @@ -78,7 +84,7 @@ func HashFromByteSlicesIterative(input [][]byte) []byte { wp := 0 // write position for rp < size { if rp+1 < size { - items[wp] = innerHash(items[rp], items[rp+1]) + items[wp] = innerHashOpt(sha, items[rp], items[rp+1]) rp += 2 } else { items[wp] = items[rp] diff --git a/crypto/sr25519/batch.go b/crypto/sr25519/batch.go new file mode 100644 index 000000000..462728598 --- /dev/null +++ b/crypto/sr25519/batch.go @@ -0,0 +1,46 @@ +package sr25519 + +import ( + "fmt" + + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" + + "github.com/tendermint/tendermint/crypto" +) + +var _ crypto.BatchVerifier = &BatchVerifier{} + +// BatchVerifier implements batch verification for sr25519. +type BatchVerifier struct { + *sr25519.BatchVerifier +} + +func NewBatchVerifier() crypto.BatchVerifier { + return &BatchVerifier{sr25519.NewBatchVerifier()} +} + +func (b *BatchVerifier) Add(key crypto.PubKey, msg, signature []byte) error { + pk, ok := key.(PubKey) + if !ok { + return fmt.Errorf("sr25519: pubkey is not sr25519") + } + + var srpk sr25519.PublicKey + if err := srpk.UnmarshalBinary(pk); err != nil { + return fmt.Errorf("sr25519: invalid public key: %w", err) + } + + var sig sr25519.Signature + if err := sig.UnmarshalBinary(signature); err != nil { + return fmt.Errorf("sr25519: unable to decode signature: %w", err) + } + + st := signingCtx.NewTranscriptBytes(msg) + b.BatchVerifier.Add(&srpk, st, &sig) + + return nil +} + +func (b *BatchVerifier) Verify() (bool, []bool) { + return b.BatchVerifier.Verify(crypto.CReader()) +} diff --git a/crypto/sr25519/bench_test.go b/crypto/sr25519/bench_test.go index 0561eff72..086a899c0 100644 --- a/crypto/sr25519/bench_test.go +++ b/crypto/sr25519/bench_test.go @@ -1,9 +1,12 @@ package sr25519 import ( + "fmt" "io" "testing" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/internal/benchmarking" ) @@ -24,3 +27,42 @@ func BenchmarkVerification(b *testing.B) { priv := GenPrivKey() benchmarking.BenchmarkVerification(b, priv) } + +func BenchmarkVerifyBatch(b *testing.B) { + msg := []byte("BatchVerifyTest") + + for _, sigsCount := range []int{1, 8, 64, 1024} { + sigsCount := sigsCount + b.Run(fmt.Sprintf("sig-count-%d", sigsCount), func(b *testing.B) { + // Pre-generate all of the keys, and signatures, but do not + // benchmark key-generation and signing. + pubs := make([]crypto.PubKey, 0, sigsCount) + sigs := make([][]byte, 0, sigsCount) + for i := 0; i < sigsCount; i++ { + priv := GenPrivKey() + sig, _ := priv.Sign(msg) + pubs = append(pubs, priv.PubKey().(PubKey)) + sigs = append(sigs, sig) + } + b.ResetTimer() + + b.ReportAllocs() + // NOTE: dividing by n so that metrics are per-signature + for i := 0; i < b.N/sigsCount; i++ { + // The benchmark could just benchmark the Verify() + // routine, but there is non-trivial overhead associated + // with BatchVerifier.Add(), which should be included + // in the benchmark. + v := NewBatchVerifier() + for i := 0; i < sigsCount; i++ { + err := v.Add(pubs[i], msg, sigs[i]) + require.NoError(b, err) + } + + if ok, _ := v.Verify(); !ok { + b.Fatal("signature set failed batch verification") + } + } + }) + } +} diff --git a/crypto/sr25519/encoding.go b/crypto/sr25519/encoding.go index 41570b5d0..c0a8a7925 100644 --- a/crypto/sr25519/encoding.go +++ b/crypto/sr25519/encoding.go @@ -1,23 +1,13 @@ package sr25519 -import ( - "github.com/tendermint/tendermint/crypto" - tmjson "github.com/tendermint/tendermint/libs/json" -) - -var _ crypto.PrivKey = PrivKey{} +import tmjson "github.com/tendermint/tendermint/libs/json" const ( PrivKeyName = "tendermint/PrivKeySr25519" PubKeyName = "tendermint/PubKeySr25519" - - // SignatureSize is the size of an Edwards25519 signature. Namely the size of a compressed - // Sr25519 point, and a field element. Both of which are 32 bytes. - SignatureSize = 64 ) func init() { - tmjson.RegisterType(PubKey{}, PubKeyName) tmjson.RegisterType(PrivKey{}, PrivKeyName) } diff --git a/crypto/sr25519/privkey.go b/crypto/sr25519/privkey.go index e77ca375c..2cee783bc 100644 --- a/crypto/sr25519/privkey.go +++ b/crypto/sr25519/privkey.go @@ -1,76 +1,126 @@ package sr25519 import ( - "crypto/subtle" + "encoding/json" "fmt" "io" - "github.com/tendermint/tendermint/crypto" + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" - schnorrkel "github.com/ChainSafe/go-schnorrkel" + "github.com/tendermint/tendermint/crypto" ) -// PrivKeySize is the number of bytes in an Sr25519 private key. -const PrivKeySize = 32 +var ( + _ crypto.PrivKey = PrivKey{} -// PrivKeySr25519 implements crypto.PrivKey. -type PrivKey []byte + signingCtx = sr25519.NewSigningContext([]byte{}) +) + +const ( + // PrivKeySize is the number of bytes in an Sr25519 private key. + PrivKeySize = 32 + + KeyType = "sr25519" +) + +// PrivKey implements crypto.PrivKey. +type PrivKey struct { + msk sr25519.MiniSecretKey + kp *sr25519.KeyPair +} // Bytes returns the byte representation of the PrivKey. func (privKey PrivKey) Bytes() []byte { - return []byte(privKey) + if privKey.kp == nil { + return nil + } + return privKey.msk[:] } // Sign produces a signature on the provided message. func (privKey PrivKey) Sign(msg []byte) ([]byte, error) { - var p [PrivKeySize]byte - copy(p[:], privKey) - miniSecretKey, err := schnorrkel.NewMiniSecretKeyFromRaw(p) - if err != nil { - return []byte{}, err - } - secretKey := miniSecretKey.ExpandEd25519() - - signingContext := schnorrkel.NewSigningContext([]byte{}, msg) - - sig, err := secretKey.Sign(signingContext) - if err != nil { - return []byte{}, err + if privKey.kp == nil { + return nil, fmt.Errorf("sr25519: uninitialized private key") } - sigBytes := sig.Encode() - return sigBytes[:], nil + st := signingCtx.NewTranscriptBytes(msg) + + sig, err := privKey.kp.Sign(crypto.CReader(), st) + if err != nil { + return nil, fmt.Errorf("sr25519: failed to sign message: %w", err) + } + + sigBytes, err := sig.MarshalBinary() + if err != nil { + return nil, fmt.Errorf("sr25519: failed to serialize signature: %w", err) + } + + return sigBytes, nil } // PubKey gets the corresponding public key from the private key. func (privKey PrivKey) PubKey() crypto.PubKey { - var p [PrivKeySize]byte - copy(p[:], privKey) - miniSecretKey, err := schnorrkel.NewMiniSecretKeyFromRaw(p) - if err != nil { - panic(fmt.Sprintf("Invalid private key: %v", err)) + if privKey.kp == nil { + panic("sr25519: uninitialized private key") } - secretKey := miniSecretKey.ExpandEd25519() - pubkey, err := secretKey.Public() + b, err := privKey.kp.PublicKey().MarshalBinary() if err != nil { - panic(fmt.Sprintf("Could not generate public key: %v", err)) + panic("sr25519: failed to serialize public key: " + err.Error()) } - key := pubkey.Encode() - return PubKey(key[:]) + + return PubKey(b) } // Equals - you probably don't need to use this. // Runs in constant time based on length of the keys. func (privKey PrivKey) Equals(other crypto.PrivKey) bool { - if otherEd, ok := other.(PrivKey); ok { - return subtle.ConstantTimeCompare(privKey[:], otherEd[:]) == 1 + if otherSr, ok := other.(PrivKey); ok { + return privKey.msk.Equal(&otherSr.msk) } return false } func (privKey PrivKey) Type() string { - return keyType + return KeyType +} + +func (privKey PrivKey) MarshalJSON() ([]byte, error) { + var b []byte + + // Handle uninitialized private keys gracefully. + if privKey.kp != nil { + b = privKey.Bytes() + } + + return json.Marshal(b) +} + +func (privKey *PrivKey) UnmarshalJSON(data []byte) error { + for i := range privKey.msk { + privKey.msk[i] = 0 + } + privKey.kp = nil + + var b []byte + if err := json.Unmarshal(data, &b); err != nil { + return fmt.Errorf("sr25519: failed to deserialize JSON: %w", err) + } + if len(b) == 0 { + return nil + } + + msk, err := sr25519.NewMiniSecretKeyFromBytes(b) + if err != nil { + return err + } + + sk := msk.ExpandEd25519() + + privKey.msk = *msk + privKey.kp = sk.KeyPair() + + return nil } // GenPrivKey generates a new sr25519 private key. @@ -81,19 +131,18 @@ func GenPrivKey() PrivKey { } // genPrivKey generates a new sr25519 private key using the provided reader. -func genPrivKey(rand io.Reader) PrivKey { - var seed [64]byte - - out := make([]byte, 64) - _, err := io.ReadFull(rand, out) +func genPrivKey(rng io.Reader) PrivKey { + msk, err := sr25519.GenerateMiniSecretKey(rng) if err != nil { - panic(err) + panic("sr25519: failed to generate MiniSecretKey: " + err.Error()) } - copy(seed[:], out) + sk := msk.ExpandEd25519() - key := schnorrkel.NewMiniSecretKey(seed).ExpandEd25519().Encode() - return key[:] + return PrivKey{ + msk: *msk, + kp: sk.KeyPair(), + } } // GenPrivKeyFromSecret hashes the secret with SHA2, and uses @@ -102,9 +151,14 @@ func genPrivKey(rand io.Reader) PrivKey { // if it's derived from user input. func GenPrivKeyFromSecret(secret []byte) PrivKey { seed := crypto.Sha256(secret) // Not Ripemd160 because we want 32 bytes. - var bz [PrivKeySize]byte - copy(bz[:], seed) - privKey, _ := schnorrkel.NewMiniSecretKeyFromRaw(bz) - key := privKey.ExpandEd25519().Encode() - return key[:] + + var privKey PrivKey + if err := privKey.msk.UnmarshalBinary(seed); err != nil { + panic("sr25519: failed to deserialize MiniSecretKey: " + err.Error()) + } + + sk := privKey.msk.ExpandEd25519() + privKey.kp = sk.KeyPair() + + return privKey } diff --git a/crypto/sr25519/pubkey.go b/crypto/sr25519/pubkey.go index 87805cacb..27d5917d8 100644 --- a/crypto/sr25519/pubkey.go +++ b/crypto/sr25519/pubkey.go @@ -4,25 +4,30 @@ import ( "bytes" "fmt" + "github.com/oasisprotocol/curve25519-voi/primitives/sr25519" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" - - schnorrkel "github.com/ChainSafe/go-schnorrkel" ) var _ crypto.PubKey = PubKey{} -// PubKeySize is the number of bytes in an Sr25519 public key. const ( + // PubKeySize is the number of bytes in an Sr25519 public key. PubKeySize = 32 - keyType = "sr25519" + + // SignatureSize is the size of a Sr25519 signature in bytes. + SignatureSize = 64 ) -// PubKeySr25519 implements crypto.PubKey for the Sr25519 signature scheme. +// PubKey implements crypto.PubKey for the Sr25519 signature scheme. type PubKey []byte // Address is the SHA256-20 of the raw pubkey bytes. func (pubKey PubKey) Address() crypto.Address { + if len(pubKey) != PubKeySize { + panic("pubkey is incorrect size") + } return crypto.Address(tmhash.SumTruncated(pubKey[:])) } @@ -31,47 +36,35 @@ func (pubKey PubKey) Bytes() []byte { return []byte(pubKey) } -func (pubKey PubKey) VerifySignature(msg []byte, sig []byte) bool { - // make sure we use the same algorithm to sign - if len(sig) != SignatureSize { - return false +// Equals - checks that two public keys are the same time +// Runs in constant time based on length of the keys. +func (pubKey PubKey) Equals(other crypto.PubKey) bool { + if otherSr, ok := other.(PubKey); ok { + return bytes.Equal(pubKey[:], otherSr[:]) } - var sig64 [SignatureSize]byte - copy(sig64[:], sig) - publicKey := &(schnorrkel.PublicKey{}) - var p [PubKeySize]byte - copy(p[:], pubKey) - err := publicKey.Decode(p) - if err != nil { + return false +} + +func (pubKey PubKey) VerifySignature(msg []byte, sigBytes []byte) bool { + var srpk sr25519.PublicKey + if err := srpk.UnmarshalBinary(pubKey); err != nil { return false } - signingContext := schnorrkel.NewSigningContext([]byte{}, msg) - - signature := &(schnorrkel.Signature{}) - err = signature.Decode(sig64) - if err != nil { + var sig sr25519.Signature + if err := sig.UnmarshalBinary(sigBytes); err != nil { return false } - return publicKey.Verify(signature, signingContext) + st := signingCtx.NewTranscriptBytes(msg) + return srpk.Verify(st, &sig) } func (pubKey PubKey) String() string { return fmt.Sprintf("PubKeySr25519{%X}", []byte(pubKey)) } -// Equals - checks that two public keys are the same time -// Runs in constant time based on length of the keys. -func (pubKey PubKey) Equals(other crypto.PubKey) bool { - if otherEd, ok := other.(PubKey); ok { - return bytes.Equal(pubKey[:], otherEd[:]) - } - return false -} - func (pubKey PubKey) Type() string { - return keyType - + return KeyType } diff --git a/crypto/sr25519/sr25519_test.go b/crypto/sr25519/sr25519_test.go index 1efe31cad..de5c125f4 100644 --- a/crypto/sr25519/sr25519_test.go +++ b/crypto/sr25519/sr25519_test.go @@ -1,6 +1,8 @@ package sr25519_test import ( + "encoding/base64" + "encoding/json" "testing" "github.com/stretchr/testify/assert" @@ -11,7 +13,6 @@ import ( ) func TestSignAndValidateSr25519(t *testing.T) { - privKey := sr25519.GenPrivKey() pubKey := privKey.PubKey() @@ -29,3 +30,69 @@ func TestSignAndValidateSr25519(t *testing.T) { assert.False(t, pubKey.VerifySignature(msg, sig)) } + +func TestBatchSafe(t *testing.T) { + v := sr25519.NewBatchVerifier() + vFail := sr25519.NewBatchVerifier() + for i := 0; i <= 38; i++ { + priv := sr25519.GenPrivKey() + pub := priv.PubKey() + + var msg []byte + if i%2 == 0 { + msg = []byte("easter") + } else { + msg = []byte("egg") + } + + sig, err := priv.Sign(msg) + require.NoError(t, err) + + err = v.Add(pub, msg, sig) + require.NoError(t, err) + + switch i % 2 { + case 0: + err = vFail.Add(pub, msg, sig) + case 1: + msg[2] ^= byte(0x01) + err = vFail.Add(pub, msg, sig) + } + require.NoError(t, err) + } + + ok, valid := v.Verify() + require.True(t, ok, "failed batch verification") + for i, ok := range valid { + require.Truef(t, ok, "sig[%d] should be marked valid", i) + } + + ok, valid = vFail.Verify() + require.False(t, ok, "succeeded batch verification (invalid batch)") + for i, ok := range valid { + expected := (i % 2) == 0 + require.Equalf(t, expected, ok, "sig[%d] should be %v", i, expected) + } +} + +func TestJSON(t *testing.T) { + privKey := sr25519.GenPrivKey() + + t.Run("PrivKey", func(t *testing.T) { + b, err := json.Marshal(privKey) + require.NoError(t, err) + + // b should be the base64 encoded MiniSecretKey, enclosed by doublequotes. + b64 := base64.StdEncoding.EncodeToString(privKey.Bytes()) + b64 = "\"" + b64 + "\"" + require.Equal(t, []byte(b64), b) + + var privKey2 sr25519.PrivKey + err = json.Unmarshal(b, &privKey2) + require.NoError(t, err) + require.Len(t, privKey2.Bytes(), sr25519.PrivKeySize) + require.EqualValues(t, privKey.Bytes(), privKey2.Bytes()) + }) + + // PubKeys are just []byte, so there is no special handling. +} diff --git a/docs/rfc/README.md b/docs/rfc/README.md index a0918cc64..83ff9551d 100644 --- a/docs/rfc/README.md +++ b/docs/rfc/README.md @@ -61,5 +61,6 @@ sections. - [RFC-021: The Future of the Socket Protocol](./rfc-021-socket-protocol.md) - [RFC-023: Semi-permanent Testnet](./rfc-023-semi-permanent-testnet.md) - [RFC-024: Block Structure Consolidation](./rfc-024-block-structure-consolidation.md) +- [RFC-025: Application Defined Transaction Storage](./rfc-025-support-app-side-mempool.md) diff --git a/docs/rfc/rfc-025-support-app-side-mempool.md b/docs/rfc/rfc-025-support-app-side-mempool.md new file mode 100644 index 000000000..6dfa83a0a --- /dev/null +++ b/docs/rfc/rfc-025-support-app-side-mempool.md @@ -0,0 +1,301 @@ +# RFC 25: Support Application Defined Transaction Storage (app-side mempools) + +## Changelog + +- Aug 17, 2022: initial draft (@williambanfield) +- Aug 19, 2022: updated draft (@williambanfield) + +## Abstract + +With the release of ABCI++, specifically the `PrepareProposal` call, the utility +of the Tendermint mempool becomes much less clear. This RFC discusses possible +changes that should be considered to Tendermint to better support applications +that intend to use `PrepareProposal` to implement much more powerful transaction +ordering and filtering functionality than Tendermint can provide. It proposes +scoping down the responsibilities of Tendermint to suit this new use case. + +## Background + +Tendermint currently ships with a data structure it calls the +[mempool][mempool-link]. The mempool's primary function is to store pending +valid transactions. Tendermint uses the contents of the mempool in two main +ways: 1) to gossip these pending transactions to other nodes on a Tendermint +network and 2) to select transactions to be included in a proposed block. Before +ABCI++, when proposing a block Tendermint selects the next set of transactions +from the mempool that fit within block and proposes them. + +There are a few issues with this data structure. These include issues of how +transaction validity is defined, how transactions should be ordered and selected +for inclusion in the next block, and when a transaction should start or stop +being gossiped. The creation of `PrepareProposal` in ABCI++ adds the additional +issue of unclear ownership over which entity, Tendermint or the ABCI +application, is responsible for selecting the transactions to be included in +a proposed block. + +None of these issues of validity, ordering, and gossiping having simple, +one-size fits all solutions. Different applications will have different +preferences and needs for each of them. The current Tendermint mempool attempts +to strike a balance but is quite prescriptive about these questions. We can +better support a varied range of applications by simplifying the current mempool +and by reducing and clarifying its scope of responsibilities. + +## Discussion + +### The mempool is a leaky abstraction and handles too many concerns + +The current mempool is a leaky abstraction. Presently, Tendermint's mempool keeps +track of a multitude of details that primarily service concerns of the application. + +#### Gas + +The mempool keeps track of Gas, a proxy for how computationally expensive it +will be to execute a transaction. As discussed in [RFC-011](https://github.com/tendermint/tendermint/blob/2313f358003d0c4d9d0e7705b4632d819dfb0d92/docs/rfc/rfc-011-delete-gas.md), this metadata is +not a concern of Tendermint's. Tendermint does not execute transactions. This +data is stored within Tendermint's mempool along with the maximum gas the application +will permit to be used in a block so that Tendermint's mempool can enforce +transaction validity using it: transactions that exceed the configured maximum +are rejected from the mempool. How much 'Gas' a transaction consumes and if that +precludes it from execution by the application is a validity condition imposed +by the application, not Tendermint. It is an application abstraction that leaks +into the Tendermint mempool. + +#### Sender + +The Tendermint mempool stores a `sender` string metadata for each transaction +it receives. The mempool only stores one transaction per sender at any time. +The `sender` metadata is populated by the application during `CheckTx`. +`Sender` uniqueness is enforced separately on each node's mempool. Nothing +prevents multiple transactions with the same `sender` from existing in separate +mempools on the network. + +While multiple transactions from the same sender on a network is a shortcoming +of the `sender` abstraction, the issue posed by sender to the mempool is that +`sender` uniqueness is a condition of transaction validity that is otherwise +meaningless to Tendermint. The `sender` field allows the application to +influence which transactions Tendermint will include next in a block. However, +with the advent of `PrepareProposal`, the application can select directly and +this `sender` field is of marginal benefit. Additionally, applications require +much more expressive validity conditions than just `sender` uniqueness. + +#### Adding additional semantics to the mempool + +The Tendermint mempool is relied upon by every ABCI application. Changing its +code to incorporate new features or update its behavior affects all of +Tendermint's downstream consumers. New applications frequently need ways of +sorting pending transactions and imposing transaction validity conditions. This +data structure cannot change quickly to meet the shifting needs of new +consumers while also maintaining a stable API for the applications that are +already successfully running on top of Tendermint. New strategies for sorting +and validating pending transactions would be best implemented outside of +Tendermint, where creating new semantics does not risk disrupting the existing +users. + +### Tendermint's scope of responsibility + +#### What should Tendermint be responsible for? + +Tendermint's responsibilities should be as narrowly scoped as possible to allow +the code base to be useful for many developers and maintainable by the core +team. + +The Tendermint node maintains a P2P network over which pending transactions, +proposed blocks, votes and other messages are sent. Tendermint, using these +messages, comes to consensus on the proposed blocks and delivers their contents +to the application in an orderly fashion. + +In this description of Tendermint, its only responsibility, in terms of pending +transactions, is to _gossip_ them over its P2P network. Any additional logic +surrounding validity, ordering etc. requires an understanding of the meaning of +the transaction that Tendermint does not and _should not_ have. + +#### What should the application be responsible for? + +Transaction contents have semantic meaning to the ABCI application. Pending +transactions are valid and have execution priority in relationship to the +current state of application. While Tendermint is clearly responsible for the +action of gossiping the transaction, it cannot decide when to start or stop +gossiping any given transaction. While only valid transactions should be +gossiped, as stated, it cannot appropriately make decisions about transaction +validity beyond simple heuristics. The application therefore should be +responsible for defining pending transaction validity, determining when to start +or stop gossiping a transaction, and for selecting which transaction should be +contained within a block. + +### How can Tendermint best be designed for this responsibility? + +With the understanding that Tendermint's responsibility is to gossip the set of +transactions that the application currently considers valid and high priority, +we can update its API and data structures accordingly. With the creation of +`PrepareProposal`, the mempool may be able to drop its responsibility to select +transactions for a block; It can be primarily responsible for gossiping and +nothing else. + +#### Goodbye mempool, hello GossipList + +The mempool contains many structures to retain, order, and select the set of +transactions to gossip and to propose. These mempool structures could be +completely replaced with a single list that allows Tendermint to fulfill the +previously stated responsibility. This proposed list, the `GossipList`, would +simply contain the set of transactions that Tendermint is responsible for +gossiping at the moment. This `GossipList` would be updated by the application +at a set of defined junctures and Tendermint would never add to it or remove +from it without input from the application. Tendermint would impose _no_ +validity constraints on the contents of this list and would not attempt to +remove items unless instructed to. + +### Mock API of the GossipList + +Outlined below is a proposed API for this data structure. These calls would be +added to the ABCI API and would come to replace the current `CheckTx` call. + +#### `OfferPendingTransaction` + +`OfferPendingTransaction` replaces the `CheckTx` call that is invoked when +Tendermint receives a submitted or gossiped transaction. The `GossipList` will +invoke `OfferPendingTransaction` on _every_ transaction sent to Tendermint that +does not match one of the transactions already in the `GossipList`. The mempool +currently drops gossiped transactions before `CheckTx` is called if the +transaction is considered invalid for a Tendermint-defined reason such as +inclusion in the mempool 'cache' or it overflows the max transaction size. + +The application can indicate if the transaction should be added to the +`GossipList` via `ResponseOfferPendingTransaction`'s `GossipStatus` field. If +the `GossipList` is full, the application must list a transaction to remove from +`GossipList`, otherwise the transaction will not be added. In this way, +a transaction will _never_ leave the list unless the application removes it from +the list explicitly. + +```proto +message RequestOfferPendingTransaction { + bytes tx = 1; + int64 gossip_list_max_size = 2; + int64 gossip_list_current_size = 3; +} + +message ResponseOfferPendingTransaction { + GossipStatus gossip_status = 1; + enum GossipStatus { + UNKNOWN = 0; + GOSSIP = 1; + NO_GOSSIP = 2; + } + repeated bytes removals =2 +} +``` + +#### `UpdateTransactionGossipList` + +`UpdateTransactionGossipList` would be a simple method that allows the +application to exactly set the contents of the `GossipList`. Tendermint would +call `UpdateTransactionGossipList` on the application, which would respond with +the list of all transactions to gossip. The contents of the `GossipList` would +be completely replaced with the contents provided by the application in +`UpdateTransactionGossipList`. + +```proto +message UpdateTransactionGossipListRequest { + int64 max_size = 1; // application cannot provide more than `max_size` transactions. +} + +message UpdateTransactionGossipListResponse { + repeated bytes = 1; +} +``` + +This new `ABCI` method would serve multiple functions. First, it would replace +the re-`CheckTx` calls made by Tendermint after each block is committed. After +each block is committed, Tendermint currently passes the entire contents of the +mempool to the application one-by-one via `CheckTx` calls with `CheckTxType` set +to `RECHECK`. The application, in this way, can then inspect the entire mempool +and remove any transactions that became invalid as a result of the most recent +block being committed. + +`UpdateTransactionGossipList` would completely replace this set of re-`CheckTx` +calls. After each block is committed, Tendermint would call +`UpdateTransactionGossipList` and the application would be responsible for +exactly providing the set of transactions for Tendermint to maintain. The IPC +overhead here would be roughly equivalent to the re-`CheckTx` overhead, as the +entire contents of the gossip structure is communicated, but, in the +`UpdateTransactionGossipList` call, the application sends transactions instead +of Tendermint. + +This new method would _also_ replace the mempool's `Update` API. The `Update` +method on the mempool receives the list of transactions that were just executed +as part of the most recent height and removes them from the mempool. The +`GossipList` would have no such method and instead, the application would become +responsible for setting the contents after each block via +`UpdateTransactionGossipList`. This gives the application more control over when +to start and stop gossiping transactions than it has at the moment. In this +call, the application can completely replace the `GossipList`. + +This also complements the `PrepareProposal` call nicely, because a transaction +introduced via `PrepareProposal` may be semantically equivalent to a transaction +present in Tendermint's mempool in a way that Tendermint cannot detect. The +mempool `Update` call only compares transaction hashes, +`UpdateTransactionGossipList` allows the application to easily compare on +transaction contents as well. + +As a nice benefit, it also allows the application to easily continue gossiping +of a transaction that was just executed in the block. Applications may wish to +execute the same transaction multiple times, which the mempool `Update` call +makes very cumbersome by clearing transactions that have the same contents of +those that were just executed. + +### Tendermint startup + +On Tendermint startup, the `GossipList` would be completely empty. It does not +persist transactions and is an in-memory only data structure. To populate the +`GossipList` on startup, Tendermint will issue an `UpdateTransactionGossipList` +call to the application to request the application provide it with a list of +transactions to fill the gossip list. + +### Additional benefits of this API + +#### No more confusing mempool cache + +The current Tendermint mempool stores a [cache][cache-when-clear] of transaction +hashes that should not be accepted into the mempool. When a transaction is sent +to the mempool but is present in the cache the transaction is dropped without +ever being sent to the application via `CheckTx`. This cache is intended to help +the application guard against receiving the same invalid transaction over and +over. However, this means that presence or absence from the mempool cache +becomes a condition of validity for pending transactions. + +Being placed in this cache has serious consequences for a proposed transaction, +but the rules for when a transaction should be placed in this cache are unclear. +So unclear in fact, that conditions for when to include a transaction in this +cache have been completely reversed by different commits +([1][update-remove-from-cache],[2][update-keep-in-cache]) on the Tendermint +project. Additional github issues have noted that it's very ambiguous as to +[when the cache should be cleared][cache-when-clear] and whether or not the +cache should allow [previously invalid transactions][later-valid] to later +become valid. There is no one-size-fits all solution to these problems. +Different applications need very different behavior, so this should ultimately +not be the responsibility of Tendermint. Implementing the `GossipList` clears +Tendermint of this responsibility. + +#### Improved guarantees about the set of transactions being gossiped + +As discussed in the [Mock API](#mock-api-of-the-gossiplist) section, the +`GossipList` only adds and removes or replaces transactions in the `GossipList` +when the application says to. Under this design, the contents of this list are +never ambiguous. The list contains exactly what the application most recently +told Tendermint to gossip, nothing more nothing less. + +### Additional considerations + +This document leaves a few aspects unconsidered that should be understood before +future designs are made in this area: + +1. Impact of duplicating transactions in both the `GossipList` and within the + application. +2. Transition plan and feasibility of migrating applications to the new API. + +## References + +[mempool-cache]:https://github.com/tendermint/tendermint/blob/c8302c5fcb7f1ffafdefc5014a26047df1d27c99/mempool/v1/mempool.go#L41 +[cache-when-clear]:https://github.com/tendermint/tendermint/issues/7723 +[update-remove-from-cache]:https://github.com/tendermint/tendermint/pull/233 +[update-keep-in-cache]:https://github.com/tendermint/tendermint/issues/2855 +[later-valid]:https://github.com/tendermint/tendermint/issues/458 +[mempool-link]:https://github.com/tendermint/tendermint/blob/c8302c5fcb7f1ffafdefc5014a26047df1d27c99/mempool/mempool.go#L30 diff --git a/docs/versions b/docs/versions index b2c43f548..ca9805668 100644 --- a/docs/versions +++ b/docs/versions @@ -1,3 +1,4 @@ main main +v0.37.x v0.37 v0.33.x v0.33 v0.34.x v0.34 diff --git a/evidence/verify.go b/evidence/verify.go index c20cb0a2d..528589421 100644 --- a/evidence/verify.go +++ b/evidence/verify.go @@ -21,7 +21,6 @@ func (evpool *Pool) verify(evidence types.Evidence) error { state = evpool.State() height = state.LastBlockHeight evidenceParams = state.ConsensusParams.Evidence - ageNumBlocks = height - evidence.Height() ) // verify the time of the evidence @@ -34,10 +33,9 @@ func (evpool *Pool) verify(evidence types.Evidence) error { return fmt.Errorf("evidence has a different time to the block it is associated with (%v != %v)", evidence.Time(), evTime) } - ageDuration := state.LastBlockTime.Sub(evTime) - // check that the evidence hasn't expired - if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks { + // checking if evidence is expired calculated using the block evidence time and height + if IsEvidenceExpired(height, state.LastBlockTime, evidence.Height(), evTime, evidenceParams) { return fmt.Errorf( "evidence from height %d (created at: %v) is too old; min height is %d and evidence can not be older than %v", evidence.Height(), @@ -284,3 +282,14 @@ func getSignedHeader(blockStore BlockStore, height int64) (*types.SignedHeader, Commit: commit, }, nil } + +// check that the evidence hasn't expired +func IsEvidenceExpired(heightNow int64, timeNow time.Time, heightEv int64, timeEv time.Time, evidenceParams types.EvidenceParams) bool { + ageDuration := timeNow.Sub(timeEv) + ageNumBlocks := heightNow - heightEv + + if ageDuration > evidenceParams.MaxAgeDuration && ageNumBlocks > evidenceParams.MaxAgeNumBlocks { + return true + } + return false +} diff --git a/go.mod b/go.mod index 9d1259336..e3022a8b6 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.18 require ( github.com/BurntSushi/toml v1.2.0 - github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d github.com/adlio/schema v1.3.3 github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/fortytw2/leaktest v1.3.0 @@ -15,9 +14,8 @@ require ( github.com/golangci/golangci-lint v1.49.0 github.com/google/orderedcode v0.0.1 github.com/gorilla/websocket v1.5.0 - github.com/gtank/merlin v0.1.1 github.com/informalsystems/tm-load-test v1.0.0 - github.com/lib/pq v1.10.6 + github.com/lib/pq v1.10.7 github.com/libp2p/go-buffer-pool v0.1.0 github.com/minio/highwayhash v1.0.2 github.com/ory/dockertest v3.3.5+incompatible @@ -31,16 +29,16 @@ require ( github.com/sasha-s/go-deadlock v0.3.1 github.com/snikch/goodman v0.0.0-20171125024755-10e37e294daa github.com/spf13/cobra v1.5.0 - github.com/spf13/viper v1.12.0 + github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.8.0 github.com/tendermint/tm-db v0.6.6 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa - golang.org/x/net v0.0.0-20220726230323-06994584191e - google.golang.org/grpc v1.49.0 + golang.org/x/net v0.0.0-20220812174116-3211cb980234 + google.golang.org/grpc v1.50.0 ) require ( - github.com/bufbuild/buf v1.7.0 + github.com/bufbuild/buf v1.8.0 github.com/creachadair/taskgroup v0.3.2 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 ) @@ -48,11 +46,12 @@ require ( require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 github.com/btcsuite/btcd/btcutil v1.1.2 - github.com/cosmos/gogoproto v1.4.1 - github.com/gofrs/uuid v4.2.0+incompatible + github.com/cosmos/gogoproto v1.4.2 + github.com/gofrs/uuid v4.3.0+incompatible github.com/google/uuid v1.3.0 + github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae github.com/vektra/mockery/v2 v2.14.0 - gonum.org/v1/gonum v0.8.2 + gonum.org/v1/gonum v0.12.0 google.golang.org/protobuf v1.28.1 ) @@ -78,16 +77,15 @@ require ( github.com/bombsimon/wsl/v3 v3.3.0 // indirect github.com/breml/bidichk v0.2.3 // indirect github.com/breml/errchkjson v0.3.0 // indirect - github.com/bufbuild/connect-go v0.2.0 // indirect + github.com/bufbuild/connect-go v0.4.0 // indirect github.com/butuzov/ireturn v0.1.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charithe/durationcheck v0.0.9 // indirect github.com/chavacava/garif v0.0.0-20220630083739-93517212f375 // indirect - github.com/containerd/containerd v1.6.6 // indirect + github.com/containerd/containerd v1.6.8 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/containerd/typeurl v1.0.2 // indirect - github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/curioswitch/go-reassign v0.1.2 // indirect github.com/daixiang0/gci v0.6.3 // indirect @@ -143,16 +141,15 @@ require ( github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/gtank/ristretto255 v0.1.2 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect github.com/jgautheron/goconst v1.5.1 // indirect - github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect + github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d // indirect github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect @@ -178,11 +175,10 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect github.com/mgechev/revive v1.2.3 // indirect - github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/buildkit v0.10.3 // indirect - github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/moricho/tparallel v0.2.1 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/nakabonne/nestif v0.3.1 // indirect @@ -194,7 +190,7 @@ require ( github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/opencontainers/runc v1.1.3 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect @@ -231,7 +227,7 @@ require ( github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect github.com/stbenjam/no-sprintf-host-port v0.1.1 // indirect github.com/stretchr/objx v0.4.0 // indirect - github.com/subosito/gotenv v1.4.0 // indirect + github.com/subosito/gotenv v1.4.1 // indirect github.com/sylvia7788/contextcheck v1.0.6 // indirect github.com/tdakkota/asciicheck v0.1.1 // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect @@ -248,22 +244,22 @@ require ( gitlab.com/bosi/decorder v0.2.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0 // indirect - go.opentelemetry.io/otel v1.8.0 // indirect - go.opentelemetry.io/otel/trace v1.8.0 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0 // indirect + go.opentelemetry.io/otel v1.9.0 // indirect + go.opentelemetry.io/otel/trace v1.9.0 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect + go.uber.org/zap v1.22.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect golang.org/x/exp/typeparams v0.0.0-20220613132600-b0d781184e0d // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect + golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/genproto v0.0.0-20220725144611-272f38e5d71b // indirect - gopkg.in/ini.v1 v1.66.6 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.3.3 // indirect diff --git a/go.sum b/go.sum index dfce61c67..0a7b98bea 100644 --- a/go.sum +++ b/go.sum @@ -56,7 +56,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= @@ -170,10 +169,10 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/buf v1.7.0 h1:uWRjhIXcrWkzIkA5TqXGyJbF51VW54QJsQZ3nwaes5Q= -github.com/bufbuild/buf v1.7.0/go.mod h1:Go40fMAF46PnPLC7jJgTQhAI95pmC0+VtxFKVC0qLq0= -github.com/bufbuild/connect-go v0.2.0 h1:WuMI/jLiJIhysHWvLWlxRozV67mGjCOUuDSl/lkDVic= -github.com/bufbuild/connect-go v0.2.0/go.mod h1:4efZ2eXFENwd4p7tuLaL9m0qtTsCOzuBvrohvRGevDM= +github.com/bufbuild/buf v1.8.0 h1:53qJ3QY/KOHwSjWgCQYkQaR3jGWst7aOfTXnFe8e+VQ= +github.com/bufbuild/buf v1.8.0/go.mod h1:tBzKkd1fzCcBV6KKSO7zo3rlhk3o1YQ0F2tQKSC2aNU= +github.com/bufbuild/connect-go v0.4.0 h1:fIMyUYG8mXSTH+nnlOx9KmRUf3mBF0R2uKK+BQBoOHE= +github.com/bufbuild/connect-go v0.4.0/go.mod h1:ZEtBnQ7J/m7bvWOW+H8T/+hKQCzPVfhhhICuvtcnjlI= github.com/butuzov/ireturn v0.1.1 h1:QvrO2QF2+/Cx1WA/vETCIYBKtRjc30vesdoPUNo1EbY= github.com/butuzov/ireturn v0.1.1/go.mod h1:Wh6Zl3IMtTpaIKbmwzqi6olnM9ptYQxxVacMsOEFPoc= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -216,8 +215,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= -github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= +github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= @@ -236,10 +235,9 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU= github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= -github.com/cosmos/gogoproto v1.4.1 h1:WoyH+0/jbCTzpKNvyav5FL1ZTWsp1im1MxEpJEzKUB8= -github.com/cosmos/gogoproto v1.4.1/go.mod h1:Ac9lzL4vFpBMcptJROQ6dQ4M3pOEK5Z/l0Q9p+LoCr4= +github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw= +github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -396,8 +394,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= -github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= -github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= +github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -548,9 +546,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= -github.com/gtank/merlin v0.1.1 h1:eQ90iG7K9pOhtereWsmyRJ6RAwcP4tHTDBHXNg+u5is= github.com/gtank/merlin v0.1.1/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= -github.com/gtank/ristretto255 v0.1.2 h1:JEqUCPA1NvLq5DwYtuzigd7ss8fwbYay9fi4/5uMzcc= github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIvY4OmlYW69o= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -605,8 +601,9 @@ github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOc github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/informalsystems/tm-load-test v1.0.0 h1:e1IeUw8701HWCMuOM1vLM/XcpH2Lrb88GNWdFAPDmmA= @@ -620,8 +617,8 @@ github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7H github.com/jhump/gopoet v0.0.0-20190322174617-17282ff210b3/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= github.com/jhump/gopoet v0.1.0/go.mod h1:me9yfT6IJSlOL3FCfrg+L6yzUEZ+5jW6WHt4Sk+UPUI= github.com/jhump/goprotoc v0.5.0/go.mod h1:VrbvcYrQOrTi3i0Vf+m+oqQWk9l72mjkJCYo7UvLHRQ= -github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f h1:BNuUg9k2EiJmlMwjoef3e8vZLHplbVw6DrjGFjLL+Yo= -github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f/go.mod h1:qr2b5kx4HbFS7/g4uYO5qv9ei8303JMsC7ESbYiqr2Q= +github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d h1:1BLWxsvcb5w9/vGjtyEo//r3dwEPNg7z73nbQ/XV4/s= +github.com/jhump/protocompile v0.0.0-20220812162104-d108583e055d/go.mod h1:qr2b5kx4HbFS7/g4uYO5qv9ei8303JMsC7ESbYiqr2Q= github.com/jhump/protoreflect v1.11.0/go.mod h1:U7aMIjN0NWq9swDP7xDdoMfRHb35uiuTd3Z9nFXJf5E= github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b h1:izTof8BKh/nE1wrKOrloNA5q4odOarjf+Xpe+4qow98= github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= @@ -692,8 +689,8 @@ github.com/leonklingele/grouper v1.1.0 h1:tC2y/ygPbMFSBOs3DcyaEMKnnwH7eYKzohOtRr github.com/leonklingele/grouper v1.1.0/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= @@ -746,7 +743,6 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= -github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94= github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= @@ -769,8 +765,8 @@ github.com/moby/buildkit v0.10.3 h1:/dGykD8FW+H4p++q5+KqKEo6gAkYKyBQHdawdjVwVAU= github.com/moby/buildkit v0.10.3/go.mod h1:jxeOuly98l9gWHai0Ojrbnczrk/rf+o9/JqNhY+UCSo= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 h1:dcztxKSvZ4Id8iPpHERQBbIJfabdt4wUm5qy3wOL2Zc= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= +github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -809,6 +805,8 @@ github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3L github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae h1:FatpGJD2jmJfhZiFDElaC0QhZUDQnxUeAwTGkfAHN3I= +github.com/oasisprotocol/curve25519-voi v0.0.0-20220708102147-0a8a51822cae/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -869,8 +867,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.2 h1:+jQXlF3scKIcSEKkdHzXhCTDLPFi5r1wnK6yPS+49Gw= -github.com/pelletier/go-toml/v2 v2.0.2/go.mod h1:MovirKjgVRESsAvNZlAjtFwV867yGuwRkXbG66OzopI= +github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= +github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= @@ -1058,8 +1056,8 @@ github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/y github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spf13/viper v1.12.0 h1:CZ7eSOd3kZoaYDLbXnmzgQI5RlciuXBMA+18HwHRfZQ= -github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiuKtSI= +github.com/spf13/viper v1.13.0 h1:BWSJ/M+f+3nmdz9bxB+bWX28kkALN2ok11D0rSo8EJU= +github.com/spf13/viper v1.13.0/go.mod h1:Icm2xNL3/8uyh/wFuB1jI7TiTNKp8632Nwegu+zgdYw= github.com/ssgreg/nlreturn/v2 v2.2.1 h1:X4XDI7jstt3ySqGU86YGAURbxw3oTDPK9sPEi6YEwQ0= github.com/ssgreg/nlreturn/v2 v2.2.1/go.mod h1:E/iiPB78hV7Szg2YfRgyIrk1AD6JVMTRkkxBiELzh2I= github.com/stbenjam/no-sprintf-host-port v0.1.1 h1:tYugd/yrm1O0dV+ThCbaKZh195Dfm07ysF0U6JQXczc= @@ -1081,12 +1079,11 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/subosito/gotenv v1.4.0 h1:yAzM1+SmVcz5R4tXGsNMu1jUl2aOJXoiWUCEwwnGrvs= -github.com/subosito/gotenv v1.4.0/go.mod h1:mZd6rFysKEcUhUHXJk0C/08wAgyDBFuwEYL7vWWGaGo= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/sylvia7788/contextcheck v1.0.6 h1:o2EZgVPyMKE/Mtoqym61DInKEjwEbsmyoxg3VrmjNO4= github.com/sylvia7788/contextcheck v1.0.6/go.mod h1:9XDxwvxyuKD+8N+a7Gs7bfWLityh5t70g/GjdEt2N2M= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1170,22 +1167,22 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0 h1:z6rnla1Asjzn0FrhohzIbDi4bxbtc6EMmQ7f5ZPn+pA= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.33.0/go.mod h1:y/SlJpJQPd2UzfBCj0E9Flk9FDCtTyqUmaCB41qFrWI= -go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= -go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= -go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= -go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0 h1:PNEMW4EvpNQ7SuoPFNkvbZqi1STkTPKq+8vfoMl/6AE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.34.0/go.mod h1:fk1+icoN47ytLSgkoWHLJrtVTSQ+HgmkNgPTKrk/Nsc= +go.opentelemetry.io/otel v1.9.0 h1:8WZNQFIB2a71LnANS9JeyidJKKGOOremcUtb/OtHISw= +go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= +go.opentelemetry.io/otel/trace v1.9.0 h1:oZaCNJUjWcg60VXWee8lJKlqhPbXAPB51URuR47pQYc= +go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= @@ -1197,8 +1194,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.22.0 h1:Zcye5DUgBloQ9BaT4qc9BnjOFog5TvBSAGkJ3Nf70c0= +go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1330,8 +1327,8 @@ golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220726230323-06994584191e h1:wOQNKh1uuDGRnmgF0jDxh7ctgGy/3P4rYWQRVJD4/Yg= -golang.org/x/net v0.0.0-20220726230323-06994584191e/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1358,8 +1355,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= +golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1459,8 +1457,9 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 h1:Sx/u41w+OwrInGdEckYmEuU5gHoGSL4QbDz3S9s6j4U= +golang.org/x/sys v0.0.0-20220818161305-2296e01440c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1582,9 +1581,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= @@ -1697,8 +1696,8 @@ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.50.0 h1:fPVVDxY9w++VjTZsYvXWqEf9Rqar/e+9zYfxKK+W+YU= +google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1728,8 +1727,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= -gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/internal/test/config.go b/internal/test/config.go new file mode 100644 index 000000000..85a84cec2 --- /dev/null +++ b/internal/test/config.go @@ -0,0 +1,96 @@ +package test + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/tendermint/tendermint/config" + tmos "github.com/tendermint/tendermint/libs/os" +) + +func ResetTestRoot(testName string) *config.Config { + return ResetTestRootWithChainID(testName, "") +} + +func ResetTestRootWithChainID(testName string, chainID string) *config.Config { + // create a unique, concurrency-safe test directory under os.TempDir() + rootDir, err := os.MkdirTemp("", fmt.Sprintf("%s-%s_", chainID, testName)) + if err != nil { + panic(err) + } + + config.EnsureRoot(rootDir) + + baseConfig := config.DefaultBaseConfig() + genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) + privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey) + privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState) + + if !tmos.FileExists(genesisFilePath) { + if chainID == "" { + chainID = DefaultTestChainID + } + testGenesis := fmt.Sprintf(testGenesisFmt, chainID) + tmos.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) + } + // we always overwrite the priv val + tmos.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) + tmos.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) + + config := config.TestConfig().SetRoot(rootDir) + return config +} + +var testGenesisFmt = `{ + "genesis_time": "2018-10-10T08:20:13.695936996Z", + "chain_id": "%s", + "initial_height": "1", + "consensus_params": { + "block": { + "max_bytes": "22020096", + "max_gas": "-1", + "time_iota_ms": "10" + }, + "evidence": { + "max_age_num_blocks": "100000", + "max_age_duration": "172800000000000", + "max_bytes": "1048576" + }, + "validator": { + "pub_key_types": [ + "ed25519" + ] + }, + "version": {} + }, + "validators": [ + { + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" + }, + "power": "10", + "name": "" + } + ], + "app_hash": "" +}` + +var testPrivValidatorKey = `{ + "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE=" + }, + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" + } +}` + +var testPrivValidatorState = `{ + "height": "0", + "round": 0, + "step": 0 +}` diff --git a/internal/test/genesis.go b/internal/test/genesis.go index 732adf5a3..d8047ba96 100644 --- a/internal/test/genesis.go +++ b/internal/test/genesis.go @@ -3,15 +3,14 @@ package test import ( "time" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/types" ) func GenesisDoc( - config *cfg.Config, time time.Time, validators []*types.Validator, consensusParams *types.ConsensusParams, + chainID string, ) *types.GenesisDoc { genesisValidators := make([]types.GenesisValidator, len(validators)) @@ -26,7 +25,7 @@ func GenesisDoc( return &types.GenesisDoc{ GenesisTime: time, InitialHeight: 1, - ChainID: config.ChainID(), + ChainID: chainID, Validators: genesisValidators, ConsensusParams: consensusParams, } diff --git a/libs/pubsub/example_test.go b/libs/pubsub/example_test.go index 6abd5de5c..da358be5e 100644 --- a/libs/pubsub/example_test.go +++ b/libs/pubsub/example_test.go @@ -24,7 +24,7 @@ func TestExample(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'")) + subscription, err := s.Subscribe(ctx, "example-client", query.MustCompile("abci.account.name='John'")) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Tombstone", map[string][]string{"abci.account.name": {"John"}}) require.NoError(t, err) diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index 8482a13fa..8edf12508 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -32,7 +32,7 @@ func TestSubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) assert.Equal(t, 1, s.NumClients()) @@ -78,14 +78,14 @@ func TestSubscribeWithCapacity(t *testing.T) { ctx := context.Background() assert.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, -1) + _, err = s.Subscribe(ctx, clientID, query.All, -1) require.NoError(t, err) }) assert.Panics(t, func() { - _, err = s.Subscribe(ctx, clientID, query.Empty{}, 0) + _, err = s.Subscribe(ctx, clientID, query.All, 0) require.NoError(t, err) }) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}, 1) + subscription, err := s.Subscribe(ctx, clientID, query.All, 1) require.NoError(t, err) err = s.Publish(ctx, "Aggamon") require.NoError(t, err) @@ -104,7 +104,7 @@ func TestSubscribeUnbuffered(t *testing.T) { }) ctx := context.Background() - subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.Empty{}) + subscription, err := s.SubscribeUnbuffered(ctx, clientID, query.All) require.NoError(t, err) published := make(chan struct{}) @@ -139,7 +139,7 @@ func TestSlowClientIsRemovedWithErrOutOfCapacity(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Fat Cobra") require.NoError(t, err) @@ -161,7 +161,7 @@ func TestDifferentClients(t *testing.T) { }) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, "client-1", query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Iceman", map[string][]string{"tm.events.type": {"NewBlock"}}) require.NoError(t, err) @@ -170,7 +170,7 @@ func TestDifferentClients(t *testing.T) { subscription2, err := s.Subscribe( ctx, "client-2", - query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), + query.MustCompile("tm.events.type='NewBlock' AND abci.account.name='Igor'"), ) require.NoError(t, err) err = s.PublishWithEvents( @@ -185,7 +185,7 @@ func TestDifferentClients(t *testing.T) { subscription3, err := s.Subscribe( ctx, "client-3", - query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), + query.MustCompile("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), ) require.NoError(t, err) err = s.PublishWithEvents(ctx, "Valeria Richards", map[string][]string{"tm.events.type": {"NewRoundStep"}}) @@ -227,7 +227,7 @@ func TestSubscribeDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustParse(tc.query)) + sub, err := s.Subscribe(ctx, fmt.Sprintf("client-%d", i), query.MustCompile(tc.query)) require.NoError(t, err) err = s.PublishWithEvents( @@ -260,7 +260,7 @@ func TestClientSubscribesTwice(t *testing.T) { }) ctx := context.Background() - q := query.MustParse("tm.events.type='NewBlock'") + q := query.MustCompile("tm.events.type='NewBlock'") subscription1, err := s.Subscribe(ctx, clientID, q) require.NoError(t, err) @@ -289,9 +289,9 @@ func TestUnsubscribe(t *testing.T) { }) ctx := context.Background() - subscription, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) err = s.Publish(ctx, "Nick Fury") @@ -313,12 +313,12 @@ func TestClientUnsubscribesTwice(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + _, err = s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + err = s.Unsubscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) err = s.UnsubscribeAll(ctx, clientID) assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) @@ -336,11 +336,11 @@ func TestResubscribe(t *testing.T) { }) ctx := context.Background() - _, err = s.Subscribe(ctx, clientID, query.Empty{}) + _, err = s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) - err = s.Unsubscribe(ctx, clientID, query.Empty{}) + err = s.Unsubscribe(ctx, clientID, query.All) require.NoError(t, err) - subscription, err := s.Subscribe(ctx, clientID, query.Empty{}) + subscription, err := s.Subscribe(ctx, clientID, query.All) require.NoError(t, err) err = s.Publish(ctx, "Cable") @@ -360,9 +360,9 @@ func TestUnsubscribeAll(t *testing.T) { }) ctx := context.Background() - subscription1, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + subscription1, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlock'")) require.NoError(t, err) - subscription2, err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlockHeader'")) + subscription2, err := s.Subscribe(ctx, clientID, query.MustCompile("tm.events.type='NewBlockHeader'")) require.NoError(t, err) err = s.UnsubscribeAll(ctx, clientID) @@ -421,7 +421,7 @@ func benchmarkNClients(n int, b *testing.B) { subscription, err := s.Subscribe( ctx, clientID, - query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), + query.MustCompile(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), ) if err != nil { b.Fatal(err) @@ -461,7 +461,7 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { }) ctx := context.Background() - q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") + q := query.MustCompile("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") for i := 0; i < n; i++ { subscription, err := s.Subscribe(ctx, clientID, q) if err != nil { diff --git a/libs/pubsub/query/Makefile b/libs/pubsub/query/Makefile deleted file mode 100644 index e08800817..000000000 --- a/libs/pubsub/query/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -fuzzy_test: - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz - go get -u -v github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build github.com/tendermint/tendermint/libs/pubsub/query/fuzz_test - go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output - -.PHONY: fuzzy_test diff --git a/libs/pubsub/query/bench_test.go b/libs/pubsub/query/bench_test.go new file mode 100644 index 000000000..b72392533 --- /dev/null +++ b/libs/pubsub/query/bench_test.go @@ -0,0 +1,46 @@ +package query_test + +import ( + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query" +) + +const testQuery = `tm.events.type='NewBlock' AND abci.account.name='Igor'` + +var testEvents = map[string][]string{ + "tm.events.index": { + "25", + }, + "tm.events.type": { + "NewBlock", + }, + "abci.account.name": { + "Anya", "Igor", + }, +} + +func BenchmarkParseCustom(b *testing.B) { + for i := 0; i < b.N; i++ { + _, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkMatchCustom(b *testing.B) { + q, err := query.New(testQuery) + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + ok, err := q.Matches(testEvents) + if err != nil { + b.Fatal(err) + } else if !ok { + b.Error("no match") + } + } +} diff --git a/libs/pubsub/query/empty.go b/libs/pubsub/query/empty.go deleted file mode 100644 index b86b8d4e8..000000000 --- a/libs/pubsub/query/empty.go +++ /dev/null @@ -1,14 +0,0 @@ -package query - -// Empty query matches any set of events. -type Empty struct { -} - -// Matches always returns true. -func (Empty) Matches(tags map[string][]string) (bool, error) { - return true, nil -} - -func (Empty) String() string { - return "empty" -} diff --git a/libs/pubsub/query/empty_test.go b/libs/pubsub/query/empty_test.go deleted file mode 100644 index 1b6ef2828..000000000 --- a/libs/pubsub/query/empty_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -func TestEmptyQueryMatchesAnything(t *testing.T) { - q := query.Empty{} - - testCases := []struct { - query map[string][]string - }{ - {map[string][]string{}}, - {map[string][]string{"Asher": {"Roth"}}}, - {map[string][]string{"Route": {"66"}}}, - {map[string][]string{"Route": {"66"}, "Billy": {"Blue"}}}, - } - - for _, tc := range testCases { - match, err := q.Matches(tc.query) - assert.Nil(t, err) - assert.True(t, match) - } -} diff --git a/libs/pubsub/query/fuzz_test/main.go b/libs/pubsub/query/fuzz_test/main.go deleted file mode 100644 index 7a46116b5..000000000 --- a/libs/pubsub/query/fuzz_test/main.go +++ /dev/null @@ -1,30 +0,0 @@ -package fuzz_test - -import ( - "fmt" - - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -func Fuzz(data []byte) int { - sdata := string(data) - q0, err := query.New(sdata) - if err != nil { - return 0 - } - - sdata1 := q0.String() - q1, err := query.New(sdata1) - if err != nil { - panic(err) - } - - sdata2 := q1.String() - if sdata1 != sdata2 { - fmt.Printf("q0: %q\n", sdata1) - fmt.Printf("q1: %q\n", sdata2) - panic("query changed") - } - - return 1 -} diff --git a/libs/pubsub/query/parser_test.go b/libs/pubsub/query/parser_test.go deleted file mode 100644 index a08a0d16d..000000000 --- a/libs/pubsub/query/parser_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package query_test - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/tendermint/tendermint/libs/pubsub/query" -) - -// TODO: fuzzy testing? -func TestParser(t *testing.T) { - cases := []struct { - query string - valid bool - }{ - {"tm.events.type='NewBlock'", true}, - {"tm.events.type = 'NewBlock'", true}, - {"tm.events.name = ''", true}, - {"tm.events.type='TIME'", true}, - {"tm.events.type='DATE'", true}, - {"tm.events.type='='", true}, - {"tm.events.type='TIME", false}, - {"tm.events.type=TIME'", false}, - {"tm.events.type==", false}, - {"tm.events.type=NewBlock", false}, - {">==", false}, - {"tm.events.type 'NewBlock' =", false}, - {"tm.events.type>'NewBlock'", false}, - {"", false}, - {"=", false}, - {"='NewBlock'", false}, - {"tm.events.type=", false}, - - {"tm.events.typeNewBlock", false}, - {"tm.events.type'NewBlock'", false}, - {"'NewBlock'", false}, - {"NewBlock", false}, - {"", false}, - - {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, - {"tm.events.type='NewBlock' AND", false}, - {"tm.events.type='NewBlock' AN", false}, - {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, - {"AND tm.events.type='NewBlock' ", false}, - - {"abci.account.name CONTAINS 'Igor'", true}, - - {"tx.date > DATE 2013-05-03", true}, - {"tx.date < DATE 2013-05-03", true}, - {"tx.date <= DATE 2013-05-03", true}, - {"tx.date >= DATE 2013-05-03", true}, - {"tx.date >= DAT 2013-05-03", false}, - {"tx.date <= DATE2013-05-03", false}, - {"tx.date <= DATE -05-03", false}, - {"tx.date >= DATE 20130503", false}, - {"tx.date >= DATE 2013+01-03", false}, - // incorrect year, month, day - {"tx.date >= DATE 0013-01-03", false}, - {"tx.date >= DATE 2013-31-03", false}, - {"tx.date >= DATE 2013-01-83", false}, - - {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, - {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, - {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, - {"tx.date >= TIME2013-05-03T14:45:00Z", false}, - {"tx.date = IME 2013-05-03T14:45:00Z", false}, - {"tx.date = TIME 2013-05-:45:00Z", false}, - {"tx.date >= TIME 2013-05-03T14:45:00", false}, - {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, - {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, - - {"account.balance=100", true}, - {"account.balance >= 200", true}, - {"account.balance >= -300", false}, - {"account.balance >>= 400", false}, - {"account.balance=33.22.1", false}, - - {"slashing.amount EXISTS", true}, - {"slashing.amount EXISTS AND account.balance=100", true}, - {"account.balance=100 AND slashing.amount EXISTS", true}, - {"slashing EXISTS", true}, - - {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, - {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, - } - - for _, c := range cases { - _, err := query.New(c.query) - if c.valid { - assert.NoErrorf(t, err, "Query was '%s'", c.query) - } else { - assert.Errorf(t, err, "Query was '%s'", c.query) - } - } -} diff --git a/libs/pubsub/query/peg.go b/libs/pubsub/query/peg.go deleted file mode 100644 index d4961ed48..000000000 --- a/libs/pubsub/query/peg.go +++ /dev/null @@ -1,10 +0,0 @@ -package query - -// Normally I would use go run, -// but the "Code generated by" comment for peg includes the full arg0, -// which includes an unpredictable temporary directory, -// resulting in a nondeterminstic generated source file. -// Using go build is the workaround as detailed in https://github.com/pointlander/peg/issues/129. - -//go:generate go build -o ./.bin/peg github.com/pointlander/peg -//go:generate ./.bin/peg -inline -switch query.peg diff --git a/libs/pubsub/query/query.go b/libs/pubsub/query/query.go index 7495b11ac..715b749f4 100644 --- a/libs/pubsub/query/query.go +++ b/libs/pubsub/query/query.go @@ -1,504 +1,347 @@ -// Package query provides a parser for a custom query format: +// Package query implements the custom query format used to filter event +// subscriptions in Tendermint. // // abci.invoice.number=22 AND abci.invoice.owner=Ivan // -// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. -// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics -// -// It has a support for numbers (integer and floating point), dates and times. +// Query expressions can handle attribute values encoding numbers, strings, +// dates, and timestamps. The complete query grammar is described in the +// query/syntax package. package query import ( "fmt" - "reflect" "regexp" "strconv" "strings" "time" + + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -var ( - numRegex = regexp.MustCompile(`([0-9\.]+)`) -) +// All is a query that matches all events. +var All *Query -// Query holds the query string and the query parser. +// A Query is the compiled form of a query. type Query struct { - str string - parser *QueryParser + ast syntax.Query + conds []condition } -// Condition represents a single condition within a query and consists of composite key -// (e.g. "tx.gas"), operator (e.g. "=") and operand (e.g. "7"). -type Condition struct { - CompositeKey string - Op Operator - Operand interface{} -} - -// New parses the given string and returns a query or error if the string is -// invalid. -func New(s string) (*Query, error) { - p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} - if err := p.Init(); err != nil { - return nil, err - } - if err := p.Parse(); err != nil { - return nil, err - } - return &Query{str: s, parser: p}, nil -} - -// MustParse turns the given string into a query or panics; for tests or others -// cases where you know the string is valid. -func MustParse(s string) *Query { - q, err := New(s) +// New parses and compiles the query expression into an executable query. +func New(query string) (*Query, error) { + ast, err := syntax.Parse(query) if err != nil { - panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + return nil, err + } + return Compile(ast) +} + +// MustCompile compiles the query expression into an executable query. +// In case of error, MustCompile will panic. +// +// This is intended for use in program initialization; use query.New if you +// need to check errors. +func MustCompile(query string) *Query { + q, err := New(query) + if err != nil { + panic(err) } return q } -// String returns the original string. -func (q *Query) String() string { - return q.str +// Compile compiles the given query AST so it can be used to match events. +func Compile(ast syntax.Query) (*Query, error) { + conds := make([]condition, len(ast)) + for i, q := range ast { + cond, err := compileCondition(q) + if err != nil { + return nil, fmt.Errorf("compile %s: %w", q, err) + } + conds[i] = cond + } + return &Query{ast: ast, conds: conds}, nil } -// Operator is an operator that defines some kind of relation between composite key and -// operand (equality, etc.). -type Operator uint8 +func ExpandEvents(flattenedEvents map[string][]string) []types.Event { + events := make([]types.Event, 0) -const ( - // "<=" - OpLessEqual Operator = iota - // ">=" - OpGreaterEqual - // "<" - OpLess - // ">" - OpGreater - // "=" - OpEqual - // "CONTAINS"; used to check if a string contains a certain sub string. - OpContains - // "EXISTS"; used to check if a certain event attribute is present. - OpExists -) + for composite, values := range flattenedEvents { + tokens := strings.Split(composite, ".") -const ( - // DateLayout defines a layout for all dates (`DATE date`) - DateLayout = "2006-01-02" - // TimeLayout defines a layout for all times (`TIME time`) - TimeLayout = time.RFC3339 -) - -// Conditions returns a list of conditions. It returns an error if there is any -// error with the provided grammar in the Query. -func (q *Query) Conditions() ([]Condition, error) { - var ( - eventAttr string - op Operator - ) - - conditions := make([]Condition, 0) - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - - case ruleexists: - op = OpExists - conditions = append(conditions, Condition{eventAttr, op, nil}) - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - conditions = append(conditions, Condition{eventAttr, op, valueWithoutSingleQuotes}) - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) + attrs := make([]types.EventAttribute, len(values)) + for i, v := range values { + attrs[i] = types.EventAttribute{ + Key: tokens[len(tokens)-1], + Value: v, } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return nil, err - } - - conditions = append(conditions, Condition{eventAttr, op, value}) } + + events = append(events, types.Event{ + Type: strings.Join(tokens[:len(tokens)-1], "."), + Attributes: attrs, + }) } - return conditions, nil + return events } -// Matches returns true if the query matches against any event in the given set -// of events, false otherwise. For each event, a match exists if the query is -// matched against *any* value in a slice of values. An error is returned if -// any attempted event match returns an error. -// -// For example, query "name=John" matches events = {"name": ["John", "Eric"]}. -// More examples could be found in parser_test.go and query_test.go. +// Matches satisfies part of the pubsub.Query interface. This implementation +// never reports an error. A nil *Query matches all events. func (q *Query) Matches(events map[string][]string) (bool, error) { - if len(events) == 0 { - return false, nil + if q == nil { + return true, nil } - - var ( - eventAttr string - op Operator - ) - - buffer, begin, end := q.parser.Buffer, 0, 0 - - // tokens must be in the following order: - - // tag ("tx.gas") -> operator ("=") -> operand ("7") - for _, token := range q.parser.Tokens() { - switch token.pegRule { - case rulePegText: - begin, end = int(token.begin), int(token.end) - - case ruletag: - eventAttr = buffer[begin:end] - - case rulele: - op = OpLessEqual - - case rulege: - op = OpGreaterEqual - - case rulel: - op = OpLess - - case ruleg: - op = OpGreater - - case ruleequal: - op = OpEqual - - case rulecontains: - op = OpContains - case ruleexists: - op = OpExists - if strings.Contains(eventAttr, ".") { - // Searching for a full "type.attribute" event. - _, ok := events[eventAttr] - if !ok { - return false, nil - } - } else { - foundEvent := false - - loop: - for compositeKey := range events { - if strings.Index(compositeKey, eventAttr) == 0 { - foundEvent = true - break loop - } - } - if !foundEvent { - return false, nil - } - } - - case rulevalue: - // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") - valueWithoutSingleQuotes := buffer[begin+1 : end-1] - - // see if the triplet (event attribute, operator, operand) matches any event - // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - match, err := match(eventAttr, op, reflect.ValueOf(valueWithoutSingleQuotes), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case rulenumber: - number := buffer[begin:end] - if strings.ContainsAny(number, ".") { // if it looks like a floating-point number - value, err := strconv.ParseFloat(number, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } else { - value, err := strconv.ParseInt(number, 10, 64) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", - err, number, - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - - case ruletime: - value, err := time.Parse(TimeLayout, buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - - case ruledate: - value, err := time.Parse("2006-01-02", buffer[begin:end]) - if err != nil { - err = fmt.Errorf( - "got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", - err, buffer[begin:end], - ) - return false, err - } - - match, err := match(eventAttr, op, reflect.ValueOf(value), events) - if err != nil { - return false, err - } - - if !match { - return false, nil - } - } - } - - return true, nil + return q.matchesEvents(ExpandEvents(events)), nil } -// match returns true if the given triplet (attribute, operator, operand) matches -// any value in an event for that attribute. If any match fails with an error, -// that error is returned. -// -// First, it looks up the key in the events and if it finds one, tries to compare -// all the values from it to the operand using the operator. -// -// "tx.gas", "=", "7", {"tx": [{"gas": 7, "ID": "4AE393495334"}]} -func match(attr string, op Operator, operand reflect.Value, events map[string][]string) (bool, error) { - // look up the tag from the query in tags - values, ok := events[attr] - if !ok { - return false, nil +// String matches part of the pubsub.Query interface. +func (q *Query) String() string { + if q == nil { + return "" } - - for _, value := range values { - // return true if any value in the set of the event's values matches - match, err := matchValue(value, op, operand) - if err != nil { - return false, err - } - - if match { - return true, nil - } - } - - return false, nil + return q.ast.String() } -// matchValue will attempt to match a string value against an operator an -// operand. A boolean is returned representing the match result. It will return -// an error if the value cannot be parsed and matched against the operand type. -func matchValue(value string, op Operator, operand reflect.Value) (bool, error) { - switch operand.Kind() { - case reflect.Struct: // time - operandAsTime := operand.Interface().(time.Time) +// Syntax returns the syntax tree representation of q. +func (q *Query) Syntax() syntax.Query { + if q == nil { + return nil + } + return q.ast +} - // try our best to convert value from events to time.Time - var ( - v time.Time - err error - ) - - if strings.ContainsAny(value, "T") { - v, err = time.Parse(TimeLayout, value) - } else { - v, err = time.Parse(DateLayout, value) +// matchesEvents reports whether all the conditions match the given events. +func (q *Query) matchesEvents(events []types.Event) bool { + for _, cond := range q.conds { + if !cond.matchesAny(events) { + return false } - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to time.Time: %w", value, err) + } + return len(events) != 0 +} + +// A condition is a compiled match condition. A condition matches an event if +// the event has the designated type, contains an attribute with the given +// name, and the match function returns true for the attribute value. +type condition struct { + tag string // e.g., "tx.hash" + match func(s string) bool +} + +// findAttr returns a slice of attribute values from event matching the +// condition tag, and reports whether the event type strictly equals the +// condition tag. +func (c condition) findAttr(event types.Event) ([]string, bool) { + if !strings.HasPrefix(c.tag, event.Type) { + return nil, false // type does not match tag + } else if len(c.tag) == len(event.Type) { + return nil, true // type == tag + } + var vals []string + for _, attr := range event.Attributes { + fullName := event.Type + "." + attr.Key + if fullName == c.tag { + vals = append(vals, attr.Value) } + } + return vals, false +} - switch op { - case OpLessEqual: - return (v.Before(operandAsTime) || v.Equal(operandAsTime)), nil - case OpGreaterEqual: - return (v.Equal(operandAsTime) || v.After(operandAsTime)), nil - case OpLess: - return v.Before(operandAsTime), nil - case OpGreater: - return v.After(operandAsTime), nil - case OpEqual: - return v.Equal(operandAsTime), nil +// matchesAny reports whether c matches at least one of the given events. +func (c condition) matchesAny(events []types.Event) bool { + for _, event := range events { + if c.matchesEvent(event) { + return true } + } + return false +} - case reflect.Float64: - var v float64 - - operandFloat64 := operand.Interface().(float64) - filteredValue := numRegex.FindString(value) - - // try our best to convert value from tags to float64 - v, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) +// matchesEvent reports whether c matches the given event. +func (c condition) matchesEvent(event types.Event) bool { + vs, tagEqualsType := c.findAttr(event) + if len(vs) == 0 { + // As a special case, a condition tag that exactly matches the event type + // is matched against an empty string. This allows existence checks to + // work for type-only queries. + if tagEqualsType { + return c.match("") } + return false + } - switch op { - case OpLessEqual: - return v <= operandFloat64, nil - case OpGreaterEqual: - return v >= operandFloat64, nil - case OpLess: - return v < operandFloat64, nil - case OpGreater: - return v > operandFloat64, nil - case OpEqual: - return v == operandFloat64, nil + // At this point, we have candidate values. + for _, v := range vs { + if c.match(v) { + return true } + } + return false +} - case reflect.Int64: - var v int64 +func compileCondition(cond syntax.Condition) (condition, error) { + out := condition{tag: cond.Tag} - operandInt := operand.Interface().(int64) - filteredValue := numRegex.FindString(value) + // Handle existence checks separately to simplify the logic below for + // comparisons that take arguments. + if cond.Op == syntax.TExists { + out.match = func(string) bool { return true } + return out, nil + } - // if value looks like float, we try to parse it as float - if strings.ContainsAny(filteredValue, ".") { - v1, err := strconv.ParseFloat(filteredValue, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to float64: %w", filteredValue, err) - } + // All the other operators require an argument. + if cond.Arg == nil { + return condition{}, fmt.Errorf("missing argument for %v", cond.Op) + } - v = int64(v1) - } else { - var err error - // try our best to convert value from tags to int64 - v, err = strconv.ParseInt(filteredValue, 10, 64) - if err != nil { - return false, fmt.Errorf("failed to convert value %v from event attribute to int64: %w", filteredValue, err) - } - } - - switch op { - case OpLessEqual: - return v <= operandInt, nil - case OpGreaterEqual: - return v >= operandInt, nil - case OpLess: - return v < operandInt, nil - case OpGreater: - return v > operandInt, nil - case OpEqual: - return v == operandInt, nil - } - - case reflect.String: - switch op { - case OpEqual: - return value == operand.String(), nil - case OpContains: - return strings.Contains(value, operand.String()), nil - } + // Precompile the argument value matcher. + argType := cond.Arg.Type + var argValue interface{} + switch argType { + case syntax.TString: + argValue = cond.Arg.Value() + case syntax.TNumber: + argValue = cond.Arg.Number() + case syntax.TTime, syntax.TDate: + argValue = cond.Arg.Time() default: - return false, fmt.Errorf("unknown kind of operand %v", operand.Kind()) + return condition{}, fmt.Errorf("unknown argument type %v", argType) } - return false, nil + mcons := opTypeMap[cond.Op][argType] + if mcons == nil { + return condition{}, fmt.Errorf("invalid op/arg combination (%v, %v)", cond.Op, argType) + } + out.match = mcons(argValue) + return out, nil +} + +// TODO(creachadair): The existing implementation allows anything number shaped +// to be treated as a number. This preserves the parts of that behavior we had +// tests for, but we should probably get rid of that. +var extractNum = regexp.MustCompile(`^\d+(\.\d+)?`) + +func parseNumber(s string) (float64, error) { + return strconv.ParseFloat(extractNum.FindString(s), 64) +} + +// A map of operator ⇒ argtype ⇒ match-constructor. +// An entry does not exist if the combination is not valid. +// +// Disable the dupl lint for this map. The result isn't even correct. +// +//nolint:dupl +var opTypeMap = map[syntax.Token]map[syntax.Token]func(interface{}) func(string) bool{ + syntax.TContains: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { + return strings.Contains(s, v.(string)) + } + }, + }, + syntax.TEq: { + syntax.TString: func(v interface{}) func(string) bool { + return func(s string) bool { return s == v.(string) } + }, + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w == v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Equal(v.(time.Time)) + } + }, + }, + syntax.TLt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w < v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.Before(v.(time.Time)) + } + }, + }, + syntax.TLeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w <= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.After(v.(time.Time)) + } + }, + }, + syntax.TGt: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w > v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && ts.After(v.(time.Time)) + } + }, + }, + syntax.TGeq: { + syntax.TNumber: func(v interface{}) func(string) bool { + return func(s string) bool { + w, err := parseNumber(s) + return err == nil && w >= v.(float64) + } + }, + syntax.TDate: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseDate(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + syntax.TTime: func(v interface{}) func(string) bool { + return func(s string) bool { + ts, err := syntax.ParseTime(s) + return err == nil && !ts.Before(v.(time.Time)) + } + }, + }, } diff --git a/libs/pubsub/query/query.peg b/libs/pubsub/query/query.peg deleted file mode 100644 index e2cfd0826..000000000 --- a/libs/pubsub/query/query.peg +++ /dev/null @@ -1,35 +0,0 @@ -package query - -type QueryParser Peg { -} - -e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !. - -condition <- tag ' '* (le ' '* (number / time / date) - / ge ' '* (number / time / date) - / l ' '* (number / time / date) - / g ' '* (number / time / date) - / equal ' '* (number / time / date / value) - / contains ' '* value - / exists - ) - -tag <- < (![ \t\n\r\\()"'=><] .)+ > -value <- < '\'' (!["'] .)* '\''> -number <- < ('0' - / [1-9] digit* ('.' digit*)?) > -digit <- [0-9] -time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') > -date <- "DATE " < year '-' month '-' day > -year <- ('1' / '2') digit digit digit -month <- ('0' / '1') digit -day <- ('0' / '1' / '2' / '3') digit -and <- "AND" - -equal <- "=" -contains <- "CONTAINS" -exists <- "EXISTS" -le <- "<=" -ge <- ">=" -l <- "<" -g <- ">" diff --git a/libs/pubsub/query/query.peg.go b/libs/pubsub/query/query.peg.go deleted file mode 100644 index e2d160652..000000000 --- a/libs/pubsub/query/query.peg.go +++ /dev/null @@ -1,1637 +0,0 @@ -package query - -// Code generated by ./.bin/peg -inline -switch query.peg DO NOT EDIT. - -import ( - "fmt" - "io" - "os" - "sort" - "strconv" - "strings" -) - -const endSymbol rune = 1114112 - -/* The rule types inferred from the grammar are below. */ -type pegRule uint8 - -const ( - ruleUnknown pegRule = iota - rulee - rulecondition - ruletag - rulevalue - rulenumber - ruledigit - ruletime - ruledate - ruleyear - rulemonth - ruleday - ruleand - ruleequal - rulecontains - ruleexists - rulele - rulege - rulel - ruleg - rulePegText -) - -var rul3s = [...]string{ - "Unknown", - "e", - "condition", - "tag", - "value", - "number", - "digit", - "time", - "date", - "year", - "month", - "day", - "and", - "equal", - "contains", - "exists", - "le", - "ge", - "l", - "g", - "PegText", -} - -type token32 struct { - pegRule - begin, end uint32 -} - -func (t *token32) String() string { - return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) -} - -type node32 struct { - token32 - up, next *node32 -} - -func (node *node32) print(w io.Writer, pretty bool, buffer string) { - var print func(node *node32, depth int) - print = func(node *node32, depth int) { - for node != nil { - for c := 0; c < depth; c++ { - fmt.Fprintf(w, " ") - } - rule := rul3s[node.pegRule] - quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) - if !pretty { - fmt.Fprintf(w, "%v %v\n", rule, quote) - } else { - fmt.Fprintf(w, "\x1B[36m%v\x1B[m %v\n", rule, quote) - } - if node.up != nil { - print(node.up, depth+1) - } - node = node.next - } - } - print(node, 0) -} - -func (node *node32) Print(w io.Writer, buffer string) { - node.print(w, false, buffer) -} - -func (node *node32) PrettyPrint(w io.Writer, buffer string) { - node.print(w, true, buffer) -} - -type tokens32 struct { - tree []token32 -} - -func (t *tokens32) Trim(length uint32) { - t.tree = t.tree[:length] -} - -func (t *tokens32) Print() { - for _, token := range t.tree { - fmt.Println(token.String()) - } -} - -func (t *tokens32) AST() *node32 { - type element struct { - node *node32 - down *element - } - tokens := t.Tokens() - var stack *element - for _, token := range tokens { - if token.begin == token.end { - continue - } - node := &node32{token32: token} - for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { - stack.node.next = node.up - node.up = stack.node - stack = stack.down - } - stack = &element{node: node, down: stack} - } - if stack != nil { - return stack.node - } - return nil -} - -func (t *tokens32) PrintSyntaxTree(buffer string) { - t.AST().Print(os.Stdout, buffer) -} - -func (t *tokens32) WriteSyntaxTree(w io.Writer, buffer string) { - t.AST().Print(w, buffer) -} - -func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { - t.AST().PrettyPrint(os.Stdout, buffer) -} - -func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { - tree, i := t.tree, int(index) - if i >= len(tree) { - t.tree = append(tree, token32{pegRule: rule, begin: begin, end: end}) - return - } - tree[i] = token32{pegRule: rule, begin: begin, end: end} -} - -func (t *tokens32) Tokens() []token32 { - return t.tree -} - -type QueryParser struct { - Buffer string - buffer []rune - rules [21]func() bool - parse func(rule ...int) error - reset func() - Pretty bool - tokens32 -} - -func (p *QueryParser) Parse(rule ...int) error { - return p.parse(rule...) -} - -func (p *QueryParser) Reset() { - p.reset() -} - -type textPosition struct { - line, symbol int -} - -type textPositionMap map[int]textPosition - -func translatePositions(buffer []rune, positions []int) textPositionMap { - length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 - sort.Ints(positions) - -search: - for i, c := range buffer { - if c == '\n' { - line, symbol = line+1, 0 - } else { - symbol++ - } - if i == positions[j] { - translations[positions[j]] = textPosition{line, symbol} - for j++; j < length; j++ { - if i != positions[j] { - continue search - } - } - break search - } - } - - return translations -} - -type parseError struct { - p *QueryParser - max token32 -} - -func (e *parseError) Error() string { - tokens, err := []token32{e.max}, "\n" - positions, p := make([]int, 2*len(tokens)), 0 - for _, token := range tokens { - positions[p], p = int(token.begin), p+1 - positions[p], p = int(token.end), p+1 - } - translations := translatePositions(e.p.buffer, positions) - format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" - if e.p.Pretty { - format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" - } - for _, token := range tokens { - begin, end := int(token.begin), int(token.end) - err += fmt.Sprintf(format, - rul3s[token.pegRule], - translations[begin].line, translations[begin].symbol, - translations[end].line, translations[end].symbol, - strconv.Quote(string(e.p.buffer[begin:end]))) - } - - return err -} - -func (p *QueryParser) PrintSyntaxTree() { - if p.Pretty { - p.tokens32.PrettyPrintSyntaxTree(p.Buffer) - } else { - p.tokens32.PrintSyntaxTree(p.Buffer) - } -} - -func (p *QueryParser) WriteSyntaxTree(w io.Writer) { - p.tokens32.WriteSyntaxTree(w, p.Buffer) -} - -func (p *QueryParser) SprintSyntaxTree() string { - var bldr strings.Builder - p.WriteSyntaxTree(&bldr) - return bldr.String() -} - -func Pretty(pretty bool) func(*QueryParser) error { - return func(p *QueryParser) error { - p.Pretty = pretty - return nil - } -} - -func Size(size int) func(*QueryParser) error { - return func(p *QueryParser) error { - p.tokens32 = tokens32{tree: make([]token32, 0, size)} - return nil - } -} -func (p *QueryParser) Init(options ...func(*QueryParser) error) error { - var ( - max token32 - position, tokenIndex uint32 - buffer []rune - ) - for _, option := range options { - err := option(p) - if err != nil { - return err - } - } - p.reset = func() { - max = token32{} - position, tokenIndex = 0, 0 - - p.buffer = []rune(p.Buffer) - if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { - p.buffer = append(p.buffer, endSymbol) - } - buffer = p.buffer - } - p.reset() - - _rules := p.rules - tree := p.tokens32 - p.parse = func(rule ...int) error { - r := 1 - if len(rule) > 0 { - r = rule[0] - } - matches := p.rules[r]() - p.tokens32 = tree - if matches { - p.Trim(tokenIndex) - return nil - } - return &parseError{p, max} - } - - add := func(rule pegRule, begin uint32) { - tree.Add(rule, begin, position, tokenIndex) - tokenIndex++ - if begin != position && position > max.end { - max = token32{rule, begin, position} - } - } - - matchDot := func() bool { - if buffer[position] != endSymbol { - position++ - return true - } - return false - } - - /*matchChar := func(c byte) bool { - if buffer[position] == c { - position++ - return true - } - return false - }*/ - - /*matchRange := func(lower byte, upper byte) bool { - if c := buffer[position]; c >= lower && c <= upper { - position++ - return true - } - return false - }*/ - - _rules = [...]func() bool{ - nil, - /* 0 e <- <('"' condition (' '+ and ' '+ condition)* '"' !.)> */ - func() bool { - position0, tokenIndex0 := position, tokenIndex - { - position1 := position - if buffer[position] != rune('"') { - goto l0 - } - position++ - if !_rules[rulecondition]() { - goto l0 - } - l2: - { - position3, tokenIndex3 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l4: - { - position5, tokenIndex5 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l5 - } - position++ - goto l4 - l5: - position, tokenIndex = position5, tokenIndex5 - } - { - position6 := position - { - position7, tokenIndex7 := position, tokenIndex - if buffer[position] != rune('a') { - goto l8 - } - position++ - goto l7 - l8: - position, tokenIndex = position7, tokenIndex7 - if buffer[position] != rune('A') { - goto l3 - } - position++ - } - l7: - { - position9, tokenIndex9 := position, tokenIndex - if buffer[position] != rune('n') { - goto l10 - } - position++ - goto l9 - l10: - position, tokenIndex = position9, tokenIndex9 - if buffer[position] != rune('N') { - goto l3 - } - position++ - } - l9: - { - position11, tokenIndex11 := position, tokenIndex - if buffer[position] != rune('d') { - goto l12 - } - position++ - goto l11 - l12: - position, tokenIndex = position11, tokenIndex11 - if buffer[position] != rune('D') { - goto l3 - } - position++ - } - l11: - add(ruleand, position6) - } - if buffer[position] != rune(' ') { - goto l3 - } - position++ - l13: - { - position14, tokenIndex14 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l14 - } - position++ - goto l13 - l14: - position, tokenIndex = position14, tokenIndex14 - } - if !_rules[rulecondition]() { - goto l3 - } - goto l2 - l3: - position, tokenIndex = position3, tokenIndex3 - } - if buffer[position] != rune('"') { - goto l0 - } - position++ - { - position15, tokenIndex15 := position, tokenIndex - if !matchDot() { - goto l15 - } - goto l0 - l15: - position, tokenIndex = position15, tokenIndex15 - } - add(rulee, position1) - } - return true - l0: - position, tokenIndex = position0, tokenIndex0 - return false - }, - /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('E' | 'e') exists) | (&('=') (equal ' '* ((&('\'') value) | (&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ - func() bool { - position16, tokenIndex16 := position, tokenIndex - { - position17 := position - { - position18 := position - { - position19 := position - { - position22, tokenIndex22 := position, tokenIndex - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l22 - } - position++ - case '>': - if buffer[position] != rune('>') { - goto l22 - } - position++ - case '=': - if buffer[position] != rune('=') { - goto l22 - } - position++ - case '\'': - if buffer[position] != rune('\'') { - goto l22 - } - position++ - case '"': - if buffer[position] != rune('"') { - goto l22 - } - position++ - case ')': - if buffer[position] != rune(')') { - goto l22 - } - position++ - case '(': - if buffer[position] != rune('(') { - goto l22 - } - position++ - case '\\': - if buffer[position] != rune('\\') { - goto l22 - } - position++ - case '\r': - if buffer[position] != rune('\r') { - goto l22 - } - position++ - case '\n': - if buffer[position] != rune('\n') { - goto l22 - } - position++ - case '\t': - if buffer[position] != rune('\t') { - goto l22 - } - position++ - default: - if buffer[position] != rune(' ') { - goto l22 - } - position++ - } - } - - goto l16 - l22: - position, tokenIndex = position22, tokenIndex22 - } - if !matchDot() { - goto l16 - } - l20: - { - position21, tokenIndex21 := position, tokenIndex - { - position24, tokenIndex24 := position, tokenIndex - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l24 - } - position++ - case '>': - if buffer[position] != rune('>') { - goto l24 - } - position++ - case '=': - if buffer[position] != rune('=') { - goto l24 - } - position++ - case '\'': - if buffer[position] != rune('\'') { - goto l24 - } - position++ - case '"': - if buffer[position] != rune('"') { - goto l24 - } - position++ - case ')': - if buffer[position] != rune(')') { - goto l24 - } - position++ - case '(': - if buffer[position] != rune('(') { - goto l24 - } - position++ - case '\\': - if buffer[position] != rune('\\') { - goto l24 - } - position++ - case '\r': - if buffer[position] != rune('\r') { - goto l24 - } - position++ - case '\n': - if buffer[position] != rune('\n') { - goto l24 - } - position++ - case '\t': - if buffer[position] != rune('\t') { - goto l24 - } - position++ - default: - if buffer[position] != rune(' ') { - goto l24 - } - position++ - } - } - - goto l21 - l24: - position, tokenIndex = position24, tokenIndex24 - } - if !matchDot() { - goto l21 - } - goto l20 - l21: - position, tokenIndex = position21, tokenIndex21 - } - add(rulePegText, position19) - } - add(ruletag, position18) - } - l26: - { - position27, tokenIndex27 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l27 - } - position++ - goto l26 - l27: - position, tokenIndex = position27, tokenIndex27 - } - { - position28, tokenIndex28 := position, tokenIndex - { - position30 := position - if buffer[position] != rune('<') { - goto l29 - } - position++ - if buffer[position] != rune('=') { - goto l29 - } - position++ - add(rulele, position30) - } - l31: - { - position32, tokenIndex32 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l32 - } - position++ - goto l31 - l32: - position, tokenIndex = position32, tokenIndex32 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l29 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l29 - } - default: - if !_rules[rulenumber]() { - goto l29 - } - } - } - - goto l28 - l29: - position, tokenIndex = position28, tokenIndex28 - { - position35 := position - if buffer[position] != rune('>') { - goto l34 - } - position++ - if buffer[position] != rune('=') { - goto l34 - } - position++ - add(rulege, position35) - } - l36: - { - position37, tokenIndex37 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l37 - } - position++ - goto l36 - l37: - position, tokenIndex = position37, tokenIndex37 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l34 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l34 - } - default: - if !_rules[rulenumber]() { - goto l34 - } - } - } - - goto l28 - l34: - position, tokenIndex = position28, tokenIndex28 - { - switch buffer[position] { - case 'E', 'e': - { - position40 := position - { - position41, tokenIndex41 := position, tokenIndex - if buffer[position] != rune('e') { - goto l42 - } - position++ - goto l41 - l42: - position, tokenIndex = position41, tokenIndex41 - if buffer[position] != rune('E') { - goto l16 - } - position++ - } - l41: - { - position43, tokenIndex43 := position, tokenIndex - if buffer[position] != rune('x') { - goto l44 - } - position++ - goto l43 - l44: - position, tokenIndex = position43, tokenIndex43 - if buffer[position] != rune('X') { - goto l16 - } - position++ - } - l43: - { - position45, tokenIndex45 := position, tokenIndex - if buffer[position] != rune('i') { - goto l46 - } - position++ - goto l45 - l46: - position, tokenIndex = position45, tokenIndex45 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l45: - { - position47, tokenIndex47 := position, tokenIndex - if buffer[position] != rune('s') { - goto l48 - } - position++ - goto l47 - l48: - position, tokenIndex = position47, tokenIndex47 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l47: - { - position49, tokenIndex49 := position, tokenIndex - if buffer[position] != rune('t') { - goto l50 - } - position++ - goto l49 - l50: - position, tokenIndex = position49, tokenIndex49 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l49: - { - position51, tokenIndex51 := position, tokenIndex - if buffer[position] != rune('s') { - goto l52 - } - position++ - goto l51 - l52: - position, tokenIndex = position51, tokenIndex51 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l51: - add(ruleexists, position40) - } - case '=': - { - position53 := position - if buffer[position] != rune('=') { - goto l16 - } - position++ - add(ruleequal, position53) - } - l54: - { - position55, tokenIndex55 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l55 - } - position++ - goto l54 - l55: - position, tokenIndex = position55, tokenIndex55 - } - { - switch buffer[position] { - case '\'': - if !_rules[rulevalue]() { - goto l16 - } - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - case '>': - { - position57 := position - if buffer[position] != rune('>') { - goto l16 - } - position++ - add(ruleg, position57) - } - l58: - { - position59, tokenIndex59 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l59 - } - position++ - goto l58 - l59: - position, tokenIndex = position59, tokenIndex59 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - case '<': - { - position61 := position - if buffer[position] != rune('<') { - goto l16 - } - position++ - add(rulel, position61) - } - l62: - { - position63, tokenIndex63 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l63 - } - position++ - goto l62 - l63: - position, tokenIndex = position63, tokenIndex63 - } - { - switch buffer[position] { - case 'D', 'd': - if !_rules[ruledate]() { - goto l16 - } - case 'T', 't': - if !_rules[ruletime]() { - goto l16 - } - default: - if !_rules[rulenumber]() { - goto l16 - } - } - } - - default: - { - position65 := position - { - position66, tokenIndex66 := position, tokenIndex - if buffer[position] != rune('c') { - goto l67 - } - position++ - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 - if buffer[position] != rune('C') { - goto l16 - } - position++ - } - l66: - { - position68, tokenIndex68 := position, tokenIndex - if buffer[position] != rune('o') { - goto l69 - } - position++ - goto l68 - l69: - position, tokenIndex = position68, tokenIndex68 - if buffer[position] != rune('O') { - goto l16 - } - position++ - } - l68: - { - position70, tokenIndex70 := position, tokenIndex - if buffer[position] != rune('n') { - goto l71 - } - position++ - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l70: - { - position72, tokenIndex72 := position, tokenIndex - if buffer[position] != rune('t') { - goto l73 - } - position++ - goto l72 - l73: - position, tokenIndex = position72, tokenIndex72 - if buffer[position] != rune('T') { - goto l16 - } - position++ - } - l72: - { - position74, tokenIndex74 := position, tokenIndex - if buffer[position] != rune('a') { - goto l75 - } - position++ - goto l74 - l75: - position, tokenIndex = position74, tokenIndex74 - if buffer[position] != rune('A') { - goto l16 - } - position++ - } - l74: - { - position76, tokenIndex76 := position, tokenIndex - if buffer[position] != rune('i') { - goto l77 - } - position++ - goto l76 - l77: - position, tokenIndex = position76, tokenIndex76 - if buffer[position] != rune('I') { - goto l16 - } - position++ - } - l76: - { - position78, tokenIndex78 := position, tokenIndex - if buffer[position] != rune('n') { - goto l79 - } - position++ - goto l78 - l79: - position, tokenIndex = position78, tokenIndex78 - if buffer[position] != rune('N') { - goto l16 - } - position++ - } - l78: - { - position80, tokenIndex80 := position, tokenIndex - if buffer[position] != rune('s') { - goto l81 - } - position++ - goto l80 - l81: - position, tokenIndex = position80, tokenIndex80 - if buffer[position] != rune('S') { - goto l16 - } - position++ - } - l80: - add(rulecontains, position65) - } - l82: - { - position83, tokenIndex83 := position, tokenIndex - if buffer[position] != rune(' ') { - goto l83 - } - position++ - goto l82 - l83: - position, tokenIndex = position83, tokenIndex83 - } - if !_rules[rulevalue]() { - goto l16 - } - } - } - - } - l28: - add(rulecondition, position17) - } - return true - l16: - position, tokenIndex = position16, tokenIndex16 - return false - }, - /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('\'') '\'') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ - nil, - /* 3 value <- <<('\'' (!('"' / '\'') .)* '\'')>> */ - func() bool { - position85, tokenIndex85 := position, tokenIndex - { - position86 := position - { - position87 := position - if buffer[position] != rune('\'') { - goto l85 - } - position++ - l88: - { - position89, tokenIndex89 := position, tokenIndex - { - position90, tokenIndex90 := position, tokenIndex - { - position91, tokenIndex91 := position, tokenIndex - if buffer[position] != rune('"') { - goto l92 - } - position++ - goto l91 - l92: - position, tokenIndex = position91, tokenIndex91 - if buffer[position] != rune('\'') { - goto l90 - } - position++ - } - l91: - goto l89 - l90: - position, tokenIndex = position90, tokenIndex90 - } - if !matchDot() { - goto l89 - } - goto l88 - l89: - position, tokenIndex = position89, tokenIndex89 - } - if buffer[position] != rune('\'') { - goto l85 - } - position++ - add(rulePegText, position87) - } - add(rulevalue, position86) - } - return true - l85: - position, tokenIndex = position85, tokenIndex85 - return false - }, - /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ - func() bool { - position93, tokenIndex93 := position, tokenIndex - { - position94 := position - { - position95 := position - { - position96, tokenIndex96 := position, tokenIndex - if buffer[position] != rune('0') { - goto l97 - } - position++ - goto l96 - l97: - position, tokenIndex = position96, tokenIndex96 - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l93 - } - position++ - l98: - { - position99, tokenIndex99 := position, tokenIndex - if !_rules[ruledigit]() { - goto l99 - } - goto l98 - l99: - position, tokenIndex = position99, tokenIndex99 - } - { - position100, tokenIndex100 := position, tokenIndex - if buffer[position] != rune('.') { - goto l100 - } - position++ - l102: - { - position103, tokenIndex103 := position, tokenIndex - if !_rules[ruledigit]() { - goto l103 - } - goto l102 - l103: - position, tokenIndex = position103, tokenIndex103 - } - goto l101 - l100: - position, tokenIndex = position100, tokenIndex100 - } - l101: - } - l96: - add(rulePegText, position95) - } - add(rulenumber, position94) - } - return true - l93: - position, tokenIndex = position93, tokenIndex93 - return false - }, - /* 5 digit <- <[0-9]> */ - func() bool { - position104, tokenIndex104 := position, tokenIndex - { - position105 := position - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l104 - } - position++ - add(ruledigit, position105) - } - return true - l104: - position, tokenIndex = position104, tokenIndex104 - return false - }, - /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ - func() bool { - position106, tokenIndex106 := position, tokenIndex - { - position107 := position - { - position108, tokenIndex108 := position, tokenIndex - if buffer[position] != rune('t') { - goto l109 - } - position++ - goto l108 - l109: - position, tokenIndex = position108, tokenIndex108 - if buffer[position] != rune('T') { - goto l106 - } - position++ - } - l108: - { - position110, tokenIndex110 := position, tokenIndex - if buffer[position] != rune('i') { - goto l111 - } - position++ - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 - if buffer[position] != rune('I') { - goto l106 - } - position++ - } - l110: - { - position112, tokenIndex112 := position, tokenIndex - if buffer[position] != rune('m') { - goto l113 - } - position++ - goto l112 - l113: - position, tokenIndex = position112, tokenIndex112 - if buffer[position] != rune('M') { - goto l106 - } - position++ - } - l112: - { - position114, tokenIndex114 := position, tokenIndex - if buffer[position] != rune('e') { - goto l115 - } - position++ - goto l114 - l115: - position, tokenIndex = position114, tokenIndex114 - if buffer[position] != rune('E') { - goto l106 - } - position++ - } - l114: - if buffer[position] != rune(' ') { - goto l106 - } - position++ - { - position116 := position - if !_rules[ruleyear]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[rulemonth]() { - goto l106 - } - if buffer[position] != rune('-') { - goto l106 - } - position++ - if !_rules[ruleday]() { - goto l106 - } - if buffer[position] != rune('T') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - if buffer[position] != rune(':') { - goto l106 - } - position++ - if !_rules[ruledigit]() { - goto l106 - } - if !_rules[ruledigit]() { - goto l106 - } - { - position117, tokenIndex117 := position, tokenIndex - { - position119, tokenIndex119 := position, tokenIndex - if buffer[position] != rune('-') { - goto l120 - } - position++ - goto l119 - l120: - position, tokenIndex = position119, tokenIndex119 - if buffer[position] != rune('+') { - goto l118 - } - position++ - } - l119: - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - if buffer[position] != rune(':') { - goto l118 - } - position++ - if !_rules[ruledigit]() { - goto l118 - } - if !_rules[ruledigit]() { - goto l118 - } - goto l117 - l118: - position, tokenIndex = position117, tokenIndex117 - if buffer[position] != rune('Z') { - goto l106 - } - position++ - } - l117: - add(rulePegText, position116) - } - add(ruletime, position107) - } - return true - l106: - position, tokenIndex = position106, tokenIndex106 - return false - }, - /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ - func() bool { - position121, tokenIndex121 := position, tokenIndex - { - position122 := position - { - position123, tokenIndex123 := position, tokenIndex - if buffer[position] != rune('d') { - goto l124 - } - position++ - goto l123 - l124: - position, tokenIndex = position123, tokenIndex123 - if buffer[position] != rune('D') { - goto l121 - } - position++ - } - l123: - { - position125, tokenIndex125 := position, tokenIndex - if buffer[position] != rune('a') { - goto l126 - } - position++ - goto l125 - l126: - position, tokenIndex = position125, tokenIndex125 - if buffer[position] != rune('A') { - goto l121 - } - position++ - } - l125: - { - position127, tokenIndex127 := position, tokenIndex - if buffer[position] != rune('t') { - goto l128 - } - position++ - goto l127 - l128: - position, tokenIndex = position127, tokenIndex127 - if buffer[position] != rune('T') { - goto l121 - } - position++ - } - l127: - { - position129, tokenIndex129 := position, tokenIndex - if buffer[position] != rune('e') { - goto l130 - } - position++ - goto l129 - l130: - position, tokenIndex = position129, tokenIndex129 - if buffer[position] != rune('E') { - goto l121 - } - position++ - } - l129: - if buffer[position] != rune(' ') { - goto l121 - } - position++ - { - position131 := position - if !_rules[ruleyear]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[rulemonth]() { - goto l121 - } - if buffer[position] != rune('-') { - goto l121 - } - position++ - if !_rules[ruleday]() { - goto l121 - } - add(rulePegText, position131) - } - add(ruledate, position122) - } - return true - l121: - position, tokenIndex = position121, tokenIndex121 - return false - }, - /* 8 year <- <(('1' / '2') digit digit digit)> */ - func() bool { - position132, tokenIndex132 := position, tokenIndex - { - position133 := position - { - position134, tokenIndex134 := position, tokenIndex - if buffer[position] != rune('1') { - goto l135 - } - position++ - goto l134 - l135: - position, tokenIndex = position134, tokenIndex134 - if buffer[position] != rune('2') { - goto l132 - } - position++ - } - l134: - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - if !_rules[ruledigit]() { - goto l132 - } - add(ruleyear, position133) - } - return true - l132: - position, tokenIndex = position132, tokenIndex132 - return false - }, - /* 9 month <- <(('0' / '1') digit)> */ - func() bool { - position136, tokenIndex136 := position, tokenIndex - { - position137 := position - { - position138, tokenIndex138 := position, tokenIndex - if buffer[position] != rune('0') { - goto l139 - } - position++ - goto l138 - l139: - position, tokenIndex = position138, tokenIndex138 - if buffer[position] != rune('1') { - goto l136 - } - position++ - } - l138: - if !_rules[ruledigit]() { - goto l136 - } - add(rulemonth, position137) - } - return true - l136: - position, tokenIndex = position136, tokenIndex136 - return false - }, - /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ - func() bool { - position140, tokenIndex140 := position, tokenIndex - { - position141 := position - { - switch buffer[position] { - case '3': - if buffer[position] != rune('3') { - goto l140 - } - position++ - case '2': - if buffer[position] != rune('2') { - goto l140 - } - position++ - case '1': - if buffer[position] != rune('1') { - goto l140 - } - position++ - default: - if buffer[position] != rune('0') { - goto l140 - } - position++ - } - } - - if !_rules[ruledigit]() { - goto l140 - } - add(ruleday, position141) - } - return true - l140: - position, tokenIndex = position140, tokenIndex140 - return false - }, - /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ - nil, - /* 12 equal <- <'='> */ - nil, - /* 13 contains <- <(('c' / 'C') ('o' / 'O') ('n' / 'N') ('t' / 'T') ('a' / 'A') ('i' / 'I') ('n' / 'N') ('s' / 'S'))> */ - nil, - /* 14 exists <- <(('e' / 'E') ('x' / 'X') ('i' / 'I') ('s' / 'S') ('t' / 'T') ('s' / 'S'))> */ - nil, - /* 15 le <- <('<' '=')> */ - nil, - /* 16 ge <- <('>' '=')> */ - nil, - /* 17 l <- <'<'> */ - nil, - /* 18 g <- <'>'> */ - nil, - nil, - } - p.rules = _rules - return nil -} diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index d511e7fab..71a05e239 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -1,222 +1,407 @@ package query_test import ( + "encoding/json" "fmt" + "sort" + "strings" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/libs/pubsub" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) -func TestMatches(t *testing.T) { +var _ pubsub.Query = (*query.Query)(nil) + +// Example events from the OpenAPI documentation: +// +// https://github.com/tendermint/tendermint/blob/master/rpc/openapi/openapi.yaml +// +// Redactions: +// +// - Add an explicit "tm" event for the built-in attributes. +// - Remove Index fields (not relevant to tests). +// - Add explicit balance values (to use in tests). +var apiEvents = map[string][]string{ + "tm.event": { + "Tx", + }, + "tm.hash": { + "XYZ", + }, + "tm.height": { + "5", + }, + "rewards.withdraw.address": { + "AddrA", + "AddrB", + }, + "rewards.withdraw.source": { + "SrcX", + "SrcY", + }, + "rewards.withdraw.amount": { + "100", + "45", + }, + "rewards.withdraw.balance": { + "1500", + "999", + }, + "transfer.sender": { + "AddrC", + }, + "transfer.recipient": { + "AddrD", + }, + "transfer.amount": { + "160", + }, +} + +var apiTypeEvents = []types.Event{ + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "event", + Value: "Tx", + }, + }, + }, + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "hash", + Value: "XYZ", + }, + }, + }, + { + Type: "tm", + Attributes: []types.EventAttribute{ + { + Key: "height", + Value: "5", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "address", + Value: "AddrA", + }, + { + Key: "address", + Value: "AddrB", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "source", + Value: "SrcX", + }, + { + Key: "source", + Value: "SrcY", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "amount", + Value: "100", + }, + { + Key: "amount", + Value: "45", + }, + }, + }, + { + Type: "rewards.withdraw", + Attributes: []types.EventAttribute{ + { + Key: "balance", + Value: "1500", + }, + { + Key: "balance", + Value: "999", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "sender", + Value: "AddrC", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "recipient", + Value: "AddrD", + }, + }, + }, + { + Type: "transfer", + Attributes: []types.EventAttribute{ + { + Key: "amount", + Value: "160", + }, + }, + }, +} + +func TestCompiledMatches(t *testing.T) { var ( txDate = "2017-01-01" txTime = "2018-05-03T14:45:00Z" ) + //nolint:lll testCases := []struct { - s string - events map[string][]string - err bool - matches bool - matchErr bool + s string + events map[string][]string + matches bool }{ - {"tm.events.type='NewBlock'", map[string][]string{"tm.events.type": {"NewBlock"}}, false, true, false}, - {"tx.gas > 7", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8stake"}}, false, true, false}, - {"transfer.amount > 7", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 7.043", map[string][]string{"transfer.amount": {"8.045stake"}}, false, true, false}, - {"transfer.amount > 8.045", map[string][]string{"transfer.amount": {"8.045stake"}}, false, false, false}, - {"tx.gas > 7 AND tx.gas < 9", map[string][]string{"tx.gas": {"8"}}, false, true, false}, - {"body.weight >= 3.5", map[string][]string{"body.weight": {"3.5"}}, false, true, false}, - {"account.balance < 1000.0", map[string][]string{"account.balance": {"900"}}, false, true, false}, - {"apples.kg <= 4", map[string][]string{"apples.kg": {"4.0"}}, false, true, false}, - {"body.weight >= 4.5", map[string][]string{"body.weight": {fmt.Sprintf("%v", float32(4.5))}}, false, true, false}, - { - "oranges.kg < 4 AND watermellons.kg > 10", - map[string][]string{"oranges.kg": {"3"}, "watermellons.kg": {"12"}}, - false, - true, - false, - }, - {"peaches.kg < 4", map[string][]string{"peaches.kg": {"5"}}, false, false, false}, - { - "tx.date > DATE 2017-01-01", - map[string][]string{"tx.date": {time.Now().Format(query.DateLayout)}}, - false, - true, - false, - }, - {"tx.date = DATE 2017-01-01", map[string][]string{"tx.date": {txDate}}, false, true, false}, - {"tx.date = DATE 2018-01-01", map[string][]string{"tx.date": {txDate}}, false, false, false}, - { - "tx.time >= TIME 2013-05-03T14:45:00Z", - map[string][]string{"tx.time": {time.Now().Format(query.TimeLayout)}}, - false, - true, - false, - }, - {"tx.time = TIME 2013-05-03T14:45:00Z", map[string][]string{"tx.time": {txTime}}, false, false, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Igor,Ivan"}}, false, true, false}, - {"abci.owner.name CONTAINS 'Igor'", map[string][]string{"abci.owner.name": {"Pavel,Ivan"}}, false, false, false}, - {"abci.owner.name = 'Igor'", map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, false, true, false}, - { - "abci.owner.name = 'Ivan'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - true, - false, - }, - { - "abci.owner.name = 'Ivan' AND abci.owner.name = 'John'", - map[string][]string{"abci.owner.name": {"Igor", "Ivan"}}, - false, - false, - false, - }, - { - "tm.events.type='NewBlock'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewBlock' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - true, - false, - }, - { - "tm.events.type='NewHeader' AND app.name = 'fuzzed'", - map[string][]string{"tm.events.type": {"NewBlock"}, "app.name": {"fuzzed"}}, - false, - false, - false, - }, - {"slash EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"sl EXISTS", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"6000"}}, - false, - true, - false, - }, - {"slash.reason EXISTS AND slash.power > 1000", - map[string][]string{"slash.reason": {"missing_signature"}, "slash.power": {"500"}}, - false, - false, - false, - }, - {"slash.reason EXISTS", - map[string][]string{"transfer.recipient": {"cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz"}, - "transfer.sender": {"cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5"}}, - false, - false, - false, - }, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`), + true}, + {`tx.gas > 7`, + newTestEvents(`tx|gas=8`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8stake`), + true}, + {`transfer.amount > 7`, + newTestEvents(`transfer|amount=8.045`), + true}, + {`transfer.amount > 7.043`, + newTestEvents(`transfer|amount=8.045stake`), + true}, + {`transfer.amount > 8.045`, + newTestEvents(`transfer|amount=8.045stake`), + false}, + {`tx.gas > 7 AND tx.gas < 9`, + newTestEvents(`tx|gas=8`), + true}, + {`body.weight >= 3.5`, + newTestEvents(`body|weight=3.5`), + true}, + {`account.balance < 1000.0`, + newTestEvents(`account|balance=900`), + true}, + {`apples.kg <= 4`, + newTestEvents(`apples|kg=4.0`), + true}, + {`body.weight >= 4.5`, + newTestEvents(`body|weight=4.5`), + true}, + {`oranges.kg < 4 AND watermellons.kg > 10`, + newTestEvents(`oranges|kg=3`, `watermellons|kg=12`), + true}, + {`peaches.kg < 4`, + newTestEvents(`peaches|kg=5`), + false}, + {`tx.date > DATE 2017-01-01`, + newTestEvents(`tx|date=` + time.Now().Format(syntax.DateFormat)), + true}, + {`tx.date = DATE 2017-01-01`, + newTestEvents(`tx|date=` + txDate), + true}, + {`tx.date = DATE 2018-01-01`, + newTestEvents(`tx|date=` + txDate), + false}, + {`tx.time >= TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + time.Now().Format(syntax.TimeFormat)), + true}, + {`tx.time = TIME 2013-05-03T14:45:00Z`, + newTestEvents(`tx|time=` + txTime), + false}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name CONTAINS 'Igor'`, + newTestEvents(`abci|owner.name=Pavel|owner.name=Ivan`), + false}, + {`abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'Igor'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + true}, + {`abci.owner.name = 'Ivan' AND abci.owner.name = 'John'`, + newTestEvents(`abci|owner.name=Igor|owner.name=Ivan`), + false}, + {`tm.events.type='NewBlock'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewBlock' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + true}, + {`tm.events.type='NewHeader' AND app.name = 'fuzzed'`, + newTestEvents(`tm|events.type=NewBlock`, `app|name=fuzzed`), + false}, + {`slash EXISTS`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=6000`), + true}, + {`slash.reason EXISTS AND slash.power > 1000`, + newTestEvents(`slash|reason=missing_signature|power=500`), + false}, + {`slash.reason EXISTS`, + newTestEvents(`transfer|recipient=cosmos1gu6y2a0ffteesyeyeesk23082c6998xyzmt9mz|sender=cosmos1crje20aj4gxdtyct7z3knxqry2jqt2fuaey6u5`), + false}, + + // Test cases based on the OpenAPI examples. + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA'`, + apiEvents, true}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrA' AND rewards.withdraw.source = 'SrcY'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrA'`, + apiEvents, false}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrC'`, + apiEvents, true}, + {`tm.event = 'Tx' AND transfer.sender = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.address = 'AddrZ'`, + apiEvents, false}, + {`tm.event = 'Tx' AND rewards.withdraw.source = 'W'`, + apiEvents, false}, } - for _, tc := range testCases { - q, err := query.New(tc.s) - if !tc.err { - require.Nil(t, err) - } - require.NotNil(t, q, "Query '%s' should not be nil", tc.s) + // NOTE: The original implementation allowed arbitrary prefix matches on + // attribute tags, e.g., "sl" would match "slash". + // + // That is weird and probably wrong: "foo.ba" should not match "foo.bar", + // or there is no way to distinguish the case where there were two values + // for "foo.bar" or one value each for "foo.ba" and "foo.bar". + // + // Apart from a single test case, I could not find any attested usage of + // this implementation detail. It isn't documented in the OpenAPI docs and + // is not shown in any of the example inputs. + // + // On that basis, I removed that test case. This implementation still does + // correctly handle variable type/attribute splits ("x", "y.z" / "x.y", "z") + // since that was required by the original "flattened" event representation. - if tc.matches { - match, err := q.Matches(tc.events) - assert.Nil(t, err, "Query '%s' should not error on match %v", tc.s, tc.events) - assert.True(t, match, "Query '%s' should match %v", tc.s, tc.events) - } else { - match, err := q.Matches(tc.events) - assert.Equal(t, tc.matchErr, err != nil, "Unexpected error for query '%s' match %v", tc.s, tc.events) - assert.False(t, match, "Query '%s' should not match %v", tc.s, tc.events) - } + for i, tc := range testCases { + t.Run(fmt.Sprintf("%02d", i+1), func(t *testing.T) { + c, err := query.New(tc.s) + if err != nil { + t.Fatalf("NewCompiled %#q: unexpected error: %v", tc.s, err) + } + + got, err := c.Matches(tc.events) + if err != nil { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got error %v", + tc.s, tc.events, err) + } + if got != tc.matches { + t.Errorf("Query: %#q\nInput: %+v\nMatches: got %v, want %v", + tc.s, tc.events, got, tc.matches) + } + }) } } -func TestMustParse(t *testing.T) { - assert.Panics(t, func() { query.MustParse("=") }) - assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) +func sortEvents(events []types.Event) []types.Event { + sort.Slice(events, func(i, j int) bool { + if events[i].Type == events[j].Type { + return events[i].Attributes[0].Key < events[j].Attributes[0].Key + } + return events[i].Type < events[j].Type + }) + return events } -func TestConditions(t *testing.T) { - txTime, err := time.Parse(time.RFC3339, "2013-05-03T14:45:00Z") +func TestExpandEvents(t *testing.T) { + expanded := query.ExpandEvents(apiEvents) + bz, err := json.Marshal(sortEvents(expanded)) require.NoError(t, err) - - testCases := []struct { - s string - conditions []query.Condition - }{ - { - s: "tm.events.type='NewBlock'", - conditions: []query.Condition{ - {CompositeKey: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}, - }, - }, - { - s: "tx.gas > 7 AND tx.gas < 9", - conditions: []query.Condition{ - {CompositeKey: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, - {CompositeKey: "tx.gas", Op: query.OpLess, Operand: int64(9)}, - }, - }, - { - s: "tx.time >= TIME 2013-05-03T14:45:00Z", - conditions: []query.Condition{ - {CompositeKey: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}, - }, - }, - { - s: "slashing EXISTS", - conditions: []query.Condition{ - {CompositeKey: "slashing", Op: query.OpExists}, - }, - }, - } - - for _, tc := range testCases { - q, err := query.New(tc.s) - require.Nil(t, err) - - c, err := q.Conditions() - require.NoError(t, err) - assert.Equal(t, tc.conditions, c) + bz2, err := json.Marshal(sortEvents(apiTypeEvents)) + require.NoError(t, err) + if string(bz) != string(bz2) { + t.Errorf("got %s, want %v", string(bz), string(bz2)) } } + +func TestAllMatchesAll(t *testing.T) { + events := newTestEvents( + ``, + `Asher|Roth=`, + `Route|66=`, + `Rilly|Blue=`, + ) + keys := make([]string, 0) + for k := range events { + keys = append(keys, k) + } + for _, key := range keys { + delete(events, key) + match, err := query.All.Matches(events) + if err != nil { + t.Errorf("Matches failed: %v", err) + } else if !match { + t.Errorf("Did not match on %+v ", events) + } + } +} + +// newTestEvent constructs an Event message from a template string. +// The format is "type|attr1=val1|attr2=val2|...". +func addNewTestEvent(events map[string][]string, s string) { + parts := strings.Split(s, "|") + key := parts[0] + for _, kv := range parts[1:] { + k, v := splitKV(kv) + k = key + "." + k + events[k] = append(events[k], v) + } +} + +// newTestEvents constructs a slice of Event messages by applying newTestEvent +// to each element of ss. +func newTestEvents(ss ...string) map[string][]string { + events := make(map[string][]string) + for _, s := range ss { + addNewTestEvent(events, s) + } + return events +} + +func splitKV(s string) (key, value string) { + kv := strings.SplitN(s, "=", 2) + return kv[0], kv[1] +} diff --git a/libs/pubsub/query/syntax/doc.go b/libs/pubsub/query/syntax/doc.go new file mode 100644 index 000000000..c6bedc381 --- /dev/null +++ b/libs/pubsub/query/syntax/doc.go @@ -0,0 +1,33 @@ +// Package syntax defines a scanner and parser for the Tendermint event filter +// query language. A query selects events by their types and attribute values. +// +// # Grammar +// +// The grammar of the query language is defined by the following EBNF: +// +// query = conditions EOF +// conditions = condition {"AND" condition} +// condition = tag comparison +// comparison = equal / order / contains / "EXISTS" +// equal = "=" (date / number / time / value) +// order = cmp (date / number / time) +// contains = "CONTAINS" value +// cmp = "<" / "<=" / ">" / ">=" +// +// The lexical terms are defined here using RE2 regular expression notation: +// +// // The name of an event attribute (type.value) +// tag = #'\w+(\.\w+)*' +// +// // A datestamp (YYYY-MM-DD) +// date = #'DATE \d{4}-\d{2}-\d{2}' +// +// // A number with optional fractional parts (0, 10, 3.25) +// number = #'\d+(\.\d+)?' +// +// // An RFC3339 timestamp (2021-11-23T22:04:19-09:00) +// time = #'TIME \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([-+]\d{2}:\d{2}|Z)' +// +// // A quoted literal string value ('a b c') +// value = #'\'[^\']*\'' +package syntax diff --git a/libs/pubsub/query/syntax/parser.go b/libs/pubsub/query/syntax/parser.go new file mode 100644 index 000000000..a100ec79c --- /dev/null +++ b/libs/pubsub/query/syntax/parser.go @@ -0,0 +1,213 @@ +package syntax + +import ( + "fmt" + "io" + "math" + "strconv" + "strings" + "time" +) + +// Parse parses the specified query string. It is shorthand for constructing a +// parser for s and calling its Parse method. +func Parse(s string) (Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Query is the root of the parse tree for a query. A query is the conjunction +// of one or more conditions. +type Query []Condition + +func (q Query) String() string { + ss := make([]string, len(q)) + for i, cond := range q { + ss[i] = cond.String() + } + return strings.Join(ss, " AND ") +} + +// A Condition is a single conditional expression, consisting of a tag, a +// comparison operator, and an optional argument. The type of the argument +// depends on the operator. +type Condition struct { + Tag string + Op Token + Arg *Arg + + opText string +} + +func (c Condition) String() string { + s := c.Tag + " " + c.opText + if c.Arg != nil { + return s + " " + c.Arg.String() + } + return s +} + +// An Arg is the argument of a comparison operator. +type Arg struct { + Type Token + text string +} + +func (a *Arg) String() string { + if a == nil { + return "" + } + switch a.Type { + case TString: + return "'" + a.text + "'" + case TTime: + return "TIME " + a.text + case TDate: + return "DATE " + a.text + default: + return a.text + } +} + +// Number returns the value of the argument text as a number, or a NaN if the +// text does not encode a valid number value. +func (a *Arg) Number() float64 { + if a == nil { + return -1 + } + v, err := strconv.ParseFloat(a.text, 64) + if err == nil && v >= 0 { + return v + } + return math.NaN() +} + +// Time returns the value of the argument text as a time, or the zero value if +// the text does not encode a timestamp or datestamp. +func (a *Arg) Time() time.Time { + var ts time.Time + if a == nil { + return ts + } + var err error + switch a.Type { + case TDate: + ts, err = ParseDate(a.text) + case TTime: + ts, err = ParseTime(a.text) + } + if err == nil { + return ts + } + return time.Time{} +} + +// Value returns the value of the argument text as a string, or "". +func (a *Arg) Value() string { + if a == nil { + return "" + } + return a.text +} + +// Parser is a query expression parser. The grammar for query expressions is +// defined in the syntax package documentation. +type Parser struct { + scanner *Scanner +} + +// NewParser constructs a new parser that reads the input from r. +func NewParser(r io.Reader) *Parser { + return &Parser{scanner: NewScanner(r)} +} + +// Parse parses the complete input and returns the resulting query. +func (p *Parser) Parse() (Query, error) { + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds := []Condition{cond} + for p.scanner.Next() != io.EOF { + if tok := p.scanner.Token(); tok != TAnd { + return nil, fmt.Errorf("offset %d: got %v, want %v", p.scanner.Pos(), tok, TAnd) + } + cond, err := p.parseCond() + if err != nil { + return nil, err + } + conds = append(conds, cond) + } + return conds, nil +} + +// parseCond parses a conditional expression: tag OP value. +func (p *Parser) parseCond() (Condition, error) { + var cond Condition + if err := p.require(TTag); err != nil { + return cond, err + } + cond.Tag = p.scanner.Text() + if err := p.require(TLeq, TGeq, TLt, TGt, TEq, TContains, TExists); err != nil { + return cond, err + } + cond.Op = p.scanner.Token() + cond.opText = p.scanner.Text() + + var err error + switch cond.Op { + case TLeq, TGeq, TLt, TGt: + err = p.require(TNumber, TTime, TDate) + case TEq: + err = p.require(TNumber, TTime, TDate, TString) + case TContains: + err = p.require(TString) + case TExists: + // no argument + return cond, nil + default: + return cond, fmt.Errorf("offset %d: unexpected operator %v", p.scanner.Pos(), cond.Op) + } + if err != nil { + return cond, err + } + cond.Arg = &Arg{Type: p.scanner.Token(), text: p.scanner.Text()} + return cond, nil +} + +// require advances the scanner and requires that the resulting token is one of +// the specified token types. +func (p *Parser) require(tokens ...Token) error { + if err := p.scanner.Next(); err != nil { + return fmt.Errorf("offset %d: %w", p.scanner.Pos(), err) + } + got := p.scanner.Token() + for _, tok := range tokens { + if tok == got { + return nil + } + } + return fmt.Errorf("offset %d: got %v, wanted %s", p.scanner.Pos(), got, tokLabel(tokens)) +} + +// tokLabel makes a human-readable summary string for the given token types. +func tokLabel(tokens []Token) string { + if len(tokens) == 1 { + return tokens[0].String() + } + last := len(tokens) - 1 + ss := make([]string, len(tokens)-1) + for i, tok := range tokens[:last] { + ss[i] = tok.String() + } + return strings.Join(ss, ", ") + " or " + tokens[last].String() +} + +// ParseDate parses s as a date string in the format used by DATE values. +func ParseDate(s string) (time.Time, error) { + return time.Parse("2006-01-02", s) +} + +// ParseTime parses s as a timestamp in the format used by TIME values. +func ParseTime(s string) (time.Time, error) { + return time.Parse(time.RFC3339, s) +} diff --git a/libs/pubsub/query/syntax/scanner.go b/libs/pubsub/query/syntax/scanner.go new file mode 100644 index 000000000..332e3f7b1 --- /dev/null +++ b/libs/pubsub/query/syntax/scanner.go @@ -0,0 +1,312 @@ +package syntax + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "time" + "unicode" +) + +// Token is the type of a lexical token in the query grammar. +type Token byte + +const ( + TInvalid = iota // invalid or unknown token + TTag // field tag: x.y + TString // string value: 'foo bar' + TNumber // number: 0, 15.5, 100 + TTime // timestamp: TIME yyyy-mm-ddThh:mm:ss([-+]hh:mm|Z) + TDate // datestamp: DATE yyyy-mm-dd + TAnd // operator: AND + TContains // operator: CONTAINS + TExists // operator: EXISTS + TEq // operator: = + TLt // operator: < + TLeq // operator: <= + TGt // operator: > + TGeq // operator: >= + + // Do not reorder these values without updating the scanner code. +) + +var tString = [...]string{ + TInvalid: "invalid token", + TTag: "tag", + TString: "string", + TNumber: "number", + TTime: "timestamp", + TDate: "datestamp", + TAnd: "AND operator", + TContains: "CONTAINS operator", + TExists: "EXISTS operator", + TEq: "= operator", + TLt: "< operator", + TLeq: "<= operator", + TGt: "> operator", + TGeq: ">= operator", +} + +func (t Token) String() string { + v := int(t) + if v > len(tString) { + return "unknown token type" + } + return tString[v] +} + +const ( + // TimeFormat is the format string used for timestamp values. + TimeFormat = time.RFC3339 + + // DateFormat is the format string used for datestamp values. + DateFormat = "2006-01-02" +) + +// Scanner reads lexical tokens of the query language from an input stream. +// Each call to Next advances the scanner to the next token, or reports an +// error. +type Scanner struct { + r *bufio.Reader + buf bytes.Buffer + tok Token + err error + + pos, last, end int +} + +// NewScanner constructs a new scanner that reads from r. +func NewScanner(r io.Reader) *Scanner { return &Scanner{r: bufio.NewReader(r)} } + +// Next advances s to the next token in the input, or reports an error. At the +// end of input, Next returns io.EOF. +func (s *Scanner) Next() error { + s.buf.Reset() + s.pos = s.end + s.tok = TInvalid + s.err = nil + + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } + if unicode.IsSpace(ch) { + s.pos = s.end + continue // skip whitespace + } + if '0' <= ch && ch <= '9' { + return s.scanNumber(ch) + } else if isTagRune(ch) { + return s.scanTagLike(ch) + } + switch ch { + case '\'': + return s.scanString(ch) + case '<', '>', '=': + return s.scanCompare(ch) + default: + return s.invalid(ch) + } + } +} + +// Token returns the type of the current input token. +func (s *Scanner) Token() Token { return s.tok } + +// Text returns the text of the current input token. +func (s *Scanner) Text() string { return s.buf.String() } + +// Pos returns the start offset of the current token in the input. +func (s *Scanner) Pos() int { return s.pos } + +// Err returns the last error reported by Next, if any. +func (s *Scanner) Err() error { return s.err } + +// scanNumber scans for numbers with optional fractional parts. +// Examples: 0, 1, 3.14 +func (s *Scanner) scanNumber(first rune) error { + s.buf.WriteRune(first) + if err := s.scanWhile(isDigit); err != nil { + return err + } + + ch, err := s.rune() + if err != nil && err != io.EOF { + return err + } + if ch == '.' { + s.buf.WriteRune(ch) + if err := s.scanWhile(isDigit); err != nil { + return err + } + } else { + s.unrune() + } + s.tok = TNumber + return nil +} + +func (s *Scanner) scanString(first rune) error { + // discard opening quote + for { + ch, err := s.rune() + if err != nil { + return s.fail(err) + } else if ch == first { + // discard closing quote + s.tok = TString + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) scanCompare(first rune) error { + s.buf.WriteRune(first) + switch first { + case '=': + s.tok = TEq + return nil + case '<': + s.tok = TLt + case '>': + s.tok = TGt + default: + return s.invalid(first) + } + + ch, err := s.rune() + if err == io.EOF { + return nil // the assigned token is correct + } else if err != nil { + return s.fail(err) + } + if ch == '=' { + s.buf.WriteRune(ch) + s.tok++ // depends on token order + return nil + } + s.unrune() + return nil +} + +func (s *Scanner) scanTagLike(first rune) error { + s.buf.WriteRune(first) + var hasSpace bool + for { + ch, err := s.rune() + if err == io.EOF { + break + } else if err != nil { + return s.fail(err) + } + if !isTagRune(ch) { + hasSpace = ch == ' ' // to check for TIME, DATE + break + } + s.buf.WriteRune(ch) + } + + text := s.buf.String() + switch text { + case "TIME": + if hasSpace { + return s.scanTimestamp() + } + s.tok = TTag + case "DATE": + if hasSpace { + return s.scanDatestamp() + } + s.tok = TTag + case "AND": + s.tok = TAnd + case "EXISTS": + s.tok = TExists + case "CONTAINS": + s.tok = TContains + default: + s.tok = TTag + } + s.unrune() + return nil +} + +func (s *Scanner) scanTimestamp() error { + s.buf.Reset() // discard "TIME" label + if err := s.scanWhile(isTimeRune); err != nil { + return err + } + if ts, err := time.Parse(TimeFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid TIME value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("timestamp year %d out of range", ts.Year())) + } + s.tok = TTime + return nil +} + +func (s *Scanner) scanDatestamp() error { + s.buf.Reset() // discard "DATE" label + if err := s.scanWhile(isDateRune); err != nil { + return err + } + if ts, err := time.Parse(DateFormat, s.buf.String()); err != nil { + return s.fail(fmt.Errorf("invalid DATE value: %w", err)) + } else if y := ts.Year(); y < 1900 || y > 2999 { + return s.fail(fmt.Errorf("datestamp year %d out of range", ts.Year())) + } + s.tok = TDate + return nil +} + +func (s *Scanner) scanWhile(ok func(rune) bool) error { + for { + ch, err := s.rune() + if err == io.EOF { + return nil + } else if err != nil { + return s.fail(err) + } else if !ok(ch) { + s.unrune() + return nil + } + s.buf.WriteRune(ch) + } +} + +func (s *Scanner) rune() (rune, error) { + ch, nb, err := s.r.ReadRune() + s.last = nb + s.end += nb + return ch, err +} + +func (s *Scanner) unrune() { + _ = s.r.UnreadRune() + s.end -= s.last +} + +func (s *Scanner) fail(err error) error { + s.err = err + return err +} + +func (s *Scanner) invalid(ch rune) error { + return s.fail(fmt.Errorf("invalid input %c at offset %d", ch, s.end)) +} + +func isDigit(r rune) bool { return '0' <= r && r <= '9' } + +func isTagRune(r rune) bool { + return r == '.' || r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r) +} + +func isTimeRune(r rune) bool { + return strings.ContainsRune("-T:+Z", r) || isDigit(r) +} + +func isDateRune(r rune) bool { return isDigit(r) || r == '-' } diff --git a/libs/pubsub/query/syntax/syntax_test.go b/libs/pubsub/query/syntax/syntax_test.go new file mode 100644 index 000000000..ac95fd8b1 --- /dev/null +++ b/libs/pubsub/query/syntax/syntax_test.go @@ -0,0 +1,190 @@ +package syntax_test + +import ( + "io" + "reflect" + "strings" + "testing" + + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" +) + +func TestScanner(t *testing.T) { + tests := []struct { + input string + want []syntax.Token + }{ + // Empty inputs + {"", nil}, + {" ", nil}, + {"\t\n ", nil}, + + // Numbers + {`0 123`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + {`0.32 3.14`, []syntax.Token{syntax.TNumber, syntax.TNumber}}, + + // Tags + {`foo foo.bar`, []syntax.Token{syntax.TTag, syntax.TTag}}, + + // Strings (values) + {` '' x 'x' 'x y'`, []syntax.Token{syntax.TString, syntax.TTag, syntax.TString, syntax.TString}}, + {` 'you are not your job' `, []syntax.Token{syntax.TString}}, + + // Comparison operators + {`< <= = > >=`, []syntax.Token{ + syntax.TLt, syntax.TLeq, syntax.TEq, syntax.TGt, syntax.TGeq, + }}, + + // Mixed values of various kinds. + {`x AND y`, []syntax.Token{syntax.TTag, syntax.TAnd, syntax.TTag}}, + {`x.y CONTAINS 'z'`, []syntax.Token{syntax.TTag, syntax.TContains, syntax.TString}}, + {`foo EXISTS`, []syntax.Token{syntax.TTag, syntax.TExists}}, + {`and AND`, []syntax.Token{syntax.TTag, syntax.TAnd}}, + + // Timestamp + {`TIME 2021-11-23T15:16:17Z`, []syntax.Token{syntax.TTime}}, + + // Datestamp + {`DATE 2021-11-23`, []syntax.Token{syntax.TDate}}, + } + + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + var got []syntax.Token + for s.Next() == nil { + got = append(got, s.Token()) + } + if err := s.Err(); err != io.EOF { + t.Errorf("Next: unexpected error: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Logf("Scanner input: %q", test.input) + t.Errorf("Wrong tokens:\ngot: %+v\nwant: %+v", got, test.want) + } + } +} + +func TestScannerErrors(t *testing.T) { + tests := []struct { + input string + }{ + {`'incomplete string`}, + {`-23`}, + {`&`}, + {`DATE xyz-pdq`}, + {`DATE xyzp-dq-zv`}, + {`DATE 0000-00-00`}, + {`DATE 0000-00-000`}, + {`DATE 2021-01-99`}, + {`TIME 2021-01-01T34:56:78Z`}, + {`TIME 2021-01-99T14:56:08Z`}, + {`TIME 2021-01-99T34:56:08`}, + {`TIME 2021-01-99T34:56:11+3`}, + } + for _, test := range tests { + s := syntax.NewScanner(strings.NewReader(test.input)) + if err := s.Next(); err == nil { + t.Errorf("Next: got %v (%#q), want error", s.Token(), s.Text()) + } + } +} + +// These parser tests were copied from the original implementation of the query +// parser, and are preserved here as a compatibility check. +func TestParseValid(t *testing.T) { + tests := []struct { + input string + valid bool + }{ + {"tm.events.type='NewBlock'", true}, + {"tm.events.type = 'NewBlock'", true}, + {"tm.events.name = ''", true}, + {"tm.events.type='TIME'", true}, + {"tm.events.type='DATE'", true}, + {"tm.events.type='='", true}, + {"tm.events.type='TIME", false}, + {"tm.events.type=TIME'", false}, + {"tm.events.type==", false}, + {"tm.events.type=NewBlock", false}, + {">==", false}, + {"tm.events.type 'NewBlock' =", false}, + {"tm.events.type>'NewBlock'", false}, + {"", false}, + {"=", false}, + {"='NewBlock'", false}, + {"tm.events.type=", false}, + + {"tm.events.typeNewBlock", false}, + {"tm.events.type'NewBlock'", false}, + {"'NewBlock'", false}, + {"NewBlock", false}, + {"", false}, + + {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, + {"tm.events.type='NewBlock' AND", false}, + {"tm.events.type='NewBlock' AN", false}, + {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, + {"AND tm.events.type='NewBlock' ", false}, + + {"abci.account.name CONTAINS 'Igor'", true}, + + {"tx.date > DATE 2013-05-03", true}, + {"tx.date < DATE 2013-05-03", true}, + {"tx.date <= DATE 2013-05-03", true}, + {"tx.date >= DATE 2013-05-03", true}, + {"tx.date >= DAT 2013-05-03", false}, + {"tx.date <= DATE2013-05-03", false}, + {"tx.date <= DATE -05-03", false}, + {"tx.date >= DATE 20130503", false}, + {"tx.date >= DATE 2013+01-03", false}, + // incorrect year, month, day + {"tx.date >= DATE 0013-01-03", false}, + {"tx.date >= DATE 2013-31-03", false}, + {"tx.date >= DATE 2013-01-83", false}, + + {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, + {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, + {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME2013-05-03T14:45:00Z", false}, + {"tx.date = IME 2013-05-03T14:45:00Z", false}, + {"tx.date = TIME 2013-05-:45:00Z", false}, + {"tx.date >= TIME 2013-05-03T14:45:00", false}, + {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, + {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, + + {"account.balance=100", true}, + {"account.balance >= 200", true}, + {"account.balance >= -300", false}, + {"account.balance >>= 400", false}, + {"account.balance=33.22.1", false}, + + {"slashing.amount EXISTS", true}, + {"slashing.amount EXISTS AND account.balance=100", true}, + {"account.balance=100 AND slashing.amount EXISTS", true}, + {"slashing EXISTS", true}, + + {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, + {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, + } + + for _, test := range tests { + q, err := syntax.Parse(test.input) + if test.valid != (err == nil) { + t.Errorf("Parse %#q: valid %v got err=%v", test.input, test.valid, err) + } + + // For valid queries, check that the query round-trips. + if test.valid { + qstr := q.String() + r, err := syntax.Parse(qstr) + if err != nil { + t.Errorf("Reparse %#q failed: %v", qstr, err) + } + if rstr := r.String(); rstr != qstr { + t.Errorf("Reparse diff\nold: %#q\nnew: %#q", qstr, rstr) + } + } + } +} diff --git a/light/client_test.go b/light/client_test.go index 4a7503d7d..09067c9d9 100644 --- a/light/client_test.go +++ b/light/client_test.go @@ -12,6 +12,7 @@ import ( dbm "github.com/tendermint/tm-db" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/light" "github.com/tendermint/tendermint/light/provider" @@ -21,7 +22,7 @@ import ( ) const ( - chainID = "test" + chainID = test.DefaultTestChainID ) var ( diff --git a/light/example_test.go b/light/example_test.go index 5bd604b12..8423cefad 100644 --- a/light/example_test.go +++ b/light/example_test.go @@ -30,10 +30,7 @@ func ExampleClient_Update() { } defer os.RemoveAll(dbDir) - var ( - config = rpctest.GetConfig() - chainID = config.ChainID() - ) + var config = rpctest.GetConfig() primary, err := httpp.New(chainID, config.RPC.ListenAddress) if err != nil { @@ -98,10 +95,7 @@ func ExampleClient_VerifyLightBlockAtHeight() { } defer os.RemoveAll(dbDir) - var ( - config = rpctest.GetConfig() - chainID = config.ChainID() - ) + var config = rpctest.GetConfig() primary, err := httpp.New(chainID, config.RPC.ListenAddress) if err != nil { diff --git a/mempool/v0/clist_mempool_test.go b/mempool/v0/clist_mempool_test.go index 533d24d16..ca5d1d269 100644 --- a/mempool/v0/clist_mempool_test.go +++ b/mempool/v0/clist_mempool_test.go @@ -21,6 +21,7 @@ import ( abciserver "github.com/tendermint/tendermint/abci/server" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/libs/service" @@ -34,7 +35,7 @@ import ( type cleanupFunc func() func newMempoolWithAppMock(cc proxy.ClientCreator, client abciclient.Client) (*CListMempool, cleanupFunc, error) { - conf := config.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfigMock(cc, conf, client) return mp, cu, nil @@ -57,7 +58,7 @@ func newMempoolWithAppAndConfigMock(cc proxy.ClientCreator, } func newMempoolWithApp(cc proxy.ClientCreator) (*CListMempool, cleanupFunc) { - conf := config.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfig(cc, conf) return mp, cu @@ -532,7 +533,7 @@ func TestMempoolTxsBytes(t *testing.T) { app := kvstore.NewInMemoryApplication() cc := proxy.NewLocalClientCreator(app) - cfg := config.ResetTestRoot("mempool_test") + cfg := test.ResetTestRoot("mempool_test") cfg.Mempool.MaxTxsBytes = 100 mp, cleanup := newMempoolWithAppAndConfig(cc, cfg) @@ -633,7 +634,7 @@ func TestMempoolRemoteAppConcurrency(t *testing.T) { } }) - cfg := config.ResetTestRoot("mempool_test") + cfg := test.ResetTestRoot("mempool_test") mp, cleanup := newMempoolWithAppAndConfig(proxy.NewRemoteClientCreator(sockPath, "socket", true), cfg) defer cleanup() diff --git a/mempool/v1/mempool_test.go b/mempool/v1/mempool_test.go index 74db1813a..d873dcfa0 100644 --- a/mempool/v1/mempool_test.go +++ b/mempool/v1/mempool_test.go @@ -18,7 +18,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" @@ -80,7 +80,7 @@ func setup(t testing.TB, cacheSize int, options ...TxMempoolOption) *TxMempool { app := &application{kvstore.NewInMemoryApplication()} cc := proxy.NewLocalClientCreator(app) - cfg := config.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|")) + cfg := test.ResetTestRoot(strings.ReplaceAll(t.Name(), "/", "|")) cfg.Mempool.CacheSize = cacheSize appConnMem, err := cc.NewABCIClient() diff --git a/mempool/v1/reactor_test.go b/mempool/v1/reactor_test.go index d0025e28e..a114d4fc6 100644 --- a/mempool/v1/reactor_test.go +++ b/mempool/v1/reactor_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/internal/test" cfg "github.com/tendermint/tendermint/config" @@ -128,7 +129,7 @@ func mempoolLogger() log.Logger { } func newMempoolWithApp(cc proxy.ClientCreator) (*TxMempool, func()) { - conf := cfg.ResetTestRoot("mempool_test") + conf := test.ResetTestRoot("mempool_test") mp, cu := newMempoolWithAppAndConfig(cc, conf) return mp, cu diff --git a/node/doc.go b/node/doc.go index 3a145c573..3b4e9b71b 100644 --- a/node/doc.go +++ b/node/doc.go @@ -31,7 +31,7 @@ To replace the built-in p2p.Reactor, use the CustomReactors option: dbProvider, metricsProvider, logger, - CustomReactors(map[string]p2p.Reactor{"BLOCKCHAIN": customBlockchainReactor}), + CustomReactors(map[string]p2p.Reactor{"BLOCKSYNC": customBlocksyncReactor}), ) The list of existing reactors can be found in CustomReactors documentation. diff --git a/node/node.go b/node/node.go index 4966ba68d..067ae39a2 100644 --- a/node/node.go +++ b/node/node.go @@ -146,7 +146,7 @@ type blockSyncReactor interface { // result in replacing it with the custom one. // // - MEMPOOL -// - BLOCKCHAIN +// - BLOCKSYNC // - CONSENSUS // - EVIDENCE // - PEX @@ -303,7 +303,7 @@ func createAndStartIndexerService( blockIndexer = &blockidxnull.BlockerIndexer{} } - indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus) + indexerService := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) indexerService.SetLogger(logger.With("module", "txindex")) if err := indexerService.Start(); err != nil { @@ -335,8 +335,10 @@ func logNodeStartupInfo(state sm.State, pubKey crypto.PubKey, logger, consensusL // Log the version info. logger.Info("Version info", "tendermint_version", version.TMCoreSemVer, + "abci", version.ABCISemVer, "block", version.BlockProtocol, "p2p", version.P2PProtocol, + "commit_hash", version.TMGitCommitHash, ) // If the state and software differ in block version, at least log it. @@ -439,7 +441,7 @@ func createEvidenceReactor(config *cfg.Config, dbProvider DBProvider, return evidenceReactor, evidencePool, nil } -func createBlockchainReactor(config *cfg.Config, +func createBlocksyncReactor(config *cfg.Config, state sm.State, blockExec *sm.BlockExecutor, blockStore *store.BlockStore, @@ -455,7 +457,7 @@ func createBlockchainReactor(config *cfg.Config, return nil, fmt.Errorf("unknown fastsync version %s", config.BlockSync.Version) } - bcReactor.SetLogger(logger.With("module", "blockchain")) + bcReactor.SetLogger(logger.With("module", "blocksync")) return bcReactor, nil } @@ -582,7 +584,7 @@ func createSwitch(config *cfg.Config, ) sw.SetLogger(p2pLogger) sw.AddReactor("MEMPOOL", mempoolReactor) - sw.AddReactor("BLOCKCHAIN", bcReactor) + sw.AddReactor("BLOCKSYNC", bcReactor) sw.AddReactor("CONSENSUS", consensusReactor) sw.AddReactor("EVIDENCE", evidenceReactor) sw.AddReactor("STATESYNC", stateSyncReactor) @@ -801,20 +803,21 @@ func NewNode(config *cfg.Config, return nil, err } - // make block executor for consensus and blockchain reactors to execute blocks + // make block executor for consensus and blocksync reactors to execute blocks blockExec := sm.NewBlockExecutor( stateStore, logger.With("module", "state"), proxyApp.Consensus(), mempool, evidencePool, + blockStore, sm.BlockExecutorWithMetrics(smMetrics), ) - // Make BlockchainReactor. Don't start block sync if we're doing a state sync first. - bcReactor, err := createBlockchainReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger) + // Make BlocksyncReactor. Don't start block sync if we're doing a state sync first. + bcReactor, err := createBlocksyncReactor(config, state, blockExec, blockStore, blockSync && !stateSync, logger) if err != nil { - return nil, fmt.Errorf("could not create blockchain reactor: %w", err) + return nil, fmt.Errorf("could not create blocksync reactor: %w", err) } // Make ConsensusReactor. Don't enable fully if doing a state sync and/or block sync first. @@ -987,7 +990,7 @@ func (n *Node) OnStart() error { if n.stateSync { bcR, ok := n.bcReactor.(blockSyncReactor) if !ok { - return fmt.Errorf("this blockchain reactor does not support switching from state sync") + return fmt.Errorf("this blocksync reactor does not support switching from state sync") } err := startStateSync(n.stateSyncReactor, bcR, n.consensusReactor, n.stateSyncProvider, n.config.StateSync, n.config.BlockSyncMode, n.stateStore, n.blockStore, n.stateSyncGenesis) @@ -1384,6 +1387,11 @@ func LoadStateFromDBOrGenesisDocProvider( if err != nil { return sm.State{}, nil, err } + + err = genDoc.ValidateAndComplete() + if err != nil { + return sm.State{}, nil, fmt.Errorf("error in genesis doc: %w", err) + } // save genesis doc to prevent a certain class of user errors (e.g. when it // was changed, accidentally or not). Also good for audit trail. if err := saveGenesisDoc(stateDB, genDoc); err != nil { diff --git a/node/node_test.go b/node/node_test.go index 006f80c05..bb0508929 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -18,6 +18,7 @@ import ( cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/evidence" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" tmrand "github.com/tendermint/tendermint/libs/rand" mempl "github.com/tendermint/tendermint/mempool" @@ -35,7 +36,7 @@ import ( ) func TestNodeStartStop(t *testing.T) { - config := cfg.ResetTestRoot("node_node_test") + config := test.ResetTestRoot("node_node_test") defer os.RemoveAll(config.RootDir) // create & start node @@ -97,7 +98,7 @@ func TestSplitAndTrimEmpty(t *testing.T) { } func TestNodeDelayedStart(t *testing.T) { - config := cfg.ResetTestRoot("node_delayed_start_test") + config := test.ResetTestRoot("node_delayed_start_test") defer os.RemoveAll(config.RootDir) now := tmtime.Now() @@ -115,7 +116,7 @@ func TestNodeDelayedStart(t *testing.T) { } func TestNodeSetAppVersion(t *testing.T) { - config := cfg.ResetTestRoot("node_app_version_test") + config := test.ResetTestRoot("node_app_version_test") defer os.RemoveAll(config.RootDir) // create & start node @@ -137,7 +138,7 @@ func TestNodeSetAppVersion(t *testing.T) { func TestNodeSetPrivValTCP(t *testing.T) { addr := "tcp://" + testFreeAddr(t) - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = addr @@ -150,7 +151,7 @@ func TestNodeSetPrivValTCP(t *testing.T) { signerServer := privval.NewSignerServer( dialerEndpoint, - config.ChainID(), + test.DefaultTestChainID, types.NewMockPV(), ) @@ -171,7 +172,7 @@ func TestNodeSetPrivValTCP(t *testing.T) { func TestPrivValidatorListenAddrNoProtocol(t *testing.T) { addrNoPrefix := testFreeAddr(t) - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = addrNoPrefix @@ -183,7 +184,7 @@ func TestNodeSetPrivValIPC(t *testing.T) { tmpfile := "/tmp/kms." + tmrand.Str(6) + ".sock" defer os.Remove(tmpfile) // clean up - config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config := test.ResetTestRoot("node_priv_val_tcp_test") defer os.RemoveAll(config.RootDir) config.BaseConfig.PrivValidatorListenAddr = "unix://" + tmpfile @@ -196,7 +197,7 @@ func TestNodeSetPrivValIPC(t *testing.T) { pvsc := privval.NewSignerServer( dialerEndpoint, - config.ChainID(), + test.DefaultTestChainID, types.NewMockPV(), ) @@ -223,7 +224,7 @@ func testFreeAddr(t *testing.T) string { // create a proposal block using real and full // mempool and evidence pool and validate it. func TestCreateProposalBlock(t *testing.T) { - config := cfg.ResetTestRoot("node_create_proposal") + config := test.ResetTestRoot("node_create_proposal") defer os.RemoveAll(config.RootDir) cc := proxy.NewLocalClientCreator(kvstore.NewInMemoryApplication()) proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) @@ -305,6 +306,7 @@ func TestCreateProposalBlock(t *testing.T) { proxyApp.Consensus(), mempool, evidencePool, + blockStore, ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) @@ -334,7 +336,7 @@ func TestCreateProposalBlock(t *testing.T) { } func TestMaxProposalBlockSize(t *testing.T) { - config := cfg.ResetTestRoot("node_create_proposal") + config := test.ResetTestRoot("node_create_proposal") defer os.RemoveAll(config.RootDir) cc := proxy.NewLocalClientCreator(kvstore.NewInMemoryApplication()) proxyApp := proxy.NewAppConns(cc, proxy.NopMetrics()) @@ -376,6 +378,8 @@ func TestMaxProposalBlockSize(t *testing.T) { ) } + blockStore := store.NewBlockStore(dbm.NewMemDB()) + // fill the mempool with one txs just below the maximum size txLength := int(types.MaxDataBytesNoEvidence(maxBytes, 1)) tx := tmrand.Bytes(txLength - 4) // to account for the varint @@ -388,6 +392,7 @@ func TestMaxProposalBlockSize(t *testing.T) { proxyApp.Consensus(), mempool, sm.EmptyEvidencePool{}, + blockStore, ) commit := types.NewCommit(height-1, 0, types.BlockID{}, nil) @@ -410,7 +415,7 @@ func TestMaxProposalBlockSize(t *testing.T) { } func TestNodeNewNodeCustomReactors(t *testing.T) { - config := cfg.ResetTestRoot("node_new_node_custom_reactors_test") + config := test.ResetTestRoot("node_new_node_custom_reactors_test") defer os.RemoveAll(config.RootDir) cr := p2pmock.NewReactor() @@ -422,7 +427,7 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { RecvMessageCapacity: 100, }, } - customBlockchainReactor := p2pmock.NewReactor() + customBlocksyncReactor := p2pmock.NewReactor() nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) require.NoError(t, err) @@ -435,7 +440,7 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { DefaultDBProvider, DefaultMetricsProvider(config.Instrumentation), log.TestingLogger(), - CustomReactors(map[string]p2p.Reactor{"FOO": cr, "BLOCKCHAIN": customBlockchainReactor}), + CustomReactors(map[string]p2p.Reactor{"FOO": cr, "BLOCKSYNC": customBlocksyncReactor}), ) require.NoError(t, err) @@ -446,8 +451,8 @@ func TestNodeNewNodeCustomReactors(t *testing.T) { assert.True(t, cr.IsRunning()) assert.Equal(t, cr, n.Switch().Reactor("FOO")) - assert.True(t, customBlockchainReactor.IsRunning()) - assert.Equal(t, customBlockchainReactor, n.Switch().Reactor("BLOCKCHAIN")) + assert.True(t, customBlocksyncReactor.IsRunning()) + assert.Equal(t, customBlocksyncReactor, n.Switch().Reactor("BLOCKSYNC")) channels := n.NodeInfo().(p2p.DefaultNodeInfo).Channels assert.Contains(t, channels, mempl.MempoolChannel) diff --git a/p2p/conn/evil_secret_connection_test.go b/p2p/conn/evil_secret_connection_test.go index 320a60ba1..455934e4c 100644 --- a/p2p/conn/evil_secret_connection_test.go +++ b/p2p/conn/evil_secret_connection_test.go @@ -7,7 +7,7 @@ import ( "testing" gogotypes "github.com/cosmos/gogoproto/types" - "github.com/gtank/merlin" + "github.com/oasisprotocol/curve25519-voi/primitives/merlin" "github.com/stretchr/testify/assert" "golang.org/x/crypto/chacha20poly1305" @@ -208,9 +208,7 @@ func (c *evilConn) signChallenge() []byte { const challengeSize = 32 var challenge [challengeSize]byte - challengeSlice := transcript.ExtractBytes(labelSecretConnectionMac, challengeSize) - - copy(challenge[:], challengeSlice[0:challengeSize]) + transcript.ExtractBytes(challenge[:], labelSecretConnectionMac) sendAead, err := chacha20poly1305.New(sendSecret[:]) if err != nil { diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index 626f2599c..1f95df37f 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -14,8 +14,8 @@ import ( "time" gogotypes "github.com/cosmos/gogoproto/types" - "github.com/gtank/merlin" pool "github.com/libp2p/go-buffer-pool" + "github.com/oasisprotocol/curve25519-voi/primitives/merlin" "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/curve25519" "golang.org/x/crypto/hkdf" @@ -38,16 +38,16 @@ const ( aeadSizeOverhead = 16 // overhead of poly 1305 authentication tag aeadKeySize = chacha20poly1305.KeySize aeadNonceSize = chacha20poly1305.NonceSize + + labelEphemeralLowerPublicKey = "EPHEMERAL_LOWER_PUBLIC_KEY" + labelEphemeralUpperPublicKey = "EPHEMERAL_UPPER_PUBLIC_KEY" + labelDHSecret = "DH_SECRET" + labelSecretConnectionMac = "SECRET_CONNECTION_MAC" ) var ( ErrSmallOrderRemotePubKey = errors.New("detected low order point from remote peer") - labelEphemeralLowerPublicKey = []byte("EPHEMERAL_LOWER_PUBLIC_KEY") - labelEphemeralUpperPublicKey = []byte("EPHEMERAL_UPPER_PUBLIC_KEY") - labelDHSecret = []byte("DH_SECRET") - labelSecretConnectionMac = []byte("SECRET_CONNECTION_MAC") - secretConnKeyAndChallengeGen = []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN") ) @@ -132,9 +132,7 @@ func MakeSecretConnection(conn io.ReadWriteCloser, locPrivKey crypto.PrivKey) (* const challengeSize = 32 var challenge [challengeSize]byte - challengeSlice := transcript.ExtractBytes(labelSecretConnectionMac, challengeSize) - - copy(challenge[:], challengeSlice[0:challengeSize]) + transcript.ExtractBytes(challenge[:], labelSecretConnectionMac) sendAead, err := chacha20poly1305.New(sendSecret[:]) if err != nil { diff --git a/p2p/errors.go b/p2p/errors.go index 3650a7a0a..4fc915292 100644 --- a/p2p/errors.go +++ b/p2p/errors.go @@ -145,6 +145,13 @@ func (e ErrTransportClosed) Error() string { return "transport has been closed" } +// ErrPeerRemoval is raised when attempting to remove a peer results in an error. +type ErrPeerRemoval struct{} + +func (e ErrPeerRemoval) Error() string { + return "peer removal failed" +} + //------------------------------------------------------------------- type ErrNetAddressNoID struct { diff --git a/p2p/metrics.gen.go b/p2p/metrics.gen.go index b2b0b25c8..98fb0121f 100644 --- a/p2p/metrics.gen.go +++ b/p2p/metrics.gen.go @@ -25,25 +25,25 @@ func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { Subsystem: MetricsSubsystem, Name: "peer_receive_bytes_total", Help: "Number of bytes received from a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id", "chID")).With(labelsAndValues...), PeerSendBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_send_bytes_total", Help: "Number of bytes sent to a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id", "chID")).With(labelsAndValues...), PeerPendingSendBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_pending_send_bytes", Help: "Pending bytes to be sent to a given peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id")).With(labelsAndValues...), NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "num_txs", Help: "Number of transactions submitted by each peer.", - }, labels).With(labelsAndValues...), + }, append(labels, "peer_id")).With(labelsAndValues...), } } diff --git a/p2p/metrics.go b/p2p/metrics.go index 67d6ae668..7e21870c7 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -17,11 +17,11 @@ type Metrics struct { // Number of peers. Peers metrics.Gauge // Number of bytes received from a given peer. - PeerReceiveBytesTotal metrics.Counter + PeerReceiveBytesTotal metrics.Counter `metrics_labels:"peer_id,chID"` // Number of bytes sent to a given peer. - PeerSendBytesTotal metrics.Counter + PeerSendBytesTotal metrics.Counter `metrics_labels:"peer_id,chID"` // Pending bytes to be sent to a given peer. - PeerPendingSendBytes metrics.Gauge + PeerPendingSendBytes metrics.Gauge `metrics_labels:"peer_id"` // Number of transactions submitted by each peer. - NumTxs metrics.Gauge + NumTxs metrics.Gauge `metrics_labels:"peer_id"` } diff --git a/p2p/mock/peer.go b/p2p/mock/peer.go index 59f6e0f4a..10254c343 100644 --- a/p2p/mock/peer.go +++ b/p2p/mock/peer.go @@ -68,3 +68,5 @@ func (mp *Peer) RemoteIP() net.IP { return mp.ip } func (mp *Peer) SocketAddr() *p2p.NetAddress { return mp.addr } func (mp *Peer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } func (mp *Peer) CloseConn() error { return nil } +func (mp *Peer) SetRemovalFailed() {} +func (mp *Peer) GetRemovalFailed() bool { return false } diff --git a/p2p/mocks/peer.go b/p2p/mocks/peer.go index e195c78bb..a9151c7d8 100644 --- a/p2p/mocks/peer.go +++ b/p2p/mocks/peer.go @@ -53,6 +53,20 @@ func (_m *Peer) Get(_a0 string) interface{} { return r0 } +// GetRemovalFailed provides a mock function with given fields: +func (_m *Peer) GetRemovalFailed() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + // ID provides a mock function with given fields: func (_m *Peer) ID() p2p.ID { ret := _m.Called() @@ -244,6 +258,11 @@ func (_m *Peer) SetLogger(_a0 log.Logger) { _m.Called(_a0) } +// SetRemovalFailed provides a mock function with given fields: +func (_m *Peer) SetRemovalFailed() { + _m.Called() +} + // SocketAddr provides a mock function with given fields: func (_m *Peer) SocketAddr() *p2p.NetAddress { ret := _m.Called() diff --git a/p2p/peer.go b/p2p/peer.go index 751ca3cf2..d8d61a7a0 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -39,6 +39,9 @@ type Peer interface { Set(string, interface{}) Get(string) interface{} + + SetRemovalFailed() + GetRemovalFailed() bool } //---------------------------------------------------------- @@ -117,6 +120,9 @@ type peer struct { metrics *Metrics metricsTicker *time.Ticker + + // When removal of a peer fails, we set this flag + removalAttemptFailed bool } type PeerOption func(*peer) @@ -316,6 +322,14 @@ func (p *peer) CloseConn() error { return p.peerConn.conn.Close() } +func (p *peer) SetRemovalFailed() { + p.removalAttemptFailed = true +} + +func (p *peer) GetRemovalFailed() bool { + return p.removalAttemptFailed +} + //--------------------------------------------------- // methods only used for testing // TODO: can we remove these? diff --git a/p2p/peer_set.go b/p2p/peer_set.go index 38dff7a9f..30bcc4d32 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -47,6 +47,9 @@ func (ps *PeerSet) Add(peer Peer) error { if ps.lookup[peer.ID()] != nil { return ErrSwitchDuplicatePeerID{peer.ID()} } + if peer.GetRemovalFailed() { + return ErrPeerRemoval{} + } index := len(ps.list) // Appending is safe even with other goroutines @@ -107,6 +110,12 @@ func (ps *PeerSet) Remove(peer Peer) bool { item := ps.lookup[peer.ID()] if item == nil { + // Removing the peer has failed so we set a flag to mark that a removal was attempted. + // This can happen when the peer add routine from the switch is running in + // parallel to the receive routine of MConn. + // There is an error within MConn but the switch has not actually added the peer to the peer set yet. + // Setting this flag will prevent a peer from being added to a node's peer set afterwards. + peer.SetRemovalFailed() return false } diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index b61b43f10..db3d9261e 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -32,6 +32,8 @@ func (mp *mockPeer) RemoteIP() net.IP { return mp.ip } func (mp *mockPeer) SocketAddr() *NetAddress { return nil } func (mp *mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } func (mp *mockPeer) CloseConn() error { return nil } +func (mp *mockPeer) SetRemovalFailed() {} +func (mp *mockPeer) GetRemovalFailed() bool { return false } // Returns a mock peer func newMockPeer(ip net.IP) *mockPeer { diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index 95936a43c..2b8071041 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -5,9 +5,9 @@ package pex import ( - crand "crypto/rand" "encoding/binary" "fmt" + "hash" "math" "math/rand" "net" @@ -104,15 +104,18 @@ type addrBook struct { filePath string key string // random prefix for bucket placement routabilityStrict bool - hashKey []byte + hasher hash.Hash64 wg sync.WaitGroup } -func newHashKey() []byte { - result := make([]byte, highwayhash.Size) - crand.Read(result) //nolint:errcheck // ignore error - return result +func mustNewHasher() hash.Hash64 { + key := crypto.CRandBytes(highwayhash.Size) + hasher, err := highwayhash.New64(key) + if err != nil { + panic(err) + } + return hasher } // NewAddrBook creates a new address book. @@ -126,7 +129,6 @@ func NewAddrBook(filePath string, routabilityStrict bool) AddrBook { badPeers: make(map[p2p.ID]*knownAddress), filePath: filePath, routabilityStrict: routabilityStrict, - hashKey: newHashKey(), } am.init() am.BaseService = *service.NewBaseService(nil, "AddrBook", am) @@ -147,6 +149,7 @@ func (a *addrBook) init() { for i := range a.bucketsOld { a.bucketsOld[i] = make(map[string]*knownAddress) } + a.hasher = mustNewHasher() } // OnStart implements Service. @@ -938,10 +941,7 @@ func groupKeyFor(na *p2p.NetAddress, routabilityStrict bool) string { } func (a *addrBook) hash(b []byte) ([]byte, error) { - hasher, err := highwayhash.New64(a.hashKey) - if err != nil { - return nil, err - } - hasher.Write(b) - return hasher.Sum(nil), nil + a.hasher.Reset() + a.hasher.Write(b) + return a.hasher.Sum(nil), nil } diff --git a/p2p/pex/bench_test.go b/p2p/pex/bench_test.go new file mode 100644 index 000000000..13c37f7b1 --- /dev/null +++ b/p2p/pex/bench_test.go @@ -0,0 +1,24 @@ +package pex + +import ( + "testing" + + "github.com/tendermint/tendermint/p2p" +) + +func BenchmarkAddrBook_hash(b *testing.B) { + book := &addrBook{ + ourAddrs: make(map[string]struct{}), + privateIDs: make(map[p2p.ID]struct{}), + addrLookup: make(map[p2p.ID]*knownAddress), + badPeers: make(map[p2p.ID]*knownAddress), + filePath: "", + routabilityStrict: true, + } + book.init() + msg := []byte(`foobar`) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = book.hash(msg) + } +} diff --git a/p2p/switch.go b/p2p/switch.go index 3214de223..884fd883e 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -370,6 +370,10 @@ func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { // https://github.com/tendermint/tendermint/issues/3338 if sw.peers.Remove(peer) { sw.metrics.Peers.Add(float64(-1)) + } else { + // Removal of the peer has failed. The function above sets a flag within the peer to mark this. + // We keep this message here as information to the developer. + sw.Logger.Debug("error on peer removal", ",", "peer", peer.ID()) } } @@ -824,6 +828,12 @@ func (sw *Switch) addPeer(p Peer) error { // so that if Receive errors, we will find the peer and remove it. // Add should not err since we already checked peers.Has(). if err := sw.peers.Add(p); err != nil { + switch err.(type) { + case ErrPeerRemoval: + sw.Logger.Error("Error starting peer ", + " err ", "Peer has already errored and removal was attempted.", + "peer", p.ID()) + } return err } sw.metrics.Peers.Add(float64(1)) diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 2fa467891..9d5466df7 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -836,3 +836,16 @@ func BenchmarkSwitchBroadcast(b *testing.B) { b.Logf("success: %v, failure: %v", numSuccess, numFailure) } + +func TestSwitchRemovalErr(t *testing.T) { + + sw1, sw2 := MakeSwitchPair(t, func(i int, sw *Switch) *Switch { + return initSwitchFunc(i, sw) + }) + assert.Equal(t, len(sw1.Peers().List()), 1) + p := sw1.Peers().List()[0] + + sw2.StopPeerForError(p, fmt.Errorf("peer should error")) + + assert.Equal(t, sw2.peers.Add(p).Error(), ErrPeerRemoval{}.Error()) +} diff --git a/rpc/client/evidence_test.go b/rpc/client/evidence_test.go index a813d3912..ca4e0567e 100644 --- a/rpc/client/evidence_test.go +++ b/rpc/client/evidence_test.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/crypto/ed25519" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/tendermint/tendermint/privval" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" @@ -117,7 +118,7 @@ func makeEvidences( func TestBroadcastEvidence_DuplicateVoteEvidence(t *testing.T) { var ( config = rpctest.GetConfig() - chainID = config.ChainID() + chainID = test.DefaultTestChainID pv = privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) ) diff --git a/rpc/jsonrpc/client/ws_client.go b/rpc/jsonrpc/client/ws_client.go index 09b41888f..5a4839b04 100644 --- a/rpc/jsonrpc/client/ws_client.go +++ b/rpc/jsonrpc/client/ws_client.go @@ -89,8 +89,10 @@ func NewWS(remoteAddr, endpoint string, options ...func(*WSClient)) (*WSClient, if err != nil { return nil, err } - // default to ws protocol, unless wss is explicitly specified - if parsedURL.Scheme != protoWSS { + // default to ws protocol, unless wss or https is specified + if parsedURL.Scheme == protoHTTPS { + parsedURL.Scheme = protoWSS + } else if parsedURL.Scheme != protoWSS { parsedURL.Scheme = protoWS } diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index ebf2e14d5..1b88dec51 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -9,6 +9,7 @@ import ( "time" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" cfg "github.com/tendermint/tendermint/config" @@ -91,7 +92,7 @@ func makeAddrs() (string, string, string) { func createConfig() *cfg.Config { pathname := makePathname() - c := cfg.ResetTestRoot(pathname) + c := test.ResetTestRoot(pathname) // and we use random ports to run in parallel tm, rpc, grpc := makeAddrs() diff --git a/spec/abci/abci++_methods.md b/spec/abci/abci++_methods.md index 1218c05ce..0da22a69c 100644 --- a/spec/abci/abci++_methods.md +++ b/spec/abci/abci++_methods.md @@ -431,8 +431,8 @@ title: Methods there are other transactions with higher priority, then it should not include it in `ResponsePrepareProposal.txs`. However, this will not remove `tx` from the mempool. * If the Application wants to add a new transaction to the proposed block, then the - Application includes it in `ResponsePrepareProposal.txs`. In this case, Tendermint - will also add the transaction to the mempool. + Application includes it in `ResponsePrepareProposal.txs`. Tendermint will not add + the transaction to the mempool. * The Application should be aware that removing and adding transactions may compromise _traceability_. > Consider the following example: the Application transforms a client-submitted diff --git a/spec/core/data_structures.md b/spec/core/data_structures.md index a3d1a4de7..1e8ad8b25 100644 --- a/spec/core/data_structures.md +++ b/spec/core/data_structures.md @@ -269,8 +269,8 @@ func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { if !bytes.Equal(pubKey.Address(), vote.ValidatorAddress) { return ErrVoteInvalidValidatorAddress } - - if !pubKey.VerifyBytes(types.VoteSignBytes(chainID), vote.Signature) { + v := vote.ToProto() + if !pubKey.VerifyBytes(types.VoteSignBytes(chainID, v), vote.Signature) { return ErrVoteInvalidSignature } return nil diff --git a/spec/ivy-proofs/domain_model.ivy b/spec/ivy-proofs/domain_model.ivy index 0f12f7288..1fd3cc99e 100644 --- a/spec/ivy-proofs/domain_model.ivy +++ b/spec/ivy-proofs/domain_model.ivy @@ -131,13 +131,14 @@ object nset = { # the type of node sets object classic_bft = { relation quorum_intersection private { - definition [quorum_intersection_def] quorum_intersection = forall Q1,Q2. exists N. well_behaved(N) & nset.member(N, Q1) & nset.member(N, Q2) # every two quorums have a well-behaved node in common + definition [quorum_intersection_def] quorum_intersection = forall Q1,Q2. nset.is_quorum(Q1) & nset.is_quorum(Q2) + -> exists N. well_behaved(N) & nset.member(N, Q1) & nset.member(N, Q2) # every two quorums have a well-behaved node in common } } trusted isolate accountable_bft = { # this is our baseline assumption about quorums: private { - property [max_2f_byzantine] exists N . well_behaved(N) & nset.member(N,Q) # every quorum has a well-behaved member + property [max_2f_byzantine] nset.is_quorum(Q) -> exists N . well_behaved(N) & nset.member(N,Q) # every quorum has a well-behaved member } } diff --git a/state/execution.go b/state/execution.go index 7db82dc31..aef814894 100644 --- a/state/execution.go +++ b/state/execution.go @@ -24,6 +24,9 @@ type BlockExecutor struct { // save state, validators, consensus params, abci responses here store Store + // use blockstore for the pruning functions. + blockStore BlockStore + // execute the app against this proxyApp proxy.AppConnConsensus @@ -56,16 +59,18 @@ func NewBlockExecutor( proxyApp proxy.AppConnConsensus, mempool mempool.Mempool, evpool EvidencePool, + blockStore BlockStore, options ...BlockExecutorOption, ) *BlockExecutor { res := &BlockExecutor{ - store: stateStore, - proxyApp: proxyApp, - eventBus: types.NopEventBus{}, - mempool: mempool, - evpool: evpool, - logger: logger, - metrics: NopMetrics(), + store: stateStore, + proxyApp: proxyApp, + eventBus: types.NopEventBus{}, + mempool: mempool, + evpool: evpool, + logger: logger, + metrics: NopMetrics(), + blockStore: blockStore, } for _, option := range options { @@ -181,16 +186,16 @@ func (blockExec *BlockExecutor) ValidateBlock(state State, block *types.Block) e // ApplyBlock validates the block against the state, executes it against the app, // fires the relevant events, commits the app, and saves the new state and responses. -// It returns the new state and the block height to retain (pruning older blocks). +// It returns the new state. // It's the only function that needs to be called // from outside this package to process and commit an entire block. // It takes a blockID to avoid recomputing the parts hash. func (blockExec *BlockExecutor) ApplyBlock( state State, blockID types.BlockID, block *types.Block, -) (State, int64, error) { +) (State, error) { if err := validateBlock(state, block); err != nil { - return state, 0, ErrInvalidBlock(err) + return state, ErrInvalidBlock(err) } startTime := time.Now().UnixNano() @@ -200,14 +205,14 @@ func (blockExec *BlockExecutor) ApplyBlock( endTime := time.Now().UnixNano() blockExec.metrics.BlockProcessingTime.Observe(float64(endTime-startTime) / 1000000) if err != nil { - return state, 0, ErrProxyAppConn(err) + return state, ErrProxyAppConn(err) } fail.Fail() // XXX // Save the results before we commit. if err := blockExec.store.SaveFinalizeBlockResponse(block.Height, abciResponse); err != nil { - return state, 0, err + return state, err } fail.Fail() // XXX @@ -215,12 +220,12 @@ func (blockExec *BlockExecutor) ApplyBlock( // validate the validator updates and convert to tendermint types err = validateValidatorUpdates(abciResponse.ValidatorUpdates, state.ConsensusParams.Validator) if err != nil { - return state, 0, fmt.Errorf("error in validator updates: %w", err) + return state, fmt.Errorf("error in validator updates: %v", err) } validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponse.ValidatorUpdates) if err != nil { - return state, 0, err + return state, err } if len(validatorUpdates) > 0 { blockExec.logger.Debug("updates to validators", "updates", types.ValidatorListString(validatorUpdates)) @@ -233,13 +238,13 @@ func (blockExec *BlockExecutor) ApplyBlock( // Update the state with the block and responses. state, err = updateState(state, blockID, &block.Header, abciResponse, validatorUpdates) if err != nil { - return state, 0, fmt.Errorf("failed to update state: %w", err) + return state, fmt.Errorf("commit failed for application: %v", err) } // Lock mempool, commit app state, update mempoool. retainHeight, err := blockExec.Commit(state, block, abciResponse) if err != nil { - return state, 0, fmt.Errorf("commit failed for application: %w", err) + return state, fmt.Errorf("commit failed for application: %v", err) } // Update evpool with the latest state. @@ -250,16 +255,26 @@ func (blockExec *BlockExecutor) ApplyBlock( // Update the app hash and save the state. state.AppHash = abciResponse.AgreedAppData if err := blockExec.store.Save(state); err != nil { - return state, 0, err + return state, err } fail.Fail() // XXX + // Prune old heights, if requested by ABCI app. + if retainHeight > 0 { + pruned, err := blockExec.pruneBlocks(retainHeight, state) + if err != nil { + blockExec.logger.Error("failed to prune blocks", "retain_height", retainHeight, "err", err) + } else { + blockExec.logger.Debug("pruned blocks", "pruned", pruned, "retain_height", retainHeight) + } + } + // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay fireEvents(blockExec.logger, blockExec.eventBus, block, blockID, abciResponse, validatorUpdates) - return state, retainHeight, nil + return state, nil } // Commit locks the mempool, runs the ABCI Commit message, and updates the @@ -586,3 +601,21 @@ func ExecCommitBlock( return resp.AgreedAppData, nil } + +func (blockExec *BlockExecutor) pruneBlocks(retainHeight int64, state State) (uint64, error) { + base := blockExec.blockStore.Base() + if retainHeight <= base { + return 0, nil + } + + amountPruned, prunedHeaderHeight, err := blockExec.blockStore.PruneBlocks(retainHeight, state) + if err != nil { + return 0, fmt.Errorf("failed to prune block store: %w", err) + } + + err = blockExec.Store().PruneStates(base, retainHeight, prunedHeaderHeight) + if err != nil { + return 0, fmt.Errorf("failed to prune state store: %w", err) + } + return amountPruned, nil +} diff --git a/state/execution_test.go b/state/execution_test.go index 949d98601..69d2dd932 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -25,9 +25,11 @@ import ( pmocks "github.com/tendermint/tendermint/proxy/mocks" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" "github.com/tendermint/tendermint/version" + dbm "github.com/tendermint/tm-db" ) var ( @@ -47,6 +49,8 @@ func TestApplyBlock(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardFinalizeBlockResponses: false, }) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + mp := &mpmocks.Mempool{} mp.On("Lock").Return() mp.On("Unlock").Return() @@ -59,16 +63,15 @@ func TestApplyBlock(t *testing.T) { mock.Anything, mock.Anything).Return(nil) blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, sm.EmptyEvidencePool{}) + mp, sm.EmptyEvidencePool{}, blockStore) block := makeBlock(state, 1, new(types.Commit)) bps, err := block.MakePartSet(testPartSize) require.NoError(t, err) blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()} - state, retainHeight, err := blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) - assert.EqualValues(t, retainHeight, 1) // TODO check state and mempool assert.EqualValues(t, 1, state.Version.Consensus.App, "App version wasn't updated") @@ -231,8 +234,10 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor(stateStore, log.TestingLogger(), proxyApp.Consensus(), - mp, evpool) + mp, evpool, blockStore) block := makeBlock(state, 1, new(types.Commit)) block.Evidence = types.EvidenceData{Evidence: ev} @@ -242,9 +247,8 @@ func TestFinalizeBlockByzantineValidators(t *testing.T) { blockID = types.BlockID{Hash: block.Hash(), PartSetHeader: bps.Header()} - state, retainHeight, err := blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) - assert.EqualValues(t, retainHeight, 1) // TODO check state and mempool assert.Equal(t, abciMb, app.Misbehavior) @@ -268,7 +272,7 @@ func TestProcessProposal(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardFinalizeBlockResponses: false, }) - + blockStore := store.NewBlockStore(dbm.NewMemDB()) eventBus := types.NewEventBus() err = eventBus.Start() require.NoError(t, err) @@ -279,6 +283,7 @@ func TestProcessProposal(t *testing.T) { proxyApp.Consensus(), new(mpmocks.Mempool), sm.EmptyEvidencePool{}, + blockStore, ) block0 := makeBlock(state, height-1, new(types.Commit)) @@ -488,12 +493,14 @@ func TestEndBlockValidatorUpdates(t *testing.T) { mock.Anything).Return(nil) mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs{}) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) eventBus := types.NewEventBus() @@ -522,7 +529,7 @@ func TestEndBlockValidatorUpdates(t *testing.T) { {PubKey: pk, Power: 10}, } - state, _, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) require.Nil(t, err) // test new validator was added to NextValidators if assert.Equal(t, state.Validators.Size()+1, state.NextValidators.Size()) { @@ -562,12 +569,14 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { stateStore := sm.NewStore(stateDB, sm.StoreOptions{ DiscardFinalizeBlockResponses: false, }) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), new(mpmocks.Mempool), sm.EmptyEvidencePool{}, + blockStore, ) block := makeBlock(state, 1, new(types.Commit)) @@ -582,7 +591,7 @@ func TestEndBlockValidatorUpdatesResultingInEmptySet(t *testing.T) { {PubKey: vp, Power: 0}, } - assert.NotPanics(t, func() { state, _, err = blockExec.ApplyBlock(state, blockID, block) }) + assert.NotPanics(t, func() { state, err = blockExec.ApplyBlock(state, blockID, block) }) assert.NotNil(t, err) assert.NotEmpty(t, state.NextValidators.Validators) } @@ -614,12 +623,14 @@ func TestEmptyPrepareProposal(t *testing.T) { mock.Anything).Return(nil) mp.On("ReapMaxBytesMaxGas", mock.Anything, mock.Anything).Return(types.Txs{}) + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -655,12 +666,14 @@ func TestPrepareProposalTxsAllIncluded(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -706,12 +719,14 @@ func TestPrepareProposalReorderTxs(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -759,12 +774,14 @@ func TestPrepareProposalErrorOnTooManyTxs(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.NewNopLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) @@ -807,12 +824,14 @@ func TestPrepareProposalErrorOnPrepareProposalError(t *testing.T) { require.NoError(t, err) defer proxyApp.Stop() //nolint:errcheck // ignore for tests + blockStore := store.NewBlockStore(dbm.NewMemDB()) blockExec := sm.NewBlockExecutor( stateStore, log.NewNopLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) pa, _ := state.Validators.GetByIndex(0) commit, err := makeValidCommit(height, types.BlockID{}, state.Validators, privVals) diff --git a/state/helpers_test.go b/state/helpers_test.go index 0ae279327..0e9652d2c 100644 --- a/state/helpers_test.go +++ b/state/helpers_test.go @@ -66,7 +66,7 @@ func makeAndApplyGoodBlock(state sm.State, height int64, lastCommit *types.Commi } blockID := types.BlockID{Hash: block.Hash(), PartSetHeader: partSet.Header()} - state, _, err = blockExec.ApplyBlock(state, blockID, block) + state, err = blockExec.ApplyBlock(state, blockID, block) if err != nil { return state, types.BlockID{}, err } diff --git a/state/indexer/block/kv/kv.go b/state/indexer/block/kv/kv.go index 50a918fad..1a33ab2fe 100644 --- a/state/indexer/block/kv/kv.go +++ b/state/indexer/block/kv/kv.go @@ -13,6 +13,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/types" ) @@ -86,10 +87,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, default: } - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("failed to parse query conditions: %w", err) - } + conditions := q.Syntax() // If there is an exact height query, return the result immediately // (if it exists). @@ -153,7 +151,7 @@ func (idx *BlockerIndexer) Search(ctx context.Context, q *query.Query) ([]int64, continue } - startKey, err := orderedcode.Append(nil, c.CompositeKey, fmt.Sprintf("%v", c.Operand)) + startKey, err := orderedcode.Append(nil, c.Tag, c.Arg.Value()) if err != nil { return nil, err } @@ -321,7 +319,7 @@ LOOP: // matched. func (idx *BlockerIndexer) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHeights map[string][]byte, firstRun bool, @@ -336,7 +334,7 @@ func (idx *BlockerIndexer) match( tmpHeights := make(map[string][]byte) switch { - case c.Op == query.OpEqual: + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(idx.store, startKeyBz) if err != nil { return nil, fmt.Errorf("failed to create prefix iterator: %w", err) @@ -355,8 +353,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpExists: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TExists: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -382,8 +380,8 @@ func (idx *BlockerIndexer) match( return nil, err } - case c.Op == query.OpContains: - prefix, err := orderedcode.Append(nil, c.CompositeKey) + case c.Op == syntax.TContains: + prefix, err := orderedcode.Append(nil, c.Tag) if err != nil { return nil, err } @@ -400,7 +398,7 @@ func (idx *BlockerIndexer) match( continue } - if strings.Contains(eventValue, c.Operand.(string)) { + if strings.Contains(eventValue, c.Arg.Value()) { tmpHeights[string(it.Value())] = it.Value() } diff --git a/state/indexer/block/kv/kv_test.go b/state/indexer/block/kv/kv_test.go index 2ce358fa3..930ec9e4e 100644 --- a/state/indexer/block/kv/kv_test.go +++ b/state/indexer/block/kv/kv_test.go @@ -82,39 +82,39 @@ func TestBlockIndexer(t *testing.T) { results []int64 }{ "block.height = 100": { - q: query.MustParse("block.height = 100"), + q: query.MustCompile(`block.height = 100`), results: []int64{}, }, "block.height = 5": { - q: query.MustParse("block.height = 5"), + q: query.MustCompile(`block.height = 5`), results: []int64{5}, }, "begin_event.key1 = 'value1'": { - q: query.MustParse("begin_event.key1 = 'value1'"), + q: query.MustCompile(`begin_event.key1 = 'value1'`), results: []int64{}, }, "begin_event.proposer = 'FCAA001'": { - q: query.MustParse("begin_event.proposer = 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer = 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, "end_event.foo <= 5": { - q: query.MustParse("end_event.foo <= 5"), + q: query.MustCompile(`end_event.foo <= 5`), results: []int64{2, 4}, }, "end_event.foo >= 100": { - q: query.MustParse("end_event.foo >= 100"), + q: query.MustCompile(`end_event.foo >= 100`), results: []int64{1}, }, "block.height > 2 AND end_event.foo <= 8": { - q: query.MustParse("block.height > 2 AND end_event.foo <= 8"), + q: query.MustCompile(`block.height > 2 AND end_event.foo <= 8`), results: []int64{4, 6, 8}, }, "begin_event.proposer CONTAINS 'FFFFFFF'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FFFFFFF'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FFFFFFF'`), results: []int64{}, }, "begin_event.proposer CONTAINS 'FCAA001'": { - q: query.MustParse("begin_event.proposer CONTAINS 'FCAA001'"), + q: query.MustCompile(`begin_event.proposer CONTAINS 'FCAA001'`), results: []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, }, } diff --git a/state/indexer/block/kv/util.go b/state/indexer/block/kv/util.go index 05d6fc45c..fff88046c 100644 --- a/state/indexer/block/kv/util.go +++ b/state/indexer/block/kv/util.go @@ -6,8 +6,7 @@ import ( "strconv" "github.com/google/orderedcode" - - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/types" ) @@ -86,10 +85,10 @@ func parseValueFromEventKey(key []byte) (string, error) { return eventValue, nil } -func lookForHeight(conditions []query.Condition) (int64, bool) { +func lookForHeight(conditions []syntax.Condition) (int64, bool) { for _, c := range conditions { - if c.CompositeKey == types.BlockHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64), true + if c.Tag == types.BlockHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()), true } } diff --git a/state/indexer/query_range.go b/state/indexer/query_range.go index b4edf53c5..4c026955d 100644 --- a/state/indexer/query_range.go +++ b/state/indexer/query_range.go @@ -3,7 +3,7 @@ package indexer import ( "time" - "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" ) // QueryRanges defines a mapping between a composite event key and a QueryRange. @@ -77,32 +77,32 @@ func (qr QueryRange) UpperBoundValue() interface{} { // LookForRanges returns a mapping of QueryRanges and the matching indexes in // the provided query conditions. -func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes []int) { +func LookForRanges(conditions []syntax.Condition) (ranges QueryRanges, indexes []int) { ranges = make(QueryRanges) for i, c := range conditions { if IsRangeOperation(c.Op) { - r, ok := ranges[c.CompositeKey] + r, ok := ranges[c.Tag] if !ok { - r = QueryRange{Key: c.CompositeKey} + r = QueryRange{Key: c.Tag} } switch c.Op { - case query.OpGreater: - r.LowerBound = c.Operand + case syntax.TGt: + r.LowerBound = conditionArg(c) - case query.OpGreaterEqual: + case syntax.TGeq: r.IncludeLowerBound = true - r.LowerBound = c.Operand + r.LowerBound = conditionArg(c) - case query.OpLess: - r.UpperBound = c.Operand + case syntax.TLt: + r.UpperBound = conditionArg(c) - case query.OpLessEqual: + case syntax.TLeq: r.IncludeUpperBound = true - r.UpperBound = c.Operand + r.UpperBound = conditionArg(c) } - ranges[c.CompositeKey] = r + ranges[c.Tag] = r indexes = append(indexes, i) } } @@ -112,12 +112,26 @@ func LookForRanges(conditions []query.Condition) (ranges QueryRanges, indexes [] // IsRangeOperation returns a boolean signifying if a query Operator is a range // operation or not. -func IsRangeOperation(op query.Operator) bool { +func IsRangeOperation(op syntax.Token) bool { switch op { - case query.OpGreater, query.OpGreaterEqual, query.OpLess, query.OpLessEqual: + case syntax.TGt, syntax.TGeq, syntax.TLt, syntax.TLeq: return true default: return false } } + +func conditionArg(c syntax.Condition) interface{} { + if c.Arg == nil { + return nil + } + switch c.Arg.Type { + case syntax.TNumber: + return int64(c.Arg.Number()) + case syntax.TTime, syntax.TDate: + return c.Arg.Time() + default: + return c.Arg.Value() // string + } +} diff --git a/state/indexer/sink/psql/psql_test.go b/state/indexer/sink/psql/psql_test.go index 68d32dcf6..9572a2b33 100644 --- a/state/indexer/sink/psql/psql_test.go +++ b/state/indexer/sink/psql/psql_test.go @@ -19,6 +19,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" // Register the Postgres database driver. @@ -196,6 +197,55 @@ func TestIndexing(t *testing.T) { err = indexer.IndexTxEvents([]*abci.TxResult{txResult}) require.NoError(t, err) }) + + t.Run("IndexerService", func(t *testing.T) { + indexer := &EventSink{store: testDB(), chainID: chainID} + + // event bus + eventBus := types.NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + t.Cleanup(func() { + if err := eventBus.Stop(); err != nil { + t.Error(err) + } + }) + + service := txindex.NewIndexerService(indexer.TxIndexer(), indexer.BlockIndexer(), eventBus, true) + err = service.Start() + require.NoError(t, err) + t.Cleanup(func() { + if err := service.Stop(); err != nil { + t.Error(err) + } + }) + + // publish block with txs + err = eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + Header: types.Header{Height: 1}, + NumTxs: int64(2), + }) + require.NoError(t, err) + txResult1 := &abci.TxResult{ + Height: 1, + Index: uint32(0), + Tx: types.Tx("foo"), + Result: abci.ResponseDeliverTx{Code: 0}, + } + err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult1}) + require.NoError(t, err) + txResult2 := &abci.TxResult{ + Height: 1, + Index: uint32(1), + Tx: types.Tx("bar"), + Result: abci.ResponseDeliverTx{Code: 1}, + } + err = eventBus.PublishEventTx(types.EventDataTx{TxResult: *txResult2}) + require.NoError(t, err) + + time.Sleep(100 * time.Millisecond) + require.True(t, service.IsRunning()) + }) } func TestStop(t *testing.T) { diff --git a/state/mocks/block_store.go b/state/mocks/block_store.go index f93f45447..d449f6711 100644 --- a/state/mocks/block_store.go +++ b/state/mocks/block_store.go @@ -4,6 +4,7 @@ package mocks import ( mock "github.com/stretchr/testify/mock" + state "github.com/tendermint/tendermint/state" types "github.com/tendermint/tendermint/types" ) @@ -27,6 +28,20 @@ func (_m *BlockStore) Base() int64 { return r0 } +// DeleteLatestBlock provides a mock function with given fields: +func (_m *BlockStore) DeleteLatestBlock() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Height provides a mock function with given fields: func (_m *BlockStore) Height() int64 { ret := _m.Called() @@ -169,25 +184,32 @@ func (_m *BlockStore) LoadSeenCommit(height int64) *types.Commit { return r0 } -// PruneBlocks provides a mock function with given fields: height -func (_m *BlockStore) PruneBlocks(height int64) (uint64, error) { - ret := _m.Called(height) +// PruneBlocks provides a mock function with given fields: height, _a1 +func (_m *BlockStore) PruneBlocks(height int64, _a1 state.State) (uint64, int64, error) { + ret := _m.Called(height, _a1) var r0 uint64 - if rf, ok := ret.Get(0).(func(int64) uint64); ok { - r0 = rf(height) + if rf, ok := ret.Get(0).(func(int64, state.State) uint64); ok { + r0 = rf(height, _a1) } else { r0 = ret.Get(0).(uint64) } - var r1 error - if rf, ok := ret.Get(1).(func(int64) error); ok { - r1 = rf(height) + var r1 int64 + if rf, ok := ret.Get(1).(func(int64, state.State) int64); ok { + r1 = rf(height, _a1) } else { - r1 = ret.Error(1) + r1 = ret.Get(1).(int64) } - return r0, r1 + var r2 error + if rf, ok := ret.Get(2).(func(int64, state.State) error); ok { + r2 = rf(height, _a1) + } else { + r2 = ret.Error(2) + } + + return r0, r1, r2 } // SaveBlock provides a mock function with given fields: block, blockParts, seenCommit diff --git a/state/mocks/store.go b/state/mocks/store.go index a6c6bb916..bf533c3d0 100644 --- a/state/mocks/store.go +++ b/state/mocks/store.go @@ -197,13 +197,13 @@ func (_m *Store) LoadValidators(_a0 int64) (*types.ValidatorSet, error) { return r0, r1 } -// PruneStates provides a mock function with given fields: _a0, _a1 -func (_m *Store) PruneStates(_a0 int64, _a1 int64) error { - ret := _m.Called(_a0, _a1) +// PruneStates provides a mock function with given fields: _a0, _a1, _a2 +func (_m *Store) PruneStates(_a0 int64, _a1 int64, _a2 int64) error { + ret := _m.Called(_a0, _a1, _a2) var r0 error - if rf, ok := ret.Get(0).(func(int64, int64) error); ok { - r0 = rf(_a0, _a1) + if rf, ok := ret.Get(0).(func(int64, int64, int64) error); ok { + r0 = rf(_a0, _a1, _a2) } else { r0 = ret.Error(0) } diff --git a/state/rollback.go b/state/rollback.go index c4686015b..57bb276b8 100644 --- a/state/rollback.go +++ b/state/rollback.go @@ -12,7 +12,7 @@ import ( // Rollback overwrites the current Tendermint state (height n) with the most // recent previous state (height n - 1). // Note that this function does not affect application state. -func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { +func Rollback(bs BlockStore, ss Store, removeBlock bool) (int64, []byte, error) { invalidState, err := ss.Load() if err != nil { return -1, nil, err @@ -24,9 +24,14 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { height := bs.Height() // NOTE: persistence of state and blocks don't happen atomically. Therefore it is possible that - // when the user stopped the node the state wasn't updated but the blockstore was. In this situation - // we don't need to rollback any state and can just return early + // when the user stopped the node the state wasn't updated but the blockstore was. Discard the + // pending block before continuing. if height == invalidState.LastBlockHeight+1 { + if removeBlock { + if err := bs.DeleteLatestBlock(); err != nil { + return -1, nil, fmt.Errorf("failed to remove final block from blockstore: %w", err) + } + } return invalidState.LastBlockHeight, invalidState.AppHash, nil } @@ -108,5 +113,13 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { return -1, nil, fmt.Errorf("failed to save rolled back state: %w", err) } + // If removeBlock is true then also remove the block associated with the previous state. + // This will mean both the last state and last block height is equal to n - 1 + if removeBlock { + if err := bs.DeleteLatestBlock(); err != nil { + return -1, nil, fmt.Errorf("failed to remove final block from blockstore: %w", err) + } + } + return rolledBackState.LastBlockHeight, rolledBackState.AppHash, nil } diff --git a/state/rollback_test.go b/state/rollback_test.go index d4c15f60f..0e6dc1946 100644 --- a/state/rollback_test.go +++ b/state/rollback_test.go @@ -3,6 +3,7 @@ package state_test import ( "crypto/rand" "testing" + "time" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" @@ -13,6 +14,7 @@ import ( tmversion "github.com/tendermint/tendermint/proto/tendermint/version" "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" "github.com/tendermint/tendermint/version" ) @@ -50,6 +52,7 @@ func TestRollback(t *testing.T) { BlockID: initialState.LastBlockID, Header: types.Header{ Height: initialState.LastBlockHeight, + Time: initialState.LastBlockTime, AppHash: crypto.CRandBytes(tmhash.Size), LastBlockID: makeBlockIDRandom(), LastResultsHash: initialState.LastResultsHash, @@ -61,6 +64,7 @@ func TestRollback(t *testing.T) { Height: nextState.LastBlockHeight, AppHash: initialState.AppHash, LastBlockID: block.BlockID, + Time: nextState.LastBlockTime, LastResultsHash: nextState.LastResultsHash, }, } @@ -69,7 +73,7 @@ func TestRollback(t *testing.T) { blockStore.On("Height").Return(nextHeight) // rollback the state - rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore) + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore, false) require.NoError(t, err) require.EqualValues(t, height, rollbackHeight) require.EqualValues(t, initialState.AppHash, rollbackHash) @@ -81,6 +85,122 @@ func TestRollback(t *testing.T) { require.EqualValues(t, initialState, loadedState) } +func TestRollbackHard(t *testing.T) { + const height int64 = 100 + blockStore := store.NewBlockStore(dbm.NewMemDB()) + stateStore := state.NewStore(dbm.NewMemDB(), state.StoreOptions{DiscardABCIResponses: false}) + + valSet, _ := types.RandValidatorSet(5, 10) + + params := types.DefaultConsensusParams() + params.Version.App = 10 + now := time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC) + + block := &types.Block{ + Header: types.Header{ + Version: tmversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: "test-chain", + Time: now, + Height: height, + AppHash: crypto.CRandBytes(tmhash.Size), + LastBlockID: makeBlockIDRandom(), + LastCommitHash: crypto.CRandBytes(tmhash.Size), + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: valSet.Hash(), + NextValidatorsHash: valSet.CopyIncrementProposerPriority(1).Hash(), + ConsensusHash: params.Hash(), + LastResultsHash: crypto.CRandBytes(tmhash.Size), + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + }, + LastCommit: &types.Commit{Height: height - 1}, + } + + partSet, err := block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockStore.SaveBlock(block, partSet, &types.Commit{Height: block.Height}) + + currState := state.State{ + Version: tmstate.Version{ + Consensus: block.Header.Version, + Software: version.TMCoreSemVer, + }, + LastBlockHeight: block.Height, + LastBlockTime: block.Time, + AppHash: crypto.CRandBytes(tmhash.Size), + LastValidators: valSet, + Validators: valSet.CopyIncrementProposerPriority(1), + NextValidators: valSet.CopyIncrementProposerPriority(2), + ConsensusParams: *params, + LastHeightConsensusParamsChanged: height + 1, + LastHeightValidatorsChanged: height + 1, + LastResultsHash: crypto.CRandBytes(tmhash.Size), + } + require.NoError(t, stateStore.Bootstrap(currState)) + + nextBlock := &types.Block{ + Header: types.Header{ + Version: tmversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: block.ChainID, + Time: block.Time, + Height: currState.LastBlockHeight + 1, + AppHash: currState.AppHash, + LastBlockID: types.BlockID{Hash: block.Hash(), PartSetHeader: partSet.Header()}, + LastCommitHash: crypto.CRandBytes(tmhash.Size), + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: valSet.CopyIncrementProposerPriority(1).Hash(), + NextValidatorsHash: valSet.CopyIncrementProposerPriority(2).Hash(), + ConsensusHash: params.Hash(), + LastResultsHash: currState.LastResultsHash, + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + }, + LastCommit: &types.Commit{Height: currState.LastBlockHeight}, + } + + nextPartSet, err := nextBlock.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockStore.SaveBlock(nextBlock, nextPartSet, &types.Commit{Height: nextBlock.Height}) + + rollbackHeight, rollbackHash, err := state.Rollback(blockStore, stateStore, true) + require.NoError(t, err) + require.Equal(t, rollbackHeight, currState.LastBlockHeight) + require.Equal(t, rollbackHash, currState.AppHash) + + // state should not have been changed + loadedState, err := stateStore.Load() + require.NoError(t, err) + require.Equal(t, currState, loadedState) + + // resave the same block + blockStore.SaveBlock(nextBlock, nextPartSet, &types.Commit{Height: nextBlock.Height}) + + params.Version.App = 11 + + nextState := state.State{ + Version: tmstate.Version{ + Consensus: block.Header.Version, + Software: version.TMCoreSemVer, + }, + LastBlockHeight: nextBlock.Height, + LastBlockTime: nextBlock.Time, + AppHash: crypto.CRandBytes(tmhash.Size), + LastValidators: valSet.CopyIncrementProposerPriority(1), + Validators: valSet.CopyIncrementProposerPriority(2), + NextValidators: valSet.CopyIncrementProposerPriority(3), + ConsensusParams: *params, + LastHeightConsensusParamsChanged: nextBlock.Height + 1, + LastHeightValidatorsChanged: nextBlock.Height + 1, + LastResultsHash: crypto.CRandBytes(tmhash.Size), + } + require.NoError(t, stateStore.Save(nextState)) + + rollbackHeight, rollbackHash, err = state.Rollback(blockStore, stateStore, true) + require.NoError(t, err) + require.Equal(t, rollbackHeight, currState.LastBlockHeight) + require.Equal(t, rollbackHash, currState.AppHash) +} + func TestRollbackNoState(t *testing.T) { stateStore := state.NewStore(dbm.NewMemDB(), state.StoreOptions{ @@ -88,7 +208,7 @@ func TestRollbackNoState(t *testing.T) { }) blockStore := &mocks.BlockStore{} - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Contains(t, err.Error(), "no state found") } @@ -101,7 +221,7 @@ func TestRollbackNoBlocks(t *testing.T) { blockStore.On("LoadBlockMeta", height).Return(nil) blockStore.On("LoadBlockMeta", height-1).Return(nil) - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Contains(t, err.Error(), "block at height 99 not found") } @@ -112,7 +232,7 @@ func TestRollbackDifferentStateHeight(t *testing.T) { blockStore := &mocks.BlockStore{} blockStore.On("Height").Return(height + 2) - _, _, err := state.Rollback(blockStore, stateStore) + _, _, err := state.Rollback(blockStore, stateStore, false) require.Error(t, err) require.Equal(t, err.Error(), "statestore height (100) is not one below or equal to blockstore height (102)") } @@ -138,6 +258,7 @@ func setupStateStore(t *testing.T, height int64) state.Store { AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), LastBlockHeight: height, + LastBlockTime: time.Now(), LastValidators: valSet, Validators: valSet.CopyIncrementProposerPriority(1), NextValidators: valSet.CopyIncrementProposerPriority(2), diff --git a/state/services.go b/state/services.go index 6e24af036..0473b43b2 100644 --- a/state/services.go +++ b/state/services.go @@ -26,7 +26,7 @@ type BlockStore interface { SaveBlock(block *types.Block, blockParts *types.PartSet, seenCommit *types.Commit) - PruneBlocks(height int64) (uint64, error) + PruneBlocks(height int64, state State) (uint64, int64, error) LoadBlockByHash(hash []byte) *types.Block LoadBlockMetaByHash(hash []byte) *types.BlockMeta @@ -34,6 +34,8 @@ type BlockStore interface { LoadBlockCommit(height int64) *types.Commit LoadSeenCommit(height int64) *types.Commit + + DeleteLatestBlock() error } //----------------------------------------------------------------------------- diff --git a/state/state.go b/state/state.go index 3e0d89392..51ce5a3f8 100644 --- a/state/state.go +++ b/state/state.go @@ -317,7 +317,7 @@ func MakeGenesisDocFromFile(genDocFile string) (*types.GenesisDoc, error) { func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { err := genDoc.ValidateAndComplete() if err != nil { - return State{}, fmt.Errorf("error in genesis file: %v", err) + return State{}, fmt.Errorf("error in genesis doc: %w", err) } var validatorSet, nextValidatorSet *types.ValidatorSet diff --git a/state/state_test.go b/state/state_test.go index c820fb7d4..dc14b6bbb 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -14,9 +14,9 @@ import ( dbm "github.com/tendermint/tm-db" abci "github.com/tendermint/tendermint/abci/types" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto/ed25519" cryptoenc "github.com/tendermint/tendermint/crypto/encoding" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -24,7 +24,7 @@ import ( // setupTestCase does setup common to all test cases. func setupTestCase(t *testing.T) (func(t *testing.T), dbm.DB, sm.State) { - config := cfg.ResetTestRoot("state_") + config := test.ResetTestRoot("state_") dbType := dbm.BackendType(config.DBBackend) stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) stateStore := sm.NewStore(stateDB, sm.StoreOptions{ diff --git a/state/store.go b/state/store.go index 53a40e8c9..bcb293d45 100644 --- a/state/store.go +++ b/state/store.go @@ -70,8 +70,8 @@ type Store interface { SaveFinalizeBlockResponse(int64, *abci.ResponseFinalizeBlock) error // Bootstrap is used for bootstrapping state when not starting from a initial height. Bootstrap(State) error - // PruneStates takes the height from which to start prning and which height stop at - PruneStates(int64, int64) error + // PruneStates takes the height from which to start pruning and which height stop at + PruneStates(int64, int64, int64) error // Close closes the connection with the database Close() error } @@ -237,14 +237,15 @@ func (store dbStore) Bootstrap(state State) error { // encoding not preserving ordering: https://github.com/tendermint/tendermint/issues/4567 // This will cause some old states to be left behind when doing incremental partial prunes, // specifically older checkpoints and LastHeightChanged targets. -func (store dbStore) PruneStates(from int64, to int64) error { +func (store dbStore) PruneStates(from int64, to int64, evidenceThresholdHeight int64) error { if from <= 0 || to <= 0 { return fmt.Errorf("from height %v and to height %v must be greater than 0", from, to) } if from >= to { return fmt.Errorf("from height %v must be lower than to height %v", from, to) } - valInfo, err := loadValidatorsInfo(store.db, to) + + valInfo, err := loadValidatorsInfo(store.db, min(to, evidenceThresholdHeight)) if err != nil { return fmt.Errorf("validators at height %v not found: %w", to, err) } @@ -298,12 +299,14 @@ func (store dbStore) PruneStates(from int64, to int64) error { return err } } - } else { + } else if h < evidenceThresholdHeight { err = batch.Delete(calcValidatorsKey(h)) if err != nil { return err } } + // else we keep the validator set because we might need + // it later on for evidence verification if keepParams[h] { p, err := store.loadConsensusParamsInfo(h) @@ -547,7 +550,7 @@ func loadValidatorsInfo(db dbm.DB, height int64) (*tmstate.ValidatorsInfo, error } if len(buf) == 0 { - return nil, errors.New("no last ABCI response has been persisted") + return nil, errors.New("value retrieved from db is empty") } v := new(tmstate.ValidatorsInfo) @@ -635,7 +638,7 @@ func (store dbStore) loadConsensusParamsInfo(height int64) (*tmstate.ConsensusPa return nil, err } if len(buf) == 0 { - return nil, errors.New("no last ABCI response has been persisted") + return nil, errors.New("value retrieved from db is empty") } paramsInfo := new(tmstate.ConsensusParamsInfo) @@ -677,3 +680,10 @@ func (store dbStore) saveConsensusParamsInfo(nextHeight, changeHeight int64, par func (store dbStore) Close() error { return store.db.Close() } + +func min(a int64, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/state/store_test.go b/state/store_test.go index 460705a78..ec042886f 100644 --- a/state/store_test.go +++ b/state/store_test.go @@ -11,9 +11,9 @@ import ( dbm "github.com/tendermint/tm-db" abci "github.com/tendermint/tendermint/abci/types" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/internal/test" tmrand "github.com/tendermint/tendermint/libs/rand" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -49,7 +49,7 @@ func TestStoreLoadValidators(t *testing.T) { func BenchmarkLoadValidators(b *testing.B) { const valSetSize = 100 - config := cfg.ResetTestRoot("state_") + config := test.ResetTestRoot("state_") defer os.RemoveAll(config.RootDir) dbType := dbm.BackendType(config.DBBackend) stateDB, err := dbm.NewDB("state", dbType, config.DBDir()) @@ -87,23 +87,25 @@ func BenchmarkLoadValidators(b *testing.B) { func TestPruneStates(t *testing.T) { testcases := map[string]struct { - makeHeights int64 - pruneFrom int64 - pruneTo int64 - expectErr bool - expectVals []int64 - expectParams []int64 - expectABCI []int64 + makeHeights int64 + pruneFrom int64 + pruneTo int64 + evidenceThresholdHeight int64 + expectErr bool + expectVals []int64 + expectParams []int64 + expectABCI []int64 }{ - "error on pruning from 0": {100, 0, 5, true, nil, nil, nil}, - "error when from > to": {100, 3, 2, true, nil, nil, nil}, - "error when from == to": {100, 3, 3, true, nil, nil, nil}, - "error when to does not exist": {100, 1, 101, true, nil, nil, nil}, - "prune all": {100, 1, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}}, - "prune some": {10, 2, 8, false, []int64{1, 3, 8, 9, 10}, + "error on pruning from 0": {100, 0, 5, 100, true, nil, nil, nil}, + "error when from > to": {100, 3, 2, 2, true, nil, nil, nil}, + "error when from == to": {100, 3, 3, 3, true, nil, nil, nil}, + "error when to does not exist": {100, 1, 101, 101, true, nil, nil, nil}, + "prune all": {100, 1, 100, 100, false, []int64{93, 100}, []int64{95, 100}, []int64{100}}, + "prune some": {10, 2, 8, 8, false, []int64{1, 3, 8, 9, 10}, []int64{1, 5, 8, 9, 10}, []int64{1, 8, 9, 10}}, - "prune across checkpoint": {100001, 1, 100001, false, []int64{99993, 100000, 100001}, + "prune across checkpoint": {100001, 1, 100001, 100001, false, []int64{99993, 100000, 100001}, []int64{99995, 100001}, []int64{100001}}, + "prune when evidence height < height": {20, 1, 18, 17, false, []int64{13, 17, 18, 19, 20}, []int64{15, 18, 19, 20}, []int64{18, 19, 20}}, } for name, tc := range testcases { tc := tc @@ -162,7 +164,7 @@ func TestPruneStates(t *testing.T) { } // Test assertions - err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo) + err := stateStore.PruneStates(tc.pruneFrom, tc.pruneTo, tc.evidenceThresholdHeight) if tc.expectErr { require.Error(t, err) return diff --git a/state/txindex/indexer_service.go b/state/txindex/indexer_service.go index a12937ef1..119d2df1d 100644 --- a/state/txindex/indexer_service.go +++ b/state/txindex/indexer_service.go @@ -3,7 +3,6 @@ package txindex import ( "context" - abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/service" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/types" @@ -20,9 +19,10 @@ const ( type IndexerService struct { service.BaseService - txIdxr TxIndexer - blockIdxr indexer.BlockIndexer - eventBus *types.EventBus + txIdxr TxIndexer + blockIdxr indexer.BlockIndexer + eventBus *types.EventBus + terminateOnError bool } // NewIndexerService returns a new service instance. @@ -30,9 +30,10 @@ func NewIndexerService( txIdxr TxIndexer, blockIdxr indexer.BlockIndexer, eventBus *types.EventBus, + terminateOnError bool, ) *IndexerService { - is := &IndexerService{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus} + is := &IndexerService{txIdxr: txIdxr, blockIdxr: blockIdxr, eventBus: eventBus, terminateOnError: terminateOnError} is.BaseService = *service.NewBaseService(nil, "IndexerService", is) return is } @@ -76,22 +77,36 @@ func (is *IndexerService) OnStart() error { "index", txResult.Index, "err", err, ) + + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } } if err := is.blockIdxr.Index(eventNewBlockEvents); err != nil { is.Logger.Error("failed to index block", "height", height, "err", err) + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } else { - is.Logger.Info("indexed block", "height", height) - } - - batch.Ops, err = DeduplicateBatch(batch.Ops, is.txIdxr) - if err != nil { - is.Logger.Error("deduplicate batch", "height", height) + is.Logger.Info("indexed block exents", "height", height) } if err = is.txIdxr.AddBatch(batch); err != nil { is.Logger.Error("failed to index block txs", "height", height, "err", err) + if is.terminateOnError { + if err := is.Stop(); err != nil { + is.Logger.Error("failed to stop", "err", err) + } + return + } } else { is.Logger.Debug("indexed block txs", "height", height, "num_txs", numTxs) } @@ -106,45 +121,3 @@ func (is *IndexerService) OnStop() { _ = is.eventBus.UnsubscribeAll(context.Background(), subscriber) } } - -// DeduplicateBatch consider the case of duplicate txs. -// if the current one under investigation is NOT OK, then we need to check -// whether there's a previously indexed tx. -// SKIP the current tx if the previously indexed record is found and successful. -func DeduplicateBatch(ops []*abci.TxResult, txIdxr TxIndexer) ([]*abci.TxResult, error) { - result := make([]*abci.TxResult, 0, len(ops)) - - // keep track of successful txs in this block in order to suppress latter ones being indexed. - var successfulTxsInThisBlock = make(map[string]struct{}) - - for _, txResult := range ops { - hash := types.Tx(txResult.Tx).Hash() - - if txResult.Result.IsOK() { - successfulTxsInThisBlock[string(hash)] = struct{}{} - } else { - // if it already appeared in current block and was successful, skip. - if _, found := successfulTxsInThisBlock[string(hash)]; found { - continue - } - - // check if this tx hash is already indexed - old, err := txIdxr.Get(hash) - - // if db op errored - // Not found is not an error - if err != nil { - return nil, err - } - - // if it's already indexed in an older block and was successful, skip. - if old != nil && old.Result.Code == abci.CodeTypeOK { - continue - } - } - - result = append(result, txResult) - } - - return result, nil -} diff --git a/state/txindex/indexer_service_test.go b/state/txindex/indexer_service_test.go index d1460f8e3..5df9bb322 100644 --- a/state/txindex/indexer_service_test.go +++ b/state/txindex/indexer_service_test.go @@ -32,7 +32,7 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { txIndexer := kv.NewTxIndex(store) blockIndexer := blockidxkv.New(db.NewPrefixDB(store, []byte("block_events"))) - service := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus) + service := txindex.NewIndexerService(txIndexer, blockIndexer, eventBus, false) service.SetLogger(log.TestingLogger()) err = service.Start() require.NoError(t, err) @@ -91,164 +91,3 @@ func TestIndexerServiceIndexesBlocks(t *testing.T) { require.NoError(t, err) require.Equal(t, txResult2, res) } - -func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { - var mockTx = types.Tx("MOCK_TX_HASH") - - testCases := []struct { - name string - tx1 abci.TxResult - tx2 abci.TxResult - expSkip bool // do we expect the second tx to be skipped by tx indexer - }{ - {"skip, previously successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - true, - }, - {"not skip, previously unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - false, - }, - {"not skip, both successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK, - }, - }, - false, - }, - {"not skip, both unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 2, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - false, - }, - {"skip, same block, previously successful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK, - }, - }, - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - true, - }, - {"not skip, same block, previously unsuccessful", - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK + 1, - }, - }, - abci.TxResult{ - Height: 1, - Index: 0, - Tx: mockTx, - Result: abci.ExecTxResult{ - Code: abci.CodeTypeOK, - }, - }, - false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - indexer := kv.NewTxIndex(db.NewMemDB()) - - if tc.tx1.Height != tc.tx2.Height { - // index the first tx - err := indexer.AddBatch(&txindex.Batch{ - Ops: []*abci.TxResult{&tc.tx1}, - }) - require.NoError(t, err) - - // check if the second one should be skipped. - ops, err := txindex.DeduplicateBatch([]*abci.TxResult{&tc.tx2}, indexer) - require.NoError(t, err) - - if tc.expSkip { - require.Empty(t, ops) - } else { - require.Equal(t, []*abci.TxResult{&tc.tx2}, ops) - } - } else { - // same block - ops := []*abci.TxResult{&tc.tx1, &tc.tx2} - ops, err := txindex.DeduplicateBatch(ops, indexer) - require.NoError(t, err) - if tc.expSkip { - // the second one is skipped - require.Equal(t, []*abci.TxResult{&tc.tx1}, ops) - } else { - require.Equal(t, []*abci.TxResult{&tc.tx1, &tc.tx2}, ops) - } - } - }) - } -} diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 28a26664f..48033eeba 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -13,6 +13,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/pubsub/query" + "github.com/tendermint/tendermint/libs/pubsub/query/syntax" "github.com/tendermint/tendermint/state/indexer" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" @@ -101,12 +102,30 @@ func (txi *TxIndex) AddBatch(b *txindex.Batch) error { // that indexed from the tx's events is a composite of the event type and the // respective attribute's key delimited by a "." (eg. "account.number"). // Any event with an empty type is not indexed. +// +// If a transaction is indexed with the same hash as a previous transaction, it will +// be overwritten unless the tx result was NOT OK and the prior result was OK i.e. +// more transactions that successfully executed overwrite transactions that failed +// or successful yet older transactions. func (txi *TxIndex) Index(result *abci.TxResult) error { b := txi.store.NewBatch() defer b.Close() hash := types.Tx(result.Tx).Hash() + if !result.Result.IsOK() { + oldResult, err := txi.Get(hash) + if err != nil { + return err + } + + // if the new transaction failed and it's already indexed in an older block and was successful + // we skip it as we want users to get the older successful transaction when they query. + if oldResult != nil && oldResult.Result.Code == abci.CodeTypeOK { + return nil + } + } + // index tx by events err := txi.indexEvents(result, hash, b) if err != nil { @@ -185,10 +204,7 @@ func (txi *TxIndex) Search(ctx context.Context, q *query.Query) ([]*abci.TxResul filteredHashes := make(map[string][]byte) // get a list of conditions (like "tx.height > 5") - conditions, err := q.Conditions() - if err != nil { - return nil, fmt.Errorf("error during parsing conditions from query: %w", err) - } + conditions := q.Syntax() // if there is a hash condition, return the result immediately hash, ok, err := lookForHash(conditions) @@ -275,10 +291,10 @@ RESULTS_LOOP: return results, nil } -func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) { +func lookForHash(conditions []syntax.Condition) (hash []byte, ok bool, err error) { for _, c := range conditions { - if c.CompositeKey == types.TxHashKey { - decoded, err := hex.DecodeString(c.Operand.(string)) + if c.Tag == types.TxHashKey { + decoded, err := hex.DecodeString(c.Arg.Value()) return decoded, true, err } } @@ -286,10 +302,10 @@ func lookForHash(conditions []query.Condition) (hash []byte, ok bool, err error) } // lookForHeight returns a height if there is an "height=X" condition. -func lookForHeight(conditions []query.Condition) (height int64) { +func lookForHeight(conditions []syntax.Condition) (height int64) { for _, c := range conditions { - if c.CompositeKey == types.TxHeightKey && c.Op == query.OpEqual { - return c.Operand.(int64) + if c.Tag == types.TxHeightKey && c.Op == syntax.TEq { + return int64(c.Arg.Number()) } } return 0 @@ -302,7 +318,7 @@ func lookForHeight(conditions []query.Condition) (height int64) { // NOTE: filteredHashes may be empty if no previous condition has matched. func (txi *TxIndex) match( ctx context.Context, - c query.Condition, + c syntax.Condition, startKeyBz []byte, filteredHashes map[string][]byte, firstRun bool, @@ -315,8 +331,8 @@ func (txi *TxIndex) match( tmpHashes := make(map[string][]byte) - switch c.Op { - case query.OpEqual: + switch { + case c.Op == syntax.TEq: it, err := dbm.IteratePrefix(txi.store, startKeyBz) if err != nil { panic(err) @@ -338,10 +354,10 @@ func (txi *TxIndex) match( panic(err) } - case query.OpExists: + case c.Op == syntax.TExists: // XXX: can't use startKeyBz here because c.Operand is nil // (e.g. "account.owner//" won't match w/ a single row) - it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, startKey(c.Tag)) if err != nil { panic(err) } @@ -362,11 +378,11 @@ func (txi *TxIndex) match( panic(err) } - case query.OpContains: + case c.Op == syntax.TContains: // XXX: startKey does not apply here. // For example, if startKey = "account.owner/an/" and search query = "account.owner CONTAINS an" // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" - it, err := dbm.IteratePrefix(txi.store, startKey(c.CompositeKey)) + it, err := dbm.IteratePrefix(txi.store, startKey(c.Tag)) if err != nil { panic(err) } @@ -377,8 +393,7 @@ func (txi *TxIndex) match( if !isTagKey(it.Key()) { continue } - - if strings.Contains(extractValueFromKey(it.Key()), c.Operand.(string)) { + if strings.Contains(extractValueFromKey(it.Key()), c.Arg.Value()) { tmpHashes[string(it.Value())] = it.Value() } @@ -557,11 +572,11 @@ func keyForHeight(result *abci.TxResult) []byte { )) } -func startKeyForCondition(c query.Condition, height int64) []byte { +func startKeyForCondition(c syntax.Condition, height int64) []byte { if height > 0 { - return startKey(c.CompositeKey, c.Operand, height) + return startKey(c.Tag, c.Arg.Value(), height) } - return startKey(c.CompositeKey, c.Operand) + return startKey(c.Tag, c.Arg.Value()) } func startKey(fields ...interface{}) []byte { diff --git a/state/txindex/kv/kv_bench_test.go b/state/txindex/kv/kv_bench_test.go index 170994f07..00cfde227 100644 --- a/state/txindex/kv/kv_bench_test.go +++ b/state/txindex/kv/kv_bench_test.go @@ -60,7 +60,7 @@ func BenchmarkTxSearch(b *testing.B) { } } - txQuery := query.MustParse("transfer.address = 'address_43' AND transfer.amount = 50") + txQuery := query.MustCompile(`transfer.address = 'address_43' AND transfer.amount = 50`) b.ResetTimer() diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 4819cde22..6fcd69186 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -126,7 +126,7 @@ func TestTxSearch(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) assert.NoError(t, err) assert.Len(t, results, tc.resultsLength) @@ -152,7 +152,7 @@ func TestTxSearchWithCancelation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - results, err := indexer.Search(ctx, query.MustParse("account.number = 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number = 1`)) assert.NoError(t, err) assert.Empty(t, results) } @@ -225,7 +225,7 @@ func TestTxSearchDeprecatedIndexing(t *testing.T) { for _, tc := range testCases { tc := tc t.Run(tc.q, func(t *testing.T) { - results, err := indexer.Search(ctx, query.MustParse(tc.q)) + results, err := indexer.Search(ctx, query.MustCompile(tc.q)) require.NoError(t, err) for _, txr := range results { for _, tr := range tc.results { @@ -249,7 +249,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) assert.Len(t, results, 1) @@ -258,6 +258,103 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { } } +func TestTxIndexDuplicatePreviouslySuccessful(t *testing.T) { + var mockTx = types.Tx("MOCK_TX_HASH") + + testCases := []struct { + name string + tx1 *abci.TxResult + tx2 *abci.TxResult + expOverwrite bool // do we expect the second tx to overwrite the first tx + }{ + { + "don't overwrite as a non-zero code was returned and the previous tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + false, + }, + { + "overwrite as the previous tx was also unsuccessful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK + 1, + }, + }, + true, + }, + { + "overwrite as the most recent tx was successful", + &abci.TxResult{ + Height: 1, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + &abci.TxResult{ + Height: 2, + Index: 0, + Tx: mockTx, + Result: abci.ResponseDeliverTx{ + Code: abci.CodeTypeOK, + }, + }, + true, + }, + } + + hash := mockTx.Hash() + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + indexer := NewTxIndex(db.NewMemDB()) + + // index the first tx + err := indexer.Index(tc.tx1) + require.NoError(t, err) + + // index the same tx with different results + err = indexer.Index(tc.tx2) + require.NoError(t, err) + + res, err := indexer.Get(hash) + require.NoError(t, err) + + if tc.expOverwrite { + require.Equal(t, tc.tx2, res) + } else { + require.Equal(t, tc.tx1, res) + } + }) + } +} + func TestTxSearchMultipleTxs(t *testing.T) { indexer := NewTxIndex(db.NewMemDB()) @@ -306,7 +403,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { ctx := context.Background() - results, err := indexer.Search(ctx, query.MustParse("account.number >= 1")) + results, err := indexer.Search(ctx, query.MustCompile(`account.number >= 1`)) assert.NoError(t, err) require.Len(t, results, 3) diff --git a/state/validation_test.go b/state/validation_test.go index 705b13997..7fc092bfa 100644 --- a/state/validation_test.go +++ b/state/validation_test.go @@ -17,8 +17,10 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/mocks" + "github.com/tendermint/tendermint/store" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" + dbm "github.com/tendermint/tm-db" ) const validationTestsStopHeight int64 = 10 @@ -44,12 +46,15 @@ func TestValidateBlockHeader(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) @@ -129,12 +134,15 @@ func TestValidateBlockCommit(t *testing.T) { mock.Anything, mock.Anything).Return(nil) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, sm.EmptyEvidencePool{}, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) wrongSigsCommit := types.NewCommit(1, 0, types.BlockID{}, nil) @@ -268,12 +276,15 @@ func TestValidateBlockEvidence(t *testing.T) { mock.Anything, mock.Anything).Return(nil) state.ConsensusParams.Evidence.MaxBytes = 1000 + blockStore := store.NewBlockStore(dbm.NewMemDB()) + blockExec := sm.NewBlockExecutor( stateStore, log.TestingLogger(), proxyApp.Consensus(), mp, evpool, + blockStore, ) lastCommit := types.NewCommit(0, 0, types.BlockID{}, nil) diff --git a/statesync/syncer.go b/statesync/syncer.go index 0724eb6cc..f8b5e5125 100644 --- a/statesync/syncer.go +++ b/statesync/syncer.go @@ -117,7 +117,7 @@ func (s *syncer) AddSnapshot(peer p2p.Peer, snapshot *snapshot) (bool, error) { } if added { s.logger.Info("Discovered new snapshot", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) } return added, nil } @@ -144,7 +144,7 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat } if discoveryTime > 0 { - s.logger.Info("sync any", "msg", log.NewLazySprintf("Discovering snapshots for %v", discoveryTime)) + s.logger.Info("Discovering snapshots", "discoverTime", discoveryTime) time.Sleep(discoveryTime) } @@ -189,18 +189,18 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat case errors.Is(err, errRetrySnapshot): chunks.RetryAll() s.logger.Info("Retrying snapshot", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) continue case errors.Is(err, errTimeout): s.snapshots.Reject(snapshot) s.logger.Error("Timed out waiting for snapshot chunks, rejected snapshot", - "height", snapshot.Height, "format", snapshot.Format, "hash", snapshot.Hash) + "height", snapshot.Height, "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) case errors.Is(err, errRejectSnapshot): s.snapshots.Reject(snapshot) s.logger.Info("Snapshot rejected", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) case errors.Is(err, errRejectFormat): s.snapshots.RejectFormat(snapshot.Format) @@ -208,7 +208,7 @@ func (s *syncer) SyncAny(discoveryTime time.Duration, retryHook func()) (sm.Stat case errors.Is(err, errRejectSender): s.logger.Info("Snapshot senders rejected", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) for _, peer := range s.snapshots.GetPeers(snapshot) { s.snapshots.RejectPeer(peer.ID()) s.logger.Info("Snapshot sender rejected", "peer", peer.ID()) @@ -308,7 +308,7 @@ func (s *syncer) Sync(snapshot *snapshot, chunks *chunkQueue) (sm.State, *types. // Done! 🎉 s.logger.Info("Snapshot restored", "height", snapshot.Height, "format", snapshot.Format, - "hash", snapshot.Hash) + "hash", log.NewLazySprintf("%X", snapshot.Hash)) return state, commit, nil } @@ -317,7 +317,7 @@ func (s *syncer) Sync(snapshot *snapshot, chunks *chunkQueue) (sm.State, *types. // response, or nil if the snapshot was accepted. func (s *syncer) offerSnapshot(snapshot *snapshot) error { s.logger.Info("Offering snapshot to ABCI app", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) resp, err := s.conn.OfferSnapshot(context.TODO(), &abci.RequestOfferSnapshot{ Snapshot: &abci.Snapshot{ Height: snapshot.Height, @@ -334,7 +334,7 @@ func (s *syncer) offerSnapshot(snapshot *snapshot) error { switch resp.Result { case abci.ResponseOfferSnapshot_ACCEPT: s.logger.Info("Snapshot accepted, restoring", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) return nil case abci.ResponseOfferSnapshot_ABORT: return errAbort @@ -462,7 +462,7 @@ func (s *syncer) requestChunk(snapshot *snapshot, chunk uint32) { peer := s.snapshots.GetPeer(snapshot) if peer == nil { s.logger.Error("No valid peers found for snapshot", "height", snapshot.Height, - "format", snapshot.Format, "hash", snapshot.Hash) + "format", snapshot.Format, "hash", log.NewLazySprintf("%X", snapshot.Hash)) return } s.logger.Debug("Requesting snapshot chunk", "height", snapshot.Height, diff --git a/store/store.go b/store/store.go index 866965ac6..c56622e47 100644 --- a/store/store.go +++ b/store/store.go @@ -7,9 +7,11 @@ import ( "github.com/cosmos/gogoproto/proto" dbm "github.com/tendermint/tm-db" + "github.com/tendermint/tendermint/evidence" tmsync "github.com/tendermint/tendermint/libs/sync" tmstore "github.com/tendermint/tendermint/proto/tendermint/store" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" ) @@ -264,20 +266,20 @@ func (bs *BlockStore) LoadSeenCommit(height int64) *types.Commit { return commit } -// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned. -func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { +// PruneBlocks removes block up to (but not including) a height. It returns number of blocks pruned and the evidence retain height - the height at which data needed to prove evidence must not be removed. +func (bs *BlockStore) PruneBlocks(height int64, state sm.State) (uint64, int64, error) { if height <= 0 { - return 0, fmt.Errorf("height must be greater than 0") + return 0, -1, fmt.Errorf("height must be greater than 0") } bs.mtx.RLock() if height > bs.height { bs.mtx.RUnlock() - return 0, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) + return 0, -1, fmt.Errorf("cannot prune beyond the latest height %v", bs.height) } base := bs.base bs.mtx.RUnlock() if height < base { - return 0, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", + return 0, -1, fmt.Errorf("cannot prune to height %v, it is lower than base height %v", height, base) } @@ -300,26 +302,42 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { return nil } + evidencePoint := height for h := base; h < height; h++ { + meta := bs.LoadBlockMeta(h) if meta == nil { // assume already deleted continue } - if err := batch.Delete(calcBlockMetaKey(h)); err != nil { - return 0, err + + // This logic is in place to protect data that proves malicious behavior. + // If the height is within the evidence age, we continue to persist the header and commit data. + + if evidencePoint == height && !evidence.IsEvidenceExpired(state.LastBlockHeight, state.LastBlockTime, h, meta.Header.Time, state.ConsensusParams.Evidence) { + evidencePoint = h + } + + // if height is beyond the evidence point we dont delete the header + if h < evidencePoint { + if err := batch.Delete(calcBlockMetaKey(h)); err != nil { + return 0, -1, err + } } if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { - return 0, err + return 0, -1, err } - if err := batch.Delete(calcBlockCommitKey(h)); err != nil { - return 0, err + // if height is beyond the evidence point we dont delete the commit data + if h < evidencePoint { + if err := batch.Delete(calcBlockCommitKey(h)); err != nil { + return 0, -1, err + } } if err := batch.Delete(calcSeenCommitKey(h)); err != nil { - return 0, err + return 0, -1, err } for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { if err := batch.Delete(calcBlockPartKey(h, p)); err != nil { - return 0, err + return 0, -1, err } } pruned++ @@ -328,7 +346,7 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { if pruned%1000 == 0 && pruned > 0 { err := flush(batch, h) if err != nil { - return 0, err + return 0, -1, err } batch = bs.db.NewBatch() defer batch.Close() @@ -337,9 +355,9 @@ func (bs *BlockStore) PruneBlocks(height int64) (uint64, error) { err := flush(batch, height) if err != nil { - return 0, err + return 0, -1, err } - return pruned, nil + return pruned, evidencePoint, nil } // SaveBlock persists the given block, blockParts, and seenCommit to the underlying db. @@ -521,3 +539,50 @@ func mustEncode(pb proto.Message) []byte { } return bz } + +//----------------------------------------------------------------------------- + +// DeleteLatestBlock removes the block pointed to by height, +// lowering height by one. +func (bs *BlockStore) DeleteLatestBlock() error { + bs.mtx.RLock() + targetHeight := bs.height + bs.mtx.RUnlock() + + batch := bs.db.NewBatch() + defer batch.Close() + + // delete what we can, skipping what's already missing, to ensure partial + // blocks get deleted fully. + if meta := bs.LoadBlockMeta(targetHeight); meta != nil { + if err := batch.Delete(calcBlockHashKey(meta.BlockID.Hash)); err != nil { + return err + } + for p := 0; p < int(meta.BlockID.PartSetHeader.Total); p++ { + if err := batch.Delete(calcBlockPartKey(targetHeight, p)); err != nil { + return err + } + } + } + if err := batch.Delete(calcBlockCommitKey(targetHeight)); err != nil { + return err + } + if err := batch.Delete(calcSeenCommitKey(targetHeight)); err != nil { + return err + } + // delete last, so as to not leave keys built on meta.BlockID dangling + if err := batch.Delete(calcBlockMetaKey(targetHeight)); err != nil { + return err + } + + bs.mtx.Lock() + bs.height = targetHeight - 1 + bs.mtx.Unlock() + bs.saveState() + + err := batch.WriteSync() + if err != nil { + return fmt.Errorf("failed to delete height %v: %w", targetHeight, err) + } + return nil +} diff --git a/store/store_test.go b/store/store_test.go index e77d9ead8..47c944956 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/internal/test" "github.com/tendermint/tendermint/libs/log" @@ -45,7 +44,7 @@ func makeTestCommit(height int64, timestamp time.Time) *types.Commit { } func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore, cleanupFunc) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) blockDB := dbm.NewMemDB() @@ -365,7 +364,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { } func TestLoadBaseMeta(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardFinalizeBlockResponses: false, @@ -382,12 +381,15 @@ func TestLoadBaseMeta(t *testing.T) { bs.SaveBlock(block, partSet, seenCommit) } - _, err = bs.PruneBlocks(4) + _, _, err = bs.PruneBlocks(4, state) require.NoError(t, err) baseBlock := bs.LoadBaseMeta() assert.EqualValues(t, 4, baseBlock.Header.Height) assert.EqualValues(t, 4, bs.Base()) + + require.NoError(t, bs.DeleteLatestBlock()) + require.EqualValues(t, 9, bs.Height()) } func TestLoadBlockPart(t *testing.T) { @@ -424,7 +426,7 @@ func TestLoadBlockPart(t *testing.T) { } func TestPruneBlocks(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardFinalizeBlockResponses: false, @@ -438,10 +440,10 @@ func TestPruneBlocks(t *testing.T) { assert.EqualValues(t, 0, bs.Size()) // pruning an empty store should error, even when pruning to 0 - _, err = bs.PruneBlocks(1) + _, _, err = bs.PruneBlocks(1, state) require.Error(t, err) - _, err = bs.PruneBlocks(0) + _, _, err = bs.PruneBlocks(0, state) require.Error(t, err) // make more than 1000 blocks, to test batch deletions @@ -457,27 +459,30 @@ func TestPruneBlocks(t *testing.T) { assert.EqualValues(t, 1500, bs.Height()) assert.EqualValues(t, 1500, bs.Size()) - prunedBlock := bs.LoadBlock(1199) + state.LastBlockTime = time.Date(2020, 1, 1, 1, 0, 0, 0, time.UTC) + state.LastBlockHeight = 1500 + + state.ConsensusParams.Evidence.MaxAgeNumBlocks = 400 + state.ConsensusParams.Evidence.MaxAgeDuration = 1 * time.Second // Check that basic pruning works - pruned, err := bs.PruneBlocks(1200) + pruned, evidenceRetainHeight, err := bs.PruneBlocks(1200, state) require.NoError(t, err) assert.EqualValues(t, 1199, pruned) assert.EqualValues(t, 1200, bs.Base()) assert.EqualValues(t, 1500, bs.Height()) assert.EqualValues(t, 301, bs.Size()) - assert.EqualValues(t, tmstore.BlockStoreState{ - Base: 1200, - Height: 1500, - }, LoadBlockStoreState(db)) + assert.EqualValues(t, 1100, evidenceRetainHeight) require.NotNil(t, bs.LoadBlock(1200)) require.Nil(t, bs.LoadBlock(1199)) - require.Nil(t, bs.LoadBlockByHash(prunedBlock.Hash())) - require.Nil(t, bs.LoadBlockCommit(1199)) - require.Nil(t, bs.LoadBlockMeta(1199)) - require.Nil(t, bs.LoadBlockMetaByHash(prunedBlock.Hash())) - require.Nil(t, bs.LoadBlockPart(1199, 1)) + + // The header and commit for heights 1100 onwards + // need to remain to verify evidence + require.NotNil(t, bs.LoadBlockMeta(1100)) + require.Nil(t, bs.LoadBlockMeta(1099)) + require.NotNil(t, bs.LoadBlockCommit(1100)) + require.Nil(t, bs.LoadBlockCommit(1099)) for i := int64(1); i < 1200; i++ { require.Nil(t, bs.LoadBlock(i)) @@ -487,26 +492,33 @@ func TestPruneBlocks(t *testing.T) { } // Pruning below the current base should error - _, err = bs.PruneBlocks(1199) + _, _, err = bs.PruneBlocks(1199, state) require.Error(t, err) // Pruning to the current base should work - pruned, err = bs.PruneBlocks(1200) + pruned, _, err = bs.PruneBlocks(1200, state) require.NoError(t, err) assert.EqualValues(t, 0, pruned) // Pruning again should work - pruned, err = bs.PruneBlocks(1300) + pruned, _, err = bs.PruneBlocks(1300, state) require.NoError(t, err) assert.EqualValues(t, 100, pruned) assert.EqualValues(t, 1300, bs.Base()) + // we should still have the header and the commit + // as they're needed for evidence + require.NotNil(t, bs.LoadBlockMeta(1100)) + require.Nil(t, bs.LoadBlockMeta(1099)) + require.NotNil(t, bs.LoadBlockCommit(1100)) + require.Nil(t, bs.LoadBlockCommit(1099)) + // Pruning beyond the current height should error - _, err = bs.PruneBlocks(1501) + _, _, err = bs.PruneBlocks(1501, state) require.Error(t, err) // Pruning to the current height should work - pruned, err = bs.PruneBlocks(1500) + pruned, _, err = bs.PruneBlocks(1500, state) require.NoError(t, err) assert.EqualValues(t, 200, pruned) assert.Nil(t, bs.LoadBlock(1499)) @@ -554,7 +566,7 @@ func TestLoadBlockMeta(t *testing.T) { } func TestLoadBlockMetaByHash(t *testing.T) { - config := cfg.ResetTestRoot("blockchain_reactor_test") + config := test.ResetTestRoot("blockchain_reactor_test") defer os.RemoveAll(config.RootDir) stateStore := sm.NewStore(dbm.NewMemDB(), sm.StoreOptions{ DiscardFinalizeBlockResponses: false, diff --git a/test/loadtime/cmd/report/main.go b/test/loadtime/cmd/report/main.go index 385504537..b92bf0ef5 100644 --- a/test/loadtime/cmd/report/main.go +++ b/test/loadtime/cmd/report/main.go @@ -87,7 +87,7 @@ func toCSVRecords(rs []report.Report) [][]string { } res := make([][]string, total+1) - res[0] = []string{"experiment_id", "duration_ns", "connections", "rate", "size"} + res[0] = []string{"experiment_id", "block_time", "duration_ns", "tx_hash", "connections", "rate", "size"} offset := 1 for _, r := range rs { idStr := r.ID.String() @@ -95,7 +95,7 @@ func toCSVRecords(rs []report.Report) [][]string { rateStr := strconv.FormatInt(int64(r.Rate), 10) sizeStr := strconv.FormatInt(int64(r.Size), 10) for i, v := range r.All { - res[offset+i] = []string{idStr, strconv.FormatInt(int64(v), 10), connStr, rateStr, sizeStr} + res[offset+i] = []string{idStr, strconv.FormatInt(v.BlockTime.UnixNano(), 10), strconv.FormatInt(int64(v.Duration), 10), fmt.Sprintf("%X", v.Hash), connStr, rateStr, sizeStr} } offset += len(r.All) } diff --git a/test/loadtime/report/report.go b/test/loadtime/report/report.go index 01ee1ef7e..269f63b2d 100644 --- a/test/loadtime/report/report.go +++ b/test/loadtime/report/report.go @@ -21,6 +21,13 @@ type BlockStore interface { LoadBlock(int64) *types.Block } +// DataPoint contains the set of data collected for each transaction. +type DataPoint struct { + Duration time.Duration + BlockTime time.Time + Hash []byte +} + // Report contains the data calculated from reading the timestamped transactions // of each block found in the blockstore. type Report struct { @@ -38,7 +45,7 @@ type Report struct { // All contains all data points gathered from all valid transactions. // The order of the contents of All is not guaranteed to be match the order of transactions // in the chain. - All []time.Duration + All []DataPoint // used for calculating average during report creation. sum int64 @@ -62,7 +69,7 @@ func (rs *Reports) ErrorCount() int { return rs.errorCount } -func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, conns, rate, size uint64) { +func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, bt time.Time, hash []byte, conns, rate, size uint64) { r, ok := rs.s[id] if !ok { r = Report{ @@ -75,7 +82,7 @@ func (rs *Reports) addDataPoint(id uuid.UUID, l time.Duration, conns, rate, size } rs.s[id] = r } - r.All = append(r.All, l) + r.All = append(r.All, DataPoint{Duration: l, BlockTime: bt, Hash: hash}) if l > r.Max { r.Max = l } @@ -116,11 +123,13 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { type payloadData struct { id uuid.UUID l time.Duration + bt time.Time + hash []byte connections, rate, size uint64 err error } type txData struct { - tx []byte + tx types.Tx bt time.Time } reports := &Reports{ @@ -150,10 +159,12 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { } l := b.bt.Sub(p.Time.AsTime()) - b := (*[16]byte)(p.Id) + idb := (*[16]byte)(p.Id) pdc <- payloadData{ l: l, - id: uuid.UUID(*b), + bt: b.bt, + hash: b.tx.Hash(), + id: uuid.UUID(*idb), connections: p.Connections, rate: p.Rate, size: p.Size, @@ -194,16 +205,16 @@ func GenerateFromBlockStore(s BlockStore) (*Reports, error) { reports.addError() continue } - reports.addDataPoint(pd.id, pd.l, pd.connections, pd.rate, pd.size) + reports.addDataPoint(pd.id, pd.l, pd.bt, pd.hash, pd.connections, pd.rate, pd.size) } reports.calculateAll() return reports, nil } -func toFloat(in []time.Duration) []float64 { +func toFloat(in []DataPoint) []float64 { r := make([]float64, len(in)) for i, v := range in { - r[i] = float64(int64(v)) + r[i] = float64(int64(v.Duration)) } return r } diff --git a/tools/tm-signer-harness/Makefile b/tools/tm-signer-harness/Makefile index 1c404ebf8..fc4157108 100644 --- a/tools/tm-signer-harness/Makefile +++ b/tools/tm-signer-harness/Makefile @@ -3,7 +3,7 @@ TENDERMINT_VERSION?=latest BUILD_TAGS?='tendermint' VERSION := $(shell git describe --always) -BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION) +BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(VERSION)" .DEFAULT_GOAL := build diff --git a/tools/tm-signer-harness/main.go b/tools/tm-signer-harness/main.go index 6d75abe7e..03f35bc6c 100644 --- a/tools/tm-signer-harness/main.go +++ b/tools/tm-signer-harness/main.go @@ -17,7 +17,6 @@ import ( const ( defaultAcceptRetries = 100 defaultBindAddr = "tcp://127.0.0.1:0" - defaultTMHome = "~/.tendermint" defaultAcceptDeadline = 1 defaultConnDeadline = 3 defaultExtractKeyOutput = "./signing.key" @@ -59,6 +58,7 @@ Use "tm-signer-harness help " for more information about that command.` fmt.Println("") } + defaultTMHome := internal.ExpandPath("~/.tendermint") runCmd = flag.NewFlagSet("run", flag.ExitOnError) runCmd.IntVar(&flagAcceptRetries, "accept-retries", diff --git a/types/event_bus_test.go b/types/event_bus_test.go index 96abdaf8a..1abe11bf0 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -36,7 +36,7 @@ func TestEventBusPublishEventTx(t *testing.T) { // PublishEventTx adds 3 composite keys, so the query below should work query := fmt.Sprintf("tm.event='Tx' AND tx.height=1 AND tx.hash='%X' AND testType.baz=1", tx.Hash()) - txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + txsSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -84,7 +84,7 @@ func TestEventBusPublishEventNewBlock(t *testing.T) { // PublishEventNewBlock adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlock' AND testType.baz=1" - blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + blocksSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -185,7 +185,7 @@ func TestEventBusPublishEventTxDuplicateKeys(t *testing.T) { } for i, tc := range testCases { - sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustParse(tc.query)) + sub, err := eventBus.Subscribe(context.Background(), fmt.Sprintf("client-%d", i), tmquery.MustCompile(tc.query)) require.NoError(t, err) done := make(chan struct{}) @@ -238,7 +238,7 @@ func TestEventBusPublishEventNewBlockHeader(t *testing.T) { block := MakeBlock(0, []Tx{}, nil, []Evidence{}) // PublishEventNewBlockHeader adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlockHeader'" - headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -273,7 +273,7 @@ func TestEventBusPublishEventNewBlockEvents(t *testing.T) { // PublishEventNewBlockHeader adds the tm.event compositeKey, so the query below should work query := "tm.event='NewBlockEvents'" - headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + headersSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -317,7 +317,7 @@ func TestEventBusPublishEventNewEvidence(t *testing.T) { require.NoError(t, err) query := "tm.event='NewEvidence'" - evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query)) + evSub, err := eventBus.Subscribe(context.Background(), "test", tmquery.MustCompile(query)) require.NoError(t, err) done := make(chan struct{}) @@ -354,7 +354,7 @@ func TestEventBusPublish(t *testing.T) { const numEventsExpected = 15 - sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.Empty{}, numEventsExpected) + sub, err := eventBus.Subscribe(context.Background(), "test", tmquery.All, numEventsExpected) require.NoError(t, err) done := make(chan struct{}) diff --git a/types/events.go b/types/events.go index db6860504..e6f4381e0 100644 --- a/types/events.go +++ b/types/events.go @@ -166,11 +166,11 @@ var ( ) func EventQueryTxFor(tx Tx) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTx, TxHashKey, tx.Hash())) + return tmquery.MustCompile(fmt.Sprintf("%s='%s' AND %s='%X'", EventTypeKey, EventTx, TxHashKey, tx.Hash())) } func QueryForEvent(eventType string) tmpubsub.Query { - return tmquery.MustParse(fmt.Sprintf("%s='%s'", EventTypeKey, eventType)) + return tmquery.MustCompile(fmt.Sprintf("%s='%s'", EventTypeKey, eventType)) } // BlockEventPublisher publishes all block related events diff --git a/types/events_test.go b/types/events_test.go index 12f75b74d..e4479d3ab 100644 --- a/types/events_test.go +++ b/types/events_test.go @@ -10,18 +10,18 @@ import ( func TestQueryTxFor(t *testing.T) { tx := Tx("foo") assert.Equal(t, - fmt.Sprintf("tm.event='Tx' AND tx.hash='%X'", tx.Hash()), + fmt.Sprintf("tm.event = 'Tx' AND tx.hash = '%X'", tx.Hash()), EventQueryTxFor(tx).String(), ) } func TestQueryForEvent(t *testing.T) { assert.Equal(t, - "tm.event='NewBlock'", + "tm.event = 'NewBlock'", QueryForEvent(EventNewBlock).String(), ) assert.Equal(t, - "tm.event='NewEvidence'", + "tm.event = 'NewEvidence'", QueryForEvent(EventNewEvidence).String(), ) } diff --git a/types/validation.go b/types/validation.go index b3b448004..3b33e90db 100644 --- a/types/validation.go +++ b/types/validation.go @@ -1,30 +1,132 @@ package types import ( + "errors" "fmt" - "time" + "github.com/tendermint/tendermint/crypto/batch" "github.com/tendermint/tendermint/crypto/tmhash" - tmtime "github.com/tendermint/tendermint/types/time" + tmmath "github.com/tendermint/tendermint/libs/math" ) -// ValidateTime does a basic time validation ensuring time does not drift too -// much: +/- one year. -// TODO: reduce this to eg 1 day -// NOTE: DO NOT USE in ValidateBasic methods in this package. This function -// can only be used for real time validation, like on proposals and votes -// in the consensus. If consensus is stuck, and rounds increase for more than a day, -// having only a 1-day band here could break things... -// Can't use for validating blocks because we may be syncing years worth of history. -func ValidateTime(t time.Time) error { - var ( - now = tmtime.Now() - oneYear = 8766 * time.Hour - ) - if t.Before(now.Add(-oneYear)) || t.After(now.Add(oneYear)) { - return fmt.Errorf("time drifted too much. Expected: -1 < %v < 1 year", now) +const batchVerifyThreshold = 2 + +func shouldBatchVerify(vals *ValidatorSet, commit *Commit) bool { + return len(commit.Signatures) >= batchVerifyThreshold && batch.SupportsBatchVerifier(vals.GetProposer().PubKey) +} + +// VerifyCommit verifies +2/3 of the set had signed the given commit. +// +// It checks all the signatures! While it's safe to exit as soon as we have +// 2/3+ signatures, doing so would impact incentivization logic in the ABCI +// application that depends on the LastCommitInfo sent in BeginBlock, which +// includes which validators signed. For instance, Gaia incentivizes proposers +// with a bonus for including more than +2/3 of the signatures. +func VerifyCommit(chainID string, vals *ValidatorSet, blockID BlockID, + height int64, commit *Commit) error { + // run a basic validation of the arguments + if err := verifyBasicValsAndCommit(vals, commit, height, blockID); err != nil { + return err } - return nil + + // calculate voting power needed. Note that total voting power is capped to + // 1/8th of max int64 so this operation should never overflow + votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 + + // ignore all absent signatures + ignore := func(c CommitSig) bool { return c.Absent() } + + // only count the signatures that are for the block + count := func(c CommitSig) bool { return c.ForBlock() } + + // attempt to batch verify + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, true, true) + } + + // if verification failed or is not supported then fallback to single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, true, true) +} + +// LIGHT CLIENT VERIFICATION METHODS + +// VerifyCommitLight verifies +2/3 of the set had signed the given commit. +// +// This method is primarily used by the light client and does not check all the +// signatures. +func VerifyCommitLight(chainID string, vals *ValidatorSet, blockID BlockID, + height int64, commit *Commit) error { + // run a basic validation of the arguments + if err := verifyBasicValsAndCommit(vals, commit, height, blockID); err != nil { + return err + } + + // calculate voting power needed + votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 + + // ignore all commit signatures that are not for the block + ignore := func(c CommitSig) bool { return !c.ForBlock() } + + // count all the remaining signatures + count := func(c CommitSig) bool { return true } + + // attempt to batch verify + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, false, true) + } + + // if verification failed or is not supported then fallback to single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, false, true) +} + +// VerifyCommitLightTrusting verifies that trustLevel of the validator set signed +// this commit. +// +// NOTE the given validators do not necessarily correspond to the validator set +// for this commit, but there may be some intersection. +// +// This method is primarily used by the light client and does not check all the +// signatures. +func VerifyCommitLightTrusting(chainID string, vals *ValidatorSet, commit *Commit, trustLevel tmmath.Fraction) error { + // sanity checks + if vals == nil { + return errors.New("nil validator set") + } + if trustLevel.Denominator == 0 { + return errors.New("trustLevel has zero Denominator") + } + if commit == nil { + return errors.New("nil commit") + } + + // safely calculate voting power needed. + totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), int64(trustLevel.Numerator)) + if overflow { + return errors.New("int64 overflow while calculating voting power needed. please provide smaller trustLevel numerator") + } + votingPowerNeeded := totalVotingPowerMulByNumerator / int64(trustLevel.Denominator) + + // ignore all commit signatures that are not for the block + ignore := func(c CommitSig) bool { return !c.ForBlock() } + + // count all the remaining signatures + count := func(c CommitSig) bool { return true } + + // attempt to batch verify commit. As the validator set doesn't necessarily + // correspond with the validator set that signed the block we need to look + // up by address rather than index. + if shouldBatchVerify(vals, commit) { + return verifyCommitBatch(chainID, vals, commit, + votingPowerNeeded, ignore, count, false, false) + } + + // attempt with single verification + return verifyCommitSingle(chainID, vals, commit, votingPowerNeeded, + ignore, count, false, false) } // ValidateHash returns an error if the hash is not empty, but its @@ -38,3 +140,218 @@ func ValidateHash(h []byte) error { } return nil } + +// Batch verification + +// verifyCommitBatch batch verifies commits. This routine is equivalent +// to verifyCommitSingle in behavior, just faster iff every signature in the +// batch is valid. +// +// Note: The caller is responsible for checking to see if this routine is +// usable via `shouldVerifyBatch(vals, commit)`. +func verifyCommitBatch( + chainID string, + vals *ValidatorSet, + commit *Commit, + votingPowerNeeded int64, + ignoreSig func(CommitSig) bool, + countSig func(CommitSig) bool, + countAllSignatures bool, + lookUpByIndex bool, +) error { + var ( + val *Validator + valIdx int32 + seenVals = make(map[int32]int, len(commit.Signatures)) + batchSigIdxs = make([]int, 0, len(commit.Signatures)) + talliedVotingPower int64 + ) + // attempt to create a batch verifier + bv, ok := batch.CreateBatchVerifier(vals.GetProposer().PubKey) + // re-check if batch verification is supported + if !ok || len(commit.Signatures) < batchVerifyThreshold { + // This should *NEVER* happen. + return fmt.Errorf("unsupported signature algorithm or insufficient signatures for batch verification") + } + + for idx, commitSig := range commit.Signatures { + // skip over signatures that should be ignored + if ignoreSig(commitSig) { + continue + } + + // If the vals and commit have a 1-to-1 correspondance we can retrieve + // them by index else we need to retrieve them by address + if lookUpByIndex { + val = vals.Validators[idx] + } else { + valIdx, val = vals.GetByAddress(commitSig.ValidatorAddress) + + // if the signature doesn't belong to anyone in the validator set + // then we just skip over it + if val == nil { + continue + } + + // because we are getting validators by address we need to make sure + // that the same validator doesn't commit twice + if firstIndex, ok := seenVals[valIdx]; ok { + secondIndex := idx + return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) + } + seenVals[valIdx] = idx + } + + // Validate signature. + voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) + + // add the key, sig and message to the verifier + if err := bv.Add(val.PubKey, voteSignBytes, commitSig.Signature); err != nil { + return err + } + batchSigIdxs = append(batchSigIdxs, idx) + + // If this signature counts then add the voting power of the validator + // to the tally + if countSig(commitSig) { + talliedVotingPower += val.VotingPower + } + + // if we don't need to verify all signatures and already have sufficient + // voting power we can break from batching and verify all the signatures + if !countAllSignatures && talliedVotingPower > votingPowerNeeded { + break + } + } + + // ensure that we have batched together enough signatures to exceed the + // voting power needed else there is no need to even verify + if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { + return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} + } + + // attempt to verify the batch. + ok, validSigs := bv.Verify() + if ok { + // success + return nil + } + + // one or more of the signatures is invalid, find and return the first + // invalid signature. + for i, ok := range validSigs { + if !ok { + // go back from the batch index to the commit.Signatures index + idx := batchSigIdxs[i] + sig := commit.Signatures[idx] + return fmt.Errorf("wrong signature (#%d): %X", idx, sig) + } + } + + // execution reaching here is a bug, and one of the following has + // happened: + // * non-zero tallied voting power, empty batch (impossible?) + // * bv.Verify() returned `false, []bool{true, ..., true}` (BUG) + return fmt.Errorf("BUG: batch verification failed with no invalid signatures") +} + +// Single Verification + +// verifyCommitSingle single verifies commits. +// If a key does not support batch verification, or batch verification fails this will be used +// This method is used to check all the signatures included in a commit. +// It is used in consensus for validating a block LastCommit. +// CONTRACT: both commit and validator set should have passed validate basic +func verifyCommitSingle( + chainID string, + vals *ValidatorSet, + commit *Commit, + votingPowerNeeded int64, + ignoreSig func(CommitSig) bool, + countSig func(CommitSig) bool, + countAllSignatures bool, + lookUpByIndex bool, +) error { + var ( + val *Validator + valIdx int32 + seenVals = make(map[int32]int, len(commit.Signatures)) + talliedVotingPower int64 + voteSignBytes []byte + ) + for idx, commitSig := range commit.Signatures { + if ignoreSig(commitSig) { + continue + } + + // If the vals and commit have a 1-to-1 correspondance we can retrieve + // them by index else we need to retrieve them by address + if lookUpByIndex { + val = vals.Validators[idx] + } else { + valIdx, val = vals.GetByAddress(commitSig.ValidatorAddress) + + // if the signature doesn't belong to anyone in the validator set + // then we just skip over it + if val == nil { + continue + } + + // because we are getting validators by address we need to make sure + // that the same validator doesn't commit twice + if firstIndex, ok := seenVals[valIdx]; ok { + secondIndex := idx + return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) + } + seenVals[valIdx] = idx + } + + voteSignBytes = commit.VoteSignBytes(chainID, int32(idx)) + + if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { + return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) + } + + // If this signature counts then add the voting power of the validator + // to the tally + if countSig(commitSig) { + talliedVotingPower += val.VotingPower + } + + // check if we have enough signatures and can thus exit early + if !countAllSignatures && talliedVotingPower > votingPowerNeeded { + return nil + } + } + + if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { + return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} + } + + return nil +} + +func verifyBasicValsAndCommit(vals *ValidatorSet, commit *Commit, height int64, blockID BlockID) error { + if vals == nil { + return errors.New("nil validator set") + } + + if commit == nil { + return errors.New("nil commit") + } + + if vals.Size() != len(commit.Signatures) { + return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) + } + + // Validate Height and BlockID. + if height != commit.Height { + return NewErrInvalidCommitHeight(height, commit.Height) + } + if !blockID.Equals(commit.BlockID) { + return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", + blockID, commit.BlockID) + } + + return nil +} diff --git a/types/validation_test.go b/types/validation_test.go new file mode 100644 index 000000000..d194d680e --- /dev/null +++ b/types/validation_test.go @@ -0,0 +1,261 @@ +package types + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + tmmath "github.com/tendermint/tendermint/libs/math" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" +) + +// Check VerifyCommit, VerifyCommitLight and VerifyCommitLightTrusting basic +// verification. +func TestValidatorSet_VerifyCommit_All(t *testing.T) { + var ( + round = int32(0) + height = int64(100) + + blockID = makeBlockID([]byte("blockhash"), 1000, []byte("partshash")) + chainID = "Lalande21185" + trustLevel = tmmath.Fraction{Numerator: 2, Denominator: 3} + ) + + testCases := []struct { + description string + // vote chainID + chainID string + // vote blockID + blockID BlockID + valSize int + + // height of the commit + height int64 + + // votes + blockVotes int + nilVotes int + absentVotes int + + expErr bool + }{ + {"good (batch verification)", chainID, blockID, 3, height, 3, 0, 0, false}, + {"good (single verification)", chainID, blockID, 1, height, 1, 0, 0, false}, + + {"wrong signature (#0)", "EpsilonEridani", blockID, 2, height, 2, 0, 0, true}, + {"wrong block ID", chainID, makeBlockIDRandom(), 2, height, 2, 0, 0, true}, + {"wrong height", chainID, blockID, 1, height - 1, 1, 0, 0, true}, + + {"wrong set size: 4 vs 3", chainID, blockID, 4, height, 3, 0, 0, true}, + {"wrong set size: 1 vs 2", chainID, blockID, 1, height, 2, 0, 0, true}, + + {"insufficient voting power: got 30, needed more than 66", chainID, blockID, 10, height, 3, 2, 5, true}, + {"insufficient voting power: got 0, needed more than 6", chainID, blockID, 1, height, 0, 0, 1, true}, + {"insufficient voting power: got 60, needed more than 60", chainID, blockID, 9, height, 6, 3, 0, true}, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.description, func(t *testing.T) { + _, valSet, vals := randVoteSet(tc.height, round, tmproto.PrecommitType, tc.valSize, 10) + totalVotes := tc.blockVotes + tc.absentVotes + tc.nilVotes + sigs := make([]CommitSig, totalVotes) + vi := 0 + // add absent sigs first + for i := 0; i < tc.absentVotes; i++ { + sigs[vi] = NewCommitSigAbsent() + vi++ + } + for i := 0; i < tc.blockVotes+tc.nilVotes; i++ { + + pubKey, err := vals[vi%len(vals)].GetPubKey() + require.NoError(t, err) + vote := &Vote{ + ValidatorAddress: pubKey.Address(), + ValidatorIndex: int32(vi), + Height: tc.height, + Round: round, + Type: tmproto.PrecommitType, + BlockID: tc.blockID, + Timestamp: time.Now(), + } + if i >= tc.blockVotes { + vote.BlockID = BlockID{} + } + + v := vote.ToProto() + + require.NoError(t, vals[vi%len(vals)].SignVote(tc.chainID, v)) + vote.Signature = v.Signature + + sigs[vi] = vote.CommitSig() + + vi++ + } + commit := NewCommit(tc.height, round, tc.blockID, sigs) + + err := valSet.VerifyCommit(chainID, blockID, height, commit) + if tc.expErr { + if assert.Error(t, err, "VerifyCommit") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommit") + } + } else { + assert.NoError(t, err, "VerifyCommit") + } + + err = valSet.VerifyCommitLight(chainID, blockID, height, commit) + if tc.expErr { + if assert.Error(t, err, "VerifyCommitLight") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommitLight") + } + } else { + assert.NoError(t, err, "VerifyCommitLight") + } + + // only a subsection of the tests apply to VerifyCommitLightTrusting + if totalVotes != tc.valSize || !tc.blockID.Equals(blockID) || tc.height != height { + tc.expErr = false + } + err = valSet.VerifyCommitLightTrusting(chainID, commit, trustLevel) + if tc.expErr { + if assert.Error(t, err, "VerifyCommitLightTrusting") { + assert.Contains(t, err.Error(), tc.description, "VerifyCommitLightTrusting") + } + } else { + assert.NoError(t, err, "VerifyCommitLightTrusting") + } + }) + } +} + +func TestValidatorSet_VerifyCommit_CheckAllSignatures(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 4th signature + vote := voteSet.GetByIndex(3) + v := vote.ToProto() + err = vals[3].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[3] = vote.CommitSig() + + err = valSet.VerifyCommit(chainID, blockID, h, commit) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "wrong signature (#3)") + } +} + +func TestValidatorSet_VerifyCommitLight_ReturnsAsSoonAsMajorityOfVotingPowerSigned(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 4th signature (3 signatures are enough for 2/3+) + vote := voteSet.GetByIndex(3) + v := vote.ToProto() + err = vals[3].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[3] = vote.CommitSig() + + err = valSet.VerifyCommitLight(chainID, blockID, h, commit) + assert.NoError(t, err) +} + +func TestValidatorSet_VerifyCommitLightTrusting_ReturnsAsSoonAsTrustLevelOfVotingPowerSigned(t *testing.T) { + var ( + chainID = "test_chain_id" + h = int64(3) + blockID = makeBlockIDRandom() + ) + + voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) + commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + require.NoError(t, valSet.VerifyCommit(chainID, blockID, h, commit)) + + // malleate 3rd signature (2 signatures are enough for 1/3+ trust level) + vote := voteSet.GetByIndex(2) + v := vote.ToProto() + err = vals[2].SignVote("CentaurusA", v) + require.NoError(t, err) + vote.Signature = v.Signature + commit.Signatures[2] = vote.CommitSig() + + err = valSet.VerifyCommitLightTrusting(chainID, commit, tmmath.Fraction{Numerator: 1, Denominator: 3}) + assert.NoError(t, err) +} + +func TestValidatorSet_VerifyCommitLightTrusting(t *testing.T) { + var ( + blockID = makeBlockIDRandom() + voteSet, originalValset, vals = randVoteSet(1, 1, tmproto.PrecommitType, 6, 1) + commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) + newValSet, _ = RandValidatorSet(2, 1) + ) + require.NoError(t, err) + + testCases := []struct { + valSet *ValidatorSet + err bool + }{ + // good + 0: { + valSet: originalValset, + err: false, + }, + // bad - no overlap between validator sets + 1: { + valSet: newValSet, + err: true, + }, + // good - first two are different but the rest of the same -> >1/3 + 2: { + valSet: NewValidatorSet(append(newValSet.Validators, originalValset.Validators...)), + err: false, + }, + } + + for _, tc := range testCases { + err = tc.valSet.VerifyCommitLightTrusting("test_chain_id", commit, + tmmath.Fraction{Numerator: 1, Denominator: 3}) + if tc.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + } +} + +func TestValidatorSet_VerifyCommitLightTrustingErrorsOnOverflow(t *testing.T) { + var ( + blockID = makeBlockIDRandom() + voteSet, valSet, vals = randVoteSet(1, 1, tmproto.PrecommitType, 1, MaxTotalVotingPower) + commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) + ) + require.NoError(t, err) + + err = valSet.VerifyCommitLightTrusting("test_chain_id", commit, + tmmath.Fraction{Numerator: 25, Denominator: 55}) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "int64 overflow") + } +} diff --git a/types/validator_set.go b/types/validator_set.go index 39a004b0b..04232973b 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -657,172 +657,25 @@ func (vals *ValidatorSet) UpdateWithChangeSet(changes []*Validator) error { return vals.updateWithChangeSet(changes, true) } -// VerifyCommit verifies +2/3 of the set had signed the given commit. -// -// It checks all the signatures! While it's safe to exit as soon as we have -// 2/3+ signatures, doing so would impact incentivization logic in the ABCI -// application that depends on the LastCommitInfo sent in BeginBlock, which -// includes which validators signed. For instance, Gaia incentivizes proposers -// with a bonus for including more than +2/3 of the signatures. +// VerifyCommit verifies +2/3 of the set had signed the given commit and all +// other signatures are valid func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height int64, commit *Commit) error { - - if vals.Size() != len(commit.Signatures) { - return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) - } - - // Validate Height and BlockID. - if height != commit.Height { - return NewErrInvalidCommitHeight(height, commit.Height) - } - if !blockID.Equals(commit.BlockID) { - return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", - blockID, commit.BlockID) - } - - talliedVotingPower := int64(0) - votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 - for idx, commitSig := range commit.Signatures { - if commitSig.Absent() { - continue // OK, some signatures can be absent. - } - - // The vals and commit have a 1-to-1 correspondance. - // This means we don't need the validator address or to do any lookup. - val := vals.Validators[idx] - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - // Good! - if commitSig.ForBlock() { - talliedVotingPower += val.VotingPower - } - // else { - // It's OK. We include stray signatures (~votes for nil) to measure - // validator availability. - // } - } - - if got, needed := talliedVotingPower, votingPowerNeeded; got <= needed { - return ErrNotEnoughVotingPowerSigned{Got: got, Needed: needed} - } - - return nil + return VerifyCommit(chainID, vals, blockID, height, commit) } // LIGHT CLIENT VERIFICATION METHODS // VerifyCommitLight verifies +2/3 of the set had signed the given commit. -// -// This method is primarily used by the light client and does not check all the -// signatures. func (vals *ValidatorSet) VerifyCommitLight(chainID string, blockID BlockID, height int64, commit *Commit) error { - - if vals.Size() != len(commit.Signatures) { - return NewErrInvalidCommitSignatures(vals.Size(), len(commit.Signatures)) - } - - // Validate Height and BlockID. - if height != commit.Height { - return NewErrInvalidCommitHeight(height, commit.Height) - } - if !blockID.Equals(commit.BlockID) { - return fmt.Errorf("invalid commit -- wrong block ID: want %v, got %v", - blockID, commit.BlockID) - } - - talliedVotingPower := int64(0) - votingPowerNeeded := vals.TotalVotingPower() * 2 / 3 - for idx, commitSig := range commit.Signatures { - // No need to verify absent or nil votes. - if !commitSig.ForBlock() { - continue - } - - // The vals and commit have a 1-to-1 correspondance. - // This means we don't need the validator address or to do any lookup. - val := vals.Validators[idx] - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - - talliedVotingPower += val.VotingPower - - // return as soon as +2/3 of the signatures are verified - if talliedVotingPower > votingPowerNeeded { - return nil - } - } - - return ErrNotEnoughVotingPowerSigned{Got: talliedVotingPower, Needed: votingPowerNeeded} + return VerifyCommitLight(chainID, vals, blockID, height, commit) } // VerifyCommitLightTrusting verifies that trustLevel of the validator set signed // this commit. -// -// NOTE the given validators do not necessarily correspond to the validator set -// for this commit, but there may be some intersection. -// -// This method is primarily used by the light client and does not check all the -// signatures. func (vals *ValidatorSet) VerifyCommitLightTrusting(chainID string, commit *Commit, trustLevel tmmath.Fraction) error { - // sanity check - if trustLevel.Denominator == 0 { - return errors.New("trustLevel has zero Denominator") - } - - var ( - talliedVotingPower int64 - seenVals = make(map[int32]int, len(commit.Signatures)) // validator index -> commit index - ) - - // Safely calculate voting power needed. - totalVotingPowerMulByNumerator, overflow := safeMul(vals.TotalVotingPower(), int64(trustLevel.Numerator)) - if overflow { - return errors.New("int64 overflow while calculating voting power needed. please provide smaller trustLevel numerator") - } - votingPowerNeeded := totalVotingPowerMulByNumerator / int64(trustLevel.Denominator) - - for idx, commitSig := range commit.Signatures { - // No need to verify absent or nil votes. - if !commitSig.ForBlock() { - continue - } - - // We don't know the validators that committed this block, so we have to - // check for each vote if its validator is already known. - valIdx, val := vals.GetByAddress(commitSig.ValidatorAddress) - - if val != nil { - // check for double vote of validator on the same commit - if firstIndex, ok := seenVals[valIdx]; ok { - secondIndex := idx - return fmt.Errorf("double vote from %v (%d and %d)", val, firstIndex, secondIndex) - } - seenVals[valIdx] = idx - - // Validate signature. - voteSignBytes := commit.VoteSignBytes(chainID, int32(idx)) - if !val.PubKey.VerifySignature(voteSignBytes, commitSig.Signature) { - return fmt.Errorf("wrong signature (#%d): %X", idx, commitSig.Signature) - } - - talliedVotingPower += val.VotingPower - - if talliedVotingPower > votingPowerNeeded { - return nil - } - } - } - - return ErrNotEnoughVotingPowerSigned{Got: talliedVotingPower, Needed: votingPowerNeeded} + return VerifyCommitLightTrusting(chainID, vals, commit, trustLevel) } // findPreviousProposer reverses the compare proposer priority function to find the validator diff --git a/types/validator_set_test.go b/types/validator_set_test.go index 6fbbb0885..6973fc80b 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -665,155 +664,6 @@ func TestSafeSubClip(t *testing.T) { //------------------------------------------------------------------- -// Check VerifyCommit, VerifyCommitLight and VerifyCommitLightTrusting basic -// verification. -func TestValidatorSet_VerifyCommit_All(t *testing.T) { - var ( - privKey = ed25519.GenPrivKey() - pubKey = privKey.PubKey() - v1 = NewValidator(pubKey, 1000) - vset = NewValidatorSet([]*Validator{v1}) - - chainID = "Lalande21185" - ) - - vote := examplePrecommit() - vote.ValidatorAddress = pubKey.Address() - v := vote.ToProto() - sig, err := privKey.Sign(VoteSignBytes(chainID, v)) - require.NoError(t, err) - vote.Signature = sig - - commit := NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{vote.CommitSig()}) - - vote2 := *vote - sig2, err := privKey.Sign(VoteSignBytes("EpsilonEridani", v)) - require.NoError(t, err) - vote2.Signature = sig2 - - testCases := []struct { - description string - chainID string - blockID BlockID - height int64 - commit *Commit - expErr bool - }{ - {"good", chainID, vote.BlockID, vote.Height, commit, false}, - - {"wrong signature (#0)", "EpsilonEridani", vote.BlockID, vote.Height, commit, true}, - {"wrong block ID", chainID, makeBlockIDRandom(), vote.Height, commit, true}, - {"wrong height", chainID, vote.BlockID, vote.Height - 1, commit, true}, - - {"wrong set size: 1 vs 0", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{}), true}, - - {"wrong set size: 1 vs 2", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, - []CommitSig{vote.CommitSig(), {BlockIDFlag: BlockIDFlagAbsent}}), true}, - - {"insufficient voting power: got 0, needed more than 666", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{{BlockIDFlag: BlockIDFlagAbsent}}), true}, - - {"wrong signature (#0)", chainID, vote.BlockID, vote.Height, - NewCommit(vote.Height, vote.Round, vote.BlockID, []CommitSig{vote2.CommitSig()}), true}, - } - - for _, tc := range testCases { - tc := tc - t.Run(tc.description, func(t *testing.T) { - err := vset.VerifyCommit(tc.chainID, tc.blockID, tc.height, tc.commit) - if tc.expErr { - if assert.Error(t, err, "VerifyCommit") { - assert.Contains(t, err.Error(), tc.description, "VerifyCommit") - } - } else { - assert.NoError(t, err, "VerifyCommit") - } - - err = vset.VerifyCommitLight(tc.chainID, tc.blockID, tc.height, tc.commit) - if tc.expErr { - if assert.Error(t, err, "VerifyCommitLight") { - assert.Contains(t, err.Error(), tc.description, "VerifyCommitLight") - } - } else { - assert.NoError(t, err, "VerifyCommitLight") - } - }) - } -} - -func TestValidatorSet_VerifyCommit_CheckAllSignatures(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 4th signature - vote := voteSet.GetByIndex(3) - v := vote.ToProto() - err = vals[3].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[3] = vote.CommitSig() - - err = valSet.VerifyCommit(chainID, blockID, h, commit) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "wrong signature (#3)") - } -} - -func TestValidatorSet_VerifyCommitLight_ReturnsAsSoonAsMajorityOfVotingPowerSigned(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 4th signature (3 signatures are enough for 2/3+) - vote := voteSet.GetByIndex(3) - v := vote.ToProto() - err = vals[3].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[3] = vote.CommitSig() - - err = valSet.VerifyCommitLight(chainID, blockID, h, commit) - assert.NoError(t, err) -} - -func TestValidatorSet_VerifyCommitLightTrusting_ReturnsAsSoonAsTrustLevelOfVotingPowerSigned(t *testing.T) { - var ( - chainID = "test_chain_id" - h = int64(3) - blockID = makeBlockIDRandom() - ) - - voteSet, valSet, vals := randVoteSet(h, 0, tmproto.PrecommitType, 4, 10) - commit, err := MakeCommit(blockID, h, 0, voteSet, vals, time.Now()) - require.NoError(t, err) - - // malleate 3rd signature (2 signatures are enough for 1/3+ trust level) - vote := voteSet.GetByIndex(2) - v := vote.ToProto() - err = vals[2].SignVote("CentaurusA", v) - require.NoError(t, err) - vote.Signature = v.Signature - commit.Signatures[2] = vote.CommitSig() - - err = valSet.VerifyCommitLightTrusting(chainID, commit, tmmath.Fraction{Numerator: 1, Denominator: 3}) - assert.NoError(t, err) -} - func TestEmptySet(t *testing.T) { var valList []*Validator @@ -1517,62 +1367,6 @@ func TestValSetUpdateOverflowRelated(t *testing.T) { } } -func TestValidatorSet_VerifyCommitLightTrusting(t *testing.T) { - var ( - blockID = makeBlockIDRandom() - voteSet, originalValset, vals = randVoteSet(1, 1, tmproto.PrecommitType, 6, 1) - commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) - newValSet, _ = RandValidatorSet(2, 1) - ) - require.NoError(t, err) - - testCases := []struct { - valSet *ValidatorSet - err bool - }{ - // good - 0: { - valSet: originalValset, - err: false, - }, - // bad - no overlap between validator sets - 1: { - valSet: newValSet, - err: true, - }, - // good - first two are different but the rest of the same -> >1/3 - 2: { - valSet: NewValidatorSet(append(newValSet.Validators, originalValset.Validators...)), - err: false, - }, - } - - for _, tc := range testCases { - err = tc.valSet.VerifyCommitLightTrusting("test_chain_id", commit, - tmmath.Fraction{Numerator: 1, Denominator: 3}) - if tc.err { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - } -} - -func TestValidatorSet_VerifyCommitLightTrustingErrorsOnOverflow(t *testing.T) { - var ( - blockID = makeBlockIDRandom() - voteSet, valSet, vals = randVoteSet(1, 1, tmproto.PrecommitType, 1, MaxTotalVotingPower) - commit, err = MakeCommit(blockID, 1, 1, voteSet, vals, time.Now()) - ) - require.NoError(t, err) - - err = valSet.VerifyCommitLightTrusting("test_chain_id", commit, - tmmath.Fraction{Numerator: 25, Denominator: 55}) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "int64 overflow") - } -} - func TestSafeMul(t *testing.T) { testCases := []struct { a int64 diff --git a/version/version.go b/version/version.go index 951dd0bdf..46fb32a97 100644 --- a/version/version.go +++ b/version/version.go @@ -1,18 +1,12 @@ package version -var TMCoreSemVer = TMVersionDefault - const ( // TMVersionDefault is the used as the fallback version of Tendermint Core // when not using git describe. It is formatted with semantic versioning. - TMVersionDefault = "v0.38.0-dev" + TMCoreSemVer = "0.38.0-dev" // ABCISemVer is the semantic version of the ABCI protocol - ABCISemVer = "1.0.0" - + ABCISemVer = "1.0.0" ABCIVersion = ABCISemVer -) - -var ( // P2PProtocol versions all p2p behavior and msgs. // This includes proposer selection. P2PProtocol uint64 = 8 @@ -21,3 +15,9 @@ var ( // This includes validity of blocks and state updates. BlockProtocol uint64 = 11 ) + +var ( + // TMGitCommitHash uses git rev-parse HEAD to find commit hash which is helpful + // for the engineering team when working with the tendermint binary. See Makefile + TMGitCommitHash = "" +)